diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index fcd5cb96..2466789f 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -42,3 +42,7 @@ RUN apt update \ git-lfs \ && rm -rf /var/lib/apt/lists/* ENV CMAKE_GENERATOR=Ninja + +# Set the default user. +# (See https://code.visualstudio.com/remote/advancedcontainers/add-nonroot-user) +USER $USERNAME diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6a0a1fbc..06f9e1e8 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,15 +2,8 @@ // https://code.visualstudio.com/docs/remote/devcontainerjson-reference { "name": "C++", - // Dockerfile in this VSCode DevContainer uses a cache image named `holoscan-sdk-dev` to - // speed up the build process. To rebuild the cache image before container creation, it runs: - // - // docker buildx use default # use the default builder to access all the cache images - // ./run build_image` - // - // as an initialization command. - // It also runs `./run install_gxf` to install the GXF library. - "initializeCommand": "docker buildx use default; ${localWorkspaceFolder}/run build_image; ${localWorkspaceFolder}/run install_gxf", + // Use 'initialize-command.sh' to execute initialization commands before the container is created. + "initializeCommand": ".devcontainer/initialize-command.sh", "build": { "dockerfile": "Dockerfile", "args": { @@ -22,6 +15,10 @@ "runArgs": [ "--runtime=nvidia", "--net=host", + // Current VSCode DevContainer doesn't support dynamic 'runArgs' for docker + // (see https://github.com/microsoft/vscode-remote-release/issues/3972). + // So, we need to comment out the following lines when we don't use AJA Capture Card or video device. + // // Uncomment the following line to use AJA Capture Card // "--device=/dev/ajantv20:/dev/ajantv20", // Uncomment the following line to use /dev/video0. Also, execute 'sudo chmod 666 /dev/video0' inside the container. @@ -32,14 +29,19 @@ "containerEnv": { "DISPLAY": "${localEnv:DISPLAY}", "NVIDIA_DRIVER_CAPABILITIES": "graphics,video,compute,utility,display", + // Set the following environment variables to use the same folder name as the host machine. + // This is needed to launch container from the workspace folder that is not same as the SDK source root folder. + "HOLOSCAN_PUBLIC_FOLDER": "${env:HOLOSCAN_PUBLIC_FOLDER}", }, "mounts": [ "source=/tmp/.X11-unix,target=/tmp/.X11-unix,type=bind,consistency=cached", - // Mount Vulkan driver ICD configuration (need to map two paths, the configurations files are installed to different locations - // when installing with deb packages or with run files) + // Mount Vulkan driver ICD configuration. // Needed due to https://github.com/NVIDIA/nvidia-container-toolkit/issues/16 - "source=/usr/share/vulkan/icd.d/nvidia_icd.json,target=/usr/share/vulkan/icd.d/nvidia_icd.json,type=bind,consistency=cached", - "source=/etc/vulkan/icd.d/nvidia_icd.json,target=/etc/vulkan/icd.d/nvidia_icd.json,type=bind,consistency=cached", + // The configurations files are installed to different locations depending on the installation method. + // (whether installing with deb packages or with run files) + // By the 'initializeCommand' script, the ICD configurations are copied to /tmp/holoscan_nvidia_icd.json and + // we mount it to /usr/share/vulkan/icd.d/nvidia_icd.json + "source=/tmp/holoscan_nvidia_icd.json,target=/usr/share/vulkan/icd.d/nvidia_icd.json,type=bind,consistency=cached", ], "workspaceMount": "source=${localWorkspaceFolder},target=/workspace/holoscan-sdk,type=bind,consistency=cached", "workspaceFolder": "/workspace/holoscan-sdk", diff --git a/.devcontainer/initialize-command.sh b/.devcontainer/initialize-command.sh new file mode 100755 index 00000000..4b786928 --- /dev/null +++ b/.devcontainer/initialize-command.sh @@ -0,0 +1,72 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_DIR=$(dirname "$(readlink -f "$0")") + +# Actually, VSCode will run this script in the local workspace folder but +# it's better to be explicit. + +# Get 'localWorkspaceFolder' environment variable from the script path. +localWorkspaceFolder=$(git rev-parse --show-toplevel 2> /dev/null || dirname $(dirname $(realpath -s $0))) + +# Get the holoscan sdk top directory. +TOP=$(readlink -f "${SCRIPT_DIR}/..") + +if [ "${localWorkspaceFolder}" != "${TOP}" ]; then + echo "Project (git) root is not Holoscan SDK source directory. Copying common-debian.sh from the source folder." + # In this case, project root is not Holoscan source root. + # If a file is symlinked, the symlinked file can't be copied to the container by Dockerfile. + # To prevent error, Copy common-debian.sh from the Holoscan source's .devcontainer folder to + # the project repository's .devcontainer folder. + cp -f ${TOP}/.devcontainer/library-scripts/common-debian.sh \ + ${localWorkspaceFolder}/.devcontainer/library-scripts/common-debian.sh +fi + +# We need to mount Vulkan icd.d JSON file into the container due to this issue +# (https://github.com/NVIDIA/nvidia-container-toolkit/issues/16). +# However, the location of the file is different depending on the driver installation method: +# - deb packages: /usr/share/vulkan/icd.d/nvidia_icd.json +# - .run files: /etc/vulkan/icd.d/nvidia_icd.json +# So we need to check which one exists and mount it. + +# Here, we copy the existing icd.d JSON file to the temporary directory (/tmp) and mount it to the +# location (/usr/share/vulkan/icd.d/nvidia_icd.json) that the container expects. +# It is because VSCode DevContainer doesn't support conditional mount points. +# (see https://github.com/microsoft/vscode-remote-release/issues/3972) +if [ -f /usr/share/vulkan/icd.d/nvidia_icd.json ]; then + icd_file=/usr/share/vulkan/icd.d/nvidia_icd.json +elif [ -f /etc/vulkan/icd.d/nvidia_icd.json ]; then + icd_file=/etc/vulkan/icd.d/nvidia_icd.json +else + >&2 echo "ERROR: Cannot find the Vulkan ICD file from /usr/share/vulkan/icd.d/nvidia_icd.json or /etc/vulkan/icd.d/nvidia_icd.json" + exit 1 +fi + +# Copy the file to the temporary directory with the name 'holoscan_nvidia_icd.json'. +cp ${icd_file} /tmp/holoscan_nvidia_icd.json +echo "Mounting ${icd_file} to /usr/share/vulkan/icd.d/nvidia_icd.json through /tmp/holoscan_nvidia_icd.json" + +# Dockerfile in this VSCode DevContainer uses a cache image named `holoscan-sdk-dev` to +# speed up the build process. To rebuild the cache image before container creation, it runs: +# +# docker buildx use default # use the default builder to access all the cache images +# ./run build_image` +# +# as an initialization command. +# It also runs `./run install_gxf` to install the GXF library. +docker buildx use default +${TOP}/run build_image +${TOP}/run install_gxf diff --git a/.vscode/c_cpp_properties.json b/.vscode/c_cpp_properties.json index 567dd1c5..48359173 100644 --- a/.vscode/c_cpp_properties.json +++ b/.vscode/c_cpp_properties.json @@ -4,14 +4,14 @@ { "name": "x86_64", "includePath": [ - "${workspaceFolder}/include" + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/include" ], "defines": [], "compilerPath": "/usr/bin/gcc", "cStandard": "gnu11", "cppStandard": "gnu++17", "intelliSenseMode": "linux-gcc-x64", - // "compileCommands": "${workspaceFolder}/build-debug-x86_64/compile_commands.json", + // "compileCommands": "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/build-debug-x86_64/compile_commands.json", "compileCommands": "${command:cmake.buildDirectory}/compile_commands.json", "browse": { "limitSymbolsToIncludedHeaders": true @@ -21,14 +21,14 @@ { "name": "arm64", "includePath": [ - "${workspaceFolder}/include" + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/include" ], "defines": [], "compilerPath": "/usr/bin/gcc", "cStandard": "gnu11", "cppStandard": "gnu++17", "intelliSenseMode": "linux-gcc-arm64", - // "compileCommands": "${workspaceFolder}/build-debug-arm64/compile_commands.json", + // "compileCommands": "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/build-debug-arm64/compile_commands.json", "compileCommands": "${command:cmake.buildDirectory}/compile_commands.json", "browse": { "limitSymbolsToIncludedHeaders": true diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 8844f9ba..9584b9e6 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -7,7 +7,6 @@ "akiramiyakoda.cppincludeguard", "ms-vscode.cpptools-extension-pack", "matepek.vscode-catch2-test-adapter", - "ms-vscode.makefile-tools", "ms-python.python", "ms-python.vscode-pylance", "shardulm94.trailing-spaces", diff --git a/.vscode/launch.json b/.vscode/launch.json index 527af9ad..33a2e6e0 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -6,21 +6,14 @@ // https://code.visualstudio.com/docs/editor/variables-reference "configurations": [ { - "name": "(gdb) apps/endoscopy_tool_tracking/cpp", + "name": "(gdb) examples/aja_capture/cpp", "type": "cppdbg", "request": "launch", - "program": "${command:cmake.buildDirectory}/apps/endoscopy_tool_tracking/cpp/endoscopy_tool_tracking", - "args": [ - "-severity", - "4" - ], + "program": "${command:cmake.buildDirectory}/examples/aja_capture/cpp/aja_capture", + "args": [], "stopAtEntry": false, "cwd": "${command:cmake.buildDirectory}", "environment": [ - { - "name": "LD_LIBRARY_PATH", - "value": "${command:cmake.buildDirectory}/lib:${env:LD_LIBRARY_PATH}" - }, { "name": "HOLOSCAN_LOG_LEVEL", "value": "TRACE" @@ -37,33 +30,52 @@ ] }, { - "name": "(gdb) apps/endoscopy_tool_tracking/python", + "name": "(gdb) examples/aja_capture/python", "type": "cppdbg", "request": "launch", "program": "/usr/bin/bash", // https://github.com/catchorg/Catch2/blob/devel/docs/command-line.md#specifying-which-tests-to-run "args": [ - "${workspaceFolder}/scripts/debug_python", - "${command:cmake.buildDirectory}/apps/endoscopy_tool_tracking/python/endoscopy_tool_tracking.py", + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/scripts/debug_python", + "${command:cmake.buildDirectory}/examples/aja_capture/python/aja_capture.py", ], "stopAtEntry": false, - "cwd": "${command:cmake.buildDirectory}/python/lib", + "cwd": "${command:cmake.buildDirectory}", "environment": [ - { - "name": "LD_LIBRARY_PATH", - "value": "${command:cmake.buildDirectory}/lib:${command:cmake.buildDirectory}/lib/gxf_extensions:${env:LD_LIBRARY_PATH}" - }, { "name": "HOLOSCAN_LOG_LEVEL", "value": "TRACE" }, { - "name": "HOLOSCAN_SAMPLE_DATA_PATH", - "value": "${workspaceFolder}/data" + "name": "PYTHONPATH", + "value": "${command:cmake.buildDirectory}/python/lib" }, + ], + "console": "externalTerminal", + "MIMode": "gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] + }, + { + "name": "(gdb) examples/numpy_native", + "type": "cppdbg", + "request": "launch", + "program": "/usr/bin/bash", + "args": [ + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/scripts/debug_python", + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/examples/numpy_native/convolve.py", + ], + "stopAtEntry": false, + "cwd": "${command:cmake.buildDirectory}", + "environment": [ { - "name": "HOLOSCAN_LIB_PATH", - "value": "${command:cmake.buildDirectory}/lib" + "name": "HOLOSCAN_LOG_LEVEL", + "value": "TRACE" }, { "name": "PYTHONPATH", @@ -81,21 +93,45 @@ ] }, { - "name": "(gdb) apps/ultrasound_segmentation/cpp", + "name": "(gdb) examples/cupy_native", "type": "cppdbg", "request": "launch", - "program": "${command:cmake.buildDirectory}/apps/ultrasound_segmentation/cpp/ultrasound_segmentation", + "program": "/usr/bin/bash", "args": [ - "-severity", - "4" + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/scripts/debug_python", + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/examples/cupy_native/matmul.py", ], "stopAtEntry": false, "cwd": "${command:cmake.buildDirectory}", "environment": [ { - "name": "LD_LIBRARY_PATH", - "value": "${command:cmake.buildDirectory}/lib:${env:LD_LIBRARY_PATH}" + "name": "HOLOSCAN_LOG_LEVEL", + "value": "TRACE" }, + { + "name": "PYTHONPATH", + "value": "${command:cmake.buildDirectory}/python/lib" + }, + ], + "console": "externalTerminal", + "MIMode": "gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] + }, + { + "name": "(gdb) examples/hello_world/cpp", + "type": "cppdbg", + "request": "launch", + "program": "${command:cmake.buildDirectory}/examples/hello_world/cpp/hello_world", + "args": [], + "stopAtEntry": false, + "cwd": "${command:cmake.buildDirectory}", + "environment": [ { "name": "HOLOSCAN_LOG_LEVEL", "value": "TRACE" @@ -112,34 +148,21 @@ ] }, { - "name": "(gdb) apps/ultrasound_segmentation/python", + "name": "(gdb) examples/hello_world/python", "type": "cppdbg", "request": "launch", "program": "/usr/bin/bash", - // https://github.com/catchorg/Catch2/blob/devel/docs/command-line.md#specifying-which-tests-to-run "args": [ - "${workspaceFolder}/scripts/debug_python", - "${command:cmake.buildDirectory}/apps/ultrasound_segmentation/python/ultrasound_segmentation.py", + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/scripts/debug_python", + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/examples/hello_world/python/hello_world.py", ], "stopAtEntry": false, - "cwd": "${command:cmake.buildDirectory}/python/lib", + "cwd": "${command:cmake.buildDirectory}", "environment": [ - { - "name": "LD_LIBRARY_PATH", - "value": "${command:cmake.buildDirectory}/lib:${command:cmake.buildDirectory}/lib/gxf_extensions:${env:LD_LIBRARY_PATH}" - }, { "name": "HOLOSCAN_LOG_LEVEL", "value": "TRACE" }, - { - "name": "HOLOSCAN_SAMPLE_DATA_PATH", - "value": "${workspaceFolder}/data" - }, - { - "name": "HOLOSCAN_LIB_PATH", - "value": "${command:cmake.buildDirectory}/lib" - }, { "name": "PYTHONPATH", "value": "${command:cmake.buildDirectory}/python/lib" @@ -156,21 +179,14 @@ ] }, { - "name": "(gdb) apps/multiai/cpp", + "name": "(gdb) examples/ping_simple/cpp", "type": "cppdbg", "request": "launch", - "program": "${command:cmake.buildDirectory}/apps/multiai/cpp/multiai", - "args": [ - "-severity", - "4" - ], + "program": "${command:cmake.buildDirectory}/examples/ping_simple/cpp/ping_simple", + "args": [], "stopAtEntry": false, "cwd": "${command:cmake.buildDirectory}", "environment": [ - { - "name": "LD_LIBRARY_PATH", - "value": "${command:cmake.buildDirectory}/lib:${env:LD_LIBRARY_PATH}" - }, { "name": "HOLOSCAN_LOG_LEVEL", "value": "TRACE" @@ -187,34 +203,21 @@ ] }, { - "name": "(gdb) apps/multiai/python", + "name": "(gdb) examples/ping_simple/python", "type": "cppdbg", "request": "launch", "program": "/usr/bin/bash", - // https://github.com/catchorg/Catch2/blob/devel/docs/command-line.md#specifying-which-tests-to-run "args": [ - "${workspaceFolder}/scripts/debug_python", - "${command:cmake.buildDirectory}/apps/multiai/python/multiai.py", + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/scripts/debug_python", + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/examples/ping_simple/python/ping_simple.py", ], "stopAtEntry": false, - "cwd": "${command:cmake.buildDirectory}/python/lib", + "cwd": "${command:cmake.buildDirectory}", "environment": [ - { - "name": "LD_LIBRARY_PATH", - "value": "${command:cmake.buildDirectory}/lib:${command:cmake.buildDirectory}/lib/gxf_extensions:${env:LD_LIBRARY_PATH}" - }, { "name": "HOLOSCAN_LOG_LEVEL", "value": "TRACE" }, - { - "name": "HOLOSCAN_SAMPLE_DATA_PATH", - "value": "${workspaceFolder}/data" - }, - { - "name": "HOLOSCAN_LIB_PATH", - "value": "${command:cmake.buildDirectory}/lib" - }, { "name": "PYTHONPATH", "value": "${command:cmake.buildDirectory}/python/lib" @@ -231,21 +234,14 @@ ] }, { - "name": "(gdb) examples/basic_workflow/cpp", + "name": "(gdb) examples/ping_custom_op/cpp", "type": "cppdbg", "request": "launch", - "program": "${command:cmake.buildDirectory}/examples/basic_workflow/cpp/basic_workflow", - "args": [ - "-severity", - "4" - ], + "program": "${command:cmake.buildDirectory}/examples/ping_custom_op/cpp/ping_custom_op", + "args": [], "stopAtEntry": false, "cwd": "${command:cmake.buildDirectory}", "environment": [ - { - "name": "LD_LIBRARY_PATH", - "value": "${command:cmake.buildDirectory}/lib:${env:LD_LIBRARY_PATH}" - }, { "name": "HOLOSCAN_LOG_LEVEL", "value": "TRACE" @@ -262,29 +258,21 @@ ] }, { - "name": "(gdb) examples/basic_workflow/python", + "name": "(gdb) examples/ping_custom_op/python", "type": "cppdbg", "request": "launch", "program": "/usr/bin/bash", "args": [ - "${workspaceFolder}/scripts/debug_python", - "${workspaceFolder}/examples/basic_workflow/python/basic_workflow.py", + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/scripts/debug_python", + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/examples/ping_custom_op/python/ping_custom_op.py", ], "stopAtEntry": false, - "cwd": "${command:cmake.buildDirectory}/python/lib", + "cwd": "${command:cmake.buildDirectory}", "environment": [ - { - "name": "LD_LIBRARY_PATH", - "value": "${command:cmake.buildDirectory}/lib:${env:LD_LIBRARY_PATH}" - }, { "name": "HOLOSCAN_LOG_LEVEL", "value": "TRACE" }, - { - "name": "HOLOSCAN_LIB_PATH", - "value": "${command:cmake.buildDirectory}/lib" - }, { "name": "PYTHONPATH", "value": "${command:cmake.buildDirectory}/python/lib" @@ -301,21 +289,14 @@ ] }, { - "name": "(gdb) examples/native_operator/cpp", + "name": "(gdb) examples/ping_multi_port/cpp", "type": "cppdbg", "request": "launch", - "program": "${command:cmake.buildDirectory}/examples/native_operator/cpp/ping", - "args": [ - "-severity", - "4" - ], + "program": "${command:cmake.buildDirectory}/examples/ping_multi_port/cpp/ping_multi_port", + "args": [], "stopAtEntry": false, "cwd": "${command:cmake.buildDirectory}", "environment": [ - { - "name": "LD_LIBRARY_PATH", - "value": "${command:cmake.buildDirectory}/lib:${env:LD_LIBRARY_PATH}" - }, { "name": "HOLOSCAN_LOG_LEVEL", "value": "TRACE" @@ -332,29 +313,21 @@ ] }, { - "name": "(gdb) examples/native_operator/python: ping", + "name": "(gdb) examples/ping_multi_port/python", "type": "cppdbg", "request": "launch", "program": "/usr/bin/bash", "args": [ - "${workspaceFolder}/scripts/debug_python", - "${workspaceFolder}/examples/native_operator/python/ping.py", + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/scripts/debug_python", + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/examples/ping_multi_port/python/ping_multi_port.py", ], "stopAtEntry": false, - "cwd": "${command:cmake.buildDirectory}/python/lib", + "cwd": "${command:cmake.buildDirectory}", "environment": [ - { - "name": "LD_LIBRARY_PATH", - "value": "${command:cmake.buildDirectory}/lib:${env:LD_LIBRARY_PATH}" - }, { "name": "HOLOSCAN_LOG_LEVEL", "value": "TRACE" }, - { - "name": "HOLOSCAN_LIB_PATH", - "value": "${command:cmake.buildDirectory}/lib" - }, { "name": "PYTHONPATH", "value": "${command:cmake.buildDirectory}/python/lib" @@ -371,33 +344,53 @@ ] }, { - "name": "(gdb) examples/native_operator/python: convolve", + "name": "(gdb) examples/tensor_interop/cpp", + "type": "cppdbg", + "request": "launch", + "program": "${command:cmake.buildDirectory}/examples/tensor_interop/cpp/tensor_interop", + "args": [], + "stopAtEntry": false, + "cwd": "${command:cmake.buildDirectory}", + "environment": [ + { + "name": "HOLOSCAN_LOG_LEVEL", + "value": "TRACE" + } + ], + "console": "externalTerminal", + "MIMode": "gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] + }, + { + "name": "(gdb) examples/tensor_interop/python", "type": "cppdbg", "request": "launch", "program": "/usr/bin/bash", "args": [ - "${workspaceFolder}/scripts/debug_python", - "${workspaceFolder}/examples/native_operator/python/convolve.py", + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/scripts/debug_python", + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/examples/tensor_interop/python/tensor_interop.py", ], "stopAtEntry": false, - "cwd": "${command:cmake.buildDirectory}/python/lib", + "cwd": "${command:cmake.buildDirectory}", "environment": [ - { - "name": "LD_LIBRARY_PATH", - "value": "${command:cmake.buildDirectory}/lib:${env:LD_LIBRARY_PATH}" - }, { "name": "HOLOSCAN_LOG_LEVEL", "value": "TRACE" }, - { - "name": "HOLOSCAN_LIB_PATH", - "value": "${command:cmake.buildDirectory}/lib" - }, { "name": "PYTHONPATH", "value": "${command:cmake.buildDirectory}/python/lib" }, + { + "name": "HOLOSCAN_SAMPLE_DATA_PATH", + "value": "${command:cmake.buildDirectory}/../data" + }, ], "console": "externalTerminal", "MIMode": "gdb", @@ -410,21 +403,14 @@ ] }, { - "name": "(gdb) examples/tensor_interop/cpp", + "name": "(gdb) examples/video_replayer/cpp", "type": "cppdbg", "request": "launch", - "program": "${command:cmake.buildDirectory}/examples/tensor_interop/cpp/tensor_interop", - "args": [ - "-severity", - "4" - ], + "program": "${command:cmake.buildDirectory}/examples/video_replayer/cpp/video_replayer", + "args": [], "stopAtEntry": false, "cwd": "${command:cmake.buildDirectory}", "environment": [ - { - "name": "LD_LIBRARY_PATH", - "value": "${command:cmake.buildDirectory}/lib:${env:LD_LIBRARY_PATH}" - }, { "name": "HOLOSCAN_LOG_LEVEL", "value": "TRACE" @@ -441,30 +427,21 @@ ] }, { - "name": "(gdb) examples/tensor_interop/python", + "name": "(gdb) examples/video_replayer/python", "type": "cppdbg", "request": "launch", - "preLaunchTask": "Install CuPy package", "program": "/usr/bin/bash", "args": [ - "${workspaceFolder}/scripts/debug_python", - "${workspaceFolder}/examples/tensor_interop/python/tensor_interop.py", + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/scripts/debug_python", + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/examples/video_replayer/python/video_replayer.py", ], "stopAtEntry": false, - "cwd": "${command:cmake.buildDirectory}/python/lib", + "cwd": "${command:cmake.buildDirectory}", "environment": [ - { - "name": "LD_LIBRARY_PATH", - "value": "${command:cmake.buildDirectory}/lib:${env:LD_LIBRARY_PATH}" - }, { "name": "HOLOSCAN_LOG_LEVEL", "value": "TRACE" }, - { - "name": "HOLOSCAN_LIB_PATH", - "value": "${command:cmake.buildDirectory}/lib" - }, { "name": "PYTHONPATH", "value": "${command:cmake.buildDirectory}/python/lib" @@ -485,24 +462,21 @@ ] }, { - "name": "(gdb) examples/video_sources/gxf: video_replayer", + // In .devcontainer/devcontainer.json file: + // Uncomment the following line to use /dev/video0. Also, execute 'sudo chmod 666 /dev/video0' inside the container. + // "--device=/dev/video0:/dev/video0", + "name": "(gdb) examples/v4l2_camera/gxf", "type": "cppdbg", "request": "launch", "program": "${command:cmake.buildDirectory}/bin/gxe", "args": [ "--app", - "examples/video_sources/gxf/video_replayer.yaml", + "examples/v4l2_camera/gxf/v4l2_camera.yaml", "--manifest", - "examples/video_sources/gxf/video_replayer_manifest.yaml" + "examples/v4l2_camera/gxf/v4l2_camera_manifest.yaml" ], "stopAtEntry": false, "cwd": "${command:cmake.buildDirectory}", - "environment": [ - { - "name": "LD_LIBRARY_PATH", - "value": "${command:cmake.buildDirectory}/lib:${env:LD_LIBRARY_PATH}" - } - ], "console": "externalTerminal", "MIMode": "gdb", "setupCommands": [ @@ -514,26 +488,25 @@ ] }, { - // In .devcontainer/devcontainer.json file: - // Uncomment the following line to use /dev/video0. Also, execute 'sudo chmod 666 /dev/video0' inside the container. - // "--device=/dev/video0:/dev/video0", - "name": "(gdb) examples/video_sources/gxf: v4l2_camera", + "name": "(gdb) examples/wrap_operator_as_gxf_extension", "type": "cppdbg", "request": "launch", "program": "${command:cmake.buildDirectory}/bin/gxe", "args": [ "--app", - "examples/video_sources/gxf/v4l2_camera.yaml", + "examples/wrap_operator_as_gxf_extension/gxf_app/ping.yaml", "--manifest", - "examples/video_sources/gxf/v4l2_camera_manifest.yaml" + "examples/wrap_operator_as_gxf_extension/gxf_app/test_operator_as_gxf_ext_manifest.yaml", + "--severity", + "4" ], "stopAtEntry": false, "cwd": "${command:cmake.buildDirectory}", "environment": [ { - "name": "LD_LIBRARY_PATH", - "value": "${command:cmake.buildDirectory}/lib:${env:LD_LIBRARY_PATH}" - } + "name": "HOLOSCAN_LOG_LEVEL", + "value": "TRACE" + }, ], "console": "externalTerminal", "MIMode": "gdb", @@ -551,35 +524,34 @@ "request": "launch", "program": "/usr/bin/bash", "args": [ - "${workspaceFolder}/scripts/debug_python", + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/scripts/debug_python", "-m", "pytest", "-v", - "-rP", - "-k", - // //"TestOperator and test_setup", - // "test_my_ping_app", + // "-rP", + // "-k", + // "test_application", "python/lib/tests", ], "stopAtEntry": false, "cwd": "${command:cmake.buildDirectory}", "environment": [ - { - "name": "LD_LIBRARY_PATH", - "value": "${command:cmake.buildDirectory}/lib:${env:LD_LIBRARY_PATH}" - }, { "name": "HOLOSCAN_LOG_LEVEL", "value": "TRACE" }, - { - "name": "HOLOSCAN_LIB_PATH", - "value": "${command:cmake.buildDirectory}/lib" - }, { "name": "PYTHONPATH", "value": "${command:cmake.buildDirectory}/python/lib" }, + { + "name": "HOLOSCAN_TESTS_DATA_PATH", + "value": "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/tests/data" + }, + { + "name": "HOLOSCAN_SAMPLE_DATA_PATH", + "value": "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/data" + }, ], "console": "externalTerminal", "MIMode": "gdb", @@ -595,18 +567,63 @@ "name": "(gdb) Holoviz: functional tests", "type": "cppdbg", "request": "launch", - "program": "${command:cmake.buildDirectory}/_deps/holoviz-build/tests/functional/clara_holoviz_functionaltests", + "program": "${command:cmake.buildDirectory}/_deps/holoviz-build/tests/functional/holoviz_functionaltests", "args": [ "--gtest_filter=*" ], "stopAtEntry": false, - "cwd": "${fileDirname}", - "environment": [ + "cwd": "${command:cmake.buildDirectory}", + "externalConsole": false, + "MIMode": "gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + }, + { + "description": "Set Disassembly Flavor to Intel", + "text": "-gdb-set disassembly-flavor intel", + "ignoreFailures": true + } + ] + }, + { + "name": "(gdb) Holoviz: demo", + "type": "cppdbg", + "request": "launch", + "program": "${command:cmake.buildDirectory}/_deps/holoviz-build/examples/demo/holoviz_demo", + "args": [], + "stopAtEntry": false, + "cwd": "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/modules/holoviz/examples/demo", + "externalConsole": false, + "MIMode": "gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + }, { - "name": "LD_LIBRARY_PATH", - "value": "${command:cmake.buildDirectory}/lib:${env:LD_LIBRARY_PATH}" + "description": "Set Disassembly Flavor to Intel", + "text": "-gdb-set disassembly-flavor intel", + "ignoreFailures": true } + ] + }, + { + "name": "(gdb) Holoviz: depth_map", + "type": "cppdbg", + "request": "launch", + "program": "${command:cmake.buildDirectory}/_deps/holoviz-build/examples/depth_map/holoviz_depth_map", + "args": [ + "-d", + "data/endoscopy_depth/disp_maps", + "-c", + "data/endoscopy_depth/rgb" ], + "stopAtEntry": false, + "cwd": "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}", "externalConsole": false, "MIMode": "gdb", "setupCommands": [ @@ -616,11 +633,46 @@ "ignoreFailures": true }, { - "description": "Set Disassembly Flavor to Intel", + "description": "Set Disassembly Flavor to Intel", "text": "-gdb-set disassembly-flavor intel", "ignoreFailures": true } ] }, + { + "name": "(gdb) Holoviz: examples/holoviz/python", + "type": "cppdbg", + "request": "launch", + "program": "/usr/bin/bash", + "args": [ + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/scripts/debug_python", + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/examples/holoviz/python/holoviz_geometry.py", + ], + "stopAtEntry": false, + "cwd": "${command:cmake.buildDirectory}", + "environment": [ + { + "name": "HOLOSCAN_LOG_LEVEL", + "value": "TRACE" + }, + { + "name": "PYTHONPATH", + "value": "${command:cmake.buildDirectory}/python/lib" + }, + { + "name": "HOLOSCAN_SAMPLE_DATA_PATH", + "value": "${command:cmake.buildDirectory}/../data" + }, + ], + "console": "externalTerminal", + "MIMode": "gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] + }, ] -} \ No newline at end of file +} diff --git a/.vscode/settings.json b/.vscode/settings.json index db5e8d82..c6eaceef 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -118,6 +118,7 @@ "charconv": "cpp" }, "git.alwaysSignOff": true, + "git.untrackedChanges": "separate", "editor.formatOnSave": true, "editor.formatOnSaveMode": "modifications", // "[cmake]": { @@ -126,11 +127,12 @@ "[dockerfile]": { "editor.formatOnSave": false, }, - "cmake.sourceDirectory": "${workspaceFolder}", + "cmake.configureOnOpen": true, + "cmake.sourceDirectory": "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}", // https://vector-of-bool.github.io/docs/vscode-cmake-tools/settings.html "cmake.buildTask": false, - "cmake.buildDirectory": "${workspaceFolder}/build-${variant:buildType}-${variant:platform}", - // "cmake.installPrefix": "${workspaceFolder}/install-${variant:buildType}-${variant:platform}", + "cmake.buildDirectory": "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/build-${variant:buildType}-${variant:platform}", + // "cmake.installPrefix": "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/install-${variant:buildType}-${variant:platform}", "cmake.preferredGenerators": [ "Ninja", "Unix Makefiles" @@ -138,6 +140,7 @@ "cmake.configureArgs": [ "--log-context", "-DFETCHCONTENT_QUIET=Off", + "-DHOLOSCAN_USE_CCACHE=On" // "--trace", // "--trace-expand", ], @@ -151,7 +154,6 @@ "C/C++ Include Guard.Path Depth": 0, "C/C++ Include Guard.Remove Extension": false, "sonarlint.pathToCompileCommands": "${command:cmake.buildDirectory}/compile_commands.json", - "cmake.configureOnEdit": false, // https://github.com/matepek/vscode-catch2-test-adapter/blob/master/documents/configuration/test.advancedExecutables.md "testMate.cpp.test.advancedExecutables": [ { @@ -161,7 +163,8 @@ "cwd": "${command:cmake.buildDirectory}", "env": { "LD_LIBRARY_PATH": "${command:cmake.buildDirectory}/lib:${command:cmake.buildDirectory}/lib/gxf_extensions:${env:LD_LIBRARY_PATH}", - "HOLOSCAN_TESTS_DATA_PATH": "/workspace/holoscan-sdk/tests/data", + "HOLOSCAN_TESTS_DATA_PATH": "/workspace/holoscan-sdk/${env:HOLOSCAN_PUBLIC_FOLDER}/tests/data", + "HOLOSCAN_SAMPLE_DATA_PATH": "/workspace/holoscan-sdk/${env:HOLOSCAN_PUBLIC_FOLDER}/data", } }, { @@ -171,15 +174,19 @@ "cwd": "${command:cmake.buildDirectory}", "env": { "LD_LIBRARY_PATH": "${command:cmake.buildDirectory}/lib:${command:cmake.buildDirectory}/lib/gxf_extensions:${env:LD_LIBRARY_PATH}", - "HOLOSCAN_TESTS_DATA_PATH": "/workspace/holoscan-sdk/tests/data", + "HOLOSCAN_TESTS_DATA_PATH": "/workspace/holoscan-sdk/${env:HOLOSCAN_PUBLIC_FOLDER}/tests/data", + "HOLOSCAN_SAMPLE_DATA_PATH": "/workspace/holoscan-sdk/${env:HOLOSCAN_PUBLIC_FOLDER}/data", } }, ], + "python.analysis.extraPaths": [ + // variable substitution, e.g. using ${command:cmake.buildDirectory} is not working here + "./build-debug-x86_64/python/lib" + ], "python.formatting.provider": "black", // It seems that Pytest Discovery feature in VSCode needs to be improved // - https://github.com/microsoft/vscode-python/issues/17242 // - https://github.com/microsoft/vscode-python/issues/19790 - // https://code.visualstudio.com/docs/python/testing#_test-configuration-settings // "python.testing.pytestArgs": [ // "build-debug-x86_64" diff --git a/.vscode/tasks.json b/.vscode/tasks.json index a5f91d82..1c11423a 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -6,7 +6,7 @@ "type": "shell", "command": "cmake --build ${command:cmake.buildDirectory} --config ${command:cmake.buildType} --target ${command:cmake.buildTargetName}", "options": { - "cwd": "${workspaceFolder}", + "cwd": "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}", "env": { "PATH": "${env:HOME}/.local/bin:${env:PATH}", "CUDACXX": "/usr/local/cuda/bin/nvcc", @@ -25,29 +25,9 @@ { "label": "Install", "type": "shell", - "command": "cmake --install ${command:cmake.buildDirectory} --prefix ${workspaceFolder}/install --component holoscan-core && cmake --install ${command:cmake.buildDirectory} --prefix ${workspaceFolder}/install --component holoscan-gxf_extensions && cmake --install ${command:cmake.buildDirectory} --prefix ${workspaceFolder}/install --component holoscan-apps && cmake --install ${command:cmake.buildDirectory} --prefix ${workspaceFolder}/install --component holoscan-gxf_libs && cmake --install ${command:cmake.buildDirectory} --prefix ${workspaceFolder}/install --component holoscan-gxf_bins && cmake --install ${command:cmake.buildDirectory} --prefix ${workspaceFolder}/install --component holoscan-modules && cmake --install ${command:cmake.buildDirectory} --prefix ${workspaceFolder}/install --component holoscan-dependencies && cmake --install ${command:cmake.buildDirectory} --prefix ${workspaceFolder}/install --component holoscan-python_libs", + "command": "cmake --install ${command:cmake.buildDirectory} --prefix ${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/install --component holoscan-core && cmake --install ${command:cmake.buildDirectory} --prefix ${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/install --component holoscan-gxf_extensions && cmake --install ${command:cmake.buildDirectory} --prefix ${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/install --component holoscan-examples && cmake --install ${command:cmake.buildDirectory} --prefix ${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/install --component holoscan-gxf_libs && cmake --install ${command:cmake.buildDirectory} --prefix ${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/install --component holoscan-gxf_bins && cmake --install ${command:cmake.buildDirectory} --prefix ${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/install --component holoscan-modules && cmake --install ${command:cmake.buildDirectory} --prefix ${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/install --component holoscan-dependencies && cmake --install ${command:cmake.buildDirectory} --prefix ${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/install --component holoscan-python_libs", "options": { - "cwd": "${workspaceFolder}", - "env": { - "PATH": "${env:HOME}/.local/bin:${env:PATH}", - "CUDACXX": "/usr/local/cuda/bin/nvcc", - } - }, - "presentation": { - "reveal": "always", - "focus": true - }, - "problemMatcher": [], - "group": { - "kind": "none", - } - }, - { - "label": "Install CuPy package", - "type": "shell", - "command": "python -m pip install cupy-cuda11x", - "options": { - "cwd": "${workspaceFolder}", + "cwd": "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}", "env": { "PATH": "${env:HOME}/.local/bin:${env:PATH}", "CUDACXX": "/usr/local/cuda/bin/nvcc", diff --git a/CMakeLists.txt b/CMakeLists.txt index 2ddb62c1..27b607a6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,22 +25,13 @@ set(HOLOSCAN_PACKAGE_NAME holoscan) option(BUILD_SHARED_LIBS "Build Shared Libraries" ON) option(HOLOSCAN_BUILD_MODULES "Build Holoscan SDK Modules" ON) option(HOLOSCAN_BUILD_GXF_EXTENSIONS "Build GXF Extensions" ON) -option(HOLOSCAN_BUILD_APPS "Build Holoscan SDK Apps" ON) option(HOLOSCAN_BUILD_EXAMPLES "Build Holoscan SDK Examples" ON) option(HOLOSCAN_BUILD_PYTHON "Build Holoscan SDK Python Bindings" ON) -option(HOLOSCAN_DOWNLOAD_DATASETS "Download SDK Apps Datasets" ON) +option(HOLOSCAN_DOWNLOAD_DATASETS "Download SDK Datasets" ON) option(HOLOSCAN_BUILD_TESTS "Build Holoscan SDK Tests" ON) option(HOLOSCAN_BUILD_DOCS "Build Holoscan SDK Documents" OFF) -option(HOLOSCAN_USE_CCACHE "Use ccache for building Holoscan SDK" ON) -option(HOLOSCAN_BUILD_HI_SPEED_ENDO_APP "Build Holoscan Hi-Speed Endoscopy App" OFF) - -set(HOLOSCAN_CACHE_DIR ".cache" CACHE PATH "Directory to use for caching data for CPM and CCache.\ - If relative, it is relative to the source directory (CMAKE_SOURCE_DIR).") -mark_as_advanced(HOLOSCAN_CACHE_DIR) - -# HOLOSCAN_USE_CCACHE_SKIPPED will be set to TRUE by 'include(SetupCache)' -# later in the build process, if CCache is not available. -set(HOLOSCAN_USE_CCACHE_SKIPPED FALSE) +option(HOLOSCAN_USE_CCACHE "Use ccache for building Holoscan SDK" OFF) +option(HOLOSCAN_INSTALL_EXAMPLE_SOURCE "Install the example source code" ON) # ############################################################################## # # Prerequisite statements @@ -55,35 +46,33 @@ list(APPEND CMAKE_MODULE_PATH "${HOLOSCAN_TOP}/cmake/modules") # Append local cmake dependency path CMAKE_PREFIX_PATH list(APPEND CMAKE_PREFIX_PATH "${HOLOSCAN_TOP}/cmake/deps") -# Set HOLOSCAN_BUILD_VERSION from 'VERSION' file -unset(HOLOSCAN_BUILD_VERSION CACHE) -file(STRINGS ${HOLOSCAN_TOP}/VERSION HOLOSCAN_BUILD_VERSION) - -# ############################################################################## -# # Include build utilities -# ############################################################################## +# Include build utilities (rapids) include(SuperBuildUtils) # set local dependent modules and fetch RAPIDS-cmake - include(rapids-cmake) include(rapids-cpm) include(rapids-cuda) include(rapids-export) include(rapids-find) -# Configuring CUDA Architecture/Compiler with delayed CUDA configuration -# Need to be called before project() -# Since we want to set CMAKE_CUDA_HOST_COMPILER to use CMAKE_CXX_COMPILER for non-Clang compilers, -# we first call project() without 'CUDA' and then set CMAKE_CUDA_HOST_COMPILER before calling 'enable_language(CUDA)' -include(SetupCUDA) - # ############################################################################## # # Project definition # ############################################################################## + +# Set HOLOSCAN_BUILD_VERSION from 'VERSION' file +unset(HOLOSCAN_BUILD_VERSION CACHE) +file(STRINGS ${HOLOSCAN_TOP}/VERSION HOLOSCAN_BUILD_VERSION) + project(${HOLOSCAN_PACKAGE_NAME} VERSION ${HOLOSCAN_BUILD_VERSION} DESCRIPTION "Holoscan SDK" LANGUAGES C CXX ) +include(SetupCUDA) + + +# ############################################################################## +# # Global properties (CMAKE_*) +# ############################################################################## # Setting the default paths for the build tree # Libraries (shared and static) are put by default in the lib directory @@ -91,14 +80,17 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib) # Set the default RPATH for built libraries to $ORIGIN and the lib directory -set(CMAKE_BUILD_RPATH \$ORIGIN:${CMAKE_BINARY_DIR}/lib) +set(CMAKE_BUILD_RPATH "\$ORIGIN:${CMAKE_BINARY_DIR}/lib:${CMAKE_BINARY_DIR}/lib/gxf_extensions") +set(CMAKE_INSTALL_RPATH "\$ORIGIN:\$ORIGIN/gxf_extensions") -# Delayed CUDA language enablement (to set CMAKE_CUDA_ARCHITECTURES and CMAKE_CUDA_HOST_COMPILER) -enable_language(CUDA) +# this generates a 'compile_commands.json' file which can be read by VSCode to +# configure the include paths for IntelliSense +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) -# ############################################################################## -# # Default language settings -# ############################################################################## +# Avoid 'Up-to-date' install messages +set(CMAKE_INSTALL_MESSAGE LAZY) + +# Default language settings set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED YES) set(CMAKE_CXX_EXTENSIONS NO) @@ -122,222 +114,82 @@ rapids_cmake_install_lib_dir(HOLOSCAN_INSTALL_LIB_DIR) # ############################################################################## # # Setup Cache # ############################################################################## + +set(HOLOSCAN_CACHE_DIR ".cache" CACHE PATH "Directory to use for caching data for CPM and CCache.\ + If relative, it is relative to the source directory (CMAKE_SOURCE_DIR).") +mark_as_advanced(HOLOSCAN_CACHE_DIR) + +# HOLOSCAN_USE_CCACHE_SKIPPED will be set to TRUE by 'include(SetupCache)' +# CCache is not available. +set(HOLOSCAN_USE_CCACHE_SKIPPED FALSE) include(SetupCache) # ############################################################################## # # Define dependencies # ############################################################################## -include(DefineSearchDirFor) -include(GenerateGXEApp) - include(cmake/setup_dependencies.cmake) # ############################################################################## -# # Add library: ${HOLOSCAN_PACKAGE_NAME} -# ############################################################################## -set(LIST_SOURCES - include/holoscan/core/application.hpp - include/holoscan/core/arg.hpp - include/holoscan/core/argument_setter.hpp - include/holoscan/core/common.hpp - include/holoscan/core/component_spec.hpp - include/holoscan/core/component_spec-inl.hpp - include/holoscan/core/component.hpp - include/holoscan/core/condition.hpp - include/holoscan/core/conditions/gxf/boolean.hpp - include/holoscan/core/conditions/gxf/count.hpp - include/holoscan/core/conditions/gxf/downstream_affordable.hpp - include/holoscan/core/conditions/gxf/message_available.hpp - include/holoscan/core/config.hpp - include/holoscan/core/domain/tensor.hpp - include/holoscan/core/execution_context.hpp - include/holoscan/core/executor.hpp - include/holoscan/core/executors/gxf/gxf_executor.hpp - include/holoscan/core/executors/gxf/gxf_parameter_adaptor.hpp - include/holoscan/core/expected.hpp - include/holoscan/core/forward_def.hpp - include/holoscan/core/fragment.hpp - include/holoscan/core/graph.hpp - include/holoscan/core/graphs/flow_graph.hpp - include/holoscan/core/gxf/entity.hpp - include/holoscan/core/gxf/gxf_component.hpp - include/holoscan/core/gxf/gxf_execution_context.hpp - include/holoscan/core/gxf/gxf_extension_registrar.hpp - include/holoscan/core/gxf/gxf_io_context.hpp - include/holoscan/core/gxf/gxf_operator.hpp - include/holoscan/core/gxf/gxf_resource.hpp - include/holoscan/core/gxf/gxf_tensor.hpp - include/holoscan/core/gxf/gxf_utils.hpp - include/holoscan/core/gxf/gxf_wrapper.hpp - include/holoscan/core/io_context.hpp - include/holoscan/core/io_spec.hpp - include/holoscan/core/message.hpp - include/holoscan/core/operator_spec.hpp - include/holoscan/core/operator.hpp - include/holoscan/core/parameter.hpp - include/holoscan/core/resource.hpp - include/holoscan/core/resources/gxf/allocator.hpp - include/holoscan/core/resources/gxf/block_memory_pool.hpp - include/holoscan/core/resources/gxf/cuda_stream_pool.hpp - include/holoscan/core/resources/gxf/double_buffer_receiver.hpp - include/holoscan/core/resources/gxf/double_buffer_transmitter.hpp - include/holoscan/core/resources/gxf/receiver.hpp - include/holoscan/core/resources/gxf/std_component_serializer.hpp - include/holoscan/core/resources/gxf/transmitter.hpp - include/holoscan/core/resources/gxf/unbounded_allocator.hpp - include/holoscan/core/resources/gxf/video_stream_serializer.hpp - include/holoscan/core/type_traits.hpp - include/holoscan/holoscan.hpp - include/holoscan/operators/aja_source/aja_source.hpp - include/holoscan/operators/aja_source/ntv2channel.hpp - include/holoscan/operators/bayer_demosaic/bayer_demosaic.hpp - include/holoscan/operators/custom_lstm_inference/lstm_tensor_rt_inference.hpp - include/holoscan/operators/format_converter/format_converter.hpp - include/holoscan/operators/holoviz/holoviz.hpp - include/holoscan/operators/multiai_inference/multiai_inference.hpp - include/holoscan/operators/multiai_postprocessor/multiai_postprocessor.hpp - include/holoscan/operators/segmentation_postprocessor/segmentation_postprocessor.hpp - include/holoscan/operators/stream_playback/video_stream_recorder.hpp - include/holoscan/operators/stream_playback/video_stream_replayer.hpp - include/holoscan/operators/tensor_rt/tensor_rt_inference.hpp - include/holoscan/operators/tool_tracking_postprocessor/tool_tracking_postprocessor.hpp - include/holoscan/operators/visualizer_icardio/visualizer_icardio.hpp - include/holoscan/operators/visualizer_tool_tracking/visualizer_tool_tracking.hpp - include/holoscan/std_ops.hpp - include/holoscan/utils/timer.hpp - include/holoscan/utils/yaml_parser.hpp - src/core/application.cpp - src/core/arg.cpp - src/core/argument_setter.cpp - src/core/condition.cpp - src/core/conditions/gxf/boolean.cpp - src/core/conditions/gxf/count.cpp - src/core/conditions/gxf/downstream_affordable.cpp - src/core/conditions/gxf/message_available.cpp - src/core/config.cpp - src/core/domain/tensor.cpp - src/core/executors/gxf/gxf_executor.cpp - src/core/executors/gxf/gxf_parameter_adaptor.cpp - src/core/fragment.cpp - src/core/graphs/flow_graph.cpp - src/core/gxf/entity.cpp - src/core/gxf/gxf_condition.cpp - src/core/gxf/gxf_execution_context.cpp - src/core/gxf/gxf_io_context.cpp - src/core/gxf/gxf_operator.cpp - src/core/gxf/gxf_resource.cpp - src/core/gxf/gxf_tensor.cpp - src/core/gxf/gxf_wrapper.cpp - src/core/io_spec.cpp - src/core/logger.cpp - src/core/operator.cpp - src/core/resource.cpp - src/core/resources/gxf/block_memory_pool.cpp - src/core/resources/gxf/cuda_stream_pool.cpp - src/core/resources/gxf/double_buffer_receiver.cpp - src/core/resources/gxf/double_buffer_transmitter.cpp - src/core/resources/gxf/std_component_serializer.cpp - src/core/resources/gxf/video_stream_serializer.cpp - src/operators/aja_source/aja_source.cpp - src/operators/bayer_demosaic/bayer_demosaic.cpp - src/operators/custom_lstm_inference/lstm_tensor_rt_inference.cpp - src/operators/format_converter/format_converter.cpp - src/operators/holoviz/holoviz.cpp - src/operators/multiai_inference/multiai_inference.cpp - src/operators/multiai_postprocessor/multiai_postprocessor.cpp - src/operators/segmentation_postprocessor/segmentation_postprocessor.cpp - src/operators/stream_playback/video_stream_recorder.cpp - src/operators/stream_playback/video_stream_replayer.cpp - src/operators/tensor_rt/tensor_rt_inference.cpp - src/operators/tool_tracking_postprocessor/tool_tracking_postprocessor.cpp - src/operators/visualizer_icardio/visualizer_icardio.cpp - src/operators/visualizer_tool_tracking/visualizer_tool_tracking.cpp -) - -if(HOLOSCAN_BUILD_HI_SPEED_ENDO_APP) - set(LIST_SOURCES - ${LIST_SOURCES} - include/holoscan/operators/emergent_source/emergent_source.hpp - src/operators/emergent_source/emergent_source.cpp - ) +# # Build sub modules (holoviz, holoinfer) +# ############################################################################## +if(HOLOSCAN_BUILD_MODULES) + add_subdirectory(modules) endif() -add_library(${HOLOSCAN_PACKAGE_NAME} ${LIST_SOURCES}) - -# Compile options -set_target_properties(${HOLOSCAN_PACKAGE_NAME} - PROPERTIES - CXX_STANDARD 17 - CXX_STANDARD_REQUIRED YES - CXX_EXTENSIONS NO - - # https://gitlab.kitware.com/cmake/community/-/wikis/doc/cmake/RPATH-handling - INSTALL_RPATH \$ORIGIN - SOVERSION ${PROJECT_VERSION_MAJOR} - VERSION ${PROJECT_VERSION} -) - -# Force targets using holoscan SDK to use the C++17 standard -target_compile_features(${HOLOSCAN_PACKAGE_NAME} INTERFACE cxx_std_17) - -# Note: Looks like the following line causes error on CMake 3.18.4 (it is working on 3.18.2). Keeping it for now. -set(HOLOSCAN_REQUIRED_FEATURES cxx_std_17) -target_compile_features(${HOLOSCAN_PACKAGE_NAME} PRIVATE ${HOLOSCAN_REQUIRED_FEATURES}) +# ############################################################################## +# # Build core and operators +# ############################################################################## +add_subdirectory(src) -# Use generator expression to avoid `nvcc fatal : Value '-std=c++17' is not defined for option 'Werror'` -target_compile_options(${HOLOSCAN_PACKAGE_NAME} - PRIVATE - $<$:-Werror -Wall -Wextra> -) -# Build EVT related files if HOLOSCAN_BUILD_HI_SPEED_ENDO_APP is ON. -if(HOLOSCAN_BUILD_HI_SPEED_ENDO_APP) - target_compile_definitions(${HOLOSCAN_PACKAGE_NAME} PUBLIC HOLOSCAN_BUILD_EMERGENT=1) +# ############################################################################## +# # Build GXF Extensions and apps +# ############################################################################## +if(HOLOSCAN_BUILD_GXF_EXTENSIONS) + add_subdirectory(gxf_extensions) endif() -# Skip trace logging in release build if CMAKE_BUILD_TYPE is Release. -# (See include/holoscan/core/logger.hpp) +# ############################################################################## +# # Package project +# ############################################################################## + +# strip release binaries if(CMAKE_BUILD_TYPE STREQUAL "Release") - target_compile_definitions(${HOLOSCAN_PACKAGE_NAME} PRIVATE HOLOSCAN_LOG_ACTIVE_LEVEL=1) + set(_INSTALL_TARGET "install/strip") +else() + set(_INSTALL_TARGET "install") endif() -# Link libraries -target_link_libraries(${HOLOSCAN_PACKAGE_NAME} - PUBLIC - ${CMAKE_DL_LIBS} - Threads::Threads - fmt::fmt-header-only - yaml-cpp - GXF::core - GXF::std - CUDA::cudart - - PRIVATE - spdlog::spdlog_header_only -) - -target_include_directories(${HOLOSCAN_PACKAGE_NAME} - PUBLIC - $ - $ - - # Bundled tl-expected - $ - $ - - # Bundled dlpack - $ - $ - - PRIVATE - ${HOLOSCAN_TOP}/src +# List targets to export/install +list(APPEND HOLOSCAN_INSTALL_TARGETS + logger + core + gxf_holoscan_wrapper_lib + holoviz + holoinfer + infer_utils + op_aja + op_bayer_demosaic + op_format_converter + op_holoviz + op_multiai_inference + op_multiai_postprocessor + op_ping_rx + op_ping_tx + op_segmentation_postprocessor + op_tensor_rt + op_video_stream_recorder + op_video_stream_replayer ) -# ############################################################################## -# # Package project -# ############################################################################## -set(HOLOSCAN_INSTALL_TARGETS - ${HOLOSCAN_PACKAGE_NAME} +# Add PUBLIC dependencies to our install target +# Note: required due to. However, could only export but not install? +# TODO: work on limiting PUBLIC dependencies by not including third-party +# headers in our nterface headers, using forward declaration or PIMPL +list(APPEND HOLOSCAN_INSTALL_TARGETS + yaml-cpp # needed by holoscan::core + glfw # needed by holoscan::viz + ajantv2 # needed by holoscan::ops::aja ) # Copy library files @@ -387,6 +239,18 @@ COMPONENT "holoscan-gxf_libs" # Install CMake script to build GXE applications install(FILES "${CMAKE_SOURCE_DIR}/cmake/modules/GenerateGXEAppInstall.cmake" +RENAME GenerateGXEApp.cmake +DESTINATION "${HOLOSCAN_INSTALL_LIB_DIR}/cmake/holoscan" +COMPONENT "holoscan-gxf_libs" +) + +# Install CMake script for GXF wrapping of extensions +install(FILES "${CMAKE_SOURCE_DIR}/cmake/modules/WrapOperatorAsGXFExtension.cmake" +DESTINATION "${HOLOSCAN_INSTALL_LIB_DIR}/cmake/holoscan" +COMPONENT "holoscan-gxf_libs" +) + +install(DIRECTORY "${CMAKE_SOURCE_DIR}/cmake/modules/wrap_operator_as_gxf_template" DESTINATION "${HOLOSCAN_INSTALL_LIB_DIR}/cmake/holoscan" COMPONENT "holoscan-gxf_libs" ) @@ -396,39 +260,52 @@ set(holoscan_doc_string [=[ libholoscan: Holoscan SDK C++ API ]=]) - # Defines the install export hook # We use add_library since we are installing the libraries as part of the SDK set(holoscan_install_hook_code_string [=[ -add_library(fmt::fmt-header-only INTERFACE IMPORTED) +if(NOT TARGET ONNXRuntime::ONNXRuntime) + add_library(ONNXRuntime::ONNXRuntime SHARED IMPORTED) + set_target_properties(ONNXRuntime::ONNXRuntime PROPERTIES + IMPORTED_LOCATION "${PACKAGE_PREFIX_DIR}/lib/${ONNXRUNTIME_LIBRARY_NAME}" + ) +endif() + +if(NOT TARGET fmt::fmt-header-only) + add_library(fmt::fmt-header-only INTERFACE IMPORTED) +endif() set(_GXFlibs core std multimedia cuda network npp serialization behavior_tree) foreach(gxflib IN LISTS _GXFlibs) - add_library(GXF::${gxflib} SHARED IMPORTED) - set_target_properties(GXF::${gxflib} PROPERTIES - IMPORTED_LOCATION "${PACKAGE_PREFIX_DIR}/lib/libgxf_${gxflib}.so" - IMPORTED_NO_SONAME ON - INTERFACE_INCLUDE_DIRECTORIES "${PACKAGE_PREFIX_DIR}/include;${PACKAGE_PREFIX_DIR}/include/gxf" - ) + if(NOT TARGET GXF::${gxflib}) + add_library(GXF::${gxflib} SHARED IMPORTED) + set_target_properties(GXF::${gxflib} PROPERTIES + IMPORTED_LOCATION "${PACKAGE_PREFIX_DIR}/lib/libgxf_${gxflib}.so" + IMPORTED_NO_SONAME ON + INTERFACE_INCLUDE_DIRECTORIES "${PACKAGE_PREFIX_DIR}/include;${PACKAGE_PREFIX_DIR}/include/gxf" + ) + endif() endforeach() -add_executable(GXF::gxe IMPORTED) -set_target_properties(GXF::gxe PROPERTIES - IMPORTED_LOCATION "${PACKAGE_PREFIX_DIR}/bin/gxe" -) +if(NOT TARGET GXF::gxe) + add_executable(GXF::gxe IMPORTED) + set_target_properties(GXF::gxe PROPERTIES + IMPORTED_LOCATION "${PACKAGE_PREFIX_DIR}/bin/gxe" + ) +endif() -set(HOLOSCAN_PACKAGE_NAME holoscan::holoscan) set(GXF_LIB_DIR "${PACKAGE_PREFIX_DIR}/lib") set(GXF_EXTENSIONS_DIR "${PACKAGE_PREFIX_DIR}/lib/gxf_extensions") -include("${CMAKE_CURRENT_LIST_DIR}/GenerateGXEAppInstall.cmake") +list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") ]=]) +string(PREPEND holoscan_install_hook_code_string "set(ONNXRUNTIME_LIBRARY_NAME ${ONNXRUNTIME_LIBRARY_NAME})\n") + set(holoscan_build_hook_code_string [=[ ]=]) @@ -442,7 +319,6 @@ set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME holoscan-core) rapids_export( INSTALL ${HOLOSCAN_PACKAGE_NAME} EXPORT_SET ${HOLOSCAN_PACKAGE_NAME}-exports - GLOBAL_TARGETS ${HOLOSCAN_INSTALL_TARGETS} NAMESPACE ${HOLOSCAN_PACKAGE_NAME}:: DOCUMENTATION holoscan_doc_string FINAL_CODE_BLOCK holoscan_install_hook_code_string @@ -450,13 +326,24 @@ rapids_export( rapids_export( BUILD ${HOLOSCAN_PACKAGE_NAME} EXPORT_SET ${HOLOSCAN_PACKAGE_NAME}-exports - GLOBAL_TARGETS ${HOLOSCAN_INSTALL_TARGETS} LANGUAGES C CXX CUDA NAMESPACE ${HOLOSCAN_PACKAGE_NAME}:: DOCUMENTATION holoscan_doc_string FINAL_CODE_BLOCK holoscan_build_hook_code_string ) +# Overwrite config version from rapids_export since it is not correct +# Need AnyNewerVersion +file(REMOVE "${CMAKE_BINARY_DIR}/rapids-cmake/holoscan/export/${HOLOSCAN_PACKAGE_NAME}-config-version.cmake") +write_basic_package_version_file( + "${CMAKE_BINARY_DIR}/${HOLOSCAN_PACKAGE_NAME}-config-version.cmake" + COMPATIBILITY AnyNewerVersion) + +install(FILES "${CMAKE_BINARY_DIR}/${HOLOSCAN_PACKAGE_NAME}-config-version.cmake" + RENAME "${HOLOSCAN_PACKAGE_NAME}-config-version.cmake" + DESTINATION "${HOLOSCAN_INSTALL_LIB_DIR}/cmake/holoscan" + COMPONENT holoscan-core) + # ############################################################################## # # Build Coverage # ############################################################################## @@ -472,44 +359,25 @@ if(HOLOSCAN_BUILD_COVERAGE) "${CMAKE_SOURCE_DIR}/modules/holoviz/src/*" "${CMAKE_SOURCE_DIR}/modules/holoinfer/src/*" "${CMAKE_SOURCE_DIR}/src/*" - "${CMAKE_SOURCE_DIR}/python/pybind11/*" + "${CMAKE_SOURCE_DIR}/python/src/*" PYTEST_EXECUTABLE pytest --pyargs "${CMAKE_BINARY_DIR}/python/lib/tests" || true PYTEST_EXTRACT "*holoscan*" ) endif() -# ############################################################################## -# # Build sub modules (holoviz) -# ############################################################################## -if(HOLOSCAN_BUILD_MODULES) - add_subdirectory(modules) -endif() - -# ############################################################################## -# # Build GXF Extensions and apps -# ############################################################################## -if(HOLOSCAN_BUILD_GXF_EXTENSIONS) - add_subdirectory(gxf_extensions) -endif() - # ############################################################################## # # Download datasets # ############################################################################## -if(HOLOSCAN_DOWNLOAD_DATASETS) - add_subdirectory(data) -endif() - -# ############################################################################## -# # Build apps -# ############################################################################## -if(HOLOSCAN_BUILD_APPS) - add_subdirectory(apps) -endif() +add_subdirectory(data) # ############################################################################## # # Build examples # ############################################################################## if(HOLOSCAN_BUILD_EXAMPLES) + if(HOLOSCAN_BUILD_TESTS) + include(CTest) + endif() + add_subdirectory(examples) endif() @@ -519,12 +387,24 @@ endif() if(HOLOSCAN_BUILD_TESTS) # Enable testing for the current directory and below include(CTest) # it calls 'enable_testing()' internally - superbuild_depend(gtest_rapids) add_subdirectory(tests) # add Holoviz tests add_test(NAME HOLOVIZ_FUNCTIONAL_TEST COMMAND holoscan::viz::functionaltests) + + # Set Environment variable to resolve memory allocation failure by loading TLS library + if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "aarch64") + execute_process(COMMAND nvidia-smi --query-gpu=driver_version --format=csv,noheader OUTPUT_VARIABLE driver_version) + if (NOT ${driver_version} STREQUAL "") + string(STRIP ${driver_version} driver_version) + set(ldpath "LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libnvidia-tls.so.${driver_version}") + set_property(TEST HOLOVIZ_FUNCTIONAL_TEST PROPERTY ENVIRONMENT ${ldpath}) + else() + message(WARNING "Could not retrieve driver version with nvidia-smi, can't set LD_PRELOAD for `HOLOVIZ_FUNCTIONAL_TEST`") + endif() + endif() + add_test(NAME HOLOVIZ_UNIT_TEST COMMAND holoscan::viz::unittests) endif() diff --git a/CPPLINT.cfg b/CPPLINT.cfg index 1dd8c6d7..468ff3fb 100644 --- a/CPPLINT.cfg +++ b/CPPLINT.cfg @@ -1,3 +1,3 @@ -filter=-build/header_guard,-readability/todo,-runtime/references,-build/c++11,-runtime/int,-build/include_subdir,-build/namespaces +filter=-build/header_guard,-readability/todo,-runtime/references,-build/c++11,-runtime/int,-build/include_subdir,-build/namespaces,-readability/casting exclude_files=\.cache|build|build-|install|data linelength=100 diff --git a/Dockerfile b/Dockerfile index 56b04dd6..84c52481 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:1 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -49,6 +49,11 @@ RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/nul cmake=3.22.2-0kitware1ubuntu20.04.1 \ && rm -rf /var/lib/apt/lists/* +# Install symlink missing from the container for H264 encode examples +RUN if [ $(uname -m) == "x86_64" ]; then \ + ln -sf /usr/lib/x86_64-linux-gnu/libnvidia-encode.so.1 /usr/lib/x86_64-linux-gnu/libnvidia-encode.so; \ + fi + # - This variable is consumed by all dependencies below as an environment variable (CMake 3.22+) # - We use ARG to only set it at docker build time, so it does not affect cmake builds # performed at docker run time in case users want to use a different BUILD_TYPE @@ -87,9 +92,14 @@ RUN if [ $(uname -m) == "aarch64" ]; then ARCH=aarch64; else ARCH=x64-gpu; fi \ ############################################################ # Vulkan SDK +# +# Use the SDK because we need the newer Vulkan headers and the newer shader compiler than provided +# by the Ubuntu deb packages. These are compile time dependencies, we still use the Vulkan loaded +# and the Vulkan validation layer as runtime components provided by Ubuntu packages because that's +# what the user will have on their installations. ############################################################ FROM base as vulkansdk-builder -ARG VULKAN_SDK_VERSION=1.3.216.0 +ARG VULKAN_SDK_VERSION WORKDIR /opt/vulkansdk @@ -99,15 +109,11 @@ RUN wget -nv --show-progress --progress=bar:force:noscroll \ https://sdk.lunarg.com/sdk/download/${VULKAN_SDK_VERSION}/linux/vulkansdk-linux-x86_64-${VULKAN_SDK_VERSION}.tar.gz RUN tar -xzf vulkansdk-linux-x86_64-${VULKAN_SDK_VERSION}.tar.gz RUN if [ $(uname -m) == "aarch64" ]; then \ - apt update \ - && apt install -y --no-install-recommends \ - libwayland-dev=1.18.0-1ubuntu0.1 \ - libx11-dev=2:1.6.9-2ubuntu1.2 \ - libxrandr-dev=2:1.5.2-0ubuntu1 \ - && cd ${VULKAN_SDK_VERSION} \ + cd ${VULKAN_SDK_VERSION} \ && rm -rf x86_64 \ - && ./vulkansdk shaderc glslang headers loader layers; \ -fi + && ./vulkansdk shaderc glslang headers; \ + fi + ############################################################ # dev image @@ -126,6 +132,9 @@ RUN apt update \ coverage==6.5.0 # Install apt & pip build dependencies +# libvulkan1 - for Vulkan apps (Holoviz) +# vulkan-validationlayers, spirv-tools - for Vulkan validation layer (enabled for Holoviz in debug mode) +# libegl1 - to run headless Vulkan apps RUN apt update \ && apt install --no-install-recommends -y \ libgl-dev=1.3.2-1~ubuntu0.20.04.2 \ @@ -136,13 +145,17 @@ RUN apt update \ libxinerama-dev=2:1.1.4-2 \ libxrandr-dev=2:1.5.2-0ubuntu1 \ libxxf86vm-dev=1:1.1.4-1build1 \ + libvulkan1=1.2.131.2-1 \ + vulkan-validationlayers=1.2.131.2-1 \ + spirv-tools=2020.1-2 \ + libegl1=1.3.2-1~ubuntu0.20.04.2 \ && rm -rf /var/lib/apt/lists/* # Install pip run dependencies -# Note: cupy pypi wheels not available for arm64 -RUN if [ $(uname -m) != "aarch64" ]; then \ - python3 -m pip install \ - cupy-cuda11x==11.3.0; \ +RUN if [ $(uname -m) == "aarch64" ]; then \ + python3 -m pip install cupy-cuda11x==11.3.0 -f https://pip.cupy.dev/aarch64; \ + else \ + python3 -m pip install cupy-cuda11x==11.3.0; \ fi ## COPY BUILT/DOWNLOADED dependencies in previous stages @@ -163,8 +176,19 @@ ENV CMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}:${ONNX_RUNTIME}" ARG VULKAN_SDK_VERSION ENV VULKAN_SDK=/opt/vulkansdk/${VULKAN_SDK_VERSION} COPY --from=vulkansdk-builder ${VULKAN_SDK}/x86_64/ ${VULKAN_SDK} -RUN mkdir -p /usr/share/vulkan/explicit_layer.d \ - && cp ${VULKAN_SDK}/etc/vulkan/explicit_layer.d/VkLayer_*.json /usr/share/vulkan/explicit_layer.d \ - && echo ${VULKAN_SDK}/lib > /etc/ld.so.conf.d/vulkan.conf +# We need to use the headers and shader compiler of the SDK but want to link against the +# Vulkan loader provided by the Ubuntu package. Therefore create a link in the SDK directory +# pointing to the system Vulkan loader library. +RUN rm -f ${VULKAN_SDK}/lib/libvulkan.so* \ + && ln -s /lib/$(uname -m)-linux-gnu/libvulkan.so.1 ${VULKAN_SDK}/lib/libvulkan.so +# Setup EGL for running headless Vulkan apps +RUN mkdir -p /usr/share/glvnd/egl_vendor.d \ + && echo -e "{\n\ + \"file_format_version\" : \"1.0.0\",\n\ + \"ICD\" : {\n\ + \"library_path\" : \"libEGL_nvidia.so.0\"\n\ + }\n\ +}\n" > /usr/share/glvnd/egl_vendor.d/10_nvidia.json + ENV PATH="${PATH}:${VULKAN_SDK}/bin" ENV CMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}:${VULKAN_SDK}" diff --git a/NOTICE.txt b/NOTICE.txt index 6fbea01b..2562a6d7 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1,1947 +1,109 @@ -Holoscan uses the following open source software: - - * cmake-modules - * CMake - * ColorBrewer - * CuPy - * DLPack - * expected - * fmt - * glad - * GLFW - * GoogleTest - * GXF - * yaml-cpp - * PyYAML - * nanovg - * Ninja - * AJA NTV2 SDK - * ONNX Runtime - * pybind11 - * pytest - * RAPIDS RAFT - * RAPIDS rapids-cmake - * RAPIDS rmm - * Google Fonts Roboto - * spdlog - * vscode-dev-containers - * yaml-cpp - -The full license text for each is listed below. - -================================================================================ -cmake-modules License -================================================================================ - -Copyright (c) 2012 - 2017, Lars Bilke -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - 3. Neither the name of the copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software without - specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR - ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON - ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -================================================================================ -CMake License -================================================================================ +Holoscan SDK +Copyright (c) 2021-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -CMake - Cross Platform Makefile Generator -Copyright 2000-2022 Kitware, Inc. and Contributors -All rights reserved. +This product includes software developed at NVIDIA CORPORATION (https://www.nvidia.com/). -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - -* Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -* Neither the name of Kitware, Inc. nor the names of Contributors - may be used to endorse or promote products derived from this - software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +Holoscan uses the following open source software: -================================================================================ -ColorBrewer License -================================================================================ +cmake-modules (https://github.com/bilke/cmake-modules) +Copyright (c) 2012 - 2017, Lars Bilke. All rights reserved. +Licensed under BSD-3-clause (https://github.com/bilke/cmake-modules/blob/master/CodeCoverage.cmake) -Apache-Style Software License for ColorBrewer software and ColorBrewer Color Schemes +CMake (https://gitlab.kitware.com/cmake/cmake) +Copyright 2000-2022 Kitware, Inc. and Contributors. All rights reserved. +Licensed under BSD-3 (https://gitlab.kitware.com/cmake/cmake/raw/master/Copyright.txt) +ColorBrewer (http://www.personal.psu.edu/cab38/ColorBrewer/ColorBrewer_all_schemes_RGBonly4_withPalette_and_Macro.xls) Copyright (c) 2002 Cynthia Brewer, Mark Harrower, and The Pennsylvania State University. +Licensed under Apache-2.0 (http://www.personal.psu.edu/cab38/ColorBrewer/ColorBrewer_updates.html) -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software distributed -under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -CONDITIONS OF ANY KIND, either express or implied. See the License for the -specific language governing permissions and limitations under the License. - - -================================================================================ -CuPy License -================================================================================ - +CuPy (https://pypi.org/project/cupy-cuda11x) Copyright (c) 2015 Preferred Infrastructure, Inc. Copyright (c) 2015 Preferred Networks, Inc. +Licensed under MIT (https://raw.githubusercontent.com/cupy/cupy/v11.3.0/LICENSE) -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - -================================================================================ -DLPack License -================================================================================ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and +DLPack (https://github.com/dmlc/dlpack) +Licensed under Apache-2.0 (https://github.com/dmlc/dlpack/blob/v0.7/LICENSE) - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2017 by Contributors - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -================================================================================ -expected License -================================================================================ - -Creative Commons Legal Code - -CC0 1.0 Universal - - CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE - LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN - ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS - INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES - REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS - PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM - THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED - HEREUNDER. - -Statement of Purpose - -The laws of most jurisdictions throughout the world automatically confer -exclusive Copyright and Related Rights (defined below) upon the creator -and subsequent owner(s) (each and all, an "owner") of an original work of -authorship and/or a database (each, a "Work"). - -Certain owners wish to permanently relinquish those rights to a Work for -the purpose of contributing to a commons of creative, cultural and -scientific works ("Commons") that the public can reliably and without fear -of later claims of infringement build upon, modify, incorporate in other -works, reuse and redistribute as freely as possible in any form whatsoever -and for any purposes, including without limitation commercial purposes. -These owners may contribute to the Commons to promote the ideal of a free -culture and the further production of creative, cultural and scientific -works, or to gain reputation or greater distribution for their Work in -part through the use and efforts of others. - -For these and/or other purposes and motivations, and without any -expectation of additional consideration or compensation, the person -associating CC0 with a Work (the "Affirmer"), to the extent that he or she -is an owner of Copyright and Related Rights in the Work, voluntarily -elects to apply CC0 to the Work and publicly distribute the Work under its -terms, with knowledge of his or her Copyright and Related Rights in the -Work and the meaning and intended legal effect of CC0 on those rights. - -1. Copyright and Related Rights. A Work made available under CC0 may be -protected by copyright and related or neighboring rights ("Copyright and -Related Rights"). Copyright and Related Rights include, but are not -limited to, the following: - - i. the right to reproduce, adapt, distribute, perform, display, - communicate, and translate a Work; - ii. moral rights retained by the original author(s) and/or performer(s); -iii. publicity and privacy rights pertaining to a person's image or - likeness depicted in a Work; - iv. rights protecting against unfair competition in regards to a Work, - subject to the limitations in paragraph 4(a), below; - v. rights protecting the extraction, dissemination, use and reuse of data - in a Work; - vi. database rights (such as those arising under Directive 96/9/EC of the - European Parliament and of the Council of 11 March 1996 on the legal - protection of databases, and under any national implementation - thereof, including any amended or successor version of such - directive); and -vii. other similar, equivalent or corresponding rights throughout the - world based on applicable law or treaty, and any national - implementations thereof. - -2. Waiver. To the greatest extent permitted by, but not in contravention -of, applicable law, Affirmer hereby overtly, fully, permanently, -irrevocably and unconditionally waives, abandons, and surrenders all of -Affirmer's Copyright and Related Rights and associated claims and causes -of action, whether now known or unknown (including existing as well as -future claims and causes of action), in the Work (i) in all territories -worldwide, (ii) for the maximum duration provided by applicable law or -treaty (including future time extensions), (iii) in any current or future -medium and for any number of copies, and (iv) for any purpose whatsoever, -including without limitation commercial, advertising or promotional -purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each -member of the public at large and to the detriment of Affirmer's heirs and -successors, fully intending that such Waiver shall not be subject to -revocation, rescission, cancellation, termination, or any other legal or -equitable action to disrupt the quiet enjoyment of the Work by the public -as contemplated by Affirmer's express Statement of Purpose. - -3. Public License Fallback. Should any part of the Waiver for any reason -be judged legally invalid or ineffective under applicable law, then the -Waiver shall be preserved to the maximum extent permitted taking into -account Affirmer's express Statement of Purpose. In addition, to the -extent the Waiver is so judged Affirmer hereby grants to each affected -person a royalty-free, non transferable, non sublicensable, non exclusive, -irrevocable and unconditional license to exercise Affirmer's Copyright and -Related Rights in the Work (i) in all territories worldwide, (ii) for the -maximum duration provided by applicable law or treaty (including future -time extensions), (iii) in any current or future medium and for any number -of copies, and (iv) for any purpose whatsoever, including without -limitation commercial, advertising or promotional purposes (the -"License"). The License shall be deemed effective as of the date CC0 was -applied by Affirmer to the Work. Should any part of the License for any -reason be judged legally invalid or ineffective under applicable law, such -partial invalidity or ineffectiveness shall not invalidate the remainder -of the License, and in such case Affirmer hereby affirms that he or she -will not (i) exercise any of his or her remaining Copyright and Related -Rights in the Work or (ii) assert any associated claims and causes of -action with respect to the Work, in either case contrary to Affirmer's -express Statement of Purpose. - -4. Limitations and Disclaimers. - - a. No trademark or patent rights held by Affirmer are waived, abandoned, - surrendered, licensed or otherwise affected by this document. - b. Affirmer offers the Work as-is and makes no representations or - warranties of any kind concerning the Work, express, implied, - statutory or otherwise, including without limitation warranties of - title, merchantability, fitness for a particular purpose, non - infringement, or the absence of latent or other defects, accuracy, or - the present or absence of errors, whether or not discoverable, all to - the greatest extent permissible under applicable law. - c. Affirmer disclaims responsibility for clearing rights of other persons - that may apply to the Work or any use thereof, including without - limitation any person's Copyright and Related Rights in the Work. - Further, Affirmer disclaims responsibility for obtaining any necessary - consents, permissions or other rights required for any use of the - Work. - d. Affirmer understands and acknowledges that Creative Commons is not a - party to this document and has no duty or obligation with respect to - this CC0 or use of the Work. - - -================================================================================ -fmt License -================================================================================ +expected (https://github.com/TartanLlama/expected) +Licensed under CC0-1.0 (https://github.com/TartanLlama/expected/blob/v1.0.0/COPYING) +fmt (https://github.com/fmtlib/fmt) Copyright (c) 2012 - present, Victor Zverovich +Licensed under MIT (https://github.com/fmtlib/fmt/blob/8.1.1/LICENSE.rst) -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - ---- Optional exception to the license --- - -As an exception, if, as a result of your compiling your source code, portions of this Software are embedded into a machine-executable object form of such source code, you may redistribute such embedded portions in such object form without including the above copyright and permission notices. - - -================================================================================ -glad License -================================================================================ - -The glad source code: - - The MIT License (MIT) - - Copyright (c) 2013-2021 David Herberth - - Permission is hereby granted, free of charge, to any person obtaining a copy of - this software and associated documentation files (the "Software"), to deal in - the Software without restriction, including without limitation the rights to - use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of - the Software, and to permit persons to whom the Software is furnished to do so, - subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER - IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - -The Khronos Specifications: - - Copyright (c) 2013-2020 The Khronos Group Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -The EGL Specification and various headers: - - Copyright (c) 2007-2016 The Khronos Group Inc. - - Permission is hereby granted, free of charge, to any person obtaining a - copy of this software and/or associated documentation files (the - "Materials"), to deal in the Materials without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Materials, and to - permit persons to whom the Materials are furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Materials. - - THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS. - - -================================================================================ -GLFW License -================================================================================ +glad (https://github.com/Dav1dde/glad) +Copyright (c) 2013-2021 David Herberth +Licensed under MIT (https://github.com/Dav1dde/glad/blob/v0.1.36/LICENSE) +GLFW (https://www.glfw.org/) Copyright (c) 2002-2006 Marcus Geelnard Copyright (c) 2006-2016 Camilla Berglund +Licensed under Zlib (https://github.com/glfw/glfw/blob/3.3.7/COPYING.txt) -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - -1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would - be appreciated but is not required. - -2. Altered source versions must be plainly marked as such, and must not - be misrepresented as being the original software. - -3. This notice may not be removed or altered from any source - distribution. - - -================================================================================ -GoogleTest License -================================================================================ +GoogleTest (https://github.com/google/googletest) +Copyright 2008, Google Inc. All rights reserved. +Licensed under BSD-3-Clause (https://github.com/google/googletest/blob/release-1.12.1/LICENSE) -Copyright 2008, Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -================================================================================ -GXF License -================================================================================ - -This project is provided under the following license: -================================================================================ +GXF Copyright 2021-2022, NVIDIA CORPORATION +Licensed under Apache-2.0 (http://www.apache.org/licenses/LICENSE-2.0) -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -================================================================================ - -This project uses the following software: - -yaml-cpp (Provided under following license) -================================================================================ -Copyright (c) 2008-2015 Jesse Beder. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -================================================================================ - -PyYAML (Provided under following license) -================================================================================ -Copyright (c) 2017-2020 Ingy döt Net -Copyright (c) 2006-2016 Kirill Simonov - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -================================================================================ - - -================================================================================ -nanovg License -================================================================================ - -Copyright (c) 2013 Mikko Mononen memon@inside.org - -This software is provided 'as-is', without any express or implied -warranty. In no event will the authors be held liable for any damages -arising from the use of this software. - -Permission is granted to anyone to use this software for any purpose, -including commercial applications, and to alter it and redistribute it -freely, subject to the following restrictions: - -1. The origin of this software must not be misrepresented; you must not -claim that you wrote the original software. If you use this software -in a product, an acknowledgment in the product documentation would be -appreciated but is not required. -2. Altered source versions must be plainly marked as such, and must not be -misrepresented as being the original software. -3. This notice may not be removed or altered from any source distribution. - - -================================================================================ -Ninja License -================================================================================ - - - Apache License - Version 2.0, January 2010 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -================================================================================ -AJA NTV2 SDK License -================================================================================ - -MIT License +Ninja (https://github.com/ninja-build/ninja) +Licensed under Apache 2 License (https://github.com/ninja-build/ninja/blob/master/COPYING) +AJA NTV2 SDK (https://github.com/ibstewart/ntv2) Copyright (c) 2021 AJA Video Systems +Licensed under MIT (https://github.com/ibstewart/ntv2/blob/holoscan-v0.2.0/LICENSE) -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - -================================================================================ -ONNX Runtime License -================================================================================ - -MIT License - +ONNX Runtime (https://github.com/microsoft/onnxruntime) Copyright (c) Microsoft Corporation +Licensed under MIT (https://github.com/microsoft/onnxruntime/blob/main/LICENSE) -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -================================================================================ -pybind11 License -================================================================================ - +pybind11 (https://github.com/pybind/pybind11) Copyright (c) 2016 Wenzel Jakob , All rights reserved. +Licensed under BSD-3-clause (https://github.com/pybind/pybind11/blob/v2.6.2/LICENSE) -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Please also refer to the file .github/CONTRIBUTING.md, which clarifies licensing of -external contributions to this project including patches, pull requests, etc. - -================================================================================ -pytest License -================================================================================ - -The MIT License (MIT) - +pytest (https://github.com/pytest-dev/pytest) Copyright (c) 2004 Holger Krekel and others +Licensed under MIT (https://github.com/pytest-dev/pytest/blob/main/LICENSE) -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -================================================================================ -RAPIDS RAFT License -================================================================================ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2020 NVIDIA Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -================================================================================ -RAPIDS rapids-cmake License -================================================================================ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2021 NVIDIA Corporation - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +RAPIDS RAFT (https://github.com/rapidsai/raft) +Copyright (c) 2020-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +Licensed under Apache-2.0 (https://github.com/rapidsai/raft/blob/branch-22.08/LICENSE) - http://www.apache.org/licenses/LICENSE-2.0 +RAPIDS rapids-cmake (https://github.com/rapidsai/rapids-cmake) +Copyright (c) 2020-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +Licensed under Apache-2.0 (https://github.com/rapidsai/rapids-cmake/blob/branch-22.08/LICENSE) - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +RAPIDS rmm (https://github.com/rapidsai/rmm) +Copyright (c) 2020-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +Licensed under Apache-2.0 (https://github.com/rapidsai/rmm/blob/branch-22.06/LICENSE) +Google Fonts Roboto (https://github.com/googlefonts/roboto/releases/download/v2.138/roboto-android.zip) +Licensed under Apache-2.0 (https://github.com/googlefonts/roboto/blob/v2.138/LICENSE) -================================================================================ -RAPIDS rmm License -================================================================================ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -================================================================================ -Google Fonts Roboto License -================================================================================ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -================================================================================ -spdlog License -================================================================================ - -The MIT License (MIT) - +spdlog (https://github.com/gabime/spdlog) Copyright (c) 2016 Gabi Melman. +Licensed under MIT (https://github.com/gabime/spdlog/blob/v1.10.0/LICENSE) -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +stb (https://github.com/nothings/stb) +Copyright (c) 2017 Sean Barrett +Licensed under MIT (https://github.com/nothings/stb/blob/master/LICENSE) --- NOTE: Third party dependency used by this software -- -This software depends on the fmt lib (MIT License), -and users must comply to its license: https://github.com/fmtlib/fmt/blob/master/LICENSE.rst +v4l-utils (https://nv-tegra.nvidia.com/tegra/v4l2-src/v4l2_libs.git) +Copyright (C) 1991, 1999 Free Software Foundation, Inc +Licensed under LGLP v2.1 (https://github.com/gjasny/v4l-utils/blob/master/COPYING.libv4l) +vscode-dev-containers (https://github.com/microsoft/vscode-dev-containers) +Copyright (c) Microsoft Corporation. All rights reserved. +Licensed under MIT (https://github.com/microsoft/vscode-dev-containers/blob/v0.241.1/LICENSE) -================================================================================ -vscode-dev-containers License -================================================================================ - - MIT License - - Copyright (c) Microsoft Corporation. All rights reserved. - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE - -================================================================================ -yaml-cpp License -================================================================================ +wheel-axle (https://pypi.org/project/wheel-axle/) +(C) Copyright 2022 Karellen, Inc. (https://www.karellen.co/) +Licensed under Apache-2.0 (https://github.com/karellen/wheel-axle/blob/v0.0.5/LICENSE) +yaml-cpp (https://github.com/jbeder/yaml-cpp) Copyright (c) 2008-2015 Jesse Beder. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - +Licensed under MIT (https://github.com/jbeder/yaml-cpp/blob/yaml-cpp-0.7.0/LICENSE) diff --git a/README.md b/README.md index 9996d70d..446ec080 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Holoscan SDK -The **Holoscan SDK** is part of [NVIDIA Holoscan](https://github.com/nvidia-holoscan), the AI sensor processing platform that combines hardware systems for low-latency sensor and network connectivity, optimized libraries for data processing and AI, and core microservices to run streaming, imaging, and other applications, from embedded to edge to cloud. It can be used to build streaming AI pipelines for a variety of domains, including Medical Devices, High Performance Computing at the Edge, Industrial Inspection and more. +The **Holoscan SDK** is part of [NVIDIA Holoscan](https://developer.nvidia.com/holoscan-sdk), the AI sensor processing platform that combines hardware systems for low-latency sensor and network connectivity, optimized libraries for data processing and AI, and core microservices to run streaming, imaging, and other applications, from embedded to edge to cloud. It can be used to build streaming AI pipelines for a variety of domains, including Medical Devices, High Performance Computing at the Edge, Industrial Inspection and more. > In previous releases, the prefix [`Clara`](https://developer.nvidia.com/industries/healthcare) was used to define Holoscan as a platform designed initially for [medical devices](https://www.nvidia.com/en-us/clara/developer-kits/). As Holoscan has grown, its potential to serve other areas has become apparent. With version 0.4.0, we're proud to announce that the Holoscan SDK is now officially built to be domain-agnostic and can be used to build sensor AI applications in multiple domains. Note that some of the content of the SDK (sample applications) or the documentation might still appear to be healthcare-specific pending additional updates. Going forward, domain specific content will be hosted on the [HoloHub](https://github.com/nvidia-holoscan/holohub) repository. @@ -19,6 +19,7 @@ Visit the [NGC demo website](https://demos.ngc.nvidia.com/holoscan) for a live d - [Advanced: Local environment + CMake](#advanced-local-environment-cmake) - [Utilities](#utilities) - [Code coverage](#code-coverage) + - [Linting](#linting) - [Troubleshooting](#troubleshooting) - [Repository structure](#repository-structure) @@ -30,13 +31,13 @@ Visit the [NGC demo website](https://demos.ngc.nvidia.com/holoscan) for a live d ## Using the released SDK The Holoscan SDK is available as part of the following packages: -- 🋠The [Holoscan container image on NGC](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/clara-holoscan/containers/holoscan) includes the Holoscan libraries, GXF extensions, headers, example source code, and sample datasets, as well as all the dependencies that were tested with Holoscan. It is the recommended way to run sample streaming applications, while still allowing you to create your own C++ and Python Holoscan application. -- ðŸ The [Holoscan python wheels on PyPI](https://pypi.org/project/holoscan/) (**NEW IN 0.4**) are the ideal way for Python developers to get started with the SDK, simply using `pip install holoscan`. The wheels include the necessary libraries and extensions, not including example code, built applications, nor sample datasets. -- ðŸ“¦ï¸ The [Holoscan Debian package on NGC](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/clara-holoscan/resources/holoscan_dev_deb) (**NEW IN 0.4**) includes the libraries, headers, and CMake configurations needed for both C++ and Python developers. It does not include example code, pre-built applications, nor sample datasets. +- 🋠The [Holoscan container image on NGC](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/clara-holoscan/containers/holoscan) includes the Holoscan libraries, GXF extensions, headers, example source code, and sample datasets, as well as all the dependencies that were tested with Holoscan. It is the recommended way to run the Holoscan examples, while still allowing you to create your own C++ and Python Holoscan application. +- ðŸ The [Holoscan python wheels on PyPI](https://pypi.org/project/holoscan/) are the simplest way for Python developers to get started with the SDK using `pip install holoscan`. The wheels include the SDK libraries, not the example applications or the sample datasets. +- ðŸ“¦ï¸ The [Holoscan Debian package on NGC](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/clara-holoscan/resources/holoscan_dev_deb) includes the libraries, headers, example applications and CMake configurations needed for both C++ and Python developers. It does not include sample datasets. ## Building the SDK from source -> **Disclaimer**: we only recommend building the SDK from source if you are a developer of the SDK, or need to build the SDK with debug symbols or other options not used as part of the released packages. If you want to modify some code to fit your use case, contribute to HoloHub if it's an operator or application, or file a feature or bug request. If that's not the case, prefer using [generated packages](using-the-released-sdk). +> **Disclaimer**: we only recommend building the SDK from source if you are a developer of the SDK, or need to build the SDK with debug symbols or other options not used as part of the released packages. If you want to modify some code to fit your use case, contribute to HoloHub if it's an operator or application, or file a feature or bug request. If that's not the case, prefer using [generated packages](#using-the-released-sdk). ### Prerequisites @@ -45,14 +46,15 @@ The Holoscan SDK currently supports the [Holoscan Developer Kits](https://www.nv #### For Clara AGX and NVIDIA IGX Orin Developer Kits (aarch64) Set up your developer kit: -- [Clara AGX Developer Kit User Guide](https://developer.nvidia.com/clara-agx-developer-kit-user-guide), or -- [NVIDIA IGX Orin Developer Kit User Guide](https://developer.nvidia.com/igx-orin-developer-kit-user-guide). +- [NVIDIA Clara AGX](https://developer.nvidia.com/clara-agx-developer-kit-user-guide) +- [NVIDIA IGX Orin [ES]](https://developer.nvidia.com/igx-orin-developer-kit-user-guide) +- NVIDIA IGX Orin: *coming soon* > Make sure you have joined the [Holoscan SDK Program](https://developer.nvidia.com/clara-holoscan-sdk-program) and, if needed, the [Rivermax SDK Program](https://developer.nvidia.com/nvidia-rivermax-sdk) before using the NVIDIA SDK Manager. - [SDK Manager](https://docs.nvidia.com/sdk-manager/install-with-sdkm-clara/) will install **Holopack 1.2** as well as the `nvgpuswitch.py` script. Once configured for dGPU mode, your developer kit will include the necessary components to build the SDK -- For Rivermax support (optional/local development only at this time), GPUDirect drivers need to be installed manually at this time, see the [User Guide](https://docs.nvidia.com/clara-holoscan/sdk-user-guide/additional_setup.html#setting-up-gpudirect-rdma). -- Refer to the user guide for additional steps needed to support third-party technologies, such as [AJA cards](https://docs.nvidia.com/clara-holoscan/sdk-user-guide/aja_setup.html) or [Emergent cameras](https://docs.nvidia.com/clara-holoscan/sdk-user-guide/emergent_setup.html). +- For Rivermax support (optional/local development only at this time), GPUDirect drivers need to be loaded manually at this time, see the [User Guide](https://docs.nvidia.com/clara-holoscan/sdk-user-guide/additional_setup.html#setting-up-gpudirect-rdma). +- Refer to the user guide for additional steps needed to support third-party technologies, such as [AJA cards](https://docs.nvidia.com/clara-holoscan/sdk-user-guide/aja_setup.html). - Additional dependencies are required when developing locally instead of using a containerized environment, see details in the [section below](#advanced-local-environment--cmake). #### For x86_64 systems @@ -93,18 +95,17 @@ Call the **`./run launch`** command to start and enter the development container * *Execute `./run launch --help` for more information* * *Execute `./run launch --dryrun` to see the commands that will be executed* -**Run the applications or examples** inside the container by running their respective commands listed within each directory README file: - * [reference applications](./apps) - * [examples](./examples) +Run the [**examples**](./examples) inside the container by running their respective commands listed within each directory README file. ### Cross-compilation While the development Dockerfile does not currently support true cross-compilation, you can compile the Holoscan SDK for the developer kits (arm64) from a x86_64 host using an emulation environment. 1. [Install qemu](https://docs.nvidia.com/datacenter/cloud-native/playground/x-arch.html#emulation-environment) -2. Export your target arch: `export HOLOSCAN_BUILD_PLATFORM=linux/arm64` -3. Clear your build cache: `./run clear_cache` -4. Rebuild: `./run build` +2. Clear your build cache: `./run clear_cache` +3. Rebuild for `linux/arm64` using `--platform` or `HOLOSCAN_BUILD_PLATFORM`: + * `./run build --platform linux/arm64` + * `HOLOSCAN_BUILD_PLATFORM=linux/arm64 ./run build` You can then copy the `install` folder generated by CMake to a developer kit with a configured environment or within a container to use for running and developing applications. @@ -155,17 +156,52 @@ cmake --build $build_dir -j cmake --install $build_dir --prefix $install_dir ``` -The commands to run the **[reference applications](./apps)** or the **[examples](./examples)** are then the same as in the dockerized environment, and can be found in the respective source directory READMEs. +The commands to run the [**examples**](./examples) are then the same as in the dockerized environment, and can be found in the respective source directory READMEs. ## Utilities +Some utilities are available in the [`scripts`](./scripts) folder, others closer to the built process are listed below: + ### Code coverage + To generate a code coverage report use the following command after completing setup: ```sh ./run coverage ``` -Open the file build/coverage/index.html to access the interactive coverage web tool. +Open the file `build/coverage/index.html` to access the interactive coverage web tool. + +### Linting + +Run the following command to run various linting tools on the repository: +```sh +./run lint # optional: specify directories +``` + +> Note: Run `run lint --help` to see the list of tools that are used. If a lint command fails due to a missing module or executable on your system, you can install it using `python3 -m pip install `. + +### Working with Visual Studio Code + +Visual Studio Code can be utilized to develop Holoscan SDK. The `.devcontainer` folder holds the configuration for setting up a [development container](https://code.visualstudio.com/docs/remote/containers) with all necessary tools and libraries installed. + +The `./run` script contains `vscode` and `vscode_remote` commands for launching Visual Studio Code in a container or from a remote machine, respectively. + +- To launch Visual Studio Code in a dev container, use `./run vscode`. +- To attach to an existing dev container from a remote machine, use `./run vscode_remote`. For more information, refer to the instructions from `./run vscode_remote -h`. + +Once Visual Studio Code is launched, the development container will be built and the recommended extensions will be installed automatically, along with CMake being configured. + +#### Configuring CMake in the Development Container + +For manual configuration of CMake, open the command palette (`Ctrl + Shift + P`) and run the `CMake: Configure` command. + +#### Building the Source Code in the Development Container + +The source code in the development container can be built by either pressing `Ctrl + Shift + B` or executing `Tasks: Run Build Task` from the command palette (`Ctrl + Shift + P`). + +#### Debugging the Source Code in the Development Container + +To debug the source code in the development container, open the `Run and Debug` view (`Ctrl + Shift + D`), select a debug configuration from the dropdown list, and press `F5` to initiate debugging. ## Troubleshooting @@ -223,16 +259,58 @@ Failed to open device, OPEN: Permission denied ``` This means the `/dev/video*` device is not available to the user from within docker. You can make it publicly available with `sudo chmod 666 /dev/video*` from your host. +### Cuda driver error 101 (CUDA_ERROR_INVALID_DEVICE): invalid device ordinal + +The error can happen when running in a multi-GPU environment: + +``` +[2023-02-10 10:42:31.039] [holoscan] [error] [gxf_wrapper.cpp:68] Exception occurred for operator: 'holoviz' - Cuda driver error 101 (CUDA_ERROR_INVALID_DEVICE): invalid device ordinal +2023-02-10 10:42:31.039 ERROR gxf/std/entity_executor.cpp@509: Failed to tick codelet holoviz in entity: holoviz code: GXF_FAILURE +``` + +This is due to the fact that operators in your application run on different GPUs, depending on their internal implementation, preventing data from being freely exchanged between them. + +To fix the issue, either: +1. Configure or modify your operators to copy data across the appropriate GPUs (using [`cuPointerGetAttribute`](https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__UNIFIED.html#group__CUDA__UNIFIED_1g0c28ed0aff848042bc0533110e45820c) and [`cuMemCpyPeerAsync()`](https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g82fcecb38018e64b98616a8ac30112f2) internally). This comes at a cost and should only be done when leveraging multiple GPUs is required for improving performance (parallel processing). +2. Configure or modify your operators to use the same GPU (using [`cudaSetDevice()`](https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__DEVICE.html#group__CUDART__DEVICE_1g159587909ffa0791bbe4b40187a4c6bb) internally). +3. Restrict your environment using [`CUDA_VISIBLE_DEVICES`](https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html?highlight=CUDA_VISIBLE_DEVICES#cuda-environment-variables) to only expose a single GPU: + ```sh + export CUDA_VISIBLE_DEVICES= + ``` + +Note that this generally occurs because the `HolovizOp` operator needs to use the GPU connected to the display, but other operators in the Holoscan pipeline might default to another GPU depending on their internal implementation. The index of the GPU connected to the display can be found by running `nvidia-smi` from a command prompt and looking for the GPU where the `Disp.A` value shows `On`. In the example below, the GPU `0` should be used before passing data to `HolovizOp`. + +``` ++-----------------------------------------------------------------------------+ +| NVIDIA-SMI 525.85.12 Driver Version: 525.85.12 CUDA Version: 12.0 | +|-------------------------------+----------------------+----------------------+ +| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | +| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | +| | | MIG M. | +|===============================+======================+======================| +| 0 NVIDIA RTX A5000 On | 00000000:09:00.0 On | Off | +| 30% 29C P8 11W / 230W | 236MiB / 24564MiB | 0% Default | +| | | N/A | ++-------------------------------+----------------------+----------------------+ +| 1 NVIDIA RTX A5000 On | 00000000:10:00.0 Off | Off | +| 30% 29C P8 11W / 230W | 236MiB / 24564MiB | 0% Default | +| | | N/A | ++-------------------------------+----------------------+----------------------+ +``` + +_Note: Holoviz should support copying data across CUDA devices in a future release._ ## Repository structure -The repository is organized as such: -- `apps/`: source code for the sample applications +The repository is organized as such:s - `cmake/`: CMake configuration files +- `data/`: directory where data will be downloaded - `examples/`: source code for the examples - `gxf_extensions/`: source code for the holoscan SDK gxf codelets -- `include/`: source code for the holoscan SDK +- `include/`: source code for the holoscan SDK core - `modules/`: source code for the holoscan SDK modules +- `patches/`: patch files applied to dependencies +- `python/`: python bindings for the holoscan SDK - `scripts/`: utility scripts -- `src/`: source code for the holoscan SDK +- `src/`: source code for the holoscan SDK core - `tests/`: tests for the holoscan SDK diff --git a/VERSION b/VERSION index 60a2d3e9..8f0916f7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4.0 \ No newline at end of file +0.5.0 diff --git a/apps/endoscopy_tool_tracking/README.md b/apps/endoscopy_tool_tracking/README.md deleted file mode 100644 index ccd55754..00000000 --- a/apps/endoscopy_tool_tracking/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Endoscopy Tool Tracking - -Based on a LSTM (long-short term memory) stateful model, these applications demonstrate the use of custom components for tool tracking, including composition and rendering of text, tool position, and mask (as heatmap) combined with the original video stream. - -### Requirements - -The provided applications are configured to either use the AJA capture card for input stream, or a pre-recorded endoscopy video (replayer). Follow the [setup instructions from the user guide](https://docs.nvidia.com/clara-holoscan/sdk-user-guide/aja_setup.html) to use the AJA capture card. - -### Data - -[ðŸ“¦ï¸ (NGC) Sample App Data for AI-based Endoscopy Tool Tracking](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/clara-holoscan/resources/holoscan_endoscopy_sample_data) - -### Build Instructions - -Built with the SDK, see instructions from the top level README. - -### Run Instructions - -First, go in your `build` or `install` directory (automatically done by `./run launch`). - -Then, run the commands of your choice: - -* Using a pre-recorded video - ```bash - # C++ - sed -i -e 's#^source:.*#source: replayer#' ./apps/endoscopy_tool_tracking/cpp/app_config.yaml \ - && ./apps/endoscopy_tool_tracking/cpp/endoscopy_tool_tracking - - # Python - python3 ./apps/endoscopy_tool_tracking/python/endoscopy_tool_tracking.py --source=replayer - ``` - -* Using an AJA card - ```bash - # C++ - sed -i -e 's#^source:.*#source: aja#' ./apps/endoscopy_tool_tracking/cpp/app_config.yaml \ - && ./apps/endoscopy_tool_tracking/cpp/endoscopy_tool_tracking - - # Python - python3 ./apps/endoscopy_tool_tracking/python/endoscopy_tool_tracking.py --source=aja - ``` - -> â„¹ï¸ The python app can run outside those folders if `HOLOSCAN_SAMPLE_DATA_PATH` is set in your environment (automatically done by `./run launch`). \ No newline at end of file diff --git a/apps/endoscopy_tool_tracking/cpp/CMakeLists.txt b/apps/endoscopy_tool_tracking/cpp/CMakeLists.txt deleted file mode 100644 index 9b480201..00000000 --- a/apps/endoscopy_tool_tracking/cpp/CMakeLists.txt +++ /dev/null @@ -1,53 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -add_executable(endoscopy_tool_tracking - main.cpp -) - -target_link_libraries(endoscopy_tool_tracking - PRIVATE - ${HOLOSCAN_PACKAGE_NAME} -) - -# Download the associated dataset if needed -if(HOLOSCAN_DOWNLOAD_DATASETS) - add_dependencies(endoscopy_tool_tracking endoscopy_data) -endif() - -# Copy config file -add_custom_target(endoscopy_tool_tracking_yaml - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/app_config.yaml" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/app_config.yaml" -) -add_dependencies(endoscopy_tool_tracking endoscopy_tool_tracking_yaml) - -# Set the install RPATH based on the location of the Holoscan SDK libraries -# The GXF extensions are loaded by the GXF libraries - no need to include here -file(RELATIVE_PATH install_lib_relative_path ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_SOURCE_DIR}/${HOLOSCAN_INSTALL_LIB_DIR} ) -set_target_properties(endoscopy_tool_tracking PROPERTIES INSTALL_RPATH "\$ORIGIN/${install_lib_relative_path}") - -# Get relative folder path for the app -file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) - -# Install the app -install(TARGETS endoscopy_tool_tracking - DESTINATION "${app_relative_dest_path}" - COMPONENT holoscan-apps -) -install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/app_config.yaml" - DESTINATION ${app_relative_dest_path} - COMPONENT holoscan-apps -) diff --git a/apps/endoscopy_tool_tracking/cpp/app_config.yaml b/apps/endoscopy_tool_tracking/cpp/app_config.yaml deleted file mode 100644 index 760f856e..00000000 --- a/apps/endoscopy_tool_tracking/cpp/app_config.yaml +++ /dev/null @@ -1,167 +0,0 @@ -%YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -extensions: - - libgxf_std.so - - libgxf_cuda.so - - libgxf_multimedia.so - - libgxf_serialization.so - - libaja_source.so - - libcustom_lstm_inference.so - - libformat_converter.so - - libholoviz.so - - libstream_playback.so - - libtool_tracking_postprocessor.so - -source: "replayer" # or "aja" -record_type: "none" # or "input" if you want to record input video stream, or "visualizer" if you want - # to record the visualizer output. - -aja: - width: 1920 - height: 1080 - rdma: true - enable_overlay: false - overlay_rdma: true - -replayer: - directory: "../data/endoscopy/video" - basename: "surgical_video" - frame_rate: 0 # as specified in timestamps - repeat: true # default: false - realtime: true # default: true - count: 0 # default: 0 (no frame count restriction) - -recorder_format_converter: - in_dtype: "rgba8888" - out_dtype: "rgb888" - -recorder: - directory: "/tmp" - basename: "tensor" - -format_converter_replayer: - out_tensor_name: source_video - out_dtype: "float32" - scale_min: 0.0 - scale_max: 255.0 - -format_converter_aja: - in_dtype: "rgba8888" - out_tensor_name: source_video - out_dtype: "float32" - scale_min: 0.0 - scale_max: 255.0 - resize_width: 854 - resize_height: 480 - -lstm_inference: - model_file_path: ../data/endoscopy/model/tool_loc_convlstm.onnx - engine_cache_dir: ../data/endoscopy/model/tool_loc_convlstm_engines - input_tensor_names: - - source_video - - cellstate_in - - hiddenstate_in - input_state_tensor_names: - - cellstate_in - - hiddenstate_in - input_binding_names: - - data_ph:0 # (shape=[1, 480, 854, 3], dtype=float32) <==> source_video - - cellstate_ph:0 # (shape=[1, 60, 107, 7], dtype=float32) == internal state - - hiddenstate_ph:0 # (shape=[1, 60, 107, 7], dtype=float32) == internal state - output_tensor_names: - - cellstate_out - - hiddenstate_out - - probs - - scaled_coords - - binary_masks - output_state_tensor_names: - - cellstate_out - - hiddenstate_out - output_binding_names: - - Model/net_states:0 # (shape=[ 1, 60, 107, 7], dtype=float32) - - Model/net_hidden:0 # (shape=[ 1, 60, 107, 7], dtype=float32) - - probs:0 # (shape=[1, 7], dtype=float32) - - Localize/scaled_coords:0 # (shape=[1, 7, 2], dtype=float32) - - Localize_1/binary_masks:0 # (shape=[1, 7, 60, 107], dtype=float32) - force_engine_update: false - verbose: true - max_workspace_size: 2147483648 - enable_fp16_: true - -visualizer_format_converter_replayer: - in_dtype: "rgb888" - out_dtype: "rgba8888" - # out_tensor_name: video - -tool_tracking_postprocessor: - -holoviz: - tensors: - - name: "" - type: color - opacity: 1.0 - priority: 0 - - name: mask - type: color - opacity: 1.0 - priority: 1 - - name: scaled_coords - type: crosses - opacity: 1.0 - line_width: 4 - color: [1.0, 0.0, 0.0, 1.0] - priority: 2 - - name: scaled_coords - type: text - opacity: 1.0 - priority: 3 - color: [1.0, 1.0, 1.0, 0.9] - text: - - Grasper - - Bipolar - - Hook - - Scissors - - Clipper - - Irrigator - - Spec.Bag - -holoviz_overlay: - headless: true - tensors: - - name: mask - type: color - opacity: 1.0 - priority: 1 - - name: scaled_coords - type: crosses - opacity: 1.0 - line_width: 4 - color: [1.0, 0.0, 0.0, 1.0] - priority: 2 - - name: scaled_coords - type: text - opacity: 1.0 - priority: 3 - color: [1.0, 1.0, 1.0, 0.9] - text: - - Grasper - - Bipolar - - Hook - - Scissors - - Clipper - - Irrigator - - Spec.Bag \ No newline at end of file diff --git a/apps/endoscopy_tool_tracking/cpp/main.cpp b/apps/endoscopy_tool_tracking/cpp/main.cpp deleted file mode 100644 index b3ed5946..00000000 --- a/apps/endoscopy_tool_tracking/cpp/main.cpp +++ /dev/null @@ -1,177 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -class App : public holoscan::Application { - public: - void set_source(const std::string& source) { - if (source == "aja") { is_aja_source_ = true; } - } - enum class Record { NONE, INPUT, VISUALIZER }; - void set_record(const std::string& record) { - if (record == "input") { - record_type_ = Record::INPUT; - } else if (record == "visualizer") { - record_type_ = Record::VISUALIZER; - } - } - - void compose() override { - using namespace holoscan; - - std::shared_ptr source; - std::shared_ptr recorder; - std::shared_ptr recorder_format_converter; - - const bool is_rdma = from_config("aja.rdma").as(); - const bool is_aja_overlay_enabled = - is_aja_source_ && from_config("aja.enable_overlay").as(); - uint32_t width = 0; - uint32_t height = 0; - uint64_t source_block_size = 0; - uint64_t source_num_blocks = 0; - - if (is_aja_source_) { - width = from_config("aja.width").as(); - height = from_config("aja.height").as(); - source = make_operator("aja", from_config("aja")); - source_block_size = width * height * 4 * 4; - source_num_blocks = is_rdma ? 3 : 4; - } else { - width = 854; - height = 480; - source = make_operator("replayer", from_config("replayer")); - source_block_size = width * height * 3 * 4; - source_num_blocks = 2; - } - - if (record_type_ != Record::NONE) { - if (((record_type_ == Record::INPUT) && is_aja_source_) || - (record_type_ == Record::VISUALIZER)) { - recorder_format_converter = make_operator( - "recorder_format_converter", - from_config("recorder_format_converter"), - Arg("pool") = - make_resource("pool", 1, source_block_size, source_num_blocks)); - } - recorder = make_operator("recorder", from_config("recorder")); - } - - auto format_converter = make_operator( - "format_converter", - from_config(is_aja_source_ ? "format_converter_aja" : "format_converter_replayer"), - Arg("pool") = - make_resource("pool", 1, source_block_size, source_num_blocks)); - - auto lstm_inferer = make_operator( - "lstm_inferer", - from_config("lstm_inference"), - Arg("pool") = make_resource("pool"), - Arg("cuda_stream_pool") = make_resource("cuda_stream", 0, 0, 0, 1, 5)); - - // A known issue using the BlockMemoryPool when converting the ONNX model to TRT from - // the LSTMTensorRTInferenceOp causes the inference to produce wrong values the first - // time the conversion is done inline. - // Switching from BlockMemoryPool to UnboundedAllocator or increasing the block size of the - // BlockMemoryPool seems to fix the issue. - // Using TensorRT 8.4.1 and above seems also to be fixing the issue. - // const uint64_t tool_tracking_postprocessor_block_size = 107 * 60 * 7 * 4; - // const uint64_t tool_tracking_postprocessor_num_blocks = 2; - auto tool_tracking_postprocessor = make_operator( - "tool_tracking_postprocessor", - from_config("tool_tracking_postprocessor"), - // Arg("device_allocator") = make_resource("device_allocator", - // 1, - // tool_tracking_postprocessor_block_size, - // tool_tracking_postprocessor_num_blocks), - Arg("device_allocator") = make_resource("device_allocator"), - Arg("host_allocator") = make_resource("host_allocator")); - - std::shared_ptr visualizer_allocator; - if ((record_type_ == Record::VISUALIZER) && !is_aja_source_) { - visualizer_allocator = - make_resource("allocator", 1, source_block_size, source_num_blocks); - } - std::shared_ptr visualizer = make_operator( - "holoviz", - from_config(is_aja_overlay_enabled ? "holoviz_overlay" : "holoviz"), - Arg("width") = width, - Arg("height") = height, - Arg("enable_render_buffer_input") = is_aja_overlay_enabled, - Arg("enable_render_buffer_output") = is_aja_overlay_enabled || - (record_type_ == Record::VISUALIZER), - Arg("allocator") = visualizer_allocator); - - // Flow definition - add_flow(lstm_inferer, tool_tracking_postprocessor, {{"tensor", "in"}}); - add_flow(tool_tracking_postprocessor, visualizer, {{"out", "receivers"}}); - - add_flow(source, - format_converter, - {{is_aja_source_ ? "video_buffer_output" : "output", "source_video"}}); - add_flow(format_converter, lstm_inferer); - - if (is_aja_overlay_enabled) { - // Overlay buffer flow between AJA source and visualizer - add_flow(source, visualizer, {{"overlay_buffer_output", "render_buffer_input"}}); - add_flow(visualizer, source, {{"render_buffer_output", "overlay_buffer_input"}}); - } else { - add_flow(source, visualizer, {{is_aja_source_ ? "video_buffer_output" : "output", - "receivers"}}); - } - - if (record_type_ == Record::INPUT) { - if (is_aja_source_) { - add_flow(source, recorder_format_converter, {{"video_buffer_output", "source_video"}}); - add_flow(recorder_format_converter, recorder); - } else { - add_flow(source, recorder); - } - } else if (record_type_ == Record::VISUALIZER) { - add_flow(visualizer, recorder_format_converter, {{"render_buffer_output", "source_video"}}); - add_flow(recorder_format_converter, recorder); - } - } - - private: - bool is_aja_source_ = false; - Record record_type_ = Record::NONE; -}; - -int main(int argc, char** argv) { - holoscan::load_env_log_level(); - - auto app = holoscan::make_application(); - - if (argc == 2) { - app->config(argv[1]); - } else { - auto config_path = std::filesystem::canonical(argv[0]).parent_path(); - config_path += "/app_config.yaml"; - app->config(config_path); - } - auto source = app->from_config("source").as(); - app->set_source(source); - auto record_type = app->from_config("record_type").as(); - app->set_record(record_type); - - app->run(); - - return 0; -} diff --git a/apps/endoscopy_tool_tracking/python/endoscopy_tool_tracking.py b/apps/endoscopy_tool_tracking/python/endoscopy_tool_tracking.py deleted file mode 100644 index b0fd63ea..00000000 --- a/apps/endoscopy_tool_tracking/python/endoscopy_tool_tracking.py +++ /dev/null @@ -1,250 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from argparse import ArgumentParser - -from holoscan.core import Application -from holoscan.logger import load_env_log_level -from holoscan.operators import ( - AJASourceOp, - FormatConverterOp, - HolovizOp, - LSTMTensorRTInferenceOp, - ToolTrackingPostprocessorOp, - VideoStreamRecorderOp, - VideoStreamReplayerOp, -) -from holoscan.resources import ( - BlockMemoryPool, - CudaStreamPool, - MemoryStorageType, - UnboundedAllocator, -) - -sample_data_path = os.environ.get("HOLOSCAN_SAMPLE_DATA_PATH", "../data") - - -class EndoscopyApp(Application): - def __init__(self, record_type=None, source="replayer"): - """Initialize the endoscopy tool tracking application - - Parameters - ---------- - record_type : {None, "input", "visualizer"}, optional - Set to "input" if you want to record the input video stream, or - "visualizer" if you want to record the visualizer output. - source : {"replayer", "aja"} - When set to "replayer" (the default), pre-recorded sample video data is - used as the application input. Otherwise, the video stream from an AJA - capture card is used. - """ - super().__init__() - - # set name - self.name = "Endoscopy App" - - # Optional parameters affecting the graph created by compose. - self.record_type = record_type - if record_type is not None: - if record_type not in ("input", "visualizer"): - raise ValueError("record_type must be either ('input' or 'visualizer')") - self.source = source - - def compose(self): - is_aja = self.source.lower() == "aja" - record_type = self.record_type - is_aja_overlay_enabled = False - - if is_aja: - aja_kwargs = self.kwargs("aja") - source = AJASourceOp(self, name="aja", **aja_kwargs) - - # 4 bytes/channel, 4 channels - width = aja_kwargs["width"] - height = aja_kwargs["height"] - is_rdma = aja_kwargs["rdma"] - is_aja_overlay_enabled = is_aja and aja_kwargs["enable_overlay"] - source_block_size = width * height * 4 * 4 - source_num_blocks = 3 if is_rdma else 4 - else: - width = 854 - height = 480 - video_dir = os.path.join(sample_data_path, "endoscopy", "video") - if not os.path.exists(video_dir): - raise ValueError(f"Could not find video data: {video_dir=}") - source = VideoStreamReplayerOp( - self, - name="replayer", - directory=video_dir, - **self.kwargs("replayer"), - ) - # 4 bytes/channel, 3 channels - source_block_size = width * height * 3 * 4 - source_num_blocks = 2 - - source_pool_kwargs = dict( - storage_type=MemoryStorageType.DEVICE, - block_size=source_block_size, - num_blocks=source_num_blocks, - ) - if record_type is not None: - if ((record_type == "input") and is_aja) or (record_type == "visualizer"): - recorder_format_converter = FormatConverterOp( - self, - name="recorder_format_converter", - pool=BlockMemoryPool(self, name="pool", **source_pool_kwargs), - **self.kwargs("recorder_format_converter"), - ) - recorder = VideoStreamRecorderOp( - name="recorder", fragment=self, **self.kwargs("recorder") - ) - - if is_aja: - config_key_name = "format_converter_aja" - else: - config_key_name = "format_converter_replayer" - - format_converter = FormatConverterOp( - self, - name="format_converter", - pool=BlockMemoryPool(self, name="pool", **source_pool_kwargs), - **self.kwargs(config_key_name), - ) - - lstm_cuda_stream_pool = CudaStreamPool( - self, - name="cuda_stream", - dev_id=0, - stream_flags=0, - stream_priority=0, - reserved_size=1, - max_size=5, - ) - model_file_path = os.path.join( - sample_data_path, "endoscopy", "model", "tool_loc_convlstm.onnx" - ) - engine_cache_dir = os.path.join( - sample_data_path, "endoscopy", "model", "tool_loc_convlstm_engines" - ) - lstm_inferer = LSTMTensorRTInferenceOp( - self, - name="lstm_inferer", - pool=UnboundedAllocator(self, name="pool"), - cuda_stream_pool=lstm_cuda_stream_pool, - model_file_path=model_file_path, - engine_cache_dir=engine_cache_dir, - **self.kwargs("lstm_inference"), - ) - - tool_tracking_postprocessor_block_size = 107 * 60 * 7 * 4 - tool_tracking_postprocessor_num_blocks = 2 - tool_tracking_postprocessor = ToolTrackingPostprocessorOp( - self, - name="tool_tracking_postprocessor", - device_allocator=BlockMemoryPool( - self, - name="device_allocator", - storage_type=MemoryStorageType.DEVICE, - block_size=tool_tracking_postprocessor_block_size, - num_blocks=tool_tracking_postprocessor_num_blocks, - ), - host_allocator=UnboundedAllocator(self, name="host_allocator"), - ) - - if (record_type == "visualizer") and not is_aja: - visualizer_allocator = BlockMemoryPool(self, name="allocator", **source_pool_kwargs) - else: - visualizer_allocator = None - - visualizer = HolovizOp( - self, - name="holoviz", - width=width, - height=height, - enable_render_buffer_input=is_aja_overlay_enabled, - enable_render_buffer_output=is_aja_overlay_enabled or record_type == "visualizer", - allocator=visualizer_allocator, - **self.kwargs("holoviz_overlay" if is_aja_overlay_enabled else "holoviz"), - ) - - # Flow definition - self.add_flow(lstm_inferer, tool_tracking_postprocessor, {("tensor", "in")}) - self.add_flow(tool_tracking_postprocessor, visualizer, {("out", "receivers")}) - self.add_flow( - source, - format_converter, - {("video_buffer_output" if is_aja else "output", "source_video")}, - ) - self.add_flow(format_converter, lstm_inferer) - if is_aja_overlay_enabled: - # Overlay buffer flow between AJA source and visualizer - self.add_flow(source, visualizer, {("overlay_buffer_output", "render_buffer_input")}) - self.add_flow(visualizer, source, {("render_buffer_output", "overlay_buffer_input")}) - else: - self.add_flow(source, visualizer, { - ("video_buffer_output" if is_aja else "output", "receivers")}) - if record_type == "input": - if is_aja: - self.add_flow( - source, - recorder_format_converter, - {("video_buffer_output", "source_video")}, - ) - self.add_flow(recorder_format_converter, recorder) - else: - self.add_flow(source, recorder) - elif record_type == "visualizer": - self.add_flow( - visualizer, - recorder_format_converter, - {("render_buffer_output", "source_video")}, - ) - self.add_flow(recorder_format_converter, recorder) - - -if __name__ == "__main__": - - load_env_log_level() - - # Parse args - parser = ArgumentParser(description="Endoscopy tool tracking demo application.") - parser.add_argument( - "-r", - "--record_type", - choices=["none", "input", "visualizer"], - default="none", - help="The video stream to record (default: %(default)s).", - ) - parser.add_argument( - "-s", - "--source", - choices=["replayer", "aja"], - default="replayer", - help=( - "If 'replayer', replay a prerecorded video. If 'aja' use an AJA " - "capture card as the source (default: %(default)s)." - ), - ) - args = parser.parse_args() - record_type = args.record_type - if record_type == "none": - record_type = None - - config_file = os.path.join(os.path.dirname(__file__), "endoscopy_tool_tracking.yaml") - - app = EndoscopyApp(record_type=record_type, source=args.source) - app.config(config_file) - app.run() diff --git a/apps/endoscopy_tool_tracking/python/endoscopy_tool_tracking.yaml b/apps/endoscopy_tool_tracking/python/endoscopy_tool_tracking.yaml deleted file mode 100644 index b5361d71..00000000 --- a/apps/endoscopy_tool_tracking/python/endoscopy_tool_tracking.yaml +++ /dev/null @@ -1,172 +0,0 @@ -%YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -extensions: - # The following extensions are automatically loaded upon Python App - # initialization, so they do not need to be listed here. - # This is a difference in behavior from the C++-API which currently requires - # explicitly listing these. - # - libgxf_std.so - # - libgxf_cuda.so - # - libgxf_multimedia.so - # - libgxf_serialization.so - # - libaja_source.so - # - libcustom_lstm_inference.so - # - libformat_converter.so - # - libholoviz.so - # - libstream_playback.so - # - libtool_tracking_postprocessor.so - # - libvisualizer_tool_tracking.so - -aja: - width: 1920 - height: 1080 - rdma: true - enable_overlay: false - overlay_rdma: true - -replayer: - # directory: "../data/endoscopy/video" - basename: "surgical_video" - frame_rate: 0 # as specified in timestamps - repeat: true # default: false - realtime: true # default: true - count: 0 # default: 0 (no frame count restriction) - -recorder_format_converter: - in_dtype: "rgba8888" - out_dtype: "rgb888" - -recorder: - directory: "/tmp" - basename: "tensor" - -format_converter_replayer: - out_tensor_name: source_video - out_dtype: "float32" - scale_min: 0.0 - scale_max: 255.0 - -format_converter_aja: - in_dtype: "rgba8888" - out_tensor_name: source_video - out_dtype: "float32" - scale_min: 0.0 - scale_max: 255.0 - resize_width: 854 - resize_height: 480 - -lstm_inference: - # model_file_path: ../data/endoscopy/model/tool_loc_convlstm.onnx - # engine_cache_dir: ../data/endoscopy/model/tool_loc_convlstm_engines - input_tensor_names: - - source_video - - cellstate_in - - hiddenstate_in - input_state_tensor_names: - - cellstate_in - - hiddenstate_in - input_binding_names: - - data_ph:0 # (shape=[1, 480, 854, 3], dtype=float32) <==> source_video - - cellstate_ph:0 # (shape=[1, 60, 107, 7], dtype=float32) == internal state - - hiddenstate_ph:0 # (shape=[1, 60, 107, 7], dtype=float32) == internal state - output_tensor_names: - - cellstate_out - - hiddenstate_out - - probs - - scaled_coords - - binary_masks - output_state_tensor_names: - - cellstate_out - - hiddenstate_out - output_binding_names: - - Model/net_states:0 # (shape=[ 1, 60, 107, 7], dtype=float32) - - Model/net_hidden:0 # (shape=[ 1, 60, 107, 7], dtype=float32) - - probs:0 # (shape=[1, 7], dtype=float32) - - Localize/scaled_coords:0 # (shape=[1, 7, 2], dtype=float32) - - Localize_1/binary_masks:0 # (shape=[1, 7, 60, 107], dtype=float32) - force_engine_update: false - verbose: true - max_workspace_size: 2147483648 - enable_fp16_: true - -visualizer_format_converter_replayer: - in_dtype: "rgb888" - out_dtype: "rgba8888" - out_tensor_name: video - -tool_tracking_postprocessor: - -holoviz: - tensors: - - name: "" - type: color - opacity: 1.0 - priority: 0 - - name: mask - type: color - opacity: 1.0 - priority: 1 - - name: scaled_coords - type: crosses - opacity: 1.0 - line_width: 4 - color: [1.0, 0.0, 0.0, 1.0] - priority: 2 - - name: scaled_coords - type: text - opacity: 1.0 - priority: 3 - color: [1.0, 1.0, 1.0, 0.9] - text: - - Grasper - - Bipolar - - Hook - - Scissors - - Clipper - - Irrigator - - Spec.Bag - -holoviz_overlay: - headless: true - tensors: - - name: "" - type: color - opacity: 1.0 - priority: 0 - - name: mask - type: color - opacity: 1.0 - priority: 1 - - name: scaled_coords - type: crosses - opacity: 1.0 - line_width: 4 - color: [1.0, 0.0, 0.0, 1.0] - priority: 2 - - name: scaled_coords - type: text - opacity: 1.0 - priority: 3 - color: [1.0, 1.0, 1.0, 0.9] - text: - - Grasper - - Bipolar - - Hook - - Scissors - - Clipper - - Irrigator - - Spec.Bag diff --git a/apps/hi_speed_endoscopy/README.md b/apps/hi_speed_endoscopy/README.md deleted file mode 100644 index b23c12db..00000000 --- a/apps/hi_speed_endoscopy/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# Hi-Speed Endoscopy - -The application showcases how high resolution cameras can be used to capture the scene, post-processed on GPU and displayed at high frame rate. - -### Requirements - -This application requires: -1. an Emergent Vision Technologies camera (see [setup instructions]((https://docs.nvidia.com/clara-holoscan/sdk-user-guide/emergent_setup.html) -2. a NVIDIA ConnectX SmartNIC with Rivermax SDK and drivers installed (see [prerequisites](../../README.md#prerequisites)) -3. a display with high refresh rate to keep up with the camera's framerate -4. [additional setups](https://docs.nvidia.com/clara-holoscan/sdk-user-guide/additional_setup.html) to reduce latency - -### Build Instructions - -The application is not currently supported in our containerized development environment (using the run script or Docker + CMake manually). Follow the advanced build instructions [here](../../README.md#advanced-local-environment-cmake) to build locally from source, and pass `-DHOLOSCAN_BUILD_HI_SPEED_ENDO_APP=ON` when configuring the project with CMake. - -> âš ï¸ At this time, camera controls are hardcoded within the `emergent-source` extension. To update them at the application level, the GXF extension, and the application need to be rebuilt. -For more information on the controls, refer to the [EVT Camera Attributes Manual](https://emergentvisiontec.com/resources/?tab=umg) - -### Run Instructions - -First, go in your `build` or `install` directory. Then, run the commands of your choice: - -* RDMA disabled - ```bash - # C++ - sed -i -e 's#rdma:.*#rdma: false#' ./apps/hi_speed_endoscopy/cpp/app_config.yaml \ - && sudo ./apps/hi_speed_endoscopy/cpp/hi_speed_endoscopy - - # Python - sudo -s - export PYTHONPATH=$(pwd)/python/lib/ - sed -i -e 's#rdma:.*#rdma: false#' ./apps/hi_speed_endoscopy/python/hi_speed_endoscopy.yaml \ - && python3 ./apps/hi_speed_endoscopy/python/hi_speed_endoscopy.py - exit - ``` - -* RDMA enabled - ```bash - # C++ - sed -i -e 's#rdma:.*#rdma: true#' ./apps/hi_speed_endoscopy/cpp/app_config.yaml \ - && sudo MELLANOX_RINGBUFF_FACTOR=14 ./apps/hi_speed_endoscopy/cpp/hi_speed_endoscopy - - # Python - sudo -s - export PYTHONPATH=$(pwd)/python/lib/ - sed -i -e 's#rdma:.*#rdma: true#' ./apps/hi_speed_endoscopy/python/hi_speed_endoscopy.yaml \ - && MELLANOX_RINGBUFF_FACTOR=14 python3 ./apps/hi_speed_endoscopy/python/hi_speed_endoscopy.py - exit - ``` - -> â„¹ï¸ The python app can run outside those folders if `HOLOSCAN_SAMPLE_DATA_PATH` is set in your environment - - -> â„¹ï¸ The `MELLANOX_RINGBUFF_FACTOR` is used by the EVT driver to decide how much BAR1 size memory would be used on the dGPU. It can be changed to different number based on different use cases. diff --git a/apps/hi_speed_endoscopy/cpp/app_config.yaml b/apps/hi_speed_endoscopy/cpp/app_config.yaml deleted file mode 100644 index 7013514c..00000000 --- a/apps/hi_speed_endoscopy/cpp/app_config.yaml +++ /dev/null @@ -1,44 +0,0 @@ -%YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -extensions: - - libgxf_std.so - - libgxf_cuda.so - - libgxf_multimedia.so - - libgxf_serialization.so - - libemergent_source.so - - libbayer_demosaic.so - - libholoviz.so - -emergent: - width: 4200 - height: 2160 - framerate: 240 - rdma: false - -demosaic: - generate_alpha: false - bayer_grid_pos: 2 - interpolation_mode: 0 # this is the only interpolation mode supported by NPP currently - -holoviz: - # display_name: DP-2 - width: 2560 - height: 1440 - framerate: 240 - # use_exclusive_display: true - fullscreen: true - diff --git a/apps/hi_speed_endoscopy/cpp/main.cpp b/apps/hi_speed_endoscopy/cpp/main.cpp deleted file mode 100644 index 0de46372..00000000 --- a/apps/hi_speed_endoscopy/cpp/main.cpp +++ /dev/null @@ -1,71 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -class App : public holoscan::Application { - public: - void compose() override { - using namespace holoscan; - - // Create instances of all the operators being used - std::shared_ptr source; - std::shared_ptr bayer_demosaic; - std::shared_ptr viz; - - // emergent camera is the source for this app for data acquisition in Bayer format - source = make_operator("emergent", from_config("emergent")); - - // bayer demosaic is the post processing step to convert Bayer frame to RGB format - bayer_demosaic = make_operator( - "bayer_demosaic", - from_config("demosaic"), - Arg("pool") = make_resource("pool", 1, 72576000, 2), - Arg("cuda_stream_pool") = make_resource("cuda_stream", 0, 0, 0, 1, 5)); - - // Holoviz is the visualizer being used for the peak performance - viz = make_operator("holoviz", from_config("holoviz")); - - // Create the pipeline source->bayer_demosaic->viz - add_flow(source, bayer_demosaic, {{"signal", "receiver"}}); - add_flow(bayer_demosaic, viz, {{"transmitter", "receivers"}}); - } - - private: -}; - -int main(int argc, char** argv) { - holoscan::load_env_log_level(); - - // Create an instance of App - auto app = holoscan::make_application(); - - // Read in the parameters provided at the command line - if (argc == 2) { - app->config(argv[1]); - } else { - auto config_path = std::filesystem::canonical(argv[0]).parent_path(); - config_path += "/app_config.yaml"; - app->config(config_path); - } - - // Run the App - app->run(); - - return 0; -} diff --git a/apps/hi_speed_endoscopy/python/hi_speed_endoscopy.py b/apps/hi_speed_endoscopy/python/hi_speed_endoscopy.py deleted file mode 100644 index 074a5267..00000000 --- a/apps/hi_speed_endoscopy/python/hi_speed_endoscopy.py +++ /dev/null @@ -1,86 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from argparse import ArgumentParser - -from holoscan.core import Application -from holoscan.logger import load_env_log_level -from holoscan.operators import BayerDemosaicOp, HolovizOp -from holoscan.resources import BlockMemoryPool, CudaStreamPool, MemoryStorageType - -sample_data_path = os.environ.get("HOLOSCAN_SAMPLE_DATA_PATH", "../data") - - -class HighSpeedEndoscopyApp(Application): - def __init__(self): - super().__init__() - - # set name - self.name = "High speed endoscopy app" - - def compose(self): - try: - from holoscan.operators import EmergentSourceOp - except ImportError: - raise ImportError( - "Could not import EmergentSourceOp. This application requires that the library" - "was built with Emergent SDK support." - ) - - source = EmergentSourceOp(self, name="emergent", **self.kwargs("emergent")) - - cuda_stream_pool = CudaStreamPool( - self, - dev_id=0, - stream_flags=0, - stream_priority=0, - reserved_size=1, - max_size=5, - ) - pool = BlockMemoryPool( - self, - name="pool", - storage_type=MemoryStorageType.DEVICE, - block_size=72576000, - num_blocks=2, - ) - - bayer_demosaic = BayerDemosaicOp( - self, - name="bayer_demosaic", - pool=pool, - cuda_stream_pool=cuda_stream_pool, - **self.kwargs("demosaic"), - ) - - viz = HolovizOp(self, name="holoviz", **self.kwargs("holoviz")) - - self.add_flow(source, bayer_demosaic, {("signal", "receiver")}) - self.add_flow(bayer_demosaic, viz, {("transmitter", "receivers")}) - - -if __name__ == "__main__": - - load_env_log_level() - - parser = ArgumentParser(description="High-speed endoscopy demo application.") - args = parser.parse_args() - - config_file = os.path.join(os.path.dirname(__file__), "hi_speed_endoscopy.yaml") - - app = HighSpeedEndoscopyApp() - app.config(config_file) - app.run() diff --git a/apps/hi_speed_endoscopy/python/hi_speed_endoscopy.yaml b/apps/hi_speed_endoscopy/python/hi_speed_endoscopy.yaml deleted file mode 100644 index 99c99e16..00000000 --- a/apps/hi_speed_endoscopy/python/hi_speed_endoscopy.yaml +++ /dev/null @@ -1,48 +0,0 @@ -%YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -extensions: - # The following extensions are automatically loaded upon Python App - # initialization, so they do not need to be listed here. - # This is a difference in behavior from the C++-API which currently requires - # explicitly listing these. - # - libgxf_std.so - # - libgxf_cuda.so - # - libgxf_multimedia.so - # - libgxf_serialization.so - # - libemergent_source.so - # - libbayer_demosaic.so - # - libholoviz.so - -emergent: - width: 4200 - height: 2160 - framerate: 240 - rdma: false - -demosaic: - generate_alpha: false - bayer_grid_pos: 2 - interpolation_mode: 0 # this is the only interpolation mode supported by NPP currently - -holoviz: - # display_name: DP-2 - width: 2560 - height: 1440 - framerate: 240 - # use_exclusive_display: true - fullscreen: true - diff --git a/apps/multiai/README.md b/apps/multiai/README.md deleted file mode 100644 index 7fc70694..00000000 --- a/apps/multiai/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# Multi-AI Ultrasound - -This application demonstrates how to run multiple inference pipelines in a single application by leveraging the Holoscan Inference module, a framework that facilitates designing and executing inference applications in the Holoscan SDK. - -The Multi AI operators (inference and postprocessor) use APIs from the Holoscan Inference module to extract data, initialize and execute the inference workflow, process, and transmit data for visualization. - -The applications uses models and echocardiogram data from iCardio.ai. The models include: -- a Plax chamber model, that identifies four critical linear measurements of the heart -- a Viewpoint Classifier model, that determines confidence of each frame to known 28 cardiac anatomical view as defined by the guidelines of the American Society of Echocardiography -- an Aortic Stenosis Classification model, that provides a score which determines likeability for the presence of aortic stenosis - -### Requirements - -The provided applications are configured to either use the AJA capture card for input stream, or a pre-recorded video of the echocardiogram (replayer). Follow the [setup instructions from the user guide](https://docs.nvidia.com/clara-holoscan/sdk-user-guide/aja_setup.html) to use the AJA capture card. - -### Data - -[ðŸ“¦ï¸ (NGC) Sample App Data for Multi-AI Ultrasound Pipeline](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/clara-holoscan/resources/holoscan_multi_ai_ultrasound_sample_data) - -### Build Instructions - -Built with the SDK, see instructions from the top level README. - -### Run Instructions - -First, go in your `build` or `install` directory (automatically done by `./run launch`). - -Then, run the commands of your choice: - -* Using a pre-recorded video - ```bash - # C++ - sed -i -e 's#^source:.*#source: replayer#' ./apps/multiai/cpp/app_config.yaml \ - && ./apps/multiai/cpp/multiai - - # Python - python3 ./apps/multiai/python/multiai.py --source=replayer - ``` - -* Using an AJA card - ```bash - # C++ - sed -i -e 's#^source:.*#source: aja#' ./apps/multiai/cpp/app_config.yaml \ - && ./apps/multiai/cpp/multiai - - # Python - python3 ./apps/multiai/python/multiai.py --source=aja - ``` - -> â„¹ï¸ The python app can run outside those folders if `HOLOSCAN_SAMPLE_DATA_PATH` is set in your environment (automatically done by `./run launch`). \ No newline at end of file diff --git a/apps/multiai/cpp/CMakeLists.txt b/apps/multiai/cpp/CMakeLists.txt deleted file mode 100644 index 0395d4c9..00000000 --- a/apps/multiai/cpp/CMakeLists.txt +++ /dev/null @@ -1,53 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -add_executable(multiai - main.cpp -) - -target_link_libraries(multiai - PRIVATE - ${HOLOSCAN_PACKAGE_NAME} -) - -# Download the associated dataset if needed -if(HOLOSCAN_DOWNLOAD_DATASETS) - add_dependencies(multiai multiai_ultrasound_data) -endif() - -# Copy config file -add_custom_target(multiai_yaml - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/app_config.yaml" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/app_config.yaml" -) -add_dependencies(multiai multiai_yaml) - -# Set the install RPATH based on the location of the Holoscan SDK libraries -# The GXF extensions are loaded by the GXF libraries - no need to include here -file(RELATIVE_PATH install_lib_relative_path ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_SOURCE_DIR}/${HOLOSCAN_INSTALL_LIB_DIR} ) -set_target_properties(multiai PROPERTIES INSTALL_RPATH "\$ORIGIN/${install_lib_relative_path}") - -# Get relative folder path for the app -file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) - -# Install the app -install(TARGETS "multiai" - DESTINATION "${app_relative_dest_path}" - COMPONENT "holoscan-apps" -) -install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/app_config.yaml" - DESTINATION ${app_relative_dest_path} - COMPONENT "holoscan-apps" -) diff --git a/apps/multiai/cpp/app_config.yaml b/apps/multiai/cpp/app_config.yaml deleted file mode 100644 index 0941d832..00000000 --- a/apps/multiai/cpp/app_config.yaml +++ /dev/null @@ -1,168 +0,0 @@ -%YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -extensions: - - libgxf_std.so - - libgxf_cuda.so - - libgxf_multimedia.so - - libgxf_serialization.so - - libaja_source.so - - libformat_converter.so - - libholoviz.so - - libstream_playback.so - - libmultiai_postprocessor.so - - libmultiai_inference.so - - libvisualizer_icardio.so - -source: "replayer" # or "aja" -do_record: false # or 'true' if you want to record input video stream. - -replayer: - directory: "../data/multiai_ultrasound/video" - basename: "tensor" - frame_rate: 0 # as specified in timestamps - repeat: true # default: false - realtime: true # default: true - count: 0 # default: 0 (no frame count restriction) - -aja: # AJASourceOp - width: 1920 - height: 1080 - rdma: true - enable_overlay: false - -broadcast: - -plax_cham_resized: - out_tensor_name: plax_cham_resize - out_dtype: "rgb888" - resize_width: 320 - resize_height: 320 - -plax_cham_pre: - out_tensor_name: plax_cham_pre_proc - out_dtype: "float32" - resize_width: 320 - resize_height: 320 - -aortic_ste_pre: - out_tensor_name: aortic_pre_proc - out_dtype: "float32" - resize_width: 300 - resize_height: 300 - -b_mode_pers_pre: - out_tensor_name: bmode_pre_proc - out_dtype: "float32" - resize_width: 320 - resize_height: 240 - -multiai_inference: - backend: "trt" - model_path_map: - "plax_chamber": "../data/multiai_ultrasound/models/plax_chamber.onnx" - "aortic_stenosis": "../data/multiai_ultrasound/models/aortic_stenosis.onnx" - "bmode_perspective": "../data/multiai_ultrasound/models/bmode_perspective.onnx" - pre_processor_map: - "plax_chamber": ["plax_cham_pre_proc"] - "aortic_stenosis": ["aortic_pre_proc"] - "bmode_perspective": ["bmode_pre_proc"] - inference_map: - "plax_chamber": "plax_cham_infer" - "aortic_stenosis": "aortic_infer" - "bmode_perspective": "bmode_infer" - in_tensor_names: ["plax_cham_pre_proc", "aortic_pre_proc", "bmode_pre_proc"] - out_tensor_names: ["plax_cham_infer", "aortic_infer", "bmode_infer"] - parallel_inference: true - infer_on_cpu: false - enable_fp16: false - input_on_cuda: true - output_on_cuda: true - transmit_on_cuda: true - is_engine_path: false - -multiai_postprocessor: - process_operations: - "plax_cham_infer": ["max_per_channel_scaled"] - processed_map: - "plax_cham_infer": "plax_chamber_processed" - in_tensor_names: ["plax_cham_infer", - "aortic_infer", - "bmode_infer"] - out_tensor_names : ["plax_chamber_processed"] - input_on_cuda: false - output_on_cuda: false - transmit_on_cuda: false - -visualizer_icardio: - in_tensor_names: ["plax_chamber_processed"] - out_tensor_names: ["keypoints", "keyarea_1", "keyarea_2", - "keyarea_3", "keyarea_4", "keyarea_5", "lines","logo"] - input_on_cuda: false - -holoviz: - tensors: - - name: plax_cham_resize - type: color - priority: 0 - - name: logo - type: color - priority: 0 - opacity: 0.5 - - name: keyarea_1 - type: ovals - color: [1.0, 0.0, 0.0, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keyarea_2 - type: ovals - color: [0.0, 1.0, 0.0, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keyarea_3 - type: ovals - color: [0.0, 1.0, 1.0, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keyarea_4 - type: ovals - color: [1.0, 0.5, 0.5, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keyarea_5 - type: ovals - color: [1.0, 0.0, 1.0, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keypoints - type: crosses - line_width: 4 - color: [1.0, 1.0, 1.0, 1.0] - priority: 3 - - name: lines - type: line_strip - line_width: 1 - color: [1.0, 1.0, 0.0, 1.0] - priority: 1 - window_title: "Multi AI Inference" - width: 320 - height: 320 - use_exclusive_display: false \ No newline at end of file diff --git a/apps/multiai/cpp/main.cpp b/apps/multiai/cpp/main.cpp deleted file mode 100644 index 9a74f805..00000000 --- a/apps/multiai/cpp/main.cpp +++ /dev/null @@ -1,129 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -class App : public holoscan::Application { - public: - void set_source(const std::string& source) { - if (source == "aja") { is_aja_source_ = true; } - } - - void compose() override { - using namespace holoscan; - - std::shared_ptr source; - std::shared_ptr pool_resource = make_resource("pool"); - if (is_aja_source_) { - source = make_operator("aja", from_config("aja")); - } else { - source = make_operator("replayer", from_config("replayer")); - } - - auto in_dtype = is_aja_source_ ? std::string("rgba8888") : std::string("rgb888"); - auto plax_cham_resized = make_operator("plax_cham_resized", - from_config("plax_cham_resized"), - Arg("in_dtype") = in_dtype, - Arg("pool") = pool_resource); - - auto plax_cham_pre = make_operator("plax_cham_pre", - from_config("plax_cham_pre"), - Arg("in_dtype") = in_dtype, - Arg("pool") = pool_resource); - - auto aortic_ste_pre = make_operator("aortic_ste_pre", - from_config("aortic_ste_pre"), - Arg("in_dtype") = in_dtype, - Arg("pool") = pool_resource); - - auto b_mode_pers_pre = make_operator("b_mode_pers_pre", - from_config("b_mode_pers_pre"), - Arg("in_dtype") = in_dtype, - Arg("pool") = pool_resource); - - auto multiai_inference = make_operator( - "multiai_inference", from_config("multiai_inference"), Arg("allocator") = pool_resource); - - auto multiai_postprocessor = - make_operator("multiai_postprocessor", - from_config("multiai_postprocessor"), - Arg("allocator") = pool_resource); - - auto visualizer_icardio = make_operator( - "visualizer_icardio", from_config("visualizer_icardio"), Arg("allocator") = pool_resource); - - auto holoviz = make_operator( - "holoviz", from_config("holoviz"), Arg("allocator") = pool_resource); - - // Flow definition - - if (is_aja_source_) { - const std::set> aja_ports = {{"video_buffer_output", ""}}; - add_flow(source, plax_cham_resized, aja_ports); - add_flow(source, plax_cham_pre, aja_ports); - add_flow(source, aortic_ste_pre, aja_ports); - add_flow(source, b_mode_pers_pre, aja_ports); - } else { - add_flow(source, plax_cham_resized); - add_flow(source, plax_cham_pre); - add_flow(source, aortic_ste_pre); - add_flow(source, b_mode_pers_pre); - } - - add_flow(plax_cham_resized, holoviz, {{"", "receivers"}}); - - add_flow(plax_cham_pre, multiai_inference, {{"", "receivers"}}); - add_flow(aortic_ste_pre, multiai_inference, {{"", "receivers"}}); - add_flow(b_mode_pers_pre, multiai_inference, {{"", "receivers"}}); - - add_flow(multiai_inference, multiai_postprocessor, {{"transmitter", "receivers"}}); - add_flow(multiai_postprocessor, visualizer_icardio, {{"transmitter", "receivers"}}); - - add_flow(visualizer_icardio, holoviz, {{"keypoints", "receivers"}}); - add_flow(visualizer_icardio, holoviz, {{"keyarea_1", "receivers"}}); - add_flow(visualizer_icardio, holoviz, {{"keyarea_2", "receivers"}}); - add_flow(visualizer_icardio, holoviz, {{"keyarea_3", "receivers"}}); - add_flow(visualizer_icardio, holoviz, {{"keyarea_4", "receivers"}}); - add_flow(visualizer_icardio, holoviz, {{"keyarea_5", "receivers"}}); - add_flow(visualizer_icardio, holoviz, {{"lines", "receivers"}}); - add_flow(visualizer_icardio, holoviz, {{"logo", "receivers"}}); - } - - private: - bool is_aja_source_ = false; -}; - -int main(int argc, char** argv) { - holoscan::load_env_log_level(); - - auto app = holoscan::make_application(); - - if (argc == 2) { - app->config(argv[1]); - } else { - auto config_path = std::filesystem::canonical(argv[0]).parent_path(); - config_path += "/app_config.yaml"; - app->config(config_path); - } - - auto source = app->from_config("source").as(); - app->set_source(source); - app->run(); - - return 0; -} diff --git a/apps/multiai/python/multiai.py b/apps/multiai/python/multiai.py deleted file mode 100644 index 68c78256..00000000 --- a/apps/multiai/python/multiai.py +++ /dev/null @@ -1,173 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from argparse import ArgumentParser - -from holoscan.core import Application -from holoscan.logger import load_env_log_level -from holoscan.operators import ( - AJASourceOp, - FormatConverterOp, - HolovizOp, - MultiAIInferenceOp, - MultiAIPostprocessorOp, - VideoStreamReplayerOp, - VisualizerICardioOp, -) -from holoscan.resources import UnboundedAllocator - -sample_data_path = os.environ.get("HOLOSCAN_SAMPLE_DATA_PATH", "../data") - - -class MultiAIICardio(Application): - def __init__(self, source="replayer"): - super().__init__() - - # set name - self.name = "Ultrasound App" - - # Optional parameters affecting the graph created by compose. - source = source.lower() - if source not in ["replayer", "aja"]: - raise ValueError(f"unsupported source: {source}. Please use 'replayer' or 'aja'.") - self.source = source - - def compose(self): - is_aja = self.source.lower() == "aja" - SourceClass = AJASourceOp if is_aja else VideoStreamReplayerOp - source_kwargs = self.kwargs(self.source) - if self.source == "replayer": - video_dir = os.path.join(sample_data_path, "multiai_ultrasound", "video") - if not os.path.exists(video_dir): - raise ValueError(f"Could not find video data: {video_dir=}") - source_kwargs["directory"] = video_dir - source = SourceClass(self, name=self.source, **source_kwargs) - - in_dtype = "rgba8888" if is_aja else "rgb888" - pool = UnboundedAllocator(self, name="pool") - plax_cham_resized = FormatConverterOp( - self, - name="plax_cham_resized", - pool=pool, - in_dtype=in_dtype, - **self.kwargs("plax_cham_resized"), - ) - plax_cham_pre = FormatConverterOp( - self, - name="plax_cham_pre", - pool=pool, - in_dtype=in_dtype, - **self.kwargs("plax_cham_pre"), - ) - aortic_ste_pre = FormatConverterOp( - self, - name="aortic_ste_pre", - pool=pool, - in_dtype=in_dtype, - **self.kwargs("aortic_ste_pre"), - ) - b_mode_pers_pre = FormatConverterOp( - self, - name="b_mode_pers_pre", - pool=pool, - in_dtype=in_dtype, - **self.kwargs("b_mode_pers_pre"), - ) - - model_path = os.path.join(sample_data_path, "multiai_ultrasound", "models") - model_path_map = { - "plax_chamber": os.path.join(model_path, "plax_chamber.onnx"), - "aortic_stenosis": os.path.join(model_path, "aortic_stenosis.onnx"), - "bmode_perspective": os.path.join(model_path, "bmode_perspective.onnx"), - } - for k, v in model_path_map.items(): - if not os.path.exists(v): - raise RuntimeError(f"Could not find model file: {v}") - infererence_kwargs = self.kwargs("multiai_inference") - infererence_kwargs["model_path_map"] = model_path_map - multiai_inference = MultiAIInferenceOp( - self, - name="multiai_inference", - allocator=pool, - **infererence_kwargs, - ) - - multiai_postprocessor = MultiAIPostprocessorOp( - self, - allocator=pool, - **self.kwargs("multiai_postprocessor"), - ) - visualizer_icardio = VisualizerICardioOp( - self, allocator=pool, **self.kwargs("visualizer_icardio") - ) - holoviz = HolovizOp(self, allocator=pool, name="holoviz", **self.kwargs("holoviz")) - - # connect the input to the resizer and each pre-processor - for op in [plax_cham_resized, plax_cham_pre, aortic_ste_pre, b_mode_pers_pre]: - if is_aja: - self.add_flow(source, op, {("video_buffer_output", "")}) - else: - self.add_flow(source, op) - - # connect the resized source video to the visualizer - self.add_flow(plax_cham_resized, holoviz, {("", "receivers")}) - - # connect all pre-processor outputs to the inference operator - for op in [plax_cham_pre, aortic_ste_pre, b_mode_pers_pre]: - self.add_flow(op, multiai_inference, {("", "receivers")}) - - # connect the inference output to the postprocessor - self.add_flow(multiai_inference, multiai_postprocessor, {("transmitter", "receivers")}) - - # prepare postprocessed output for visualization with holoviz - self.add_flow(multiai_postprocessor, visualizer_icardio, {("transmitter", "receivers")}) - - # connect the overlays to holoviz - visualizer_inputs = ( - "keypoints", - "keyarea_1", - "keyarea_2", - "keyarea_3", - "keyarea_4", - "keyarea_5", - "lines", - "logo" - ) - for src in visualizer_inputs: - self.add_flow(visualizer_icardio, holoviz, {(src, "receivers")}) - - -if __name__ == "__main__": - - load_env_log_level() - parser = ArgumentParser(description="Multi-AI demo application.") - parser.add_argument( - "-s", - "--source", - choices=["replayer", "aja"], - default="replayer", - help=( - "If 'replayer', replay a prerecorded video. If 'aja' use an AJA " - "capture card as the source (default: %(default)s)." - ), - ) - args = parser.parse_args() - - config_file = os.path.join(os.path.dirname(__file__), "multiai.yaml") - - app = MultiAIICardio(source=args.source) - app.config(config_file) - app.run() diff --git a/apps/multiai/python/multiai.yaml b/apps/multiai/python/multiai.yaml deleted file mode 100644 index dd5d2983..00000000 --- a/apps/multiai/python/multiai.yaml +++ /dev/null @@ -1,165 +0,0 @@ -%YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -extensions: - # These extensions are automatically loaded on import in the Python application - # - libgxf_std.so - # - libgxf_cuda.so - # - libgxf_multimedia.so - # - libgxf_serialization.so - # - libaja_source.so - # - libformat_converter.so - # - libholoviz.so - # - libstream_playback.so - # - libmultiai_postprocessor.so - # - libmultiai_inference.so - # - libvisualizer_icardio.so - -source: "replayer" # or "aja" -do_record: false # or 'true' if you want to record input video stream. - -replayer: - # directory is defined in the Python script to avoid need for a relative path here - basename: "tensor" - frame_rate: 0 # as specified in timestamps - repeat: true # default: false - realtime: true # default: true - count: 0 # default: 0 (no frame count restriction) - -aja: # AJASourceOp - width: 1920 - height: 1080 - rdma: true - enable_overlay: false - -broadcast: - -plax_cham_resized: - out_tensor_name: plax_cham_resize - out_dtype: "rgb888" - resize_width: 320 - resize_height: 320 - -plax_cham_pre: - out_tensor_name: plax_cham_pre_proc - out_dtype: "float32" - resize_width: 320 - resize_height: 320 - -aortic_ste_pre: - out_tensor_name: aortic_pre_proc - out_dtype: "float32" - resize_width: 300 - resize_height: 300 - -b_mode_pers_pre: - out_tensor_name: bmode_pre_proc - out_dtype: "float32" - resize_width: 320 - resize_height: 240 - -multiai_inference: - backend: "trt" - # model_path_map is defined in the Python script to avoid need for relative paths here - pre_processor_map: - "plax_chamber": ["plax_cham_pre_proc"] - "aortic_stenosis": ["aortic_pre_proc"] - "bmode_perspective": ["bmode_pre_proc"] - inference_map: - "plax_chamber": "plax_cham_infer" - "aortic_stenosis": "aortic_infer" - "bmode_perspective": "bmode_infer" - in_tensor_names: ["plax_cham_pre_proc", "aortic_pre_proc", "bmode_pre_proc"] - out_tensor_names: ["plax_cham_infer", "aortic_infer", "bmode_infer"] - parallel_inference: true - infer_on_cpu: false - enable_fp16: false - input_on_cuda: true - output_on_cuda: true - transmit_on_cuda: true - -multiai_postprocessor: - process_operations: - "plax_cham_infer": ["max_per_channel_scaled"] - processed_map: - "plax_cham_infer": "plax_chamber_processed" - in_tensor_names: ["plax_cham_infer", - "aortic_infer", - "bmode_infer"] - out_tensor_names : ["plax_chamber_processed"] - input_on_cuda: false - output_on_cuda: false - transmit_on_cuda: false - -visualizer_icardio: - in_tensor_names: ["plax_chamber_processed"] - out_tensor_names: ["keypoints", "keyarea_1", "keyarea_2", - "keyarea_3", "keyarea_4", "keyarea_5", "lines", "logo"] - input_on_cuda: false - -holoviz: - tensors: - - name: plax_cham_resize - type: color - priority: 0 - - name: logo - type: color - priority: 0 - opacity: 0.5 - - name: keyarea_1 - type: ovals - color: [1.0, 0.0, 0.0, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keyarea_2 - type: ovals - color: [0.0, 1.0, 0.0, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keyarea_3 - type: ovals - color: [0.0, 1.0, 1.0, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keyarea_4 - type: ovals - color: [1.0, 0.5, 0.5, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keyarea_5 - type: ovals - color: [1.0, 0.0, 1.0, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keypoints - type: crosses - line_width: 4 - color: [1.0, 1.0, 1.0, 1.0] - priority: 3 - - name: lines - type: line_strip - line_width: 1 - color: [1.0, 1.0, 0.0, 1.0] - priority: 1 - window_title: "Multi AI Inference" - width: 320 - height: 320 - use_exclusive_display: false diff --git a/apps/ultrasound_segmentation/README.md b/apps/ultrasound_segmentation/README.md deleted file mode 100644 index 9542b0a7..00000000 --- a/apps/ultrasound_segmentation/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Ultrasound Bone Scoliosis Segmentation - -Full workflow including a generic visualization of segmentation results from a spinal scoliosis segmentation model of ultrasound videos. The model used is stateless, so this workflow could be configured to adapt to any vanilla DNN model. - -### Requirements - -The provided applications are configured to either use the AJA capture card for input stream, or a pre-recorded video of the ultrasound data (replayer). Follow the [setup instructions from the user guide](https://docs.nvidia.com/clara-holoscan/sdk-user-guide/aja_setup.html) to use the AJA capture card. - -### Data - -[ðŸ“¦ï¸ (NGC) Sample App Data for AI-based Bone Scoliosis Segmentation](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/clara-holoscan/resources/holoscan_ultrasound_sample_data) - -### Build Instructions - -Built with the SDK, see instructions from the top level README. - -### Run Instructions - -First, go in your `build` or `install` directory (automatically done by `./run launch`). - -Then, run the commands of your choice: - -* Using a pre-recorded video - ```bash - # C++ - sed -i -e 's#^source:.*#source: replayer#' ./apps/ultrasound_segmentation/cpp/app_config.yaml \ - && ./apps/ultrasound_segmentation/cpp/ultrasound_segmentation - - # Python - python3 ./apps/ultrasound_segmentation/python/ultrasound_segmentation.py --source=replayer - ``` - -* Using an AJA card - ```bash - # C++ - sed -i -e 's#^source:.*#source: aja#' ./apps/ultrasound_segmentation/cpp/app_config.yaml \ - && ./apps/ultrasound_segmentation/cpp/ultrasound_segmentation - - # Python - python3 ./apps/ultrasound_segmentation/python/ultrasound_segmentation.py --source=aja - ``` - -> â„¹ï¸ The python app can run outside those folders if `HOLOSCAN_SAMPLE_DATA_PATH` is set in your environment (automatically done by `./run launch`). \ No newline at end of file diff --git a/apps/ultrasound_segmentation/cpp/CMakeLists.txt b/apps/ultrasound_segmentation/cpp/CMakeLists.txt deleted file mode 100644 index 5f8ad9c8..00000000 --- a/apps/ultrasound_segmentation/cpp/CMakeLists.txt +++ /dev/null @@ -1,53 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -add_executable(ultrasound_segmentation - main.cpp -) - -target_link_libraries(ultrasound_segmentation - PRIVATE - ${HOLOSCAN_PACKAGE_NAME} -) - -# Download the associated dataset if needed -if(HOLOSCAN_DOWNLOAD_DATASETS) - add_dependencies(ultrasound_segmentation ultrasound_data) -endif() - -# Copy config file -add_custom_target(ultrasound_segmentation_yaml - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/app_config.yaml" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/app_config.yaml" -) -add_dependencies(ultrasound_segmentation ultrasound_segmentation_yaml) - -# Set the install RPATH based on the location of the Holoscan SDK libraries -# The GXF extensions are loaded by the GXF libraries - no need to include here -file(RELATIVE_PATH install_lib_relative_path ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_SOURCE_DIR}/${HOLOSCAN_INSTALL_LIB_DIR} ) -set_target_properties(ultrasound_segmentation PROPERTIES INSTALL_RPATH "\$ORIGIN/${install_lib_relative_path}") - -# Get relative folder path for the app -file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) - -# Install the app -install(TARGETS "ultrasound_segmentation" - DESTINATION "${app_relative_dest_path}" - COMPONENT "holoscan-apps" -) -install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/app_config.yaml" - DESTINATION ${app_relative_dest_path} - COMPONENT "holoscan-apps" -) diff --git a/apps/ultrasound_segmentation/cpp/app_config.yaml b/apps/ultrasound_segmentation/cpp/app_config.yaml deleted file mode 100644 index f94a6e1b..00000000 --- a/apps/ultrasound_segmentation/cpp/app_config.yaml +++ /dev/null @@ -1,88 +0,0 @@ -%YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -extensions: - - libgxf_std.so - - libgxf_cuda.so - - libgxf_multimedia.so - - libgxf_serialization.so - - libaja_source.so - - libformat_converter.so - - libholoviz.so - - libsegmentation_postprocessor.so - - libstream_playback.so - - libtensor_rt.so - -source: "replayer" # or "aja" - -replayer: # VideoStreamReplayer - directory: "../data/ultrasound/video" - basename: "ultrasound_256x256" - frame_rate: 0 # as specified in timestamps - repeat: true # default: false - realtime: true # default: true - count: 0 # default: 0 (no frame count restriction) - -aja: # AJASourceOp - width: 1920 - height: 1080 - rdma: true - enable_overlay: false - -drop_alpha_channel: # FormatConverter - in_dtype: "rgba8888" - in_tensor_name: source_video - out_tensor_name: source_video - out_dtype: "rgb888" - -segmentation_preprocessor: # FormatConverter - # in_tensor_name: source_video - out_tensor_name: source_video - out_dtype: "float32" - resize_width: 256 - resize_height: 256 - -segmentation_inference: # TensorRtInference - model_file_path: ../data/ultrasound/model/us_unet_256x256_nhwc.onnx - engine_cache_dir: ../data/ultrasound/model/us_unet_256x256_nhwc_engines - input_tensor_names: - - source_video - input_binding_names: - - INPUT__0 - output_tensor_names: - - inference_output_tensor - output_binding_names: - - OUTPUT__0 - force_engine_update: false - verbose: true - max_workspace_size: 2147483648 - enable_fp16_: false - -segmentation_postprocessor: # Postprocessor - in_tensor_name: inference_output_tensor - network_output_type: softmax - data_format: nchw - -segmentation_visualizer: # Holoviz - color_lut: [ - [0.65, 0.81, 0.89, 0.1], - [0.2, 0.63, 0.17, 0.7], - [0.98, 0.6, 0.6, 0.7], - [0.89, 0.1, 0.11, 0.7], - [0.99, 0.75, 0.44, 0.7], - [1.0, 0.5, 0.0, 0.7], - [0.0, 0.0, 0.0, 0.1] - ] diff --git a/apps/ultrasound_segmentation/cpp/main.cpp b/apps/ultrasound_segmentation/cpp/main.cpp deleted file mode 100644 index ab1e6f42..00000000 --- a/apps/ultrasound_segmentation/cpp/main.cpp +++ /dev/null @@ -1,116 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -class App : public holoscan::Application { - public: - void set_source(const std::string& source) { - if (source == "aja") { is_aja_source_ = true; } - } - - void compose() override { - using namespace holoscan; - - std::shared_ptr source; - std::shared_ptr drop_alpha_channel; - - if (is_aja_source_) { - source = make_operator("aja", from_config("aja")); - } else { - source = make_operator("replayer", from_config("replayer")); - } - - const int width = 1920; - const int height = 1080; - const int n_channels = 4; - const int bpp = 4; - if (is_aja_source_) { - uint64_t drop_alpha_block_size = width * height * n_channels * bpp; - uint64_t drop_alpha_num_blocks = 2; - drop_alpha_channel = make_operator( - "drop_alpha_channel", - from_config("drop_alpha_channel"), - Arg("pool") = make_resource( - "pool", 1, drop_alpha_block_size, drop_alpha_num_blocks)); - } - - int width_preprocessor = 1264; - int height_preprocessor = 1080; - uint64_t preprocessor_block_size = width_preprocessor * height_preprocessor * n_channels * bpp; - uint64_t preprocessor_num_blocks = 2; - auto segmentation_preprocessor = make_operator( - "segmentation_preprocessor", - from_config("segmentation_preprocessor"), - Arg("in_tensor_name", std::string(is_aja_source_ ? "source_video" : "")), - Arg("pool") = make_resource( - "pool", 1, preprocessor_block_size, preprocessor_num_blocks)); - - auto segmentation_inference = make_operator( - "segmentation_inference", - from_config("segmentation_inference"), - Arg("pool") = make_resource("pool"), - Arg("cuda_stream_pool") = make_resource("cuda_stream", 0, 0, 0, 1, 5)); - - auto segmentation_postprocessor = make_operator( - "segmentation_postprocessor", - from_config("segmentation_postprocessor"), - Arg("allocator") = make_resource("allocator")); - - auto segmentation_visualizer = make_operator( - "segmentation_visualizer", from_config("segmentation_visualizer")); - - // Flow definition - - if (is_aja_source_) { - add_flow(source, segmentation_visualizer, {{"video_buffer_output", "receivers"}}); - add_flow(source, drop_alpha_channel, {{"video_buffer_output", ""}}); - add_flow(drop_alpha_channel, segmentation_preprocessor); - } else { - add_flow(source, segmentation_visualizer, {{"", "receivers"}}); - add_flow(source, segmentation_preprocessor); - } - - add_flow(segmentation_preprocessor, segmentation_inference); - add_flow(segmentation_inference, segmentation_postprocessor); - add_flow(segmentation_postprocessor, segmentation_visualizer, {{"", "receivers"}}); - } - - private: - bool is_aja_source_ = false; -}; - -int main(int argc, char** argv) { - holoscan::load_env_log_level(); - - auto app = holoscan::make_application(); - - if (argc == 2) { - app->config(argv[1]); - } else { - auto config_path = std::filesystem::canonical(argv[0]).parent_path(); - config_path += "/app_config.yaml"; - app->config(config_path); - } - - auto source = app->from_config("source").as(); - app->set_source(source); - app->run(); - - return 0; -} diff --git a/apps/ultrasound_segmentation/python/CMakeLists.txt b/apps/ultrasound_segmentation/python/CMakeLists.txt deleted file mode 100644 index 968e7aa5..00000000 --- a/apps/ultrasound_segmentation/python/CMakeLists.txt +++ /dev/null @@ -1,39 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Get relative folder path for the app -file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) - -# Copy ultrasound segmentation application file -add_custom_target(python_ultrasound_segmentation - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/ultrasound_segmentation.py" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/ultrasound_segmentation.py" -) -add_dependencies(${HOLOSCAN_PACKAGE_NAME} python_ultrasound_segmentation) - -# Copy ultrasound segmentation config file -add_custom_target(python_ultrasound_segmentation_yaml - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/ultrasound_segmentation.yaml" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/ultrasound_segmentation.yaml" -) -add_dependencies(${HOLOSCAN_PACKAGE_NAME} python_ultrasound_segmentation_yaml) - -# Install the app -install(FILES - "${CMAKE_CURRENT_SOURCE_DIR}/ultrasound_segmentation.py" - "${CMAKE_CURRENT_SOURCE_DIR}/ultrasound_segmentation.yaml" - DESTINATION "${app_relative_dest_path}" - COMPONENT "holoscan-apps" -) diff --git a/apps/ultrasound_segmentation/python/ultrasound_segmentation.py b/apps/ultrasound_segmentation/python/ultrasound_segmentation.py deleted file mode 100644 index 71674254..00000000 --- a/apps/ultrasound_segmentation/python/ultrasound_segmentation.py +++ /dev/null @@ -1,178 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from argparse import ArgumentParser - -from holoscan.core import Application -from holoscan.logger import load_env_log_level -from holoscan.operators import ( - AJASourceOp, - FormatConverterOp, - HolovizOp, - SegmentationPostprocessorOp, - TensorRTInferenceOp, - VideoStreamReplayerOp, -) -from holoscan.resources import ( - BlockMemoryPool, - CudaStreamPool, - MemoryStorageType, - UnboundedAllocator, -) - -sample_data_path = os.environ.get("HOLOSCAN_SAMPLE_DATA_PATH", "../data") - -model_path = os.path.join(sample_data_path, "ultrasound", "model") - -model_file_path = os.path.join(model_path, "us_unet_256x256_nhwc.onnx") -engine_cache_dir = os.path.join(model_path, "us_unet_256x256_nhwc_engines") - - -class UltrasoundApp(Application): - def __init__(self, source="replayer"): - """Initialize the ultrasound segmentation application - - Parameters - ---------- - source : {"replayer", "aja"} - When set to "replayer" (the default), pre-recorded sample video data is - used as the application input. Otherwise, the video stream from an AJA - capture card is used. - """ - - super().__init__() - - # set name - self.name = "Ultrasound App" - - # Optional parameters affecting the graph created by compose. - self.source = source - - def compose(self): - n_channels = 4 # RGBA - bpp = 4 # bytes per pixel - - is_aja = self.source.lower() == "aja" - if is_aja: - source = AJASourceOp(self, name="aja", **self.kwargs("aja")) - drop_alpha_block_size = 1920 * 1080 * n_channels * bpp - drop_alpha_num_blocks = 2 - drop_alpha_channel = FormatConverterOp( - self, - name="drop_alpha_channel", - pool=BlockMemoryPool( - self, - storage_type=MemoryStorageType.DEVICE, - block_size=drop_alpha_block_size, - num_blocks=drop_alpha_num_blocks, - ), - **self.kwargs("drop_alpha_channel"), - ) - else: - video_dir = os.path.join(sample_data_path, "ultrasound", "video") - if not os.path.exists(video_dir): - raise ValueError(f"Could not find video data: {video_dir=}") - source = VideoStreamReplayerOp( - self, name="replayer", directory=video_dir, **self.kwargs("replayer") - ) - - width_preprocessor = 1264 - height_preprocessor = 1080 - preprocessor_block_size = width_preprocessor * height_preprocessor * n_channels * bpp - preprocessor_num_blocks = 2 - segmentation_preprocessor = FormatConverterOp( - self, - name="segmentation_preprocessor", - pool=BlockMemoryPool( - self, - storage_type=MemoryStorageType.DEVICE, - block_size=preprocessor_block_size, - num_blocks=preprocessor_num_blocks, - ), - **self.kwargs("segmentation_preprocessor"), - ) - - tensorrt_cuda_stream_pool = CudaStreamPool( - self, - name="cuda_stream", - dev_id=0, - stream_flags=0, - stream_priority=0, - reserved_size=1, - max_size=5, - ) - segmentation_inference = TensorRTInferenceOp( - self, - name="segmentation_inference", - engine_cache_dir=engine_cache_dir, - model_file_path=model_file_path, - pool=UnboundedAllocator(self, name="pool"), - cuda_stream_pool=tensorrt_cuda_stream_pool, - **self.kwargs("segmentation_inference"), - ) - - segmentation_postprocessor = SegmentationPostprocessorOp( - self, - name="segmentation_postprocessor", - allocator=UnboundedAllocator(self, name="allocator"), - **self.kwargs("segmentation_postprocessor"), - ) - - segmentation_visualizer = HolovizOp( - self, - name="segmentation_visualizer", - **self.kwargs("segmentation_visualizer"), - ) - - if is_aja: - self.add_flow(source, segmentation_visualizer, {("video_buffer_output", "receivers")}) - self.add_flow(source, drop_alpha_channel, {("video_buffer_output", "")}) - self.add_flow(drop_alpha_channel, segmentation_preprocessor) - else: - self.add_flow(source, segmentation_visualizer, {("", "receivers")}) - self.add_flow(source, segmentation_preprocessor) - self.add_flow(segmentation_preprocessor, segmentation_inference) - self.add_flow(segmentation_inference, segmentation_postprocessor) - self.add_flow( - segmentation_postprocessor, - segmentation_visualizer, - {("", "receivers")}, - ) - - -if __name__ == "__main__": - - # Parse args - parser = ArgumentParser(description="Ultrasound segmentation demo application.") - parser.add_argument( - "-s", - "--source", - choices=["replayer", "aja"], - default="replayer", - help=( - "If 'replayer', replay a prerecorded video. If 'aja' use an AJA " - "capture card as the source (default: %(default)s)." - ), - ) - args = parser.parse_args() - - load_env_log_level() - - config_file = os.path.join(os.path.dirname(__file__), "ultrasound_segmentation.yaml") - - app = UltrasoundApp(source=args.source) - app.config(config_file) - app.run() diff --git a/apps/ultrasound_segmentation/python/ultrasound_segmentation.yaml b/apps/ultrasound_segmentation/python/ultrasound_segmentation.yaml deleted file mode 100644 index a899b5e9..00000000 --- a/apps/ultrasound_segmentation/python/ultrasound_segmentation.yaml +++ /dev/null @@ -1,88 +0,0 @@ -%YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -extensions: - # The following extensions are automatically loaded upon Python App - # initialization, so they do not need to be listed here. - # This is a difference in behavior from the C++-API which currently requires - # explicitly listing these. - # - libgxf_std.so - # - libgxf_cuda.so - # - libgxf_multimedia.so - # - libgxf_serialization.so - # - libaja_source.so - # - libformat_converter.so - # - libholoviz.so - # - libsegmentation_postprocessor.so - # - libstream_playback.so - # - libtensor_rt.so - -replayer: # VideoStreamReplayer - # directory: "../data/ultrasound/video" - basename: "ultrasound_256x256" - frame_rate: 0 # as specified in timestamps - repeat: true # default: false - realtime: true # default: true - count: 0 # default: 0 (no frame count restriction) - -aja: # AJASourceOp - width: 1920 - height: 1080 - rdma: true - enable_overlay: false - -drop_alpha_channel: # FormatConverter - in_dtype: "rgba8888" - in_tensor_name: source_video - out_dtype: "rgb888" - -segmentation_preprocessor: # FormatConverter - out_tensor_name: source_video - out_dtype: "float32" - resize_width: 256 - resize_height: 256 - -segmentation_inference: # TensorRtInference - # model_file_path: "../data/ultrasound/model/us_unet_256x256_nhwc.onnx" - # engine_cache_dir: "../data/ultrasound/model/us_unet_256x256_nhwc_engines" - input_tensor_names: - - source_video - input_binding_names: - - INPUT__0 - output_tensor_names: - - inference_output_tensor - output_binding_names: - - OUTPUT__0 - force_engine_update: false - verbose: true - max_workspace_size: 2147483648 - enable_fp16_: false - -segmentation_postprocessor: # Postprocessor - in_tensor_name: inference_output_tensor - network_output_type: softmax - data_format: nchw - -segmentation_visualizer: # Holoviz - color_lut: [ - [0.65, 0.81, 0.89, 0.1], - [0.2, 0.63, 0.17, 0.7], - [0.98, 0.6, 0.6, 0.7], - [0.89, 0.1, 0.11, 0.7], - [0.99, 0.75, 0.44, 0.7], - [1.0, 0.5, 0.0, 0.7], - [0.0, 0.0, 0.0, 0.1] - ] diff --git a/cmake/deps/ajantv2_rapids.cmake b/cmake/deps/ajantv2_rapids.cmake index c0c59154..5cd2da9b 100644 --- a/cmake/deps/ajantv2_rapids.cmake +++ b/cmake/deps/ajantv2_rapids.cmake @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,8 +21,8 @@ rapids_cpm_find(ajantv2 16.2.0 CPM_ARGS - GITHUB_REPOSITORY ibstewart/ntv2 - GIT_TAG holoscan-v0.2.0 + GITHUB_REPOSITORY nvidia-holoscan/ntv2 + GIT_TAG 1321a8d4c1a8de696c996d05b65e5aa2934f89d1 OPTIONS "AJA_BUILD_APPS OFF" "AJA_BUILD_DOCS OFF" @@ -41,4 +41,11 @@ if(ajantv2_ADDED) # ajantv2 is a static library so we do not need to install it. add_library(AJA::ajantv2 ALIAS ajantv2) + + # Install the headers needed for development with the SDK + install(DIRECTORY ${ajantv2_SOURCE_DIR}/ajalibraries + DESTINATION "include" + COMPONENT "holoscan-dependencies" + FILES_MATCHING PATTERN "*.h" PATTERN "*.hh" + ) endif() diff --git a/cmake/deps/gtest_rapids.cmake b/cmake/deps/gtest_rapids.cmake index 92894cf8..4c34ce7f 100644 --- a/cmake/deps/gtest_rapids.cmake +++ b/cmake/deps/gtest_rapids.cmake @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,7 +15,8 @@ # https://docs.rapids.ai/api/rapids-cmake/stable/packages/rapids_cpm_gtest.html include(${rapids-cmake-dir}/cpm/gtest.cmake) +include(${rapids-cmake-dir}/cpm/package_override.cmake) -# It uses GTest version 1.10.0 (as of rapids-cmake v22.06) -# https://github.com/rapidsai/rapids-cmake/blob/branch-22.06/rapids-cmake/cpm/versions.json +# Using GTest 1.12.1 +rapids_cpm_package_override("${CMAKE_SOURCE_DIR}/cmake/deps/rapids-cmake-packages.json") rapids_cpm_gtest() diff --git a/cmake/deps/nanovg_rapids.cmake b/cmake/deps/nanovg_rapids.cmake deleted file mode 100644 index 8b5f3086..00000000 --- a/cmake/deps/nanovg_rapids.cmake +++ /dev/null @@ -1,45 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# https://docs.rapids.ai/api/rapids-cmake/stable/command/rapids_find_package.html# -include(${rapids-cmake-dir}/cpm/find.cmake) - -# nanovg doesn't have versioning, so we provide a fake version (0.1) -rapids_cpm_find(nanovg 0.1 - GLOBAL_TARGETS nanovg - - CPM_ARGS - - GITHUB_REPOSITORY memononen/nanovg - GIT_TAG 5f65b43 - EXCLUDE_FROM_ALL -) - -if(nanovg_ADDED) - file(GLOB NANOVG_HEADERS - ${nanovg_SOURCE_DIR}/src/*.h) - - file(GLOB NANOVG_SOURCES - ${nanovg_SOURCE_DIR}/src/*.c) - - # nanovg is a static library so we do not need to install it. - add_library(nanovg STATIC ${NANOVG_SOURCES} ${NANOVG_HEADERS}) - - target_include_directories(nanovg - PUBLIC - $ - $ - ) -endif() diff --git a/cmake/deps/rapids-cmake-packages.json b/cmake/deps/rapids-cmake-packages.json new file mode 100644 index 00000000..9023b07b --- /dev/null +++ b/cmake/deps/rapids-cmake-packages.json @@ -0,0 +1,9 @@ +{ + "packages" : { + "GTest" : { + "version" : "1.12.1", + "git_url" : "https://github.com/google/googletest.git", + "git_tag" : "release-${version}" + } + } +} diff --git a/cmake/deps/yaml-cpp_rapids.cmake b/cmake/deps/yaml-cpp_rapids.cmake index b48dcb27..34e04bf5 100644 --- a/cmake/deps/yaml-cpp_rapids.cmake +++ b/cmake/deps/yaml-cpp_rapids.cmake @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -37,14 +37,14 @@ set(CMAKE_FIND_USE_PACKAGE_REGISTRY FALSE) # https://github.com/cpm-cmake/CPM.cmake/wiki/More-Snippets#yaml-cpp set(YAML_CPP_CPM_ARGS GITHUB_REPOSITORY jbeder/yaml-cpp - GIT_TAG yaml-cpp-0.6.3 + GIT_TAG yaml-cpp-0.7.0 OPTIONS "YAML_CPP_BUILD_TESTS Off" "YAML_CPP_BUILD_CONTRIB Off" "YAML_CPP_BUILD_TOOLS Off" "YAML_BUILD_SHARED_LIBS On" # Build the shared library instead of the default static library ) -rapids_cpm_find(yaml-cpp 0.6.3 +rapids_cpm_find(yaml-cpp 0.7.0 GLOBAL_TARGETS yaml-cpp BUILD_EXPORT_SET ${HOLOSCAN_PACKAGE_NAME}-exports diff --git a/cmake/modules/DefineSearchDirFor.cmake b/cmake/modules/DefineSearchDirFor.cmake deleted file mode 100644 index 44b242cb..00000000 --- a/cmake/modules/DefineSearchDirFor.cmake +++ /dev/null @@ -1,25 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Create a CMake cache variable named ${lib}_DIR and ensure it is set -# so it can be added to `CMAKE_PREFIX_PATH` to search for the package -macro(define_search_dir_for lib description) - set(${lib}_DIR CACHE PATH ${description}) - if(EXISTS ${${lib}_DIR}) - list(APPEND CMAKE_PREFIX_PATH ${${lib}_DIR}) - else() - message(FATAL_ERROR "${description} (${lib}_DIR) is not defined") - endif() -endmacro() diff --git a/cmake/modules/FindGXF.cmake b/cmake/modules/FindGXF.cmake index e48e2329..329b1ebf 100644 --- a/cmake/modules/FindGXF.cmake +++ b/cmake/modules/FindGXF.cmake @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -55,6 +55,12 @@ if(NOT HOLOSCAN_INSTALL_LIB_DIR) endif() endif() +# Need PatchELF to update the RPATH of the libs +find_program(PATCHELF_EXECUTABLE patchelf) +if(NOT PATCHELF_EXECUTABLE) + message(FATAL_ERROR "Please specify the PATCHELF executable") +endif() + # Library names list(APPEND _GXF_EXTENSIONS behavior_tree @@ -67,6 +73,10 @@ list(APPEND _GXF_EXTENSIONS serialization std tensor_rt + videoencoderio + videoencoder + videodecoderio + videodecoder ) # Common headers @@ -142,6 +152,29 @@ foreach(component IN LISTS _GXF_LIBRARIES) INTERFACE_INCLUDE_DIRECTORIES "${GXF_${component}_INCLUDE_DIRS}" ) + list(APPEND _GXF_LIB_RPATH "\$ORIGIN" "\$ORIGIN/gxf_extensions") + + # The video encoder/decoder libraries need an extra path for aarch64 + # To find the right l4t libraries + if(CMAKE_SYSTEM_PROCESSOR STREQUAL aarch64 OR CMAKE_SYSTEM_PROCESSOR STREQUAL arm64) + if(component STREQUAL videoencoderio + OR component STREQUAL videoencoder + OR component STREQUAL videodecoderio + OR component STREQUAL videodecoder) + list(APPEND _GXF_LIB_RPATH "/usr/lib/aarch64-linux-gnu/tegra/") + endif() + endif() + + # Patch RUNPATH + list(JOIN _GXF_LIB_RPATH ":" _GXF_LIB_RPATH) + execute_process(COMMAND + "${PATCHELF_EXECUTABLE}" + "--set-rpath" + "${_GXF_LIB_RPATH}" + "${gxf_component_location}" + ) + unset(_GXF_LIB_RPATH) + set(GXF_${component}_FOUND TRUE) else() set(GXF_${component}_FOUND FALSE) @@ -151,20 +184,6 @@ endforeach() unset(_GXF_EXTENSIONS) unset(_GXF_LIBRARIES) -# Sets the RPATH on GXF -find_program(PATCHELF_EXECUTABLE patchelf) -if(NOT PATCHELF_EXECUTABLE) - message(FATAL_ERROR "Please specify the PATCHELF executable") -endif() - -# Patch GXF core -get_target_property(GXF_CORE_IMPORTED_LOCATION GXF::core IMPORTED_LOCATION) -execute_process(COMMAND "${PATCHELF_EXECUTABLE}" "--set-rpath" "\$ORIGIN:\$ORIGIN/gxf_extensions" "${GXF_CORE_IMPORTED_LOCATION}") - -# Patch GXF std -get_target_property(GXF_STD_IMPORTED_LOCATION GXF::std IMPORTED_LOCATION) -execute_process(COMMAND "${PATCHELF_EXECUTABLE}" "--set-rpath" "\$ORIGIN:\$ORIGIN/gxf_extensions" "${GXF_STD_IMPORTED_LOCATION}") - # Find version if(GXF_core_INCLUDE_DIR) # Note: "kGxfCoreVersion \"(.*)\"$" does not work with a simple string @@ -178,6 +197,43 @@ if(GXF_core_INCLUDE_DIR) unset(_GXF_VERSION_LINE) endif() +# Install the Deepstream dependencies only on x86_64 +if(_public_GXF_recipe STREQUAL x86_64) + file(GLOB DS_DEPS_NVUTILS "${GXF_core_INCLUDE_DIR}/x86_64/nvutils/*.so") + file(GLOB DS_DEPS_CUVIDV4L2 "${GXF_core_INCLUDE_DIR}/x86_64/cuvidv4l2/*.so") + + foreach(dsdep IN LISTS DS_DEPS_NVUTILS DS_DEPS_CUVIDV4L2) + + # Patch RUNPATH + execute_process(COMMAND + "${PATCHELF_EXECUTABLE}" + "--set-rpath" + "\$ORIGIN" + "${dsdep}" + ) + + # Install the files + install(FILES "${dsdep}" + DESTINATION "${HOLOSCAN_INSTALL_LIB_DIR}" + COMPONENT "holoscan-gxf_libs" + ) + endforeach() + + # Create a symlink + INSTALL(CODE "execute_process( \ + COMMAND ${CMAKE_COMMAND} -E create_symlink \ + libnvv4l2.so \ + libv4l2.so.0 \ + COMMAND ${CMAKE_COMMAND} -E create_symlink \ + libnvv4lconvert.so \ + libv4lconvert.so.0 \ + WORKING_DIRECTORY \"\${CMAKE_INSTALL_PREFIX}/${HOLOSCAN_INSTALL_LIB_DIR}\" + )" + COMPONENT "holoscan-gxf_libs" + ) +endif() + + # GXE find_program(GXF_gxe_PATH NAMES gxe @@ -191,6 +247,21 @@ if(GXF_gxe_PATH) add_executable(GXF::gxe IMPORTED) endif() + # 'gxe' is read-only file, so we need to update the permissions before patching + # with patchelf (https://github.com/NixOS/nixpkgs/issues/14440). + # Note that 'file(CHMOD)' is supported since later CMake versions(>=3.19) + file(CHMOD ${GXF_gxe_PATH} + FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE + ) + + # Patch RUNPATH so that it can find libgxf_core.so library. + execute_process(COMMAND + "${PATCHELF_EXECUTABLE}" + "--set-rpath" + "\$ORIGIN:\$ORIGIN/../${HOLOSCAN_INSTALL_LIB_DIR}" + "${GXF_gxe_PATH}" + ) + set_target_properties(GXF::gxe PROPERTIES IMPORTED_LOCATION "${GXF_gxe_PATH}" ) diff --git a/cmake/modules/FindTensorRT.cmake b/cmake/modules/FindTensorRT.cmake index 6a68cbf5..544a58f4 100644 --- a/cmake/modules/FindTensorRT.cmake +++ b/cmake/modules/FindTensorRT.cmake @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,7 +15,7 @@ # Create TensorRT imported targets # -# This module defines TensorRT_FOUND if all GXF libraries are found or +# This module defines TensorRT_FOUND if the tensorRT include dir is found or # if the required libraries (COMPONENTS property in find_package) # are found. # @@ -28,7 +28,8 @@ # [2] https://cmake.org/cmake/help/latest/manual/cmake-packages.7.html#find-module-packages # Find headers -find_path(TensorRT_INCLUDE_DIR NAMES NvInferVersion.h REQUIRED) +find_path(TensorRT_INCLUDE_DIR NAMES NvInferVersion.h REQUIRED + PATHS /usr/include/x86_64-linux-gnu /usr/include/aarch64-linux-gnu) mark_as_advanced(TensorRT_INCLUDE_DIR) # Find version @@ -46,6 +47,7 @@ unset(_TRT_VERSION_FILE) # Find libs, and create the imported target macro(find_trt_library libname) + if(NOT TARGET TensorRT::${libname}) find_library(TensorRT_${libname}_LIBRARY NAMES ${libname} REQUIRED) mark_as_advanced(TensorRT_${libname}_LIBRARY) add_library(TensorRT::${libname} SHARED IMPORTED GLOBAL) @@ -53,6 +55,7 @@ macro(find_trt_library libname) IMPORTED_LOCATION "${TensorRT_${libname}_LIBRARY}" INTERFACE_SYSTEM_INCLUDE_DIRECTORIES "${TensorRT_INCLUDE_DIR}" ) + endif() endmacro() find_trt_library(nvinfer) @@ -66,5 +69,5 @@ include(FindPackageHandleStandardArgs) find_package_handle_standard_args(TensorRT FOUND_VAR TensorRT_FOUND VERSION_VAR TensorRT_VERSION - REQUIRED_VARS TensorRT_INCLUDE_DIR # no need for libs/targets, since find_library si REQUIRED + REQUIRED_VARS TensorRT_INCLUDE_DIR # no need for libs/targets, since find_library is REQUIRED ) diff --git a/cmake/modules/GenHeaderFromBinaryFile.cmake b/cmake/modules/GenHeaderFromBinaryFile.cmake new file mode 100644 index 00000000..78466c59 --- /dev/null +++ b/cmake/modules/GenHeaderFromBinaryFile.cmake @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include(CMakeParseArguments) + +# Generate a C++ header file from a binary file. +# +# The header file will contain a uint8_t array and will be generated in +# +# ${CMAKE_CURRENT_BINARY_DIR}\DIR\ARRAY_NAME.hpp +# +# The array name is build from FILE_PATH by using the file name and replacing `-`, and `.` by `_`. +# Example usage in CMakeLists.txt: +# +# add_library(fonts_target SHARED) +# gen_header_from_binary_file(TARGET fonts_target `fonts\Roboto-Bold.ttf`) +# +# This will generate the `fonts\roboto_bold_ttf.hpp` in the current binary dir. The file contains an +# array named `roboto_bold_ttf`. The file added to the project and the current binary dir is added +# to the include paths. +# To use the generated header in the code: +# +# #include +# void func() { +# cout << roboto_bold_ttf[0] << std::endl; +# } +# +# Usage: +# +# gen_header_from_binary_file (FILE_PATH [TARGET ]) +# +# ``FILE_PATH`` +# binary file to convert to a header relative to CMAKE_CURRENT_SOURCE_DIR +# ``TARGET`` +# if specified the generated header file and the include path is added to the target + +function(gen_header_from_binary_file) + set(_options ) + set(_singleargs FILE_PATH TARGET) + set(_multiargs ) + + cmake_parse_arguments(gen_header "${_options}" "${_singleargs}" "${_multiargs}" "${ARGN}") + + # read the file + set(SOURCE_FILE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/${gen_header_FILE_PATH}") + file(READ "${SOURCE_FILE_PATH}" FILE_CONTENT HEX) + + # separate into individual bytes + string(REGEX MATCHALL "([A-Fa-f0-9][A-Fa-f0-9])" FILE_CONTENT ${FILE_CONTENT}) + + # add 0x and commas + list(JOIN FILE_CONTENT ", 0x" FILE_CONTENT) + string(PREPEND FILE_CONTENT "0x") + + # build the array name + get_filename_component(FILE_ARRAY_NAME ${gen_header_FILE_PATH} NAME) + string(TOLOWER ${FILE_ARRAY_NAME} FILE_ARRAY_NAME) + string(REPLACE "-" "_" FILE_ARRAY_NAME ${FILE_ARRAY_NAME}) + string(REPLACE "." "_" FILE_ARRAY_NAME ${FILE_ARRAY_NAME}) + + # add a dependency to re-run configure when the file changes + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${SOURCE_FILE_PATH}") + + # create the header file + get_filename_component(FILE_DIRECTORY ${gen_header_FILE_PATH} DIRECTORY) + set(HEADER_FILE_NAME "${CMAKE_CURRENT_BINARY_DIR}/${FILE_DIRECTORY}/${FILE_ARRAY_NAME}.hpp") + configure_file( + "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/gen_header_from_binary_file_template/header.hpp.in" + ${HEADER_FILE_NAME} + ) + + if(gen_header_TARGET) + # add the created file to the target + target_sources(${gen_header_TARGET} + PRIVATE + ${HEADER_FILE_NAME} + ) + # also add the binary dir to include directories so the header can be found + target_include_directories(${gen_header_TARGET} + PRIVATE + $ + ) + endif() + + message(STATUS "Created header ${HEADER_FILE_NAME} from binary file '${SOURCE_FILE_PATH}'") + +endfunction() diff --git a/cmake/modules/GenerateGXEApp.cmake b/cmake/modules/GenerateGXEApp.cmake index c6a6cffc..02ec2924 100644 --- a/cmake/modules/GenerateGXEApp.cmake +++ b/cmake/modules/GenerateGXEApp.cmake @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -133,11 +133,10 @@ function(create_gxe_application) # Copy the yaml file get_filename_component(GXE_APP_YAML_NAME ${GXE_APP_YAML} NAME) cmake_path(APPEND GXE_APP_YAML_RELATIVE_PATH ${app_relative_dest_path} ${GXE_APP_YAML_NAME}) - add_custom_target(gxf_app-${GXE_APP_NAME}-yaml + add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${GXE_APP_YAML_NAME} COMMAND ${CMAKE_COMMAND} -E copy ${GXE_APP_YAML} ${CMAKE_CURRENT_BINARY_DIR}/${GXE_APP_YAML_NAME} DEPENDS ${GXE_APP_YAML} - ) - add_dependencies(${HOLOSCAN_PACKAGE_NAME} gxf_app-${GXE_APP_NAME}-yaml) + ) install(FILES ${GXE_APP_YAML} DESTINATION ${app_relative_dest_path} COMPONENT "${GXE_APP_COMPONENT}" @@ -173,10 +172,6 @@ function(create_gxe_application) VERBATIM DEPENDS "${GXE_APP_MANIFEST_ORIGINAL}" ) - add_custom_target(gxf_app-${GXE_APP_NAME}-manifest - DEPENDS "${GXE_APP_MANIFEST}" - ) - add_dependencies(${HOLOSCAN_PACKAGE_NAME} gxf_app-${GXE_APP_NAME}-manifest) install(FILES ${GXE_APP_MANIFEST} DESTINATION ${app_relative_dest_path} COMPONENT "${GXE_APP_COMPONENT}" @@ -198,6 +193,16 @@ export LD_LIBRARY_PATH=$(pwd):$(pwd)/${HOLOSCAN_INSTALL_LIB_DIR}:\${LD_LIBRARY_P PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE COMPONENT "${GXE_APP_COMPONENT}" ) + + # Create custom targets for the generated files + add_custom_target(${GXE_APP_NAME} ALL + DEPENDS + "${CMAKE_CURRENT_BINARY_DIR}/${GXE_APP_YAML_NAME}" + "${GXE_APP_MANIFEST}" + # TODO: fix name conflict dependency with `file(GENERATE)` + # Would need to find a way to only generate at build time? + # "${GXE_APP_EXECUTABLE}" + ) endfunction() # Install target to a sub directory that has the same folder structure as the source diff --git a/cmake/modules/GenerateGXEAppInstall.cmake b/cmake/modules/GenerateGXEAppInstall.cmake index ed6e8c08..c74806b2 100644 --- a/cmake/modules/GenerateGXEAppInstall.cmake +++ b/cmake/modules/GenerateGXEAppInstall.cmake @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -83,8 +83,8 @@ function(create_gxe_application) if(NOT lib) set(lib "$") endif() - elseif(GXF_EXTENSIONS_DIR) # use the full path - set(lib "${GXF_EXTENSIONS_DIR}/lib${extension}.so") + elseif(GXF_EXTENSIONS_DIR) # use the full path and check if the target exists at generation time + set(lib "$<$:$>$<$>:${GXF_EXTENSIONS_DIR}/lib${extension}.so>") else() message(FATAL_ERROR "Target not found for ${extension} and GXF_EXTENSTIONS_DIR not set") endif() diff --git a/cmake/modules/HoloscanCPack.cmake b/cmake/modules/HoloscanCPack.cmake index 682ec38d..a943f553 100644 --- a/cmake/modules/HoloscanCPack.cmake +++ b/cmake/modules/HoloscanCPack.cmake @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -36,41 +36,36 @@ set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT) # Just one package will all the components set(CPACK_DEB_COMPONENT_INSTALL 1) -set(CPACK_COMPONENTS_GROUPING ONE_PER_GROUP) -set(CPACK_DEBIAN_ENABLE_COMPONENT_DEPENDS 1) +set(CPACK_COMPONENTS_GROUPING ALL_COMPONENTS_IN_ONE) # List of the components that should be installed -set(CPACK_COMPONENTS_ALL +set(CPACK_COMPONENTS_ALL holoscan-core holoscan-gxf_extensions holoscan-gxf_libs holoscan-gxf_bins holoscan-modules holoscan-dependencies - holoscan-apps - holoscan-data + holoscan-examples holoscan-python_libs ) -set(CPACK_DEBIAN_DEV_PACKAGE_DEPENDS "cuda (>=11.6.1), cuda (<<12.0), libnvinfer-bin (>= 8.2.3), libnvinfer-plugin8 (>=8.2.3),libnvinfer8 (>= 8.2.3), libnvonnxparsers8 (>= 8.2.3)") -set(CPACK_COMPONENT_APPS_DEPENDS DEV) +set(CPACK_DEBIAN_PACKAGE_DEPENDS "cuda-cudart-dev-11-6 | cuda-cudart-dev-11-7 | cuda-cudart-dev-11-8, \ +cuda-nvcc-11-6 | cuda-nvcc-11-7 | cuda-nvcc-11-8, \ +libnpp-11-6 | libnpp-11-7 | libnpp-11-8, \ +libnvinfer-bin (>= 8.2.3), \ +libvulkan1 (>=1.2.131), libx11-6 (>=1.6.9)") include(CPack) message(STATUS "Components to pack: ${CPACK_COMPONENTS_ALL}") # Create the Dev package -cpack_add_component_group(DEV "Holoscan - Dev") -cpack_add_component(holoscan-core GROUP DEV) -cpack_add_component(holoscan-python_libs GROUP DEV) -cpack_add_component(holoscan-gxf_extensions GROUP DEV) -cpack_add_component(holoscan-gxf_libs GROUP DEV) -cpack_add_component(holoscan-gxf_bins GROUP DEV) -cpack_add_component(holoscan-modules GROUP DEV) -cpack_add_component(holoscan-dependencies GROUP DEV) - -# Create the Apps package -cpack_add_component_group(APPS "Holoscan - Sample Applications") -cpack_add_component(holoscan-apps GROUP APPS) -cpack_add_component(holoscan-data GROUP APPS) - +cpack_add_component(holoscan-core) +cpack_add_component(holoscan-python_libs) +cpack_add_component(holoscan-gxf_extensions) +cpack_add_component(holoscan-gxf_libs) +cpack_add_component(holoscan-gxf_bins) +cpack_add_component(holoscan-modules) +cpack_add_component(holoscan-dependencies) +cpack_add_component(holoscan-examples) diff --git a/cmake/modules/SetupCUDA-post.cmake b/cmake/modules/SetupCUDA-post.cmake deleted file mode 100644 index f2aac6f5..00000000 --- a/cmake/modules/SetupCUDA-post.cmake +++ /dev/null @@ -1,36 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -list(APPEND CMAKE_MESSAGE_CONTEXT "setupcuda-post") - -# If CMAKE_PROJECT__INCLUDE is chained, call the previous one -if(DEFINED HOLOSCAN_PREVIOUS_CMAKE_PROJECT_INCLUDE) - include("${HOLOSCAN_PREVIOUS_CMAKE_PROJECT_INCLUDE}") -endif() - -# Setup CMAKE_CUDA_HOST_COMPILER -if(NOT DEFINED CMAKE_CUDA_HOST_COMPILER) - message(STATUS "Setting CUDA host compiler to use the same CXX compiler defined by CMAKE_CXX_COMPILER(${CMAKE_CXX_COMPILER})") - - # Set host compiler if we are not using clang - # Clang (>=8) doesn't seem to be compatible with CUDA (>=11.4) - # - https://forums.developer.nvidia.com/t/cuda-issues-with-clang-compiler/177589 - # - https://forums.developer.nvidia.com/t/building-with-clang-cuda-11-3-0-works-but-with-cuda-11-3-1-fails-regression/182176 - if(NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") - set(CMAKE_CUDA_HOST_COMPILER "${CMAKE_CXX_COMPILER}") - endif() -endif() - -list(POP_BACK CMAKE_MESSAGE_CONTEXT) diff --git a/cmake/modules/SetupCUDA.cmake b/cmake/modules/SetupCUDA.cmake index 881c9266..904be8ea 100644 --- a/cmake/modules/SetupCUDA.cmake +++ b/cmake/modules/SetupCUDA.cmake @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,15 +13,87 @@ # See the License for the specific language governing permissions and # limitations under the License. +# In this file, we: +# 1. Set up the CMAKE_CUDA_HOST_COMPILER to use CMAKE_CXX_COMPILER if not Clang +# 2. Use enable_language(CUDA) to retrieve CMAKE_CUDA_COMPILER_ID and CMAKE_CUDA_COMPILER_VERSION +# (this requires caching and unsetting CMAKE_CUDA_ARCHITECTURES beforehand to avoid errors) +# 3. Parse and update CMAKE_CUDA_ARCHITECTURES to support NATIVE and ALL architectures, and filter +# out unsupported archs on current host. +# +# Some of this process will be simplified when updating to CMAKE 3.23+ with enhanced support for +# CMAKE_CUDA_ARCHITECTURES +# + list(APPEND CMAKE_MESSAGE_CONTEXT "buildconfig") +message(STATUS "Configuring CUDA Compiler") + +# If a toolchain file was provided, include it now to read any CMAKE_CUDA variables it may define. +if(DEFINED CMAKE_TOOLCHAIN_FILE) + include(${CMAKE_TOOLCHAIN_FILE}) +endif() + +# Setup CMAKE_CUDA_HOST_COMPILER +if(NOT DEFINED CMAKE_CUDA_HOST_COMPILER) + message(STATUS "Setting CUDA host compiler to use the same CXX compiler defined by CMAKE_CXX_COMPILER(${CMAKE_CXX_COMPILER})") + # Set host compiler if we are not using clang + # Clang (>=8) doesn't seem to be compatible with CUDA (>=11.4) + # - https://forums.developer.nvidia.com/t/cuda-issues-with-clang-compiler/177589 + # - https://forums.developer.nvidia.com/t/building-with-clang-cuda-11-3-0-works-but-with-cuda-11-3-1-fails-regression/182176 + if(NOT "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + set(CMAKE_CUDA_HOST_COMPILER "${CMAKE_CXX_COMPILER}") + endif() +endif() + +# Keep track of original CMAKE_CUDA_ARCHITECTURES and unset to avoid errors when calling +# 'enable_language(CUDA)' with CMAKE_CUDA_ARCHITECTURES set to 'ALL' or 'NATIVE' +set(CMAKE_CUDA_ARCHITECTURES_CACHE ${CMAKE_CUDA_ARCHITECTURES}) +unset(CMAKE_CUDA_ARCHITECTURES) +unset(CMAKE_CUDA_ARCHITECTURES CACHE) + +# Delayed CUDA language enablement to make CMAKE_CUDA_COMPILER_ID and CMAKE_CUDA_COMPILER_VERSION +# available for CMAKE_CUDA_ARCHITECTURES setup +enable_language(CUDA) + +# Restore original CMAKE_CUDA_ARCHITECTURES +set(CMAKE_CUDA_ARCHITECTURES ${CMAKE_CUDA_ARCHITECTURES_CACHE} CACHE STRING "CUDA architectures to build for" FORCE) +unset(CMAKE_CUDA_ARCHITECTURES_CACHE) + message(STATUS "Configuring CUDA Architecture") -function(update_cmake_cuda_architectures supported_archs) +# Function to filter archs and create PTX for last arch +function(update_cmake_cuda_architectures supported_archs warn) list(APPEND CMAKE_MESSAGE_CONTEXT "update_cmake_cuda_architectures") - if(CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA" AND CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 11.1.0) - list(REMOVE_ITEM supported_archs "86") + if(CMAKE_CUDA_COMPILER_ID STREQUAL "NVIDIA") + if(CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 12.0.0) + if ("90a" IN_LIST supported_archs AND ${warn}) + message(WARNING "sm90a not supported with nvcc < 12.0.0") + endif() + list(REMOVE_ITEM supported_archs "90a") + endif() + if(CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 11.8.0) + if ("90" IN_LIST supported_archs AND ${warn}) + message(WARNING "sm90 not supported with nvcc < 11.8.0") + endif() + if ("89" IN_LIST supported_archs AND ${warn}) + message(WARNING "sm89 not supported with nvcc < 11.8.0") + endif() + list(REMOVE_ITEM supported_archs "90") + list(REMOVE_ITEM supported_archs "89") + endif() + if(CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 11.5.0) + if ("87" IN_LIST supported_archs AND ${warn}) + message(WARNING "sm87 not supported with nvcc < 11.5.0") + endif() + list(REMOVE_ITEM supported_archs "87") + endif() + if(CMAKE_CUDA_COMPILER_VERSION VERSION_LESS 11.2.0) + if ("86" IN_LIST supported_archs AND ${warn}) + message(WARNING "sm86 not supported with nvcc < 11.2.0") + endif() + list(REMOVE_ITEM supported_archs "86") + endif() endif() # Create SASS for all architectures in the list and @@ -33,29 +105,34 @@ function(update_cmake_cuda_architectures supported_archs) set(CMAKE_CUDA_ARCHITECTURES ${supported_archs} PARENT_SCOPE) endfunction() -# Default to 'NATIVE' if no CUDA_ARCHITECTURES are specified -# Otherwise, update CMAKE_CUDA_ARCHITECTURES to the specified values (last architecture creates only PTX) +# Default to 'NATIVE' if no CUDA_ARCHITECTURES are specified. if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES OR CMAKE_CUDA_ARCHITECTURES STREQUAL "") set(CMAKE_CUDA_ARCHITECTURES "NATIVE") message(STATUS "CMAKE_CUDA_ARCHITECTURES is not defined. Using 'NATIVE' to " - "build only for locally available architecture through rapids_cuda_init_architectures(). " + "build only for locally available architecture through rapids_cuda_set_architectures(). " "Please specify -DCMAKE_CUDA_ARCHITECTURES='ALL' to support all archs.") -elseif(NOT CMAKE_CUDA_ARCHITECTURES STREQUAL "ALL" AND NOT CMAKE_CUDA_ARCHITECTURES STREQUAL "NATIVE") - message(STATUS "Requested CUDA architectures are: ${CMAKE_CUDA_ARCHITECTURES}") - update_cmake_cuda_architectures("${CMAKE_CUDA_ARCHITECTURES}") +else() + message(STATUS "Requested CUDA architectures: ${CMAKE_CUDA_ARCHITECTURES}") endif() -# https://docs.rapids.ai/api/rapids-cmake/stable/command/rapids_cuda_init_architectures.html -# - need to be called before project() -rapids_cuda_init_architectures(${HOLOSCAN_PACKAGE_NAME}) - -# Store the existing project include file path if exists, to chain with the new one -# (https://cmake.org/cmake/help/latest/variable/CMAKE_PROJECT_PROJECT-NAME_INCLUDE.html) -if(DEFINED CMAKE_PROJECT_${HOLOSCAN_PACKAGE_NAME}_INCLUDE) - set(HOLOSCAN_PREVIOUS_CMAKE_PROJECT_INCLUDE "${CMAKE_PROJECT_${HOLOSCAN_PACKAGE_NAME}_INCLUDE}") +if(CMAKE_CUDA_ARCHITECTURES STREQUAL "NATIVE") + # Let RAPIDS-CUDA handle "NATIVE" + # https://docs.rapids.ai/api/rapids-cmake/nightly/command/rapids_cuda_set_architectures.html + rapids_cuda_set_architectures(NATIVE) +elseif(CMAKE_CUDA_ARCHITECTURES STREQUAL "ALL") + # Since `rapids_cuda_init_architectures()` cannot handle all the supported architecture, + # (only 60;70;75;80;86 are considered. See https://github.com/rapidsai/rapids-cmake/blob/branch-22.08/rapids-cmake/cuda/set_architectures.cmake#L60) + # we need to have our own logic to add all architectures. + # FYI, since CMake 3.23, it supports CUDA_ARCHITECTURES (https://cmake.org/cmake/help/latest/prop_tgt/CUDA_ARCHITECTURES.html) + # Note: 72 is Xavier + # 87 is Orin + # 90a is Thor + set(CMAKE_CUDA_ARCHITECTURES "60;70;72;75;80;86;87;89;90;90a") + update_cmake_cuda_architectures("${CMAKE_CUDA_ARCHITECTURES}" FALSE) +elseif() + update_cmake_cuda_architectures("${CMAKE_CUDA_ARCHITECTURES}" TRUE) endif() -# Set post-project hook -set(CMAKE_PROJECT_${HOLOSCAN_PACKAGE_NAME}_INCLUDE "${CMAKE_CURRENT_LIST_DIR}/SetupCUDA-post.cmake") +message(STATUS "Using CUDA architectures: ${CMAKE_CUDA_ARCHITECTURES}") list(POP_BACK CMAKE_MESSAGE_CONTEXT) diff --git a/cmake/modules/WrapOperatorAsGXFExtension.cmake b/cmake/modules/WrapOperatorAsGXFExtension.cmake new file mode 100644 index 00000000..0877f279 --- /dev/null +++ b/cmake/modules/WrapOperatorAsGXFExtension.cmake @@ -0,0 +1,161 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generates a GXF extension that includes a GXF codelet which wraps the input C++ operator +# +# Inputs are: +# OPERATOR_CLASS: name of the native operator class to wrap (with namespace) +# OPERATOR_HEADER_INCLUDE: path to the native operator class header file +# OPERATOR_TARGET: cmake target of the library holding the native operator implementation +# CODELET_ID_HASH[1|2]: hash pair to register the GXF codelet +# CODELET_NAME: name of the GXF codelet class (without namespace) +# CODELET_NAMESPACE: namespace of the GXF codelet class +# CODELET DESCRIPTION: description of the GXF codelet +# CODELET_TARGET_NAME: name of the cmake target to generate for the gxf codelet library +# Note: optional, defaults to CODELET_NAME lowercase +# CODELET_TARGET_PROPERTIES: any valid cmake properties that should affect the target above. +# Note: `INCLUDE_DIRECTORIES` might be needed to point to the root of the OPERATOR_HEADER_INCLUDE path +# EXTENSION_ID_HASH[1|2]: hash pair to register the GXF extension +# EXTENSION_NAME: name of the GXF extension class (without namespace) +# EXTENSION DESCRIPTION: description of the GXF extension +# EXTENSION_AUTHOR: GXF extension author +# EXTENSION_VERSION: GXF extension version +# EXTENSION_LICENSE: GXF extension license +# EXTENSION_TARGET_NAME: name of the cmake target to generate for the gxf extension library +# Note: optional, defaults to EXTENSION_NAME lowercase +# EXTENSION_TARGET_PROPERTIES: any valid cmake properties that should affect the target above. +# +# Outputs are: +# lib.so +# lib.so + +function(wrap_operator_as_gxf_extension) + # Define arguments + list(APPEND OPTION_VARS "") + list(APPEND REQUIRED_SINGLE_VALUE_VARS + EXTENSION_ID_HASH1 + EXTENSION_ID_HASH2 + EXTENSION_NAME + EXTENSION_DESCRIPTION + EXTENSION_AUTHOR + EXTENSION_VERSION + EXTENSION_LICENSE + CODELET_ID_HASH1 + CODELET_ID_HASH2 + CODELET_NAME + CODELET_NAMESPACE + CODELET_DESCRIPTION + OPERATOR_TARGET + OPERATOR_CLASS + OPERATOR_HEADER_INCLUDE + ) + list(APPEND OPTIONAL_SINGLE_VALUE_VARS + CODELET_TARGET_NAME + EXTENSION_TARGET_NAME + ) + list(APPEND MULTI_VALUE_VARS + EXTENSION_TARGET_PROPERTIES + CODELET_TARGET_PROPERTIES + ) + + # Parse arguments + list(APPEND SINGLE_VALUE_VARS + ${REQUIRED_SINGLE_VALUE_VARS} + ${OPTIONAL_SINGLE_VALUE_VARS} + ) + cmake_parse_arguments( + ARG + "${OPTION_VARS}" + "${SINGLE_VALUE_VARS}" + "${MULTI_VALUE_VARS}" + ${ARGN} + ) + + # Remove the `ARG` prefix + foreach(VAR IN LISTS OPTION_VARS SINGLE_VALUE_VARS MULTI_VALUE_VARS) + if (DEFINED ARG_${VAR}) + set(${VAR} "${ARG_${VAR}}") + endif() + endforeach() + + # Check for required arguments + foreach(VAR IN LISTS REQUIRED_SINGLE_VALUE_VARS) + if (NOT DEFINED ${VAR}) + message(SEND_ERROR "Missing required ${VAR} argument") + endif() + endforeach() + + ## Generate additional variables + # EXTENSION_CPP + # CODELET_CPP + # CODELET_HEADER_INCLUDE + # CODELET_HEADER + # CODELET_NAME_UPPERCASE + # CODELET_NAMESPACE_UPPERCASE + set(EXTENSION_CPP_SUFFIX ) + if (CODELET_NAME STREQUAL EXTENSION_NAME) + set(EXTENSION_CPP_SUFFIX "_ext") + endif() + set(EXTENSION_CPP ${CMAKE_CURRENT_BINARY_DIR}/${EXTENSION_NAME}${EXTENSION_CPP_SUFFIX}.cpp) + set(CODELET_CPP ${CMAKE_CURRENT_BINARY_DIR}/${CODELET_NAME}.cpp) + set(CODELET_HEADER_INCLUDE ${CODELET_NAME}.hpp) + set(CODELET_HEADER ${CMAKE_CURRENT_BINARY_DIR}/${CODELET_HEADER_INCLUDE}) + string(TOUPPER ${CODELET_NAME} CODELET_NAME_UPPERCASE) + string(TOUPPER ${CODELET_NAMESPACE} CODELET_NAMESPACE_UPPERCASE) + + # Configure the source files + set(TEMPLATE_DIR "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/wrap_operator_as_gxf_template") + configure_file(${TEMPLATE_DIR}/extension.cpp.in ${EXTENSION_CPP}) + configure_file(${TEMPLATE_DIR}/codelet.cpp.in ${CODELET_CPP}) + configure_file(${TEMPLATE_DIR}/codelet.hpp.in ${CODELET_HEADER}) + + # Create codelet library + if (NOT DEFINED CODELET_TARGET_NAME) + string(TOLOWER ${CODELET_NAME} CODELET_TARGET_NAME) + endif() + message(STATUS ${CODELET_TARGET_NAME}) + add_library(${CODELET_TARGET_NAME} SHARED + ${CODELET_CPP} + ${CODELET_HEADER} + ) + target_link_libraries(${CODELET_TARGET_NAME} + PUBLIC + gxf_holoscan_wrapper_lib + ${OPERATOR_TARGET} + ) + if (DEFINED CODELET_TARGET_PROPERTIES) + set_target_properties(${CODELET_TARGET_NAME} + PROPERTIES ${CODELET_TARGET_PROPERTIES} + ) + endif() + + # Create extension library + if (NOT DEFINED EXTENSION_TARGET_NAME) + string(TOLOWER ${EXTENSION_NAME} EXTENSION_TARGET_NAME) + endif() + message(STATUS ${EXTENSION_TARGET_NAME}) + add_library(${EXTENSION_TARGET_NAME} SHARED + ${EXTENSION_CPP} + ) + target_link_libraries(${EXTENSION_TARGET_NAME} + PUBLIC ${CODELET_TARGET_NAME} + ) + if (DEFINED EXTENSION_TARGET_PROPERTIES) + set_target_properties(${EXTENSION_TARGET_NAME} + PROPERTIES ${EXTENSION_TARGET_PROPERTIES} + ) + endif() + +endfunction() diff --git a/modules/holoviz/LICENSE b/cmake/modules/gen_header_from_binary_file_template/header.hpp.in similarity index 81% rename from modules/holoviz/LICENSE rename to cmake/modules/gen_header_from_binary_file_template/header.hpp.in index d393318d..b3d17d50 100644 --- a/modules/holoviz/LICENSE +++ b/cmake/modules/gen_header_from_binary_file_template/header.hpp.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,3 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +#include + +static uint8_t @FILE_ARRAY_NAME@[] = { + @FILE_CONTENT@ +}; + diff --git a/cmake/modules/wrap_operator_as_gxf_template/codelet.cpp.in b/cmake/modules/wrap_operator_as_gxf_template/codelet.cpp.in new file mode 100644 index 00000000..65cab5b3 --- /dev/null +++ b/cmake/modules/wrap_operator_as_gxf_template/codelet.cpp.in @@ -0,0 +1,27 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "@CODELET_HEADER_INCLUDE@" +#include "@OPERATOR_HEADER_INCLUDE@" + +namespace @CODELET_NAMESPACE@ { + +@CODELET_NAME@::@CODELET_NAME@() : holoscan::gxf::OperatorWrapper() { + op_ = std::make_shared<@OPERATOR_CLASS@>(); +} + +} // namespace @CODELET_NAMESPACE@ diff --git a/cmake/modules/wrap_operator_as_gxf_template/codelet.hpp.in b/cmake/modules/wrap_operator_as_gxf_template/codelet.hpp.in new file mode 100644 index 00000000..53f16da0 --- /dev/null +++ b/cmake/modules/wrap_operator_as_gxf_template/codelet.hpp.in @@ -0,0 +1,37 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef @CODELET_NAMESPACE_UPPERCASE@_@CODELET_NAME_UPPERCASE@_HPP +#define @CODELET_NAMESPACE_UPPERCASE@_@CODELET_NAME_UPPERCASE@_HPP + +#include "gxf_holoscan_wrapper/operator_wrapper.hpp" + +namespace @CODELET_NAMESPACE@ { + +class @CODELET_NAME@ : public holoscan::gxf::OperatorWrapper { + public: + @CODELET_NAME@(); + virtual ~@CODELET_NAME@() = default; + + const char* holoscan_typename() const { + return "@OPERATOR_CLASS@"; + } +}; + +} // namespace @CODELET_NAMESPACE@ + +#endif /* @CODELET_NAMESPACE_UPPERCASE@_@CODELET_NAME_UPPERCASE@_HPP */ diff --git a/gxf_extensions/format_converter/format_converter_ext.cpp b/cmake/modules/wrap_operator_as_gxf_template/extension.cpp.in similarity index 51% rename from gxf_extensions/format_converter/format_converter_ext.cpp rename to cmake/modules/wrap_operator_as_gxf_template/extension.cpp.in index 52c12113..25b401db 100644 --- a/gxf_extensions/format_converter/format_converter_ext.cpp +++ b/cmake/modules/wrap_operator_as_gxf_template/extension.cpp.in @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,14 +14,19 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "gxf/std/extension_factory_helper.hpp" -#include "format_converter.hpp" +#include "gxf/std/extension_factory_helper.hpp" +#include "@CODELET_HEADER_INCLUDE@" GXF_EXT_FACTORY_BEGIN() -GXF_EXT_FACTORY_SET_INFO(0x6b7ee46293a142f6, 0xa92611c951965c2b, "FormatConverterExtension", - "Holoscan Format Converter extension", "NVIDIA", "0.2.0", "LICENSE"); -GXF_EXT_FACTORY_ADD(0xd8cdf281f6574461, 0x830d93ef6d1252e7, - nvidia::holoscan::formatconverter::FormatConverter, nvidia::gxf::Codelet, - "Format Converter codelet."); +GXF_EXT_FACTORY_SET_INFO(@EXTENSION_ID_HASH1@, @EXTENSION_ID_HASH2@, + "@EXTENSION_NAME@", + "@EXTENSION_DESCRIPTION@", + "@EXTENSION_AUTHOR@", + "@EXTENSION_VERSION@", + "@EXTENSION_LICENSE@"); +GXF_EXT_FACTORY_ADD(@CODELET_ID_HASH1@, @CODELET_ID_HASH2@, + @CODELET_NAMESPACE@::@CODELET_NAME@, + holoscan::gxf::OperatorWrapper, + "@CODELET_DESCRIPTION@"); GXF_EXT_FACTORY_END() diff --git a/cmake/setup_dependencies.cmake b/cmake/setup_dependencies.cmake index 6b216f20..6423c5dc 100644 --- a/cmake/setup_dependencies.cmake +++ b/cmake/setup_dependencies.cmake @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -85,6 +85,10 @@ superbuild_depend(dlpack_rapids) superbuild_depend(gxf) superbuild_depend(glfw_rapids) superbuild_depend(glad_rapids) -superbuild_depend(nanovg_rapids) superbuild_depend(tensorrt) superbuild_depend(ajantv2_rapids) + +# Testing dependencies +if(HOLOSCAN_BUILD_TESTS) + superbuild_depend(gtest_rapids) +endif() diff --git a/data/CMakeLists.txt b/data/CMakeLists.txt index 92908f58..37a8ac9b 100644 --- a/data/CMakeLists.txt +++ b/data/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,60 +15,50 @@ include(ExternalProject) -# Download the endoscopy sample data -ExternalProject_Add(endoscopy_data - URL https://api.ngc.nvidia.com/v2/resources/nvidia/clara-holoscan/holoscan_endoscopy_sample_data/versions/20221121/zip - URL_MD5 d86b6145b9c72d0063d512875f613963 - DOWNLOAD_NAME holoscan_endoscopy_sample_data_20221121.zip - DOWNLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR} - SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/endoscopy - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - EXCLUDE_FROM_ALL 1 -) +# Endoscopy sample data +add_library(endoscopy_data INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/endoscopy/video/surgical_video.264") +# Setup the installation rule +install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/endoscopy DESTINATION data COMPONENT holoscan-data) -# Download the ultrasound sample data -ExternalProject_Add(ultrasound_data - URL https://api.ngc.nvidia.com/v2/resources/nvidia/clara-holoscan/holoscan_ultrasound_sample_data/versions/20220608/zip - URL_MD5 de88ccb8660f4e5787f726d868d77f69 - DOWNLOAD_NAME holoscan_ultrasound_sample_data_20220608 - DOWNLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR} - SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/ultrasound - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - EXCLUDE_FROM_ALL 1 -) +if(HOLOSCAN_DOWNLOAD_DATASETS) + # Download the endoscopy sample data + ExternalProject_Add(endoscopy_data_download + URL https://api.ngc.nvidia.com/v2/resources/nvidia/clara-holoscan/holoscan_endoscopy_sample_data/versions/20230128/zip + URL_MD5 9732a54944589f7ca4f1337e8adf0838 + DOWNLOAD_NAME holoscan_endoscopy_sample_data_20230128.zip + DOWNLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR} + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/endoscopy + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + EXCLUDE_FROM_ALL 1 + BUILD_BYPRODUCTS "${CMAKE_CURRENT_SOURCE_DIR}/endoscopy/video/surgical_video.264" + ) -# Download the multiai sample data -ExternalProject_Add(multiai_ultrasound_data - URL https://api.ngc.nvidia.com/v2/resources/nvidia/clara-holoscan/holoscan_multi_ai_ultrasound_sample_data/versions/20221201/zip - URL_MD5 8096b080563e6a75c8e50c3b271364cd - DOWNLOAD_NAME holoscan_multi_ai_ultrasound_sample_data_20221201.zip - DOWNLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR} - SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/multiai_ultrasound - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - EXCLUDE_FROM_ALL 1 -) + # Add dependency to force the download + add_dependencies(endoscopy_data endoscopy_data_download) +endif() -# Download the colonoscopy sample data -ExternalProject_Add(colonoscopy_data - URL https://api.ngc.nvidia.com/v2/resources/nvidia/clara-holoscan/holoscan_colonoscopy_sample_data/versions/20221031/zip - URL_MD5 d94aee8ee4d847535db85725f2c6b7ac - DOWNLOAD_NAME holoscan_colonoscopy_sample_data_20221031.zip - DOWNLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR} - SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/colonoscopy - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - EXCLUDE_FROM_ALL 1 -) +# Multiai ultrasound sample data +add_library(multiai_ultrasound_data INTERFACE "${CMAKE_CURRENT_SOURCE_DIR}/multiai_ultrasound/video/icardio_input1.avi") +# Setup the installation rule +install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/multiai_ultrasound DESTINATION data COMPONENT holoscan-data) - # Setup the installation rules - install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/endoscopy DESTINATION data COMPONENT holoscan-data) - install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/ultrasound DESTINATION data COMPONENT holoscan-data) - install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/colonoscopy DESTINATION data COMPONENT holoscan-data) - install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/multiai_ultrasound DESTINATION data COMPONENT holoscan-data) +if(HOLOSCAN_DOWNLOAD_DATASETS) + # Download the multiai sample data + ExternalProject_Add(multiai_ultrasound_data_download + URL https://api.ngc.nvidia.com/v2/resources/nvidia/clara-holoscan/holoscan_multi_ai_ultrasound_sample_data/versions/20221201/zip + URL_MD5 8096b080563e6a75c8e50c3b271364cd + DOWNLOAD_NAME holoscan_multi_ai_ultrasound_sample_data_20221201.zip + DOWNLOAD_DIR ${CMAKE_CURRENT_SOURCE_DIR} + SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/multiai_ultrasound + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + EXCLUDE_FROM_ALL 1 + BUILD_BYPRODUCTS "${CMAKE_CURRENT_SOURCE_DIR}/multiai_ultrasound/video/icardio_input1.avi" + ) + + # Add dependency to force the download + add_dependencies(multiai_ultrasound_data multiai_ultrasound_data_download) +endif() diff --git a/examples/CMakeLists.min.txt.in b/examples/CMakeLists.min.txt.in new file mode 100644 index 00000000..89a57a7d --- /dev/null +++ b/examples/CMakeLists.min.txt.in @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.20) +project(holoscan_examples) + +# Finds the package holoscan +find_package(holoscan REQUIRED CONFIG + PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install") + +# Enable testing +include(CTest) + +@cmake_example_directories@ diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index bf0aecb2..59798dd2 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,9 +12,33 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -add_subdirectory(basic_workflow) +add_subdirectory(aja_capture) add_subdirectory(bring_your_own_model) -add_subdirectory(native_operator) +add_subdirectory(cupy_native) +add_subdirectory(hello_world) +add_subdirectory(holoviz) +add_subdirectory(numpy_native) +add_subdirectory(ping_simple) +add_subdirectory(ping_custom_op) +add_subdirectory(ping_multi_port) add_subdirectory(tensor_interop) -add_subdirectory(video_sources) +add_subdirectory(v4l2_camera) +add_subdirectory(video_replayer) +add_subdirectory(wrap_operator_as_gxf_extension) + +if(HOLOSCAN_INSTALL_EXAMPLE_SOURCE) +# Generates the install CMakeLists.txt to compile all the cpp examples +set(cmake_example_directories) +file(GLOB_RECURSE cmakeinstallfiles RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} */CMakeLists.min.txt) +foreach(cmakeinstallfile IN LISTS cmakeinstallfiles) + string(REPLACE "/CMakeLists.min.txt" "" cmakedirectory ${cmakeinstallfile} ) + string(APPEND cmake_example_directories "add_subdirectory(${cmakedirectory})\n") +endforeach() + +configure_file(CMakeLists.min.txt.in CMakeLists.min.txt @ONLY) + +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/CMakeLists.min.txt" + RENAME "CMakeLists.txt" + DESTINATION examples + COMPONENT holoscan-examples) +endif() diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 00000000..4e2bdd28 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,40 @@ +# Holoscan SDK Examples + +This directory contains examples to help users learn how to use the Holoscan SDK for development. See [HoloHub](https://github.com/nvidia-holoscan/holohub) to find additional reference applications. + +## Core + +The following examples demonstrate the basics of the Holoscan core API, and are ideal for new users starting with the SDK: + +1. [**Hello World**](hello_world): the simplest holoscan application running a single operator +2. **Ping**: simple application workflows with basic source/sink (Tx/Rx) + 1. [**ping_simple**](ping_simple): connecting existing operators + 2. [**ping_custom_op**](ping_custom_op): creating and connecting your own operator + 3. [**ping_multi_port**](ping_multi_port): connecting operators with multiple IO ports +3. [**Video Replayer**](video_replayer): switch the source/sink from Tx/Rx to loading a video from disk and displaying its frames + +## Visualization +* [**Holoviz**](holoviz): display overlays of various geometric primitives + +## Inference +* [**Bring-Your-Own-Model**](bring_your_own_model): create a simple inference pipeline for ML applications + + +## Working with third-party frameworks + +The following examples demonstrate how to seamlessly leverage third-party frameworks in holoscan applications: + +* [**NumPy native**](numpy_native): signal processing on the CPU using numpy arrays +* [**CuPy native**](cupy_native): basic computation on the GPU using cupy arrays + +## Sensors + +The following examples demonstrate how sensors can be used as input streams to your holoscan applications: + +* [**v4l2 camera**](v4l2_camera): for USB cameras (*GXF app, legacy*) +* [**AJA capture**](aja_capture): for AJA capture cards + +## GXF and Holoscan + +* [**Tensor interop**](tensor_interop): use the `Entity` message to pass tensors to/from Holoscan operators wrapping GXF codelets in Holoscan applications +* [**Wrap operator as GXF extension**](wrap_operator_as_gxf_extension): wrap Holoscan native operators as GXF codelets to use in GXF applications diff --git a/apps/multiai/CMakeLists.txt b/examples/aja_capture/CMakeLists.txt similarity index 75% rename from apps/multiai/CMakeLists.txt rename to examples/aja_capture/CMakeLists.txt index 221c7a5a..3b215c16 100644 --- a/apps/multiai/CMakeLists.txt +++ b/examples/aja_capture/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,3 +15,9 @@ add_subdirectory(cpp) add_subdirectory(python) + +install( + FILES README.md + DESTINATION "examples/aja_capture" + COMPONENT "holoscan-examples" +) diff --git a/examples/aja_capture/README.md b/examples/aja_capture/README.md new file mode 100644 index 00000000..94ef3821 --- /dev/null +++ b/examples/aja_capture/README.md @@ -0,0 +1,47 @@ +# AJA Capture + +Minimal example to demonstrate the use of the aja source operator to capture device input and stream to holoviz operator. + +## C++ Run instructions + +* **using deb package install or NGC container**: + ```bash + /opt/nvidia/holoscan/examples/aja_capture/cpp/aja_capture + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + ./examples/aja_capture/cpp/aja_capture + ``` +* **source (local env)**: + ```bash + ${BUILD_OR_INSTALL_DIR}/examples/aja_capture/cpp/aja_capture + ``` + +## Python Run instructions + +* **using python wheel**: + ```bash + # [Prerequisite] Download example .py file below to `APP_DIR` + # [Optional] Start the virtualenv where holoscan is installed + python3 /aja_capture.py + ``` +* **using deb package install**: + ```bash + export PYTHONPATH=/opt/nvidia/holoscan/python/lib + python3 /opt/nvidia/holoscan/examples/aja_capture/python/aja_capture.py + ``` +* **from NGC container**: + ```bash + python3 /opt/nvidia/holoscan/examples/aja_capture/python/aja_capture.py + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + python3 ./examples/aja_capture/python/aja_capture.py + ``` +* **source (local env)**: + ```bash + export PYTHONPATH=${BUILD_OR_INSTALL_DIR}/python/lib + python3 ${BUILD_OR_INSTALL_DIR}/examples/aja_capture/python/aja_capture.py + ``` diff --git a/apps/hi_speed_endoscopy/cpp/CMakeLists.txt b/examples/aja_capture/cpp/CMakeLists.txt similarity index 63% rename from apps/hi_speed_endoscopy/cpp/CMakeLists.txt rename to examples/aja_capture/cpp/CMakeLists.txt index c842ce71..f53016f2 100644 --- a/apps/hi_speed_endoscopy/cpp/CMakeLists.txt +++ b/examples/aja_capture/cpp/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,36 +13,41 @@ # See the License for the specific language governing permissions and # limitations under the License. -add_executable(hi_speed_endoscopy - main.cpp +# Create example +add_executable(aja_capture + aja_capture.cpp ) -target_link_libraries(hi_speed_endoscopy +target_link_libraries(aja_capture PRIVATE - ${HOLOSCAN_PACKAGE_NAME} + holoscan::core + holoscan::ops::aja + holoscan::ops::holoviz ) # Copy config file -add_custom_target(hi_speed_endoscopy_yaml - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/app_config.yaml" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/app_config.yaml" +add_custom_target(aja_capture_yaml + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/aja_capture.yaml" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "aja_capture.yaml" + BYPRODUCTS "aja_capture.yaml" ) -add_dependencies(hi_speed_endoscopy hi_speed_endoscopy_yaml) + +add_dependencies(aja_capture aja_capture_yaml) # Set the install RPATH based on the location of the Holoscan SDK libraries # The GXF extensions are loaded by the GXF libraries - no need to include here file(RELATIVE_PATH install_lib_relative_path ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_SOURCE_DIR}/${HOLOSCAN_INSTALL_LIB_DIR} ) -set_target_properties(hi_speed_endoscopy PROPERTIES INSTALL_RPATH "\$ORIGIN/${install_lib_relative_path}") +set_target_properties(aja_capture PROPERTIES INSTALL_RPATH "\$ORIGIN/${install_lib_relative_path}") # Get relative folder path for the app file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) # Install the app -install(TARGETS hi_speed_endoscopy +install(TARGETS aja_capture DESTINATION "${app_relative_dest_path}" - COMPONENT holoscan-apps + COMPONENT holoscan-examples ) -install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/app_config.yaml" +install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/aja_capture.yaml" DESTINATION ${app_relative_dest_path} - COMPONENT holoscan-apps + COMPONENT holoscan-examples ) diff --git a/examples/aja_capture/cpp/aja_capture.cpp b/examples/aja_capture/cpp/aja_capture.cpp new file mode 100644 index 00000000..7b32c2b5 --- /dev/null +++ b/examples/aja_capture/cpp/aja_capture.cpp @@ -0,0 +1,46 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +class App : public holoscan::Application { + public: + void compose() override { + using namespace holoscan; + + auto source = make_operator("aja", from_config("aja")); + auto visualizer = make_operator("holoviz", from_config("holoviz")); + + // Flow definition + add_flow(source, visualizer, {{"video_buffer_output", "receivers"}}); + } +}; + +int main(int argc, char** argv) { + App app; + + // Get the configuration + auto config_path = std::filesystem::canonical(argv[0]).parent_path(); + config_path /= std::filesystem::path("aja_capture.yaml"); + app.config(config_path); + + app.run(); + + return 0; +} diff --git a/examples/native_operator/cpp/app_config.yaml b/examples/aja_capture/cpp/aja_capture.yaml similarity index 65% rename from examples/native_operator/cpp/app_config.yaml rename to examples/aja_capture/cpp/aja_capture.yaml index 744377d1..c901c115 100644 --- a/examples/native_operator/cpp/app_config.yaml +++ b/examples/aja_capture/cpp/aja_capture.yaml @@ -1,5 +1,5 @@ %YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,8 +14,18 @@ # See the License for the specific language governing permissions and # limitations under the License. --- -extensions: - - libgxf_std.so +aja: + width: 1920 + height: 1080 + rdma: true + enable_overlay: false + overlay_rdma: true -mx: - multiplier: 3 +holoviz: + width: 854 + height: 480 + tensors: + - name: "" + type: color + opacity: 1.0 + priority: 0 diff --git a/examples/aja_capture/python/CMakeLists.txt b/examples/aja_capture/python/CMakeLists.txt new file mode 100644 index 00000000..4c79909a --- /dev/null +++ b/examples/aja_capture/python/CMakeLists.txt @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Get relative folder path for the app +file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) + +# Copy aja_capture application file +add_custom_target(python_aja_capture ALL + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/aja_capture.py" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "aja_capture.py" + BYPRODUCTS "aja_capture.py" +) + +# Install the app +install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/aja_capture.py" + DESTINATION "${app_relative_dest_path}" + COMPONENT "holoscan-examples" +) diff --git a/examples/aja_capture/python/aja_capture.py b/examples/aja_capture/python/aja_capture.py new file mode 100644 index 00000000..b296f577 --- /dev/null +++ b/examples/aja_capture/python/aja_capture.py @@ -0,0 +1,65 @@ +""" +SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" # noqa + +from holoscan.core import Application +from holoscan.logger import load_env_log_level +from holoscan.operators import AJASourceOp, HolovizOp + + +class AJACaptureApp(Application): + """ + Example of an application that uses the following operators: + + - AJASourceOp + - HolovizOp + + The AJASourceOp reads frames from an AJA input device and sends it to the HolovizOp. + The HolovizOp displays the frames. + """ + + def compose(self): + + width = 1920 + height = 1080 + + source = AJASourceOp( + self, + name="aja", + width=width, + height=height, + rdma=True, + enable_overlay=False, + overlay_rdma=True, + ) + + visualizer = HolovizOp( + self, + name="holoviz", + width=width, + height=height, + tensors=[{"name": "", "type": "color", "opacity": 1.0, "priority": 0}], + ) + + self.add_flow(source, visualizer, {("video_buffer_output", "receivers")}) + + +if __name__ == "__main__": + + load_env_log_level() + + app = AJACaptureApp() + app.run() diff --git a/examples/basic_workflow/CMakeLists.txt b/examples/basic_workflow/CMakeLists.txt deleted file mode 100644 index 221c7a5a..00000000 --- a/examples/basic_workflow/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -add_subdirectory(cpp) -add_subdirectory(python) diff --git a/examples/basic_workflow/README.md b/examples/basic_workflow/README.md deleted file mode 100644 index c85e639d..00000000 --- a/examples/basic_workflow/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# Basic Workflow - -Minimal example to demonstrate the use of adding components in a pipeline. The workflow in the example tracks tools in the endoscopy video sample data. - -### Data - -[ðŸ“¦ï¸ (NGC) Sample App Data for AI-based Endoscopy Tool Tracking](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/clara-holoscan/resources/holoscan_endoscopy_sample_data) - -### Build instructions - -Built with the SDK, see instructions from the top level README. - -### Run instructions - -First, go in your `build` or `install` directory (automatically done by `./run launch`). - -Then, run the commands of your choice: - -```bash -# C++ -./examples/basic_workflow/cpp/basic_workflow - -# Python -python3 ./examples/basic_workflow/python/basic_workflow.py -``` - -> â„¹ï¸ Python apps can run outside those folders if `HOLOSCAN_SAMPLE_DATA_PATH` is set in your environment (automatically done by `./run launch`). \ No newline at end of file diff --git a/examples/basic_workflow/cpp/basic_workflow.cpp b/examples/basic_workflow/cpp/basic_workflow.cpp deleted file mode 100644 index b6c12bfa..00000000 --- a/examples/basic_workflow/cpp/basic_workflow.cpp +++ /dev/null @@ -1,81 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -class App : public holoscan::Application { - public: - void compose() override { - using namespace holoscan; - - auto replayer = make_operator("replayer", from_config("replayer")); - - auto recorder = make_operator("recorder", from_config("recorder")); - - auto format_converter = make_operator( - "format_converter", - from_config("format_converter_replayer"), - Arg("pool") = make_resource("pool", 1, 854 * 480 * 3 * 4, 2)); - - auto lstm_inferer = make_operator( - "lstm_inferer", - from_config("lstm_inference"), - Arg("pool") = make_resource("pool"), - Arg("cuda_stream_pool") = make_resource("cuda_stream", 0, 0, 0, 1, 5)); - - - // A known issue using the BlockMemoryPool when converting the ONNX model to TRT from - // the LSTMTensorRTInferenceOp causes the inference to produce wrong values the first - // time the conversion is done inline. - // Switching from BlockMemoryPool to UnboundedAllocator or increasing the block size of the - // BlockMemoryPool seems to fix the issue. - // Using TensorRT 8.4.1 and above seems also to be fixing the issue. - auto tool_tracking_postprocessor = make_operator( - "tool_tracking_postprocessor", - from_config("tool_tracking_postprocessor"), - // Arg("device_allocator") = - // make_resource("device_allocator", 1, 107 * 60 * 7 * 4, 2), - Arg("device_allocator") = make_resource("device_allocator"), - Arg("host_allocator") = make_resource("host_allocator")); - - auto visualizer = make_operator("holoviz", from_config("holoviz")); - - // Flow definition - add_flow(replayer, visualizer, {{"output", "receivers"}}); - - add_flow(replayer, format_converter); - add_flow(format_converter, lstm_inferer); - add_flow(lstm_inferer, tool_tracking_postprocessor, {{"tensor", "in"}}); - add_flow(tool_tracking_postprocessor, visualizer, {{"out", "receivers"}}); - - add_flow(replayer, recorder); - } -}; - -int main(int argc, char** argv) { - App app; - - // Get the configuration - auto config_path = std::filesystem::canonical(argv[0]).parent_path(); - config_path += "/app_config.yaml"; - app.config(config_path); - - app.run(); - - return 0; -} diff --git a/examples/basic_workflow/python/basic_workflow.py b/examples/basic_workflow/python/basic_workflow.py deleted file mode 100644 index b9f15ba5..00000000 --- a/examples/basic_workflow/python/basic_workflow.py +++ /dev/null @@ -1,134 +0,0 @@ -""" -SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -SPDX-License-Identifier: Apache-2.0 - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" # noqa - -import os - -from holoscan.core import Application -from holoscan.logger import load_env_log_level -from holoscan.operators import ( - FormatConverterOp, - HolovizOp, - LSTMTensorRTInferenceOp, - ToolTrackingPostprocessorOp, - VideoStreamRecorderOp, - VideoStreamReplayerOp, -) -from holoscan.resources import ( - BlockMemoryPool, - CudaStreamPool, - MemoryStorageType, - UnboundedAllocator, -) - -sample_data_path = os.environ.get("HOLOSCAN_SAMPLE_DATA_PATH", "../../../data") - - -class EndoscopyApp(Application): - - def compose(self): - width = 854 - height = 480 - video_dir = os.path.join(sample_data_path, "endoscopy", "video") - if not os.path.exists(video_dir): - raise ValueError(f"Could not find video data: {video_dir=}") - replayer = VideoStreamReplayerOp( - self, - name="replayer", - directory=video_dir, - **self.kwargs("replayer"), - ) - # 4 bytes/channel, 3 channels - source_pool_kwargs = dict( - storage_type=MemoryStorageType.DEVICE, - block_size=width * height * 3 * 4, - num_blocks=2, - ) - recorder = VideoStreamRecorderOp( - name="recorder", fragment=self, **self.kwargs("recorder") - ) - - format_converter = FormatConverterOp( - self, - name="format_converter_replayer", - pool=BlockMemoryPool(self, name="pool", **source_pool_kwargs), - **self.kwargs("format_converter_replayer"), - ) - - lstm_cuda_stream_pool = CudaStreamPool( - self, - name="cuda_stream", - dev_id=0, - stream_flags=0, - stream_priority=0, - reserved_size=1, - max_size=5, - ) - model_file_path = os.path.join( - sample_data_path, "endoscopy", "model", "tool_loc_convlstm.onnx" - ) - engine_cache_dir = os.path.join( - sample_data_path, "endoscopy", "model", "tool_loc_convlstm_engines" - ) - lstm_inferer = LSTMTensorRTInferenceOp( - self, - name="lstm_inferer", - pool=UnboundedAllocator(self, name="pool"), - cuda_stream_pool=lstm_cuda_stream_pool, - model_file_path=model_file_path, - engine_cache_dir=engine_cache_dir, - **self.kwargs("lstm_inference"), - ) - - tool_tracking_postprocessor_block_size = 107 * 60 * 7 * 4 - tool_tracking_postprocessor_num_blocks = 2 - tool_tracking_postprocessor = ToolTrackingPostprocessorOp( - self, - name="tool_tracking_postprocessor", - device_allocator=BlockMemoryPool( - self, - name="device_allocator", - storage_type=MemoryStorageType.DEVICE, - block_size=tool_tracking_postprocessor_block_size, - num_blocks=tool_tracking_postprocessor_num_blocks, - ), - host_allocator=UnboundedAllocator(self, name="host_allocator"), - ) - - visualizer = HolovizOp( - self, name="holoviz", width=width, height=height, **self.kwargs("holoviz"), - ) - - # Flow definition - self.add_flow(replayer, visualizer, {("output", "receivers")}) - - self.add_flow(replayer, format_converter) - self.add_flow(format_converter, lstm_inferer) - self.add_flow(lstm_inferer, tool_tracking_postprocessor, {("tensor", "in")}) - self.add_flow(tool_tracking_postprocessor, visualizer, {("out", "receivers")}) - - self.add_flow(replayer, recorder) - - -if __name__ == "__main__": - - load_env_log_level() - - config_file = os.path.join(os.path.dirname(__file__), "basic_workflow.yaml") - - app = EndoscopyApp() - app.config(config_file) - app.run() diff --git a/examples/basic_workflow/python/basic_workflow.yaml b/examples/basic_workflow/python/basic_workflow.yaml deleted file mode 100644 index ef5dbc95..00000000 --- a/examples/basic_workflow/python/basic_workflow.yaml +++ /dev/null @@ -1,105 +0,0 @@ -%YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -extensions: - # Extensions are automatically loaded upon Python App initialization, so they - # do not need to be listed here. This is a difference in behavior from the - # C++-API which currently requires explicitly listing them here. - -replayer: - # we pass directory as a kwarg in the Python application instead - # directory: "../data/endoscopy/video" - basename: "surgical_video" - frame_rate: 0 # as specified in timestamps - repeat: true # default: false - realtime: true # default: true - count: 0 # default: 0 (no frame count restriction) - -recorder: - directory: "/tmp" - basename: "tensor" - -format_converter_replayer: - out_tensor_name: source_video - out_dtype: "float32" - scale_min: 0.0 - scale_max: 255.0 - -lstm_inference: - # model_file_path: ../data/endoscopy/model/tool_loc_convlstm.onnx - # engine_cache_dir: ../data/endoscopy/model/tool_loc_convlstm_engines - input_tensor_names: - - source_video - - cellstate_in - - hiddenstate_in - input_state_tensor_names: - - cellstate_in - - hiddenstate_in - input_binding_names: - - data_ph:0 # (shape=[1, 480, 854, 3], dtype=float32) <==> source_video - - cellstate_ph:0 # (shape=[1, 60, 107, 7], dtype=float32) == internal state - - hiddenstate_ph:0 # (shape=[1, 60, 107, 7], dtype=float32) == internal state - output_tensor_names: - - cellstate_out - - hiddenstate_out - - probs - - scaled_coords - - binary_masks - output_state_tensor_names: - - cellstate_out - - hiddenstate_out - output_binding_names: - - Model/net_states:0 # (shape=[ 1, 60, 107, 7], dtype=float32) - - Model/net_hidden:0 # (shape=[ 1, 60, 107, 7], dtype=float32) - - probs:0 # (shape=[1, 7], dtype=float32) - - Localize/scaled_coords:0 # (shape=[1, 7, 2], dtype=float32) - - Localize_1/binary_masks:0 # (shape=[1, 7, 60, 107], dtype=float32) - force_engine_update: false - verbose: true - max_workspace_size: 2147483648 - enable_fp16_: true - -tool_tracking_postprocessor: - -holoviz: - tensors: - - name: "" - type: color - opacity: 1.0 - priority: 0 - - name: mask - type: color - opacity: 1.0 - priority: 1 - - name: scaled_coords - type: crosses - opacity: 1.0 - line_width: 4 - color: [1.0, 0.0, 0.0, 1.0] - priority: 2 - - name: scaled_coords - type: text - opacity: 1.0 - priority: 3 - color: [1.0, 1.0, 1.0, 0.9] - text: - - Grasper - - Bipolar - - Hook - - Scissors - - Clipper - - Irrigator - - Spec.Bag diff --git a/examples/bring_your_own_model/CMakeLists.txt b/examples/bring_your_own_model/CMakeLists.txt index e243c9f0..c288384b 100644 --- a/examples/bring_your_own_model/CMakeLists.txt +++ b/examples/bring_your_own_model/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,3 +14,10 @@ # limitations under the License. add_subdirectory(python) +add_subdirectory(model) + +install( + FILES README.md + DESTINATION "examples/bring_your_own_model" + COMPONENT "holoscan-examples" +) diff --git a/examples/bring_your_own_model/README.md b/examples/bring_your_own_model/README.md index 5b89b9f3..1823edad 100644 --- a/examples/bring_your_own_model/README.md +++ b/examples/bring_your_own_model/README.md @@ -1,24 +1,42 @@ -# Bring Your Own Model - Colonoscopy - -This example shows how to use the [Bring Your Own Model](https://docs.nvidia.com/clara-holoscan/sdk-user-guide/clara_holoscan_applications.html#bring-your-own-model-byom-customizing-the-ultrasound-segmentation-application-for-your-model) (BYOM) concept for Holoscan by changing a few properties of the `ultrasound_segmentation` app to run a segmentation of polyps from a colonoscopy video input instead. - -### Data - -[ðŸ“¦ï¸ (NGC) Sample App Data for AI Colonoscopy Segmentation of Polyps](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/clara-holoscan/resources/holoscan_colonoscopy_sample_data) - -### Build instructions - -Built with the SDK, see instructions from the top level README. - -### Run instructions - -First, go in your `build` or `install` directory (automatically done by `./run launch`). - -Then, run the following: - -```bash -# Update the configurations (run again to reverse) -patch -ub -p0 -i examples/bring_your_own_model/python/colonoscopy_segmentation.patch -# Run the application -python3 ./apps/ultrasound_segmentation/python/ultrasound_segmentation.py -``` +# Bring Your Own Model - Ultrasound + +This example shows how to use the [Bring Your Own Model](https://docs.nvidia.com/clara-holoscan/sdk-user-guide/clara_holoscan_applications.html#bring-your-own-model-byom-customizing-the-ultrasound-segmentation-application-for-your-model) (BYOM) concept for Holoscan by replacing the existing identity model. + +## Data + +The following datasets are used by this example: +- [ðŸ“¦ï¸ (NGC) Sample App Data for AI-based Endoscopy Tool Tracking](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/clara-holoscan/resources/holoscan_endoscopy_sample_data) +- [(Git) Identity ONNX model](model/identity_model.onnx) + +## Run instructions + +* **using python wheel**: + ```bash + # [Prerequisite] Download NGC dataset above to `DATA_DIR` + # [Prerequisite] Download `model` and `python` folders below to `APP_DIR` + # [Optional] Start the virtualenv where holoscan is installed + export HOLOSCAN_SAMPLE_DATA_PATH= + python3 /python/byom.py + ``` +* **using deb package install**: + ```bash + # [Prerequisite] Download NGC dataset above to `DATA_DIR` + export PYTHONPATH=/opt/nvidia/holoscan/python/lib + export HOLOSCAN_SAMPLE_DATA_PATH= + python3 /opt/nvidia/holoscan/examples/bring_your_own_model/python/byom.py + ``` +* **from NGC container**: + ```bash + python3 /opt/nvidia/holoscan/examples/bring_your_own_model/python/byom.py + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + python3 ./examples/bring_your_own_model/python/byom.py + ``` +* **source (local env)**: + ```bash + export PYTHONPATH=${BUILD_OR_INSTALL_DIR}/python/lib + export HOLOSCAN_SAMPLE_DATA_PATH=${SRC_DIR}/data + python3 ${BUILD_OR_INSTALL_DIR}/examples/bring_your_own_model/python/byom.py + ``` diff --git a/examples/bring_your_own_model/model/CMakeLists.txt b/examples/bring_your_own_model/model/CMakeLists.txt new file mode 100644 index 00000000..3176eb72 --- /dev/null +++ b/examples/bring_your_own_model/model/CMakeLists.txt @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) + +# Copy configuration file into the build tree +add_custom_target(identity_model_onnx ALL + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/identity_model.onnx" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "identity_model.onnx" + BYPRODUCTS "identity_model.onnx" +) + +# Install the configuration file +install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/identity_model.onnx" + DESTINATION "${app_relative_dest_path}" + COMPONENT "holoscan-examples" +) diff --git a/examples/bring_your_own_model/model/identity_model.onnx b/examples/bring_your_own_model/model/identity_model.onnx new file mode 100644 index 00000000..ac90a0a1 --- /dev/null +++ b/examples/bring_your_own_model/model/identity_model.onnx @@ -0,0 +1,18 @@ +pytorch1.10:” +% +inputoutput +Identity_0"Identitytorch-jit-exportZ+ +input" +  +  +batch_size + +€ +€b, +output" +  +  +batch_size + +€ +€B \ No newline at end of file diff --git a/examples/bring_your_own_model/python/CMakeLists.txt b/examples/bring_your_own_model/python/CMakeLists.txt index 76efa5dc..5dbfae85 100644 --- a/examples/bring_your_own_model/python/CMakeLists.txt +++ b/examples/bring_your_own_model/python/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,20 +16,26 @@ file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) # Copy configuration file into the build tree -add_custom_target(colonoscopy_segmentation_patch_python ALL - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/colonoscopy_segmentation.patch" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/colonoscopy_segmentation.patch" +add_custom_target(python_byom ALL + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/byom.py" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "byom.py" + BYPRODUCTS "byom.py" ) -add_dependencies(colonoscopy_segmentation_patch_python ultrasound_segmentation) -# Download the associated dataset if needed -if(HOLOSCAN_DOWNLOAD_DATASETS) - add_dependencies(colonoscopy_segmentation_patch_python colonoscopy_data) -endif() +add_dependencies(python_byom endoscopy_data) + +# Copy configuration file into the build tree +add_custom_target(python_byom_yaml ALL + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/byom.yaml" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "byom.yaml" + BYPRODUCTS "byom.yaml" +) +add_dependencies(python_byom_yaml python_byom) # Install the configuration file install(FILES - "${CMAKE_CURRENT_SOURCE_DIR}/colonoscopy_segmentation.patch" + "${CMAKE_CURRENT_SOURCE_DIR}/byom.py" + "${CMAKE_CURRENT_SOURCE_DIR}/byom.yaml" DESTINATION "${app_relative_dest_path}" - COMPONENT "holoscan-apps" + COMPONENT "holoscan-examples" ) diff --git a/examples/bring_your_own_model/python/byom.py b/examples/bring_your_own_model/python/byom.py new file mode 100644 index 00000000..06662e6e --- /dev/null +++ b/examples/bring_your_own_model/python/byom.py @@ -0,0 +1,111 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from argparse import ArgumentParser + +from holoscan.core import Application +from holoscan.logger import load_env_log_level +from holoscan.operators import ( + FormatConverterOp, + HolovizOp, + MultiAIInferenceOp, + SegmentationPostprocessorOp, + VideoStreamReplayerOp, +) +from holoscan.resources import UnboundedAllocator + + +class BYOMApp(Application): + def __init__(self, data): + """Initialize the application + + Parameters + ---------- + data : Location to the data + """ + + super().__init__() + + # set name + self.name = "BYOM App" + + if data == "none": + data = os.environ.get("HOLOSCAN_SAMPLE_DATA_PATH", "../data") + + self.sample_data_path = data + + self.model_path = os.path.join(os.path.dirname(__file__), "../model") + self.model_path_map = { + "byom_model": os.path.join(self.model_path, "identity_model.onnx"), + } + + self.video_dir = os.path.join(self.sample_data_path, "endoscopy", "video") + if not os.path.exists(self.video_dir): + raise ValueError(f"Could not find video data: {self.video_dir=}") + + def compose(self): + + host_allocator = UnboundedAllocator(self, name="host_allocator") + + source = VideoStreamReplayerOp( + self, name="replayer", directory=self.video_dir, **self.kwargs("replayer") + ) + + preprocessor = FormatConverterOp( + self, name="preprocessor", pool=host_allocator, **self.kwargs("preprocessor") + ) + + inference = MultiAIInferenceOp( + self, + name="inference", + allocator=host_allocator, + model_path_map=self.model_path_map, + **self.kwargs("inference"), + ) + + postprocessor = SegmentationPostprocessorOp( + self, name="postprocessor", allocator=host_allocator, **self.kwargs("postprocessor") + ) + + viz = HolovizOp(self, name="viz", **self.kwargs("viz")) + + # Define the workflow + self.add_flow(source, viz, {("output", "receivers")}) + self.add_flow(source, preprocessor, {("output", "source_video")}) + self.add_flow(preprocessor, inference, {("tensor", "receivers")}) + self.add_flow(inference, postprocessor, {("transmitter", "in_tensor")}) + self.add_flow(postprocessor, viz, {("out_tensor", "receivers")}) + + +if __name__ == "__main__": + # Parse args + parser = ArgumentParser(description="BYOM demo application.") + parser.add_argument( + "-d", + "--data", + default="none", + help=("Set the data path"), + ) + + args = parser.parse_args() + + load_env_log_level() + + config_file = os.path.join(os.path.dirname(__file__), "byom.yaml") + + app = BYOMApp(data=args.data) + app.config(config_file) + app.run() diff --git a/examples/bring_your_own_model/python/byom.yaml b/examples/bring_your_own_model/python/byom.yaml new file mode 100644 index 00000000..9e7df1f7 --- /dev/null +++ b/examples/bring_your_own_model/python/byom.yaml @@ -0,0 +1,49 @@ +%YAML 1.2 +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +replayer: # VideoStreamReplayer + basename: "surgical_video" + frame_rate: 0 # as specified in timestamps + repeat: true # default: false + realtime: true # default: true + count: 0 # default: 0 (no frame count restriction) + +preprocessor: # FormatConverter + out_tensor_name: source_video + out_dtype: "float32" + resize_width: 512 + resize_height: 512 + +inference: # MultiaAIInference + backend: "trt" + pre_processor_map: + "byom_model": ["source_video"] + inference_map: + "byom_model": "output" + in_tensor_names: ["source_video"] + out_tensor_names: ["output"] + +postprocessor: # SegmentationPostprocessor + in_tensor_name: output + # network_output_type: None + data_format: nchw + +viz: # Holoviz + width: 854 + height: 480 + color_lut: [ + [0.65, 0.81, 0.89, 0.1], + ] diff --git a/examples/bring_your_own_model/python/colonoscopy_segmentation.patch b/examples/bring_your_own_model/python/colonoscopy_segmentation.patch deleted file mode 100644 index 0b870371..00000000 --- a/examples/bring_your_own_model/python/colonoscopy_segmentation.patch +++ /dev/null @@ -1,96 +0,0 @@ -diff '--color=auto' -ruN apps/ultrasound_segmentation/python/ultrasound_segmentation.py examples/bring_your_own_model/python/ultrasound_segmentation.py ---- apps/ultrasound_segmentation/python/ultrasound_segmentation.py 2022-12-02 15:21:28.852965988 -0500 -+++ examples/bring_your_own_model/python/ultrasound_segmentation.py 2022-12-02 15:19:46.652800278 -0500 -@@ -35,10 +35,10 @@ - - sample_data_path = os.environ.get("HOLOSCAN_SAMPLE_DATA_PATH", "../data") - --model_path = os.path.join(sample_data_path, "ultrasound", "model") -+model_path = os.path.join(sample_data_path, "colonoscopy", "model") - --model_file_path = os.path.join(model_path, "us_unet_256x256_nhwc.onnx") --engine_cache_dir = os.path.join(model_path, "us_unet_256x256_nhwc_engines") -+model_file_path = os.path.join(model_path, "colon.onnx") -+engine_cache_dir = os.path.join(model_path, "engine") - - - class UltrasoundApp(Application): -@@ -93,15 +93,15 @@ - **self.kwargs("drop_alpha_channel"), - ) - else: -- video_dir = os.path.join(sample_data_path, "ultrasound", "video") -+ video_dir = os.path.join(sample_data_path, "colonoscopy", "video") - if not os.path.exists(video_dir): - raise ValueError(f"Could not find video data: {video_dir=}") - source = VideoStreamReplayerOp( - self, name="replayer", directory=video_dir, **self.kwargs("replayer") - ) - -- width_preprocessor = 1264 -- height_preprocessor = 1080 -+ width_preprocessor = 1350 -+ height_preprocessor = 1072 - preprocessor_block_size = width_preprocessor * height_preprocessor * n_channels * bpp - preprocessor_num_blocks = 2 - segmentation_preprocessor = FormatConverterOp( -@@ -118,8 +118,8 @@ - ) - - n_channels_inference = 2 -- width_inference = 256 -- height_inference = 256 -+ width_inference = 512 -+ height_inference = 512 - bpp_inference = 4 - inference_block_size = width_inference * height_inference * n_channels_inference * bpp_inference - inference_num_blocks = 2 -diff '--color=auto' -ruN apps/ultrasound_segmentation/python/ultrasound_segmentation.yaml examples/bring_your_own_model/python/ultrasound_segmentation.yaml ---- apps/ultrasound_segmentation/python/ultrasound_segmentation.yaml 2022-12-02 15:21:30.452968543 -0500 -+++ examples/bring_your_own_model/python/ultrasound_segmentation.yaml 2022-12-02 15:19:46.652800278 -0500 -@@ -31,8 +31,8 @@ - # - libtensor_rt.so - - replayer: # VideoStreamReplayer -- # directory: "../data/ultrasound/video" -- basename: "ultrasound_256x256" -+ # directory: "../data/colonoscopy/video" -+ basename: "colonoscopy" - frame_rate: 0 # as specified in timestamps - repeat: true # default: false - realtime: true # default: true -@@ -52,12 +52,12 @@ - segmentation_preprocessor: # FormatConverter - out_tensor_name: source_video - out_dtype: "float32" -- resize_width: 256 -- resize_height: 256 -+ resize_width: 512 -+ resize_height: 512 - - segmentation_inference: # TensorRtInference -- # model_file_path: "../data/ultrasound/model/us_unet_256x256_nhwc.onnx" -- # engine_cache_dir: "../data/ultrasound/model/us_unet_256x256_nhwc_engines" -+ # model_file_path: "../data/colonoscopy/model/colon.onnx" -+ # engine_cache_dir: "../data/colonoscopy/model/engine" - input_tensor_names: - - source_video - input_binding_names: -@@ -65,7 +65,7 @@ - output_tensor_names: - - inference_output_tensor - output_binding_names: -- - OUTPUT__0 -+ - output_old - force_engine_update: false - verbose: true - max_workspace_size: 2147483648 -@@ -73,7 +73,7 @@ - - segmentation_postprocessor: # Postprocessor - in_tensor_name: inference_output_tensor -- network_output_type: softmax -+ network_output_type: sigmoid - data_format: nchw - - segmentation_visualizer: # Holoviz diff --git a/examples/cupy_native/CMakeLists.txt b/examples/cupy_native/CMakeLists.txt new file mode 100644 index 00000000..94617305 --- /dev/null +++ b/examples/cupy_native/CMakeLists.txt @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Get relative folder path for the app +file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) + +# Copy native operator matmul application +add_custom_target(python_matmul ALL + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/matmul.py" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "matmul.py" + BYPRODUCTS "matmul.py" +) + +# Install the app and doc +install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/matmul.py" + "${CMAKE_CURRENT_SOURCE_DIR}/README.md" + DESTINATION "${app_relative_dest_path}" + COMPONENT "holoscan-examples" +) + +# Testing +if(HOLOSCAN_BUILD_TESTS) + add_test(NAME EXAMPLE_PYTHON_MATMUL_TEST + COMMAND python3 matmul.py + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_PYTHON_MATMUL_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Scheduler finished.") +endif() diff --git a/examples/cupy_native/README.md b/examples/cupy_native/README.md new file mode 100644 index 00000000..e5574594 --- /dev/null +++ b/examples/cupy_native/README.md @@ -0,0 +1,34 @@ +# CuPy Native + +This minimal application multiplies two randomly generated matrices on the GPU, to showcase the use of CuPy with holoscan. + +## Run instructions + +* **using python wheel**: + ```bash + # [Prerequisite] Download example .py file below to `APP_DIR` + # [Optional] Start the virtualenv where holoscan is installed + python3 -m pip install cupy-cuda11x # Append `-f https://pip.cupy.dev/aarch64` on aarch64 + python3 /matmul.py + ``` +* **using deb package install**: + ```bash + python3 -m pip install cupy-cuda11x # Append `-f https://pip.cupy.dev/aarch64` on aarch64 + export PYTHONPATH=/opt/nvidia/holoscan/python/lib + python3 /opt/nvidia/holoscan/examples/cupy_native/matmul.py + ``` +* **from NGC container**: + ```bash + python3 /opt/nvidia/holoscan/examples/cupy_native/matmul.py + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + python3 ./examples/cupy_native/matmul.py + ``` +* **source (local env)**: + ```bash + python3 -m pip install cupy-cuda11x # Append `-f https://pip.cupy.dev/aarch64` on aarch64 + export PYTHONPATH=${BUILD_OR_INSTALL_DIR}/python/lib + python3 ${BUILD_OR_INSTALL_DIR}/examples/cupy_native/matmul.py + ``` diff --git a/examples/cupy_native/matmul.py b/examples/cupy_native/matmul.py new file mode 100644 index 00000000..fe45ba19 --- /dev/null +++ b/examples/cupy_native/matmul.py @@ -0,0 +1,82 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from holoscan.conditions import CountCondition +from holoscan.core import Application, Operator, OperatorSpec +from holoscan.logger import load_env_log_level + +try: + import cupy as cp +except ImportError: + raise ImportError("cupy must be installed to run this example.") + + +class SourceOp(Operator): + def __init__(self, *args, **kwargs): + self.rng = cp.random.default_rng() + self.static_out = self.rng.standard_normal((1000, 1000), dtype=cp.float32) + super().__init__(*args, **kwargs) + + def setup(self, spec: OperatorSpec): + spec.output("static_out") + spec.output("variable_out") + + def compute(self, op_input, op_output, context): + op_output.emit(self.rng.standard_normal((1000, 1000), dtype=cp.float32), "variable_out") + op_output.emit(self.static_out, "static_out") + + +class MatMulOp(Operator): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def setup(self, spec: OperatorSpec): + spec.input("in_static") + spec.input("in_variable") + spec.output("out") + + def compute(self, op_input, op_output, context): + mat_static = op_input.receive("in_static") + mat_dynamic = op_input.receive("in_variable") + op_output.emit(cp.matmul(mat_static, mat_dynamic), "out") + + +class SinkOp(Operator): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def setup(self, spec: OperatorSpec): + spec.input("in") + + def compute(self, op_input, op_output, context): + sig = op_input.receive("in") + print(sig) + + +class MatMulApp(Application): + def compose(self): + src = SourceOp(self, CountCondition(self, 1000), name="src_op") + matmul = MatMulOp(self, name="matmul_op") + sink = SinkOp(self, name="sink_op") + + # Connect the operators into the workflow: src -> matmul -> sink + self.add_flow(src, matmul, {("static_out", "in_static"), ("variable_out", "in_variable")}) + self.add_flow(matmul, sink) + + +if __name__ == "__main__": + load_env_log_level() + app = MatMulApp() + app.run() diff --git a/examples/hello_world/CMakeLists.txt b/examples/hello_world/CMakeLists.txt new file mode 100644 index 00000000..418501de --- /dev/null +++ b/examples/hello_world/CMakeLists.txt @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_subdirectory(cpp) +add_subdirectory(python) + +install( + FILES README.md + DESTINATION "examples/hello_world" + COMPONENT "holoscan-examples" +) diff --git a/examples/hello_world/README.md b/examples/hello_world/README.md new file mode 100644 index 00000000..5e0d1bfa --- /dev/null +++ b/examples/hello_world/README.md @@ -0,0 +1,47 @@ +# Hello World + +This example demonstrates a hello world application using the holoscan SDK. + +## C++ Run instructions + +* **using deb package install or NGC container**: + ```bash + /opt/nvidia/holoscan/examples/hello_world/cpp/hello_world + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + ./examples/hello_world/cpp/hello_world + ``` +* **source (local env)**: + ```bash + ${BUILD_OR_INSTALL_DIR}/examples/hello_world/cpp/hello_world + ``` + +## Python Run instructions + +* **using python wheel**: + ```bash + # [Prerequisite] Download example .py file below to `APP_DIR` + # [Optional] Start the virtualenv where holoscan is installed + python3 /hello_world.py + ``` +* **using deb package install**: + ```bash + export PYTHONPATH=/opt/nvidia/holoscan/python/lib + python3 /opt/nvidia/holoscan/examples/hello_world/python/hello_world.py + ``` +* **from NGC container**: + ```bash + python3 /opt/nvidia/holoscan/examples/hello_world/python/hello_world.py + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + python3 ./examples/hello_world/python/hello_world.py + ``` +* **source (local env)**: + ```bash + export PYTHONPATH=${BUILD_OR_INSTALL_DIR}/python/lib + python3 ${BUILD_OR_INSTALL_DIR}/examples/hello_world/python/hello_world.py + ``` diff --git a/examples/hello_world/cpp/CMakeLists.min.txt b/examples/hello_world/cpp/CMakeLists.min.txt new file mode 100644 index 00000000..72e3d5a5 --- /dev/null +++ b/examples/hello_world/cpp/CMakeLists.min.txt @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the \"License\"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.20) +project(holoscan_hello_world CXX) + +# Finds the package holoscan +find_package(holoscan REQUIRED CONFIG + PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install") + +add_executable(hello_world + hello_world.cpp +) + +target_link_libraries(hello_world + PRIVATE + holoscan::core +) + +# Testing +if(BUILD_TESTING) + add_test(NAME EXAMPLE_CPP_HELLO_WORLD_TEST + COMMAND hello_world + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_CPP_HELLO_WORLD_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Hello World!") +endif() diff --git a/examples/native_operator/cpp/CMakeLists.txt b/examples/hello_world/cpp/CMakeLists.txt similarity index 63% rename from examples/native_operator/cpp/CMakeLists.txt rename to examples/hello_world/cpp/CMakeLists.txt index 3dc909d0..18c9a79e 100644 --- a/examples/native_operator/cpp/CMakeLists.txt +++ b/examples/hello_world/cpp/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,53 +14,51 @@ # limitations under the License. # Create examples -add_executable(ping - ping.cpp +add_executable(hello_world + hello_world.cpp ) -target_link_libraries(ping +target_link_libraries(hello_world PRIVATE - ${HOLOSCAN_PACKAGE_NAME} + holoscan::core ) -# Copy config file to the build tree -add_custom_target(ping_yaml - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/app_config.yaml" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/app_config.yaml" -) -add_dependencies(ping ping_yaml) - # Install examples # Set the install RPATH based on the location of the Holoscan SDK libraries # The GXF extensions are loaded by the GXF libraries - no need to include here file(RELATIVE_PATH install_lib_relative_path ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_SOURCE_DIR}/${HOLOSCAN_INSTALL_LIB_DIR}) -set_target_properties(ping PROPERTIES INSTALL_RPATH "\$ORIGIN/${install_lib_relative_path}") +set_target_properties(hello_world PROPERTIES INSTALL_RPATH "\$ORIGIN/${install_lib_relative_path}") # Install following the relative folder path file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) +if(HOLOSCAN_INSTALL_EXAMPLE_SOURCE) # Install the source -install(FILES ping.cpp +install(FILES hello_world.cpp DESTINATION "${app_relative_dest_path}" - COMPONENT holoscan-apps + COMPONENT holoscan-examples ) # Install the minimal CMakeLists.txt file install(FILES CMakeLists.min.txt RENAME "CMakeLists.txt" DESTINATION "${app_relative_dest_path}" - COMPONENT holoscan-apps + COMPONENT holoscan-examples ) +endif() # Install the compiled example -install(TARGETS ping +install(TARGETS hello_world DESTINATION "${app_relative_dest_path}" - COMPONENT holoscan-apps + COMPONENT holoscan-examples ) -# Install the configuration file -install(FILES - "${CMAKE_CURRENT_SOURCE_DIR}/app_config.yaml" - DESTINATION "${app_relative_dest_path}" - COMPONENT holoscan-apps -) +# Testing +if(HOLOSCAN_BUILD_TESTS) + add_test(NAME EXAMPLE_CPP_HELLO_WORLD_TEST + COMMAND hello_world + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_CPP_HELLO_WORLD_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Hello World!") +endif() diff --git a/examples/hello_world/cpp/hello_world.cpp b/examples/hello_world/cpp/hello_world.cpp new file mode 100644 index 00000000..e8277316 --- /dev/null +++ b/examples/hello_world/cpp/hello_world.cpp @@ -0,0 +1,61 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +namespace holoscan::ops { + +class HelloWorldOp : public Operator { + public: + HOLOSCAN_OPERATOR_FORWARD_ARGS(HelloWorldOp) + + HelloWorldOp() = default; + + void setup(OperatorSpec& spec) override { + } + + void compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) override { + std::cout << std::endl; + std::cout << "Hello World!" << std::endl; + std::cout << std::endl; + } +}; + +} // namespace holoscan::ops + + +class HelloWorldApp : public holoscan::Application { + public: + void compose() override { + using namespace holoscan; + + // Define the operators + auto hello = make_operator("hello", make_condition(1)); + + // Define the one-operator workflow + add_operator(hello); + } +}; + +int main(int argc, char** argv) { + auto app = holoscan::make_application(); + app->run(); + + return 0; +} diff --git a/apps/multiai/python/CMakeLists.txt b/examples/hello_world/python/CMakeLists.txt similarity index 55% rename from apps/multiai/python/CMakeLists.txt rename to examples/hello_world/python/CMakeLists.txt index 081f0b3e..08bfe778 100644 --- a/apps/multiai/python/CMakeLists.txt +++ b/examples/hello_world/python/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,24 +16,27 @@ # Get relative folder path for the app file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) -# Copy multi-AI iCardio application -add_custom_target(python_multiai - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/multiai.py" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/multiai.py" +# Copy native operator ping application +add_custom_target(python_hello_world ALL + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/hello_world.py" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "hello_world.py" + BYPRODUCTS "hello_world.py" ) -add_dependencies(${HOLOSCAN_PACKAGE_NAME} python_multiai) - -# Copy multi-AI iCardio config file -add_custom_target(python_multiai_yaml - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/multiai.yaml" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/multiai.yaml" -) -add_dependencies(${HOLOSCAN_PACKAGE_NAME} python_multiai_yaml) # Install the app install(FILES - "${CMAKE_CURRENT_SOURCE_DIR}/multiai.py" - "${CMAKE_CURRENT_SOURCE_DIR}/multiai.yaml" + "${CMAKE_CURRENT_SOURCE_DIR}/hello_world.py" DESTINATION "${app_relative_dest_path}" - COMPONENT "holoscan-apps" + COMPONENT "holoscan-examples" ) + +# Testing +if(HOLOSCAN_BUILD_TESTS) + add_test(NAME EXAMPLE_PYTHON_HELLO_WORLD_TEST + COMMAND python3 hello_world.py + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_PYTHON_HELLO_WORLD_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Hello World!" + ) +endif() diff --git a/examples/hello_world/python/hello_world.py b/examples/hello_world/python/hello_world.py new file mode 100644 index 00000000..f5cc25ed --- /dev/null +++ b/examples/hello_world/python/hello_world.py @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from holoscan.conditions import CountCondition +from holoscan.core import Application, Operator, OperatorSpec + +# define custom Operators for use in the demo + + +class HelloWorldOp(Operator): + """Simple hello world operator. + + This operator has no ports. + + On each tick, this operator prints out hello world messages. + """ + + def setup(self, spec: OperatorSpec): + pass + + def compute(self, op_input, op_output, context): + print("") + print("Hello World!") + print("") + + +# Now define a simple application using the operator defined above + + +class HelloWorldApp(Application): + def compose(self): + # Define the operators + hello = HelloWorldOp(self, CountCondition(self, 1), name="hello") + + # Define the one-operator workflow + self.add_operator(hello) + + +if __name__ == "__main__": + app = HelloWorldApp() + app.run() diff --git a/apps/ultrasound_segmentation/CMakeLists.txt b/examples/holoviz/CMakeLists.txt similarity index 80% rename from apps/ultrasound_segmentation/CMakeLists.txt rename to examples/holoviz/CMakeLists.txt index 221c7a5a..96a7c231 100644 --- a/apps/ultrasound_segmentation/CMakeLists.txt +++ b/examples/holoviz/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,5 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -add_subdirectory(cpp) add_subdirectory(python) + +install( + FILES README.md + DESTINATION "examples/holoviz" + COMPONENT "holoscan-examples" +) diff --git a/examples/holoviz/README.md b/examples/holoviz/README.md new file mode 100644 index 00000000..250405cd --- /dev/null +++ b/examples/holoviz/README.md @@ -0,0 +1,49 @@ +# HolovizOp geometry layer usage + +As for `example/tensor_interop/python/tensor_interop.py`, this application demonstrates interoperability between a native operator (`ImageProcessingOp`) and two operators (`VideoStreamReplayerOp` and `HolovizOp`) that wrap existing C++-based operators. This application also demonstrates two additional aspects: +- capability to add multiple tensors to the message sent on the output port of a native operator (`ImageProcessingOp`) +- expected tensor shapes and arguments needed for HolovizOp in order to display overlays of various geometric primitives onto an underlying color video. + +## Data + +The following dataset is used by this example: +[ðŸ“¦ï¸ (NGC) Sample App Data for AI-based Endoscopy Tool Tracking](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/clara-holoscan/resources/holoscan_endoscopy_sample_data). + +## Run instructions + +* **using python wheel**: + ```bash + # [Prerequisite] Download NGC dataset above to `DATA_DIR` + # [Prerequisite] Download example .py file below to `APP_DIR` + # [Optional] Start the virtualenv where holoscan is installed + python3 -m pip install numpy + python3 -m pip install cupy-cuda11x # Append `-f https://pip.cupy.dev/aarch64` on aarch64 + export HOLOSCAN_SAMPLE_DATA_PATH= + python3 /holoviz_geometry.py + ``` +* **using deb package install**: + ```bash + # [Prerequisite] Download NGC dataset above to `DATA_DIR` + python3 -m pip install numpy + python3 -m pip install cupy-cuda11x # Append `-f https://pip.cupy.dev/aarch64` on aarch64 + export PYTHONPATH=/opt/nvidia/holoscan/python/lib + export HOLOSCAN_SAMPLE_DATA_PATH= + python3 /opt/nvidia/holoscan/examples/holoviz/python/holoviz_geometry.py + ``` +* **from NGC container**: + ```bash + python3 /opt/nvidia/holoscan/examples/holoviz/python/holoviz_geometry.py + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + python3 ./examples/holoviz/python/holoviz_geometry.py + ``` +* **source (local env)**: + ```bash + python3 -m pip install numpy + python3 -m pip install cupy-cuda11x # Append `-f https://pip.cupy.dev/aarch64` on aarch64 + export PYTHONPATH=${BUILD_OR_INSTALL_DIR}/python/lib + export HOLOSCAN_SAMPLE_DATA_PATH=${SRC_DIR}/data + python3 ${BUILD_OR_INSTALL_DIR}/examples/holoviz/python/holoviz_geometry.py + ``` \ No newline at end of file diff --git a/examples/basic_workflow/python/CMakeLists.txt b/examples/holoviz/python/CMakeLists.txt similarity index 53% rename from examples/basic_workflow/python/CMakeLists.txt rename to examples/holoviz/python/CMakeLists.txt index 86bffa38..26c37142 100644 --- a/examples/basic_workflow/python/CMakeLists.txt +++ b/examples/holoviz/python/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,24 +16,28 @@ # Get relative folder path for the app file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) -# Copy endoscopy tool tracking application -add_custom_target(python_basic_workflow - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/basic_workflow.py" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/basic_workflow.py" +# Copy native operator tensor_interop application +add_custom_target(python_holoviz_geometry ALL + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/holoviz_geometry.py" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "holoviz_geometry.py" + BYPRODUCTS "holoviz_geometry.py" ) -add_dependencies(${HOLOSCAN_PACKAGE_NAME} python_basic_workflow) -# Copy endoscopy tool tracking config file -add_custom_target(python_basic_workflow_yaml - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/basic_workflow.yaml" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/basic_workflow.yaml" -) -add_dependencies(${HOLOSCAN_PACKAGE_NAME} python_basic_workflow_yaml) +add_dependencies(python_holoviz_geometry endoscopy_data) # Install the app install(FILES - "${CMAKE_CURRENT_SOURCE_DIR}/basic_workflow.py" - "${CMAKE_CURRENT_SOURCE_DIR}/basic_workflow.yaml" + "${CMAKE_CURRENT_SOURCE_DIR}/holoviz_geometry.py" DESTINATION "${app_relative_dest_path}" - COMPONENT "holoscan-apps" + COMPONENT "holoscan-examples" ) + +# Testing +if(HOLOSCAN_BUILD_TESTS) + add_test(NAME EXAMPLE_PYTHON_HOLOVIZ_TEST + COMMAND python3 holoviz_geometry.py --count 10 + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_PYTHON_HOLOVIZ_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Reach end of file or playback count reaches to the limit. Stop ticking.") +endif() diff --git a/examples/holoviz/python/holoviz_geometry.py b/examples/holoviz/python/holoviz_geometry.py new file mode 100644 index 00000000..d1b515c2 --- /dev/null +++ b/examples/holoviz/python/holoviz_geometry.py @@ -0,0 +1,327 @@ +""" +SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" # no qa + +import os +from argparse import ArgumentParser + +import numpy as np + +import holoscan as hs +from holoscan.core import Application, Operator, OperatorSpec +from holoscan.gxf import Entity +from holoscan.logger import load_env_log_level +from holoscan.operators import HolovizOp, VideoStreamReplayerOp + +try: + import cupy as cp +except ImportError: + raise ImportError( + "CuPy must be installed to run this example. See " + "https://docs.cupy.dev/en/stable/install.html" + ) + +sample_data_path = os.environ.get("HOLOSCAN_SAMPLE_DATA_PATH", "../data") + + +# Define custom Operators for use in the demo +class ImageProcessingOp(Operator): + """Example of an operator processing input video (as a tensor). + + This operator has: + inputs: "input_tensor" + outputs: "output_tensor" + + The data from each input is processed by a CuPy gaussian filter and + the result is sent to the output. + + In this demo, the input and output image (2D RGB) is an 3D array of shape + (height, width, channels). + """ + + def __init__(self, fragment, *args, sigma=1.0, **kwargs): + self.count = 1 + + # Need to call the base class constructor last + super().__init__(fragment, *args, **kwargs) + + def setup(self, spec: OperatorSpec): + spec.input("input_tensor") + spec.output("outputs") + + def compute(self, op_input, op_output, context): + message = op_input.receive("input_tensor") + + input_tensor = message.get() + # dim the video by a factor of 2 (preserving uint8 dtype) + output_array = cp.asarray(input_tensor) // 2 + + # pass through the input tensor unmodified under the name "video" + out_message = Entity(context) + out_message.add(hs.as_tensor(output_array), "video") + + # Now draw various different types of geometric primitives. + # In all cases, x and y are normalized coordinates in the range [0, 1]. + # x runs from left to right and y from bottom to top. All coordinates + # should be defined using a single precision (np.float32) dtype. + + ########################################## + # Create a tensor defining four rectnagles + ########################################## + # For rectangles (bounding boxes), they are defined by a pair of + # 2-tuples defining the upper-left and lower-right coordinates of a + # box: (x1, y1), (x2, y2). + box_coords = np.asarray( + # fmt: off + [ + (0.1, 0.2), (0.8, 0.5), + (0.2, 0.4), (0.3, 0.6), + (0.3, 0.5), (0.4, 0.7), + (0.5, 0.7), (0.6, 0.9), + ], + # fmt: on + dtype=np.float32, + ) + # append initial axis of shape 1 + box_coords = box_coords[np.newaxis, :, :] + out_message.add(hs.as_tensor(box_coords), "boxes") + + ######################################## + # Create a tensor defining two triangles + ######################################## + # Each triangle is defined by a set of 3 (x, y) coordinate pairs. + triangle_coords = np.asarray( + # fmt: off + [ + (0.1, 0.8), (0.18, 0.75), (0.14, 0.66), + (0.3, 0.8), (0.38, 0.75), (0.34, 0.56), + ], + # fmt: on + dtype=np.float32, + ) + triangle_coords = triangle_coords[np.newaxis, :, :] + out_message.add(hs.as_tensor(triangle_coords), "triangles") + + ###################################### + # Create a tensor defining two crosses + ###################################### + # Each cross is defined by an (x, y, size) 3-tuple + cross_coords = np.asarray( + [ + (0.25, 0.25, 0.05), + (0.75, 0.25, 0.10), + ], + dtype=np.float32, + ) + cross_coords = cross_coords[np.newaxis, :, :] + out_message.add(hs.as_tensor(cross_coords), "crosses") + + ###################################### + # Create a tensor defining three ovals + ###################################### + # Each oval is defined by an (x, y, size_x, size_y) 4-tuple + oval_coords = np.asarray( + [ + (0.25, 0.65, 0.10, 0.05), + (0.50, 0.65, 0.02, 0.15), + (0.75, 0.65, 0.05, 0.10), + ], + dtype=np.float32, + ) + oval_coords = oval_coords[np.newaxis, :, :] + out_message.add(hs.as_tensor(oval_coords), "ovals") + + ####################################### + # Create a time-varying "points" tensor + ####################################### + # Set of (x, y) points with 50 points equally spaced along x whose y + # coordinate varies sinusoidally over time. + x = np.linspace(0, 1.0, 50, dtype=np.float32) + y = 0.8 + 0.1 * np.sin(8 * np.pi * x + self.count / 60 * 2 * np.pi) + self.count += 1 + # Stack and then add an axis so the final shape is (1, n_points, 2) + point_coords = np.stack((x, y), axis=-1)[np.newaxis, :, :] + out_message.add(hs.as_tensor(point_coords), "points") + + #################################### + # Create a tensor for "label_coords" + #################################### + # Set of two (x, y) points marking the location of text labels + label_coords = np.asarray( + [ + (0.10, 0.1), + (0.70, 0.1), + ], + dtype=np.float32, + ) + label_coords = label_coords[np.newaxis, :, :] + out_message.add(hs.as_tensor(label_coords), "label_coords") + + op_output.emit(out_message, "outputs") + + +# Now define a simple application using the operators defined above +class MyVideoProcessingApp(Application): + """Example of an application that uses the operators defined above. + + This application has the following operators: + + - VideoStreamReplayerOp + - ImageProcessingOp + - HolovizOp + + The VideoStreamReplayerOp reads a video file and sends the frames to the ImageProcessingOp. + The ImageProcessingOp processes the frames and sends the processed frames to the HolovizOp. + The HolovizOp displays the processed frames. + """ + + def __init__(self, config_count=0): + """Initialize MyVideoProcessingAPP + + Parameters + ---------- + config_count : optional + Limits the number of frames to show before the application ends. + Set to 0 by default. The video stream will not automatically stop. + Any positive integer will limit on the number of frames displayed. + """ + super().__init__() + + self.count = int(config_count) + + def compose(self): + width = 854 + height = 480 + video_dir = os.path.join(sample_data_path, "endoscopy", "video") + if not os.path.exists(video_dir): + raise ValueError(f"Could not find video data: {video_dir=}") + source = VideoStreamReplayerOp( + self, + name="replayer", + directory=video_dir, + basename="surgical_video", + frame_rate=0, # as specified in timestamps + repeat=True, # default: false + realtime=True, # default: true + count=self.count, # default: 0 (no frame count restriction) + ) + + image_processing = ImageProcessingOp( + self, + name="image_processing", + sigma=0.0, + ) + + visualizer = HolovizOp( + self, + name="holoviz", + width=width, + height=height, + tensors=[ + dict(name="video", type="color", opacity=1.0, priority=0), + # Parameters defining the rectangle primitives + dict( + name="boxes", + type="rectangles", + opacity=1.0, + color=[1.0, 0.0, 1.0, 0.5], + line_width=2, + ), + # line strip reuses the rectangle coordinates. This will make + # a connected set of line segments through the diagonals of + # each box. + dict( + name="boxes", + type="line_strip", + opacity=1.0, + color=[0.4, 0.4, 1.0, 0.7], + line_width=3, + ), + # Lines also reuses the boxes coordinates so will plot a set of + # disconnected line segments along the box diagonals. + dict( + name="boxes", + type="lines", + opacity=1.0, + color=[0.4, 1.0, 0.4, 0.7], + line_width=3, + ), + # Parameters defining the triangle primitives + dict( + name="triangles", + type="triangles", + opacity=1.0, + color=[1.0, 0.0, 0.0, 0.5], + line_width=1, + ), + # Parameters defining the crosses primitives + dict( + name="crosses", + type="crosses", + opacity=1.0, + color=[0.0, 1.0, 0.0, 1.0], + line_width=3, + ), + # Parameters defining the ovals primitives + dict( + name="ovals", + type="ovals", + opacity=0.5, + color=[1.0, 1.0, 1.0, 1.0], + line_width=2, + ), + # Parameters defining the points primitives + dict( + name="points", + type="points", + opacity=1.0, + color=[1.0, 1.0, 1.0, 1.0], + point_size=4, + ), + # Parameters defining the label_coords primitives + dict( + name="label_coords", + type="text", + opacity=1.0, + color=[1.0, 1.0, 1.0, 1.0], + point_size=4, + text=["label_1", "label_2"], + ), + ], + ) + self.add_flow(source, image_processing) + self.add_flow(image_processing, visualizer, {("outputs", "receivers")}) + + +if __name__ == "__main__": + load_env_log_level() + app = MyVideoProcessingApp() + # Parse args + parser = ArgumentParser(description="Example video processing application") + parser.add_argument( + "-c", + "--count", + default="none", + help="Set the number of frames to display the video", + ) + args = parser.parse_args() + count = args.count + if count == "none": + app = MyVideoProcessingApp() + else: + app = MyVideoProcessingApp(config_count=count) + + app.run() diff --git a/examples/native_operator/CMakeLists.txt b/examples/native_operator/CMakeLists.txt deleted file mode 100644 index 221c7a5a..00000000 --- a/examples/native_operator/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -add_subdirectory(cpp) -add_subdirectory(python) diff --git a/examples/native_operator/README.md b/examples/native_operator/README.md deleted file mode 100644 index 322e22e6..00000000 --- a/examples/native_operator/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# Native Operator - -These examples demonstrate how to use native operators (the operators that do not have an underlying, pre-compiled GXF Codelet): - -## C++ API - -This example shows the application using only native operators. There are three operators involved: - 1. a transmitter, set to transmit a sequence of even integers on port `out1` and odd integers on port `out2`. - 2. a middle operator that prints the received values, multiplies by a scalar and transmits the modified values - 3. a receiver that prints the received values to the terminal - -### Build instructions - -Built with the SDK, see instructions from the top level README. - -### Run instructions - -First, go in your `build` or `install` directory (automatically done by `./run launch`). - -Then, run: -```bash -./examples/native_operator/cpp/ping -``` - -## Python API - -- `ping.py`: This example is similar to the C++ native operator example, using Python. -- `convolve.py`: This example demonstrates a simple 1D convolution-based signal processing application, to demonstrate passing NumPy arrays between operators as Python objects. - -### Build instructions - -Built with the SDK, see instructions from the top level README. - -### Run instructions - -First, go in your `build` or `install` directory (automatically done by `./run launch`). - -Then, run the commands of your choice: - -```bash -python3 ./examples/native_operator/python/ping.py -python3 ./examples/native_operator/python/convolve.py -``` - -> â„¹ï¸ Python apps can run outside those folders if `HOLOSCAN_SAMPLE_DATA_PATH` is set in your environment (automatically done by `./run launch`). diff --git a/examples/native_operator/python/CMakeLists.txt b/examples/native_operator/python/CMakeLists.txt deleted file mode 100644 index b35d05e9..00000000 --- a/examples/native_operator/python/CMakeLists.txt +++ /dev/null @@ -1,55 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Get relative folder path for the app -file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) - -# Copy native operator ping application -add_custom_target(python_ping - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/ping.py" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/ping.py" -) -add_dependencies(${HOLOSCAN_PACKAGE_NAME} python_ping) - -# Copy native operator ping config file -add_custom_target(python_ping_yaml - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/ping.yaml" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/ping.yaml" -) -add_dependencies(${HOLOSCAN_PACKAGE_NAME} python_ping_yaml) - -# Copy native operator convolve application -add_custom_target(python_convolve - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/convolve.py" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/convolve.py" -) -add_dependencies(${HOLOSCAN_PACKAGE_NAME} python_convolve) - -# Copy native operator convolve config file -add_custom_target(python_convolve_yaml - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/convolve.yaml" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/convolve.yaml" -) -add_dependencies(${HOLOSCAN_PACKAGE_NAME} python_convolve_yaml) - -# Install the app -install(FILES - "${CMAKE_CURRENT_SOURCE_DIR}/ping.py" - "${CMAKE_CURRENT_SOURCE_DIR}/ping.yaml" - "${CMAKE_CURRENT_SOURCE_DIR}/convolve.py" - "${CMAKE_CURRENT_SOURCE_DIR}/convolve.yaml" - DESTINATION "${app_relative_dest_path}" - COMPONENT "holoscan-apps" -) diff --git a/examples/native_operator/python/ping.yaml b/examples/native_operator/python/ping.yaml deleted file mode 100644 index f34e82d9..00000000 --- a/examples/native_operator/python/ping.yaml +++ /dev/null @@ -1,18 +0,0 @@ -%YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -mx: - multiplier: 3 diff --git a/examples/numpy_native/CMakeLists.txt b/examples/numpy_native/CMakeLists.txt new file mode 100644 index 00000000..5975eaf0 --- /dev/null +++ b/examples/numpy_native/CMakeLists.txt @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Get relative folder path for the app +file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) + +# Copy native operator convolve application +add_custom_target(python_convolve ALL + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/convolve.py" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "convolve.py" + BYPRODUCTS "convolve.py" +) + +# Install the app and doc +install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/convolve.py" + "${CMAKE_CURRENT_SOURCE_DIR}/README.md" + DESTINATION "${app_relative_dest_path}" + COMPONENT "holoscan-examples" +) + +# Testing +if(HOLOSCAN_BUILD_TESTS) + add_test(NAME EXAMPLE_PYTHON_CONVOLVE_TEST + COMMAND python3 convolve.py + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_PYTHON_CONVOLVE_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "\[1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\]") +endif() diff --git a/examples/numpy_native/README.md b/examples/numpy_native/README.md new file mode 100644 index 00000000..a19bf26d --- /dev/null +++ b/examples/numpy_native/README.md @@ -0,0 +1,34 @@ +# NumPy Native + +This minimal signal processing application generates a time-varying impulse, convolves it with a boxcar kernel, and prints the result to the terminal, to showcase the use of NumPy with holoscan. + +## Run instructions + +* **using python wheel**: + ```bash + # [Prerequisite] Download example .py file below to `APP_DIR` + # [Optional] Start the virtualenv where holoscan is installed + python3 -m pip install numpy + python3 /convolve.py + ``` +* **using deb package install**: + ```bash + python3 -m pip install numpy + export PYTHONPATH=/opt/nvidia/holoscan/python/lib + python3 /opt/nvidia/holoscan/examples/numpy_native/convolve.py + ``` +* **from NGC container**: + ```bash + python3 /opt/nvidia/holoscan/examples/numpy_native/convolve.py + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + python3 ./examples/numpy_native/convolve.py + ``` +* **source (local env)**: + ```bash + python3 -m pip install numpy + export PYTHONPATH=${BUILD_OR_INSTALL_DIR}/python/lib + python3 ${BUILD_OR_INSTALL_DIR}/examples/numpy_native/convolve.py + ``` diff --git a/examples/native_operator/python/convolve.py b/examples/numpy_native/convolve.py similarity index 84% rename from examples/native_operator/python/convolve.py rename to examples/numpy_native/convolve.py index 50935b01..4f1c174e 100644 --- a/examples/native_operator/python/convolve.py +++ b/examples/numpy_native/convolve.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os - from holoscan.conditions import CountCondition from holoscan.core import Application, Operator, OperatorSpec from holoscan.logger import LogLevel, set_log_level @@ -35,6 +33,8 @@ class SignalGeneratorOp(Operator): Parameters ---------- + fragment : holoscan.core.Fragment + The Fragment (or Application) the operator belongs to. height : number The height of the signal impulse. size : number @@ -43,18 +43,17 @@ class SignalGeneratorOp(Operator): The data type of the generated signal. """ - def __init__(self, *args, height=1, size=10, dtype=np.int32, **kwargs): + def __init__(self, fragment, *args, height=1, size=10, dtype=np.int32, **kwargs): self.count = 0 self.height = height self.dtype = dtype self.size = size - super().__init__(*args, **kwargs) + super().__init__(fragment, *args, **kwargs) def setup(self, spec: OperatorSpec): spec.output("signal") def compute(self, op_input, op_output, context): - # single sample wide impulse at a time-varying position signal = np.zeros((self.size,), dtype=self.dtype) signal[self.count % signal.size] = self.height @@ -70,6 +69,8 @@ class ConvolveOp(Operator): Parameters ---------- + fragment : holoscan.core.Fragment + The Fragment (or Application) the operator belongs to. width : number The width of the boxcar kernel used in the convolution. unit_area : bool, optional @@ -80,18 +81,17 @@ class ConvolveOp(Operator): for the kernel. """ - def __init__(self, *args, width=4, unit_area=False, **kwargs): + def __init__(self, fragment, *args, width=4, unit_area=False, **kwargs): self.count = 0 self.width = width self.unit_area = unit_area - super().__init__(*args, **kwargs) + super().__init__(fragment, *args, **kwargs) def setup(self, spec: OperatorSpec): spec.input("signal_in") spec.output("signal_out") def compute(self, op_input, op_output, context): - signal = op_input.receive("signal_in") assert isinstance(signal, np.ndarray) @@ -131,9 +131,16 @@ def compose(self): self, CountCondition(self, count=24), name="generator", - **self.kwargs("signal_generator"), + height=1, + size=20, + dtype="int32", + ) + convolver = ConvolveOp( + self, + name="conv", + width=4, + unit_area=False, ) - convolver = ConvolveOp(self, name="conv", **self.kwargs("convolve")) printer = PrintSignalOp(self, name="printer") self.add_flow(signal_generator, convolver) self.add_flow(convolver, printer) @@ -143,6 +150,4 @@ def compose(self): set_log_level(LogLevel.WARN) app = ConvolveApp() - config_file = os.path.join(os.path.dirname(__file__), "convolve.yaml") - app.config(config_file) app.run() diff --git a/examples/ping_custom_op/CMakeLists.txt b/examples/ping_custom_op/CMakeLists.txt new file mode 100644 index 00000000..82c7bac7 --- /dev/null +++ b/examples/ping_custom_op/CMakeLists.txt @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_subdirectory(cpp) +add_subdirectory(python) + +install( + FILES README.md + DESTINATION "examples/ping_custom_op" + COMPONENT "holoscan-examples" +) diff --git a/examples/ping_custom_op/README.md b/examples/ping_custom_op/README.md new file mode 100644 index 00000000..3bb55b7d --- /dev/null +++ b/examples/ping_custom_op/README.md @@ -0,0 +1,52 @@ +# Ping Custom Op + +This example builds on top of the ping_simple example to demonstrate how to create your own custom operator. + +There are three operators involved in this example (tx -> mx -> rx): + 1. a transmitter, set to transmit a sequence of integes from 1-10 to it's 'out' port + 2. a middle operator that prints the received values, multiplies by a scalar and transmits the modified values + 3. a receiver that prints the received values to the terminal + +## C++ Run instructions + +* **using deb package install or NGC container**: + ```bash + /opt/nvidia/holoscan/examples/ping_custom_op/cpp/ping_custom_op + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + ./examples/ping_custom_op/cpp/ping_custom_op + ``` +* **source (local env)**: + ```bash + ${BUILD_OR_INSTALL_DIR}/examples/ping_custom_op/cpp/ping_custom_op + ``` + +## Python Run instructions + +* **using python wheel**: + ```bash + # [Prerequisite] Download example .py file below to `APP_DIR` + # [Optional] Start the virtualenv where holoscan is installed + python3 /ping_custom_op.py + ``` +* **using deb package install**: + ```bash + export PYTHONPATH=/opt/nvidia/holoscan/python/lib + python3 /opt/nvidia/holoscan/examples/ping_custom_op/python/ping_custom_op.py + ``` +* **from NGC container**: + ```bash + python3 /opt/nvidia/holoscan/examples/ping_custom_op/python/ping_custom_op.py + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + python3 ./examples/ping_custom_op/python/ping_custom_op.py + ``` +* **source (local env)**: + ```bash + export PYTHONPATH=${BUILD_OR_INSTALL_DIR}/python/lib + python3 ${BUILD_OR_INSTALL_DIR}/examples/ping_custom_op/python/ping_custom_op.py + ``` diff --git a/examples/ping_custom_op/cpp/CMakeLists.min.txt b/examples/ping_custom_op/cpp/CMakeLists.min.txt new file mode 100644 index 00000000..387ddffd --- /dev/null +++ b/examples/ping_custom_op/cpp/CMakeLists.min.txt @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the \"License\"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.20) +project(holoscan_ping_custom_op CXX) + +# Finds the package holoscan +find_package(holoscan REQUIRED CONFIG + PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install") + +add_executable(ping_custom_op + ping_custom_op.cpp +) + +target_link_libraries(ping_custom_op + PRIVATE + holoscan::core + holoscan::ops::ping_rx + holoscan::ops::ping_tx +) + +# Testing +if(BUILD_TESTING) + add_test(NAME EXAMPLE_CPP_PING_CUSTOM_OP_TEST + COMMAND ping_custom_op + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_CPP_PING_CUSTOM_OP_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Middle message value: 10" + PASS_REGULAR_EXPRESSION "Rx message value: 30" + ) +endif() diff --git a/examples/basic_workflow/cpp/CMakeLists.txt b/examples/ping_custom_op/cpp/CMakeLists.txt similarity index 57% rename from examples/basic_workflow/cpp/CMakeLists.txt rename to examples/ping_custom_op/cpp/CMakeLists.txt index 04c96956..e55e22ae 100644 --- a/examples/basic_workflow/cpp/CMakeLists.txt +++ b/examples/ping_custom_op/cpp/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,59 +13,56 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Create example -add_executable(basic_workflow - basic_workflow.cpp +# Create examples +add_executable(ping_custom_op + ping_custom_op.cpp ) - -target_link_libraries(basic_workflow - PRIVATE - ${HOLOSCAN_PACKAGE_NAME} -) - -# Copy config file to the build tree -add_custom_target(basic_workflow_yaml - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_SOURCE_DIR}/apps/endoscopy_tool_tracking/cpp/app_config.yaml" ${CMAKE_CURRENT_BINARY_DIR} +target_link_libraries(ping_custom_op + PUBLIC + holoscan::core + holoscan::ops::ping_tx + holoscan::ops::ping_rx ) -add_dependencies(basic_workflow basic_workflow_yaml) - -# Download the associated dataset if needed -if(HOLOSCAN_DOWNLOAD_DATASETS) - add_dependencies(basic_workflow endoscopy_data) -endif() - -# Install example +# Install examples # Set the install RPATH based on the location of the Holoscan SDK libraries # The GXF extensions are loaded by the GXF libraries - no need to include here file(RELATIVE_PATH install_lib_relative_path ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_SOURCE_DIR}/${HOLOSCAN_INSTALL_LIB_DIR}) -set_target_properties(basic_workflow PROPERTIES INSTALL_RPATH "\$ORIGIN/${install_lib_relative_path}") +set_target_properties(ping_custom_op PROPERTIES INSTALL_RPATH "\$ORIGIN/${install_lib_relative_path}") # Install following the relative folder path file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) +if(HOLOSCAN_INSTALL_EXAMPLE_SOURCE) # Install the source -install(FILES basic_workflow.cpp +install(FILES ping_custom_op.cpp DESTINATION "${app_relative_dest_path}" - COMPONENT holoscan-apps + COMPONENT holoscan-examples ) # Install the minimal CMakeLists.txt file install(FILES CMakeLists.min.txt RENAME "CMakeLists.txt" DESTINATION "${app_relative_dest_path}" - COMPONENT holoscan-apps -) - -# Install the configuration file -install(FILES ${CMAKE_SOURCE_DIR}/apps/endoscopy_tool_tracking/cpp/app_config.yaml - DESTINATION "${app_relative_dest_path}" - COMPONENT holoscan-apps + COMPONENT holoscan-examples ) +endif() # Install the compiled example -install(TARGETS basic_workflow +install(TARGETS ping_custom_op DESTINATION "${app_relative_dest_path}" - COMPONENT holoscan-apps + COMPONENT holoscan-examples ) + +# Testing +if(HOLOSCAN_BUILD_TESTS) + add_test(NAME EXAMPLE_CPP_PING_CUSTOM_OP_TEST + COMMAND ping_custom_op + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_CPP_PING_CUSTOM_OP_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Middle message value: 10" + PASS_REGULAR_EXPRESSION "Rx message value: 30" + ) +endif() diff --git a/examples/ping_custom_op/cpp/ping_custom_op.cpp b/examples/ping_custom_op/cpp/ping_custom_op.cpp new file mode 100644 index 00000000..4f0f4108 --- /dev/null +++ b/examples/ping_custom_op/cpp/ping_custom_op.cpp @@ -0,0 +1,73 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +namespace holoscan::ops { + +class PingMxOp : public Operator { + public: + HOLOSCAN_OPERATOR_FORWARD_ARGS(PingMxOp) + + PingMxOp() = default; + + void setup(OperatorSpec& spec) override { + spec.input("in"); + spec.output("out"); + spec.param(multiplier_, "multiplier", "Multiplier", "Multiply the input by this value", 2); + } + + void compute(InputContext& op_input, OutputContext& op_output, ExecutionContext&) override { + auto value = op_input.receive("in"); + + std::cout << "Middle message value: " << *(value.get()) << std::endl; + + // Multiply the value by the multiplier parameter + *(value.get()) *= multiplier_; + + op_output.emit(value); + }; + + private: + Parameter multiplier_; +}; + +} // namespace holoscan::ops + +class MyPingApp : public holoscan::Application { + public: + void compose() override { + using namespace holoscan; + // Define the tx, mx, rx operators, allowing tx operator to execute 10 times + auto tx = make_operator("tx", make_condition(10)); + auto mx = make_operator("mx", Arg("multiplier", 3)); + auto rx = make_operator("rx"); + + // Define the workflow: tx -> mx -> rx + add_flow(tx, mx); + add_flow(mx, rx); + } +}; + +int main(int argc, char** argv) { + auto app = holoscan::make_application(); + app->run(); + + return 0; +} diff --git a/apps/hi_speed_endoscopy/python/CMakeLists.txt b/examples/ping_custom_op/python/CMakeLists.txt similarity index 51% rename from apps/hi_speed_endoscopy/python/CMakeLists.txt rename to examples/ping_custom_op/python/CMakeLists.txt index 9be32931..1a839d57 100644 --- a/apps/hi_speed_endoscopy/python/CMakeLists.txt +++ b/examples/ping_custom_op/python/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,24 +16,28 @@ # Get relative folder path for the app file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) -# Copy high-speed endoscopy tool tracking application -add_custom_target(python_hi_speed_endoscopy - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/hi_speed_endoscopy.py" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/hi_speed_endoscopy.py" +# Copy native operator ping application +add_custom_target(python_ping_custom_op ALL + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/ping_custom_op.py" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "ping_custom_op.py" + BYPRODUCTS "ping_custom_op.py" ) -add_dependencies(${HOLOSCAN_PACKAGE_NAME} python_hi_speed_endoscopy) - -# Copy high-speed endoscopy tool tracking config file -add_custom_target(python_hi_speed_endoscopy_yaml - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/hi_speed_endoscopy.yaml" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/hi_speed_endoscopy.yaml" -) -add_dependencies(${HOLOSCAN_PACKAGE_NAME} python_hi_speed_endoscopy_yaml) # Install the app install(FILES - "${CMAKE_CURRENT_SOURCE_DIR}/hi_speed_endoscopy.py" - "${CMAKE_CURRENT_SOURCE_DIR}/hi_speed_endoscopy.yaml" + "${CMAKE_CURRENT_SOURCE_DIR}/ping_custom_op.py" DESTINATION "${app_relative_dest_path}" - COMPONENT "holoscan-apps" + COMPONENT "holoscan-examples" ) + +# Testing +if(HOLOSCAN_BUILD_TESTS) + add_test(NAME EXAMPLE_PYTHON_PING_CUSTOM_OP_TEST + COMMAND python3 ping_custom_op.py + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_PYTHON_PING_CUSTOM_OP_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Middle message value: 10" + PASS_REGULAR_EXPRESSION "Rx message value: 30" + ) +endif() diff --git a/examples/ping_custom_op/python/ping_custom_op.py b/examples/ping_custom_op/python/ping_custom_op.py new file mode 100644 index 00000000..38bb2265 --- /dev/null +++ b/examples/ping_custom_op/python/ping_custom_op.py @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from holoscan.conditions import CountCondition +from holoscan.core import Application, Operator, OperatorSpec +from holoscan.operators import PingRxOp, PingTxOp + +# define custom Operators for use in the demo + + +class PingMxOp(Operator): + """Example of an operator modifying data. + + This operator has 1 input and 1 output port: + input: "in" + output: "out" + + The data from each input is multiplied by a user-defined value. + + """ + + def setup(self, spec: OperatorSpec): + spec.input("in") + spec.output("out") + spec.param("multiplier", 2) + + def compute(self, op_input, op_output, context): + value = op_input.receive("in") + print(f"Middle message value: {value}") + + # Multiply the values by the multiplier parameter + value *= self.multiplier + + op_output.emit(value, "out") + + +# Now define a simple application using the operators defined above + + +class MyPingApp(Application): + def compose(self): + # Define the tx, mx, rx operators, allowing the tx operator to execute 10 times + tx = PingTxOp(self, CountCondition(self, 10), name="tx") + mx = PingMxOp(self, name="mx", multiplier=3) + rx = PingRxOp(self, name="rx") + + # Define the workflow: tx -> mx -> rx + self.add_flow(tx, mx) + self.add_flow(mx, rx) + + +if __name__ == "__main__": + app = MyPingApp() + app.run() diff --git a/examples/ping_multi_port/CMakeLists.txt b/examples/ping_multi_port/CMakeLists.txt new file mode 100644 index 00000000..6e9577ec --- /dev/null +++ b/examples/ping_multi_port/CMakeLists.txt @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_subdirectory(cpp) +add_subdirectory(python) + +install( + FILES README.md + DESTINATION "examples/ping_multi_port" + COMPONENT "holoscan-examples" +) diff --git a/examples/ping_multi_port/README.md b/examples/ping_multi_port/README.md new file mode 100644 index 00000000..f45b8455 --- /dev/null +++ b/examples/ping_multi_port/README.md @@ -0,0 +1,54 @@ +# Ping Multi Port + +This example builds on top of the ping_custom_op example and demonstrates how to send and receive data using +a custom data class. Additionally, this example shows how create and use operators that have multiple input +and/or output ports. + +There are three operators involved in this example: + 1. a transmitter, set to transmit a sequence of even integers on port `out1` and odd integers on port `out2`. + 2. a middle operator that prints the received values, multiplies by a scalar and transmits the modified values + 3. a receiver that prints the received values to the terminal + +## C++ Run instructions + +* **using deb package install or NGC container**: + ```bash + /opt/nvidia/holoscan/examples/ping_multi_port/cpp/ping_multi_port + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + ./examples/ping_multi_port/cpp/ping_multi_port + ``` +* **source (local env)**: + ```bash + ${BUILD_OR_INSTALL_DIR}/examples/ping_multi_port/cpp/ping_multi_port + ``` + +## Python Run instructions + +* **using python wheel**: + ```bash + # [Prerequisite] Download example .py file below to `APP_DIR` + # [Optional] Start the virtualenv where holoscan is installed + python3 /ping_multi_port.py + ``` +* **using deb package install**: + ```bash + export PYTHONPATH=/opt/nvidia/holoscan/python/lib + python3 /opt/nvidia/holoscan/examples/ping_multi_port/python/ping_multi_port.py + ``` +* **from NGC container**: + ```bash + python3 /opt/nvidia/holoscan/examples/ping_multi_port/python/ping_multi_port.py + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + python3 ./examples/ping_multi_port/python/ping_multi_port.py + ``` +* **source (local env)**: + ```bash + export PYTHONPATH=${BUILD_OR_INSTALL_DIR}/python/lib + python3 ${BUILD_OR_INSTALL_DIR}/examples/ping_multi_port/python/ping_multi_port.py + ``` diff --git a/examples/ping_multi_port/cpp/CMakeLists.min.txt b/examples/ping_multi_port/cpp/CMakeLists.min.txt new file mode 100644 index 00000000..5fbe4547 --- /dev/null +++ b/examples/ping_multi_port/cpp/CMakeLists.min.txt @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the \"License\"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.20) +project(holoscan_ping_multi_port CXX) + +# Finds the package holoscan +find_package(holoscan REQUIRED CONFIG + PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install") + +add_executable(ping_multi_port + ping_multi_port.cpp +) + +target_link_libraries(ping_multi_port + PRIVATE + holoscan::core +) + +# Testing +if(BUILD_TESTING) + add_test(NAME EXAMPLE_CPP_PING_MULTI_PORT_TEST + COMMAND ping_multi_port + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_CPP_PING_MULTI_PORT_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Rx message value1: 57") +endif() diff --git a/examples/ping_multi_port/cpp/CMakeLists.txt b/examples/ping_multi_port/cpp/CMakeLists.txt new file mode 100644 index 00000000..2f29452d --- /dev/null +++ b/examples/ping_multi_port/cpp/CMakeLists.txt @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Create examples +add_executable(ping_multi_port + ping_multi_port.cpp +) +target_link_libraries(ping_multi_port + PRIVATE + holoscan::core +) + +# Set the install RPATH based on the location of the Holoscan SDK libraries +# The GXF extensions are loaded by the GXF libraries - no need to include here +file(RELATIVE_PATH install_lib_relative_path ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_SOURCE_DIR}/${HOLOSCAN_INSTALL_LIB_DIR}) +set_target_properties(ping_multi_port PROPERTIES INSTALL_RPATH "\$ORIGIN/${install_lib_relative_path}") + +# Install following the relative folder path +file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) + +if(HOLOSCAN_INSTALL_EXAMPLE_SOURCE) +# Install the source +install(FILES ping_multi_port.cpp + DESTINATION "${app_relative_dest_path}" + COMPONENT holoscan-examples +) + +# Install the minimal CMakeLists.txt file +install(FILES CMakeLists.min.txt + RENAME "CMakeLists.txt" + DESTINATION "${app_relative_dest_path}" + COMPONENT holoscan-examples +) +endif() + +# Install the compiled example +install(TARGETS ping_multi_port + DESTINATION "${app_relative_dest_path}" + COMPONENT holoscan-examples +) + +# Testing +if(HOLOSCAN_BUILD_TESTS) + add_test(NAME EXAMPLE_CPP_PING_MULTI_PORT_TEST + COMMAND ping_multi_port + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_CPP_PING_MULTI_PORT_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Rx message value1: 57") +endif() diff --git a/examples/native_operator/cpp/ping.cpp b/examples/ping_multi_port/cpp/ping_multi_port.cpp similarity index 85% rename from examples/native_operator/cpp/ping.cpp rename to examples/ping_multi_port/cpp/ping_multi_port.cpp index 43af43fa..74de85eb 100644 --- a/examples/native_operator/cpp/ping.cpp +++ b/examples/ping_multi_port/cpp/ping_multi_port.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,9 +23,7 @@ class ValueData { explicit ValueData(int value) : data_(value) { HOLOSCAN_LOG_TRACE("ValueData::ValueData(): {}", data_); } - ~ValueData() { - HOLOSCAN_LOG_TRACE("ValueData::~ValueData(): {}", data_); - } + ~ValueData() { HOLOSCAN_LOG_TRACE("ValueData::~ValueData(): {}", data_); } void data(int value) { data_ = value; } @@ -55,14 +53,14 @@ class PingTxOp : public Operator { auto value2 = std::make_shared(index_++); op_output.emit(value2, "out2"); }; - int index_ = 0; + int index_ = 1; }; -class PingMiddleOp : public Operator { +class PingMxOp : public Operator { public: - HOLOSCAN_OPERATOR_FORWARD_ARGS(PingMiddleOp) + HOLOSCAN_OPERATOR_FORWARD_ARGS(PingMxOp) - PingMiddleOp() = default; + PingMxOp() = default; void setup(OperatorSpec& spec) override { spec.input("in1"); @@ -120,15 +118,17 @@ class PingRxOp : public Operator { } // namespace holoscan::ops -class App : public holoscan::Application { +class MyPingApp : public holoscan::Application { public: void compose() override { using namespace holoscan; + // Define the tx, mx, rx operators, allowing the tx operator to execute 10 times auto tx = make_operator("tx", make_condition(10)); - auto mx = make_operator("mx", from_config("mx")); + auto mx = make_operator("mx", Arg("multiplier", 3)); auto rx = make_operator("rx"); + // Define the workflow add_flow(tx, mx, {{"out1", "in1"}, {"out2", "in2"}}); add_flow(mx, rx, {{"out1", "receivers"}, {"out2", "receivers"}}); } @@ -136,14 +136,7 @@ class App : public holoscan::Application { int main(int argc, char** argv) { holoscan::load_env_log_level(); - - auto app = holoscan::make_application(); - - // Get the configuration - auto config_path = std::filesystem::canonical(argv[0]).parent_path(); - config_path += "/app_config.yaml"; - app->config(config_path); - + auto app = holoscan::make_application(); app->run(); return 0; diff --git a/examples/ping_multi_port/python/CMakeLists.txt b/examples/ping_multi_port/python/CMakeLists.txt new file mode 100644 index 00000000..72ef77ac --- /dev/null +++ b/examples/ping_multi_port/python/CMakeLists.txt @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Get relative folder path for the app +file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) + +# Copy native operator ping_multi_port application +add_custom_target(python_ping_multi_port ALL + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/ping_multi_port.py" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "ping_multi_port.py" + BYPRODUCTS "ping_multi_port.py" +) + +# Install the app +install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/ping_multi_port.py" + DESTINATION "${app_relative_dest_path}" + COMPONENT "holoscan-examples" +) + +# Testing +if(HOLOSCAN_BUILD_TESTS) + add_test(NAME EXAMPLE_PYTHON_PING_MULTI_PORT_TEST + COMMAND python3 ping_multi_port.py + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_PYTHON_PING_MULTI_PORT_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Rx message value1: 57") +endif() diff --git a/examples/native_operator/python/ping.py b/examples/ping_multi_port/python/ping_multi_port.py similarity index 76% rename from examples/native_operator/python/ping.py rename to examples/ping_multi_port/python/ping_multi_port.py index 3bcfcc9d..e13b5850 100644 --- a/examples/native_operator/python/ping.py +++ b/examples/ping_multi_port/python/ping_multi_port.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os - from holoscan.conditions import CountCondition from holoscan.core import Application, Operator, OperatorSpec from holoscan.logger import load_env_log_level @@ -52,10 +50,10 @@ class PingTxOp(Operator): each call to compute. """ - def __init__(self, *args, **kwargs): - self.index = 0 + def __init__(self, fragment, *args, **kwargs): + self.index = 1 # Need to call the base class constructor last - super().__init__(*args, **kwargs) + super().__init__(fragment, *args, **kwargs) def setup(self, spec: OperatorSpec): spec.output("out1") @@ -71,7 +69,7 @@ def compute(self, op_input, op_output, context): op_output.emit(value2, "out2") -class PingMiddleOp(Operator): +class PingMxOp(Operator): """Example of an operator modifying data. This operator has: @@ -79,23 +77,13 @@ class PingMiddleOp(Operator): outputs: "out1", "out2" The data from each input is multiplied by a user-defined value. - - In this demo, the `multiplier` parameter value is read from a "ping.yaml" - configuration file (near the bottom of this script), overriding the default - defined in the setup() method below. """ - def __init__(self, *args, **kwargs): - # If `self.multiplier` is set here (e.g., `self.multiplier = 4`), then - # the default value by `param()` in `setup()` will be ignored. - # (you can just call `spec.param("multiplier")` in `setup()` to use the - # default value) - # - # self.multiplier = 4 + def __init__(self, fragment, *args, **kwargs): self.count = 1 # Need to call the base class constructor last - super().__init__(*args, **kwargs) + super().__init__(fragment, *args, **kwargs) def setup(self, spec: OperatorSpec): spec.input("in1") @@ -131,10 +119,10 @@ class PingRxOp(Operator): number of inputs connected to is "receivers" port. """ - def __init__(self, *args, **kwargs): + def __init__(self, fragment, *args, **kwargs): self.count = 1 # Need to call the base class constructor last - super().__init__(*args, **kwargs) + super().__init__(fragment, *args, **kwargs) def setup(self, spec: OperatorSpec): spec.param("receivers", kind="receivers") @@ -152,13 +140,12 @@ def compute(self, op_input, op_output, context): class MyPingApp(Application): def compose(self): - # Configure the operators. Here we use CountCondition to terminate - # execution after a specific number of messages have been sent. + # Define the tx, mx, rx operators, allowing the tx operator to execute 10 times tx = PingTxOp(self, CountCondition(self, 10), name="tx") - mx = PingMiddleOp(self, self.from_config("mx"), name="mx") + mx = PingMxOp(self, name="mx", multiplier=3) rx = PingRxOp(self, name="rx") - # Connect the operators into the workflow: tx -> mx -> rx + # Define the workflow self.add_flow(tx, mx, {("out1", "in1"), ("out2", "in2")}) self.add_flow(mx, rx, {("out1", "receivers"), ("out2", "receivers")}) @@ -166,5 +153,4 @@ def compose(self): if __name__ == "__main__": load_env_log_level() app = MyPingApp() - app.config(os.path.join(os.path.dirname(__file__), "ping.yaml")) app.run() diff --git a/examples/ping_simple/CMakeLists.txt b/examples/ping_simple/CMakeLists.txt new file mode 100644 index 00000000..b963068c --- /dev/null +++ b/examples/ping_simple/CMakeLists.txt @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_subdirectory(cpp) +add_subdirectory(python) + +install( + FILES README.md + DESTINATION "examples/ping_simple" + COMPONENT "holoscan-examples" +) diff --git a/examples/ping_simple/README.md b/examples/ping_simple/README.md new file mode 100644 index 00000000..302f7254 --- /dev/null +++ b/examples/ping_simple/README.md @@ -0,0 +1,51 @@ +# Ping Simple + +This example demonstrates a simple ping application with two operators connected using add_flow(). + +There are two operators involved in this example: + 1. a transmitter, set to transmit a sequence of integers from 1-10 to it's 'out' port + 2. a receiver that prints the received values to the terminal + +## C++ Run instructions + +* **using deb package install or NGC container**: + ```bash + /opt/nvidia/holoscan/examples/ping_simple/cpp/ping_simple + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + ./examples/ping_simple/cpp/ping_simple + ``` +* **source (local env)**: + ```bash + ${BUILD_OR_INSTALL_DIR}/examples/ping_simple/cpp/ping_simple + ``` + +## Python Run instructions + +* **using python wheel**: + ```bash + # [Prerequisite] Download example .py file below to `APP_DIR` + # [Optional] Start the virtualenv where holoscan is installed + python3 /ping_simple.py + ``` +* **using deb package install**: + ```bash + export PYTHONPATH=/opt/nvidia/holoscan/python/lib + python3 /opt/nvidia/holoscan/examples/ping_simple/python/ping_simple.py + ``` +* **from NGC container**: + ```bash + python3 /opt/nvidia/holoscan/examples/ping_simple/python/ping_simple.py + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + python3 ./examples/ping_simple/python/ping_simple.py + ``` +* **source (local env)**: + ```bash + export PYTHONPATH=${BUILD_OR_INSTALL_DIR}/python/lib + python3 ${BUILD_OR_INSTALL_DIR}/examples/ping_simple/python/ping_simple.py + ``` diff --git a/examples/ping_simple/cpp/CMakeLists.min.txt b/examples/ping_simple/cpp/CMakeLists.min.txt new file mode 100644 index 00000000..a7a8274d --- /dev/null +++ b/examples/ping_simple/cpp/CMakeLists.min.txt @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the \"License\"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.20) +project(holoscan_ping_simple CXX) + +# Finds the package holoscan +find_package(holoscan REQUIRED CONFIG + PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install") + +add_executable(ping_simple + ping_simple.cpp +) + +target_link_libraries(ping_simple + PRIVATE + holoscan::core + holoscan::ops::ping_rx + holoscan::ops::ping_tx +) + +# Testing +if(BUILD_TESTING) + add_test(NAME EXAMPLE_CPP_PING_SIMPLE_TEST + COMMAND ping_simple + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_CPP_PING_SIMPLE_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Rx message value: 10") +endif() diff --git a/examples/ping_simple/cpp/CMakeLists.txt b/examples/ping_simple/cpp/CMakeLists.txt new file mode 100644 index 00000000..1a2e9450 --- /dev/null +++ b/examples/ping_simple/cpp/CMakeLists.txt @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Create examples +add_executable(ping_simple + ping_simple.cpp +) +target_link_libraries(ping_simple + PUBLIC + holoscan::core + holoscan::ops::ping_tx + holoscan::ops::ping_rx +) + +# Install examples + +# Set the install RPATH based on the location of the Holoscan SDK libraries +# The GXF extensions are loaded by the GXF libraries - no need to include here +file(RELATIVE_PATH install_lib_relative_path ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_SOURCE_DIR}/${HOLOSCAN_INSTALL_LIB_DIR}) +set_target_properties(ping_simple PROPERTIES INSTALL_RPATH "\$ORIGIN/${install_lib_relative_path}") + +# Install following the relative folder path +file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) + +if(HOLOSCAN_INSTALL_EXAMPLE_SOURCE) +# Install the source +install(FILES ping_simple.cpp + DESTINATION "${app_relative_dest_path}" + COMPONENT holoscan-examples +) + +# Install the minimal CMakeLists.txt file +install(FILES CMakeLists.min.txt + RENAME "CMakeLists.txt" + DESTINATION "${app_relative_dest_path}" + COMPONENT holoscan-examples +) +endif() + +# Install the compiled example +install(TARGETS ping_simple + DESTINATION "${app_relative_dest_path}" + COMPONENT holoscan-examples +) + +# Testing +if(HOLOSCAN_BUILD_TESTS) + add_test(NAME EXAMPLE_CPP_PING_SIMPLE_TEST + COMMAND ping_simple + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_CPP_PING_SIMPLE_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Rx message value: 10") +endif() + diff --git a/examples/ping_simple/cpp/ping_simple.cpp b/examples/ping_simple/cpp/ping_simple.cpp new file mode 100644 index 00000000..878f2415 --- /dev/null +++ b/examples/ping_simple/cpp/ping_simple.cpp @@ -0,0 +1,41 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + + +class MyPingApp : public holoscan::Application { + public: + void compose() override { + using namespace holoscan; + // Define the tx and rx operators, allowing the tx operator to execute 10 times + auto tx = make_operator("tx", make_condition(10)); + auto rx = make_operator("rx"); + + // Define the workflow: tx -> rx + add_flow(tx, rx); + } +}; + +int main(int argc, char** argv) { + auto app = holoscan::make_application(); + app->run(); + + return 0; +} diff --git a/apps/endoscopy_tool_tracking/python/CMakeLists.txt b/examples/ping_simple/python/CMakeLists.txt similarity index 50% rename from apps/endoscopy_tool_tracking/python/CMakeLists.txt rename to examples/ping_simple/python/CMakeLists.txt index 74abb8a2..5f018ad1 100644 --- a/apps/endoscopy_tool_tracking/python/CMakeLists.txt +++ b/examples/ping_simple/python/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,24 +16,27 @@ # Get relative folder path for the app file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) -# Copy endoscopy tool tracking application -add_custom_target(python_endoscopy_tool_tracking - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/endoscopy_tool_tracking.py" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/endoscopy_tool_tracking.py" +# Copy native operator ping application +add_custom_target(python_ping_simple ALL + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/ping_simple.py" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "ping_simple.py" + BYPRODUCTS "ping_simple.py" ) -add_dependencies(${HOLOSCAN_PACKAGE_NAME} python_endoscopy_tool_tracking) - -# Copy endoscopy tool tracking config file -add_custom_target(python_endoscopy_tool_tracking_yaml - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/endoscopy_tool_tracking.yaml" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/endoscopy_tool_tracking.yaml" -) -add_dependencies(${HOLOSCAN_PACKAGE_NAME} python_endoscopy_tool_tracking_yaml) # Install the app install(FILES - "${CMAKE_CURRENT_SOURCE_DIR}/endoscopy_tool_tracking.py" - "${CMAKE_CURRENT_SOURCE_DIR}/endoscopy_tool_tracking.yaml" + "${CMAKE_CURRENT_SOURCE_DIR}/ping_simple.py" DESTINATION "${app_relative_dest_path}" - COMPONENT "holoscan-apps" + COMPONENT "holoscan-examples" ) + +# Testing +if(HOLOSCAN_BUILD_TESTS) + add_test(NAME EXAMPLE_PYTHON_PING_SIMPLE_TEST + COMMAND python3 ping_simple.py + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_PYTHON_PING_SIMPLE_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Rx message value: 10" + ) +endif() diff --git a/gxf_extensions/holoviz/CMakeLists.txt b/examples/ping_simple/python/ping_simple.py similarity index 52% rename from gxf_extensions/holoviz/CMakeLists.txt rename to examples/ping_simple/python/ping_simple.py index 3ea3f8ff..b5d55ee2 100644 --- a/gxf_extensions/holoviz/CMakeLists.txt +++ b/examples/ping_simple/python/ping_simple.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,29 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -# create component library -add_library(holoviz_lib SHARED - holoviz.cpp - holoviz.hpp -) -target_link_libraries(holoviz_lib - PUBLIC - GXF::std - GXF::serialization - GXF::cuda - GXF::multimedia - yaml-cpp - clara_holoviz -) +from holoscan.conditions import CountCondition +from holoscan.core import Application +from holoscan.operators import PingRxOp, PingTxOp -# Create extension -add_library(holoviz SHARED -holoviz_ext.cpp -) -target_link_libraries(holoviz - PUBLIC - holoviz_lib -) -# Install GXF extension as a component 'holoscan-gxf_extensions' -install_gxf_extension(holoviz) +class MyPingApp(Application): + def compose(self): + # Define the tx and rx operators, allowing tx to execute 10 times + tx = PingTxOp(self, CountCondition(self, 10), name="tx") + rx = PingRxOp(self, name="rx") + + # Define the workflow: tx -> rx + self.add_flow(tx, rx) + + +if __name__ == "__main__": + app = MyPingApp() + app.run() diff --git a/examples/tensor_interop/CMakeLists.txt b/examples/tensor_interop/CMakeLists.txt index 221c7a5a..9b3431d9 100644 --- a/examples/tensor_interop/CMakeLists.txt +++ b/examples/tensor_interop/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,3 +15,9 @@ add_subdirectory(cpp) add_subdirectory(python) + +install( + FILES README.md + DESTINATION "examples/tensor_interop" + COMPONENT "holoscan-examples" +) diff --git a/examples/tensor_interop/README.md b/examples/tensor_interop/README.md index ca266852..e9effe4b 100644 --- a/examples/tensor_interop/README.md +++ b/examples/tensor_interop/README.md @@ -10,18 +10,21 @@ This application demonstrates interoperability between a native operator (`Proce Notably, the two GXF codelets have not been wrapped as Holoscan operators, but are instead registered at runtime in the `compose` method of the application. -### Build instructions - -Built with the SDK, see instructions from the top level README. - ### Run instructions -First, go in your `build` or `install` directory (automatically done by `./run launch`). - -Then, run: -```bash -./examples/tensor_interop/cpp/tensor_interop -``` +* **using deb package install or NGC container**: + ```bash + /opt/nvidia/holoscan/examples/tensor_interop/cpp/tensor_interop + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + ./examples/tensor_interop/cpp/tensor_interop + ``` +* **source (local env)**: + ```bash + ${BUILD_OR_INSTALL_DIR}/examples/tensor_interop/cpp/tensor_interop + ``` ## Python API @@ -30,21 +33,43 @@ This application demonstrates interoperability between a native operator (`Image - The output tensor (in a new entity) is sent to the `HolovizOp` operator (codelet) which gets the tensor from the entity and displays the image in the GUI. The `VideoStreamReplayerOp` operator is used to replay the video stream from the sample data. - The Holoscan Tensor object is interoperable with DLPack or array interfaces. -### Requirements - -This example requires cupy which is included in the x86_64 development container. You'll need to build cupy for arm64 if you want to run this example on the developer kits. +### Data -### Build instructions - -Built with the SDK, see instructions from the top level README. +The following dataset is used by this example: +[ðŸ“¦ï¸ (NGC) Sample App Data for AI-based Endoscopy Tool Tracking](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/clara-holoscan/resources/holoscan_endoscopy_sample_data). ### Run instructions -First, go in your `build` or `install` directory (automatically done by `./run launch`). - -Then run: -```bash -python3 ./examples/tensor_interop/python/tensor_interop.py -``` - -> â„¹ï¸ Python apps can run outside those folders if `HOLOSCAN_SAMPLE_DATA_PATH` is set in your environment (automatically done by `./run launch`). \ No newline at end of file +* **using python wheel**: + ```bash + # [Prerequisite] Download NGC dataset above to `DATA_DIR` + # [Prerequisite] Download example .py and .yaml file below to `APP_DIR` + # [Optional] Start the virtualenv where holoscan is installed + python3 -m pip install cupy-cuda11x # Append `-f https://pip.cupy.dev/aarch64` on aarch64 + export HOLOSCAN_SAMPLE_DATA_PATH= + python3 /tensor_interop.py + ``` +* **using deb package install**: + ```bash + # [Prerequisite] Download NGC dataset above to `DATA_DIR` + python3 -m pip install cupy-cuda11x # Append `-f https://pip.cupy.dev/aarch64` on aarch64 + export PYTHONPATH=/opt/nvidia/holoscan/python/lib + export HOLOSCAN_SAMPLE_DATA_PATH= + python3 /opt/nvidia/holoscan/examples/tensor_interop/python/tensor_interop.py + ``` +* **from NGC container**: + ```bash + python3 /opt/nvidia/holoscan/examples/tensor_interop/python/tensor_interop.py + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + python3 ./examples/tensor_interop/python/tensor_interop.py + ``` +* **source (local env)**: + ```bash + python3 -m pip install cupy-cuda11x # Append `-f https://pip.cupy.dev/aarch64` on aarch64 + export PYTHONPATH=${BUILD_OR_INSTALL_DIR}/python/lib + export HOLOSCAN_SAMPLE_DATA_PATH=${SRC_DIR}/data + python3 ${BUILD_OR_INSTALL_DIR}/examples/tensor_interop/python/tensor_interop.py + ``` \ No newline at end of file diff --git a/examples/tensor_interop/cpp/CMakeLists.min.txt b/examples/tensor_interop/cpp/CMakeLists.min.txt index 75fd0dee..41c2fc32 100644 --- a/examples/tensor_interop/cpp/CMakeLists.min.txt +++ b/examples/tensor_interop/cpp/CMakeLists.min.txt @@ -14,10 +14,11 @@ # limitations under the License. cmake_minimum_required(VERSION 3.20) -project(holoscan_cpp_tensor_interop CXX) +project(holoscan_tensor_interop CXX) # Finds the package holoscan -find_package(holoscan REQUIRED) +find_package(holoscan REQUIRED CONFIG + PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install") add_executable(tensor_interop tensor_interop.cpp @@ -27,12 +28,17 @@ add_executable(tensor_interop target_link_libraries(tensor_interop PRIVATE - holoscan::holoscan + holoscan::core CUDA::cudart ) -# Copy config file to the build tree -add_custom_target(tensor_interop_yaml - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_SOURCE_DIR}/app_config.yaml" ${CMAKE_CURRENT_BINARY_DIR} -) -add_dependencies(tensor_interop tensor_interop_yaml) +# Testing +if(BUILD_TESTING) + add_test(NAME EXAMPLE_CPP_TENSOR_INTEROP_TEST + COMMAND tensor_interop + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_CPP_TENSOR_INTEROP_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Scheduling 3 entities" + ) +endif() diff --git a/examples/tensor_interop/cpp/CMakeLists.txt b/examples/tensor_interop/cpp/CMakeLists.txt index 5d9f483e..323aad73 100644 --- a/examples/tensor_interop/cpp/CMakeLists.txt +++ b/examples/tensor_interop/cpp/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,17 +21,10 @@ add_executable(tensor_interop ) target_link_libraries(tensor_interop PRIVATE - ${HOLOSCAN_PACKAGE_NAME} + holoscan::core CUDA::cudart ) -# Copy config file to the build tree -add_custom_target(tensor_interop_yaml - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/app_config.yaml" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/app_config.yaml" -) -add_dependencies(tensor_interop tensor_interop_yaml) - # Install example # Set the install RPATH based on the location of the Holoscan SDK libraries @@ -45,25 +38,31 @@ file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SO # Install the target install(TARGETS tensor_interop DESTINATION "${app_relative_dest_path}" - COMPONENT holoscan-apps + COMPONENT holoscan-examples ) +if(HOLOSCAN_INSTALL_EXAMPLE_SOURCE) # Install the source install(FILES tensor_interop.cpp receive_tensor_gxf.hpp send_tensor_gxf.hpp DESTINATION "${app_relative_dest_path}" - COMPONENT holoscan-apps + COMPONENT holoscan-examples ) # Install the minimal CMakeLists.txt file install(FILES CMakeLists.min.txt RENAME "CMakeLists.txt" DESTINATION "${app_relative_dest_path}" - COMPONENT holoscan-apps + COMPONENT holoscan-examples ) +endif() -# Install the configuration file -install(FILES - "${CMAKE_CURRENT_SOURCE_DIR}/app_config.yaml" - DESTINATION "${app_relative_dest_path}" - COMPONENT holoscan-apps -) +# Testing +if(HOLOSCAN_BUILD_TESTS) + add_test(NAME EXAMPLE_CPP_TENSOR_INTEROP_TEST + COMMAND tensor_interop + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_CPP_TENSOR_INTEROP_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Scheduling 3 entities" + ) +endif() diff --git a/examples/tensor_interop/cpp/app_config.yaml b/examples/tensor_interop/cpp/app_config.yaml deleted file mode 100644 index 744377d1..00000000 --- a/examples/tensor_interop/cpp/app_config.yaml +++ /dev/null @@ -1,21 +0,0 @@ -%YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -extensions: - - libgxf_std.so - -mx: - multiplier: 3 diff --git a/examples/tensor_interop/cpp/tensor_interop.cpp b/examples/tensor_interop/cpp/tensor_interop.cpp index 38709119..7b823666 100644 --- a/examples/tensor_interop/cpp/tensor_interop.cpp +++ b/examples/tensor_interop/cpp/tensor_interop.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +22,6 @@ #include #include #include -#include #include "./receive_tensor_gxf.hpp" #include "./send_tensor_gxf.hpp" @@ -169,12 +168,6 @@ int main(int argc, char** argv) { holoscan::load_env_log_level(); auto app = holoscan::make_application(); - - // Get the configuration - auto config_path = std::filesystem::canonical(argv[0]).parent_path(); - config_path += "/app_config.yaml"; - app->config(config_path); - app->run(); return 0; diff --git a/examples/tensor_interop/python/CMakeLists.txt b/examples/tensor_interop/python/CMakeLists.txt index 281ba8bb..49062930 100644 --- a/examples/tensor_interop/python/CMakeLists.txt +++ b/examples/tensor_interop/python/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,23 +17,40 @@ file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) # Copy native operator tensor_interop application -add_custom_target(python_tensor_interop +add_custom_target(python_tensor_interop ALL COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/tensor_interop.py" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/tensor_interop.py" + DEPENDS "tensor_interop.py" + BYPRODUCTS "tensor_interop.py" ) -add_dependencies(${HOLOSCAN_PACKAGE_NAME} python_tensor_interop) # Copy endoscopy tool tracking config file add_custom_target(python_tensor_interop_yaml COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/tensor_interop.yaml" ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/tensor_interop.yaml" + DEPENDS "tensor_interop.yaml" + BYPRODUCTS "tensor_interop.yaml" ) -add_dependencies(${HOLOSCAN_PACKAGE_NAME} python_tensor_interop_yaml) +add_dependencies(python_tensor_interop python_tensor_interop_yaml endoscopy_data) # Install the app install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/tensor_interop.py" "${CMAKE_CURRENT_SOURCE_DIR}/tensor_interop.yaml" DESTINATION "${app_relative_dest_path}" - COMPONENT "holoscan-apps" + COMPONENT "holoscan-examples" ) + +# Testing +if(HOLOSCAN_BUILD_TESTS) + file(READ ${CMAKE_CURRENT_SOURCE_DIR}/tensor_interop.yaml CONFIG_STRING) + string(REPLACE "count: 0" "count: 10" CONFIG_STRING ${CONFIG_STRING}) + set(CONFIG_FILE ${CMAKE_CURRENT_BINARY_DIR}/python_tensor_interop_testing_config.yaml) + file(WRITE ${CONFIG_FILE} ${CONFIG_STRING}) + + add_test(NAME EXAMPLE_PYTHON_TENSOR_INTEROP_TEST + COMMAND python3 tensor_interop.py ${CONFIG_FILE} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_PYTHON_TENSOR_INTEROP_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Reach end of file or playback count reaches to the limit. Stop ticking." + ) +endif() diff --git a/examples/tensor_interop/python/tensor_interop.py b/examples/tensor_interop/python/tensor_interop.py index f5aaa2c6..92a84314 100644 --- a/examples/tensor_interop/python/tensor_interop.py +++ b/examples/tensor_interop/python/tensor_interop.py @@ -1,5 +1,5 @@ """ -SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,6 +16,7 @@ """ # no qa import os +import sys import holoscan as hs from holoscan.core import Application, Operator, OperatorSpec @@ -50,11 +51,11 @@ class ImageProcessingOp(Operator): (height, width, channels). """ - def __init__(self, *args, **kwargs): + def __init__(self, fragment, *args, **kwargs): self.count = 1 # Need to call the base class constructor last - super().__init__(*args, **kwargs) + super().__init__(fragment, *args, **kwargs) def setup(self, spec: OperatorSpec): spec.input("input_tensor") @@ -130,6 +131,12 @@ def compose(self): if __name__ == "__main__": load_env_log_level() + + config_file = os.path.join(os.path.dirname(__file__), "tensor_interop.yaml") + + if len(sys.argv) >= 2: + config_file = sys.argv[1] + app = MyVideoProcessingApp() - app.config(os.path.join(os.path.dirname(__file__), "tensor_interop.yaml")) + app.config(config_file) app.run() diff --git a/examples/tensor_interop/python/tensor_interop.yaml b/examples/tensor_interop/python/tensor_interop.yaml index 9431a894..1449b816 100644 --- a/examples/tensor_interop/python/tensor_interop.yaml +++ b/examples/tensor_interop/python/tensor_interop.yaml @@ -1,5 +1,5 @@ %YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,23 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. --- -extensions: - # The following extensions are automatically loaded upon Python App - # initialization, so they do not need to be listed here. - # This is a difference in behavior from the C++-API which currently requires - # explicitly listing these. - # - libgxf_std.so - # - libgxf_cuda.so - # - libgxf_multimedia.so - # - libgxf_serialization.so - # - libaja_source.so - # - libcustom_lstm_inference.so - # - libformat_converter.so - # - libholoviz.so - # - libstream_playback.so - # - libtool_tracking_postprocessor.so - # - libvisualizer_tool_tracking.so - replayer: # directory: "../data/endoscopy/video" basename: "surgical_video" diff --git a/examples/video_sources/CMakeLists.txt b/examples/v4l2_camera/CMakeLists.txt similarity index 73% rename from examples/video_sources/CMakeLists.txt rename to examples/v4l2_camera/CMakeLists.txt index cab1c812..984eeca8 100644 --- a/examples/video_sources/CMakeLists.txt +++ b/examples/v4l2_camera/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,6 +14,10 @@ # limitations under the License. # Build the applications based on GXF if enabled -if(HOLOSCAN_BUILD_GXF_EXTENSIONS) - add_subdirectory(gxf) -endif() +add_subdirectory(gxf) + +install( + FILES README.md + DESTINATION "examples/v4l2_camera" + COMPONENT "holoscan-examples" +) diff --git a/examples/v4l2_camera/README.md b/examples/v4l2_camera/README.md new file mode 100644 index 00000000..5bdeae9f --- /dev/null +++ b/examples/v4l2_camera/README.md @@ -0,0 +1,23 @@ +# V4L2 Camera + +Minimal examples using GXF YAML API to illustrate the usage of [Video4Linux](https://www.kernel.org/doc/html/v4.9/media/uapi/v4l/v4l2.html) to stream from a V4L2 node such as a USB webcam. + +## Run instructions + +* **using deb package install and NGC container**: + ```bash + cd /opt/nvidia/holoscan # for GXE to find GXF extensions + ./examples/v4l2_camera/gxf/v4l2_camera + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + ./examples/v4l2_camera/gxf/v4l2_camera + ``` +* **source (local env)**: + ```bash + cd ${BUILD_OR_INSTALL_DIR} # for GXE to find GXF extensions + ./examples/v4l2_camera/gxf/v4l2_camera + ``` + +> Note: if using a container, add `--device /dev/video0:/dev/video0` (or the ID of whatever device you'd like to use) to the `docker run` command to make your USB camera is available to the V4L2 codelet in the container. This is automatically done by `./run launch`. Also note that your container might not have permissions to open the video devices, run `sudo chmod 666 /dev/video*` to make them available. \ No newline at end of file diff --git a/examples/video_sources/gxf/CMakeLists.min.txt b/examples/v4l2_camera/gxf/CMakeLists.min.txt similarity index 70% rename from examples/video_sources/gxf/CMakeLists.min.txt rename to examples/v4l2_camera/gxf/CMakeLists.min.txt index bb691ba5..b0cc3d9e 100644 --- a/examples/video_sources/gxf/CMakeLists.min.txt +++ b/examples/v4l2_camera/gxf/CMakeLists.min.txt @@ -17,18 +17,10 @@ cmake_minimum_required(VERSION 3.20) project(holoscan_basic_workflow CXX) # Finds the package holoscan -find_package(holoscan REQUIRED) +find_package(holoscan REQUIRED CONFIG + PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install") -create_gxe_application( - NAME aja_capture - YAML aja_capture.yaml - EXTENSIONS - GXF::std - GXF::cuda - GXF::multimedia - aja_source - holoviz -) +include(GenerateGXEApp) create_gxe_application( NAME v4l2_camera @@ -39,16 +31,3 @@ create_gxe_application( opengl_renderer v4l2_source ) - -create_gxe_application( - NAME video_replayer - YAML video_replayer.yaml - EXTENSIONS - GXF::std - GXF::cuda - GXF::multimedia - GXF::serialization - format_converter - stream_playback - holoviz -) diff --git a/examples/video_sources/gxf/CMakeLists.txt b/examples/v4l2_camera/gxf/CMakeLists.txt similarity index 67% rename from examples/video_sources/gxf/CMakeLists.txt rename to examples/v4l2_camera/gxf/CMakeLists.txt index 35d8568e..d717468a 100644 --- a/examples/video_sources/gxf/CMakeLists.txt +++ b/examples/v4l2_camera/gxf/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,16 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -create_gxe_application( - NAME aja_capture - YAML aja_capture.yaml - EXTENSIONS - GXF::std - GXF::cuda - GXF::multimedia - aja_source - holoviz -) +# For create_gxe_application +include(GenerateGXEApp) create_gxe_application( NAME v4l2_camera @@ -32,27 +24,17 @@ create_gxe_application( GXF::multimedia opengl_renderer v4l2_source -) - -create_gxe_application( - NAME video_replayer - YAML video_replayer.yaml - EXTENSIONS - GXF::std - GXF::cuda - GXF::multimedia - GXF::serialization - format_converter - stream_playback - holoviz + COMPONENT holoscan-examples ) # Install following the relative folder path file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) +if(HOLOSCAN_INSTALL_EXAMPLE_SOURCE) # Install the minimal CMakeLists.txt file install(FILES CMakeLists.min.txt RENAME "CMakeLists.txt" DESTINATION "${app_relative_dest_path}" - COMPONENT holoscan-apps -) \ No newline at end of file + COMPONENT holoscan-examples +) +endif() diff --git a/examples/video_sources/gxf/v4l2_camera.yaml b/examples/v4l2_camera/gxf/v4l2_camera.yaml similarity index 94% rename from examples/video_sources/gxf/v4l2_camera.yaml rename to examples/v4l2_camera/gxf/v4l2_camera.yaml index fa0c5173..0981720b 100644 --- a/examples/video_sources/gxf/v4l2_camera.yaml +++ b/examples/v4l2_camera/gxf/v4l2_camera.yaml @@ -1,5 +1,5 @@ %YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/examples/video_replayer/CMakeLists.txt b/examples/video_replayer/CMakeLists.txt new file mode 100644 index 00000000..759f1650 --- /dev/null +++ b/examples/video_replayer/CMakeLists.txt @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_subdirectory(cpp) +add_subdirectory(python) + +install( + FILES README.md + DESTINATION "examples/video_replayer" + COMPONENT "holoscan-examples" +) diff --git a/examples/video_replayer/README.md b/examples/video_replayer/README.md new file mode 100644 index 00000000..35299e8a --- /dev/null +++ b/examples/video_replayer/README.md @@ -0,0 +1,67 @@ +# Video Replayer + +Minimal example to demonstrate the use of the video stream replayer operator to load video from disk. The video frames need to have been converted to a gxf entity format, as shown [here](../../scripts/README.md#convert_video_to_gxf_entitiespy). + +> Note: Support for H264 stream support is in progress + +## Data + +The following dataset is used by this example: +[ðŸ“¦ï¸ (NGC) Sample App Data for AI-based Endoscopy Tool Tracking](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/clara-holoscan/resources/holoscan_endoscopy_sample_data). + +## C++ Run instructions + +* **using deb package install**: + ```bash + # [Prerequisite] Download NGC dataset above to `/opt/nvidia/data` + cd /opt/nvidia/holoscan # to find dataset + ./examples/video_replayer/cpp/video_replayer + ``` +* **from NGC container**: + ```bash + cd /opt/nvidia/holoscan # to find dataset + ./examples/video_replayer/cpp/video_replayer + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + ./examples/video_replayer/cpp/video_replayer + ``` +* **source (local env)**: + ```bash + cd ${BUILD_OR_INSTALL_DIR} + ./examples/video_replayer/cpp/video_replayer + ``` + +## Python Run instructions + +* **using python wheel**: + ```bash + # [Prerequisite] Download NGC dataset above to `DATA_DIR` + # [Prerequisite] Download example .py file below to `APP_DIR` + # [Optional] Start the virtualenv where holoscan is installed + export HOLOSCAN_SAMPLE_DATA_PATH= + python3 /video_replayer.py + ``` +* **using deb package install**: + ```bash + # [Prerequisite] Download NGC dataset above to `DATA_DIR` + export PYTHONPATH=/opt/nvidia/holoscan/python/lib + export HOLOSCAN_SAMPLE_DATA_PATH= + python3 /opt/nvidia/holoscan/examples/video_replayer/python/video_replayer.py + ``` +* **from NGC container**: + ```bash + python3 /opt/nvidia/holoscan/examples/video_replayer/python/video_replayer.py + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + python3 ./examples/video_replayer/python/video_replayer.py + ``` +* **source (local env)**: + ```bash + export PYTHONPATH=${BUILD_OR_INSTALL_DIR}/python/lib + export HOLOSCAN_SAMPLE_DATA_PATH=${SRC_DIR}/data + python3 ${BUILD_OR_INSTALL_DIR}/examples/video_replayer/python/video_replayer.py + ``` diff --git a/examples/video_replayer/cpp/CMakeLists.min.txt b/examples/video_replayer/cpp/CMakeLists.min.txt new file mode 100644 index 00000000..5064e855 --- /dev/null +++ b/examples/video_replayer/cpp/CMakeLists.min.txt @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the \"License\"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.20) +project(holoscan_basic_workflow CXX) + +# Finds the package holoscan +find_package(holoscan REQUIRED CONFIG + PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install") + +# Create example +add_executable(video_replayer + video_replayer.cpp +) + +target_link_libraries(video_replayer + PRIVATE + holoscan::core + holoscan::ops::video_stream_replayer + holoscan::ops::holoviz +) + +# Copy config file +add_custom_target(video_replayer_yaml + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/video_replayer.yaml" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "video_replayer.yaml" + BYPRODUCTS "video_replayer.yaml" +) +add_dependencies(video_replayer video_replayer_yaml) + +# Testing +if(BUILD_TESTING) + file(READ ${CMAKE_CURRENT_SOURCE_DIR}/video_replayer.yaml CONFIG_STRING) + string(REPLACE "count: 0" "count: 10" CONFIG_STRING ${CONFIG_STRING}) + set(CONFIG_FILE ${CMAKE_CURRENT_BINARY_DIR}/cpp_video_replayer_config.yaml) + file(WRITE ${CONFIG_FILE} ${CONFIG_STRING}) + + add_test(NAME EXAMPLE_CPP_VIDEO_REPLAYER_TEST + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/video_replayer ${CONFIG_FILE} + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_CPP_VIDEO_REPLAYER_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Reach end of file or playback count reaches to the limit. Stop ticking." + ) +endif() diff --git a/examples/video_replayer/cpp/CMakeLists.txt b/examples/video_replayer/cpp/CMakeLists.txt new file mode 100644 index 00000000..e14e78c5 --- /dev/null +++ b/examples/video_replayer/cpp/CMakeLists.txt @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Create example +add_executable(video_replayer + video_replayer.cpp +) + +target_link_libraries(video_replayer + PRIVATE + holoscan::core + holoscan::ops::video_stream_replayer + holoscan::ops::holoviz +) + +# Copy config file +add_custom_target(video_replayer_yaml + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/video_replayer.yaml" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "video_replayer.yaml" + BYPRODUCTS "video_replayer.yaml" +) + +add_dependencies(video_replayer video_replayer_yaml endoscopy_data) + +# Set the install RPATH based on the location of the Holoscan SDK libraries +# The GXF extensions are loaded by the GXF libraries - no need to include here +file(RELATIVE_PATH install_lib_relative_path ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_SOURCE_DIR}/${HOLOSCAN_INSTALL_LIB_DIR} ) +set_target_properties(video_replayer PROPERTIES INSTALL_RPATH "\$ORIGIN/${install_lib_relative_path}") + +# Get relative folder path for the app +file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) + +if(HOLOSCAN_INSTALL_EXAMPLE_SOURCE) +# Install the source +install(FILES video_replayer.cpp + DESTINATION "${app_relative_dest_path}" + COMPONENT holoscan-examples +) + +# Install the minimal CMakeLists.txt file +install(FILES CMakeLists.min.txt + RENAME "CMakeLists.txt" + DESTINATION "${app_relative_dest_path}" + COMPONENT holoscan-examples +) +endif() + +# Install the app +install(TARGETS video_replayer + DESTINATION "${app_relative_dest_path}" + COMPONENT holoscan-examples +) +install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/video_replayer.yaml" + DESTINATION ${app_relative_dest_path} + COMPONENT holoscan-examples +) + +# Testing +if(HOLOSCAN_BUILD_TESTS) + file(READ ${CMAKE_CURRENT_SOURCE_DIR}/video_replayer.yaml CONFIG_STRING) + string(REPLACE "count: 0" "count: 10" CONFIG_STRING ${CONFIG_STRING}) + set(CONFIG_FILE ${CMAKE_CURRENT_BINARY_DIR}/cpp_video_replayer_config.yaml) + file(WRITE ${CONFIG_FILE} ${CONFIG_STRING}) + + add_test(NAME EXAMPLE_CPP_VIDEO_REPLAYER_TEST + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/video_replayer ${CONFIG_FILE} + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_CPP_VIDEO_REPLAYER_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Reach end of file or playback count reaches to the limit. Stop ticking." + ) +endif() diff --git a/examples/video_replayer/cpp/video_replayer.cpp b/examples/video_replayer/cpp/video_replayer.cpp new file mode 100644 index 00000000..6570e406 --- /dev/null +++ b/examples/video_replayer/cpp/video_replayer.cpp @@ -0,0 +1,49 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +class VideoReplayerApp : public holoscan::Application { + public: + void compose() override { + using namespace holoscan; + + // Define the replayer and holoviz operators and configure using yaml configuration + auto replayer = make_operator("replayer", from_config("replayer")); + auto visualizer = make_operator("holoviz", from_config("holoviz")); + + // Define the workflow: replayer -> holoviz + add_flow(replayer, visualizer, {{"output", "receivers"}}); + } +}; + +int main(int argc, char** argv) { + // Get the yaml configuration file + auto config_path = std::filesystem::canonical(argv[0]).parent_path(); + config_path += "/video_replayer.yaml"; + if ( argc >= 2 ) { + config_path = argv[1]; + } + + auto app = holoscan::make_application(); + app->config(config_path); + app->run(); + + return 0; +} diff --git a/examples/native_operator/python/convolve.yaml b/examples/video_replayer/cpp/video_replayer.yaml similarity index 55% rename from examples/native_operator/python/convolve.yaml rename to examples/video_replayer/cpp/video_replayer.yaml index 7b2d98f4..3fe77b94 100644 --- a/examples/native_operator/python/convolve.yaml +++ b/examples/video_replayer/cpp/video_replayer.yaml @@ -1,5 +1,5 @@ %YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,11 +14,19 @@ # See the License for the specific language governing permissions and # limitations under the License. --- -signal_generator: - height: 1 - size: 20 - dtype: int32 +replayer: + directory: "../data/endoscopy/video" + basename: "surgical_video" + frame_rate: 0 # as specified in timestamps + repeat: true # default: false + realtime: true # default: true + count: 0 # default: 0 (no frame count restriction) -convolve: - width: 4 - unit_area: false +holoviz: + width: 854 + height: 480 + tensors: + - name: "" + type: color + opacity: 1.0 + priority: 0 diff --git a/examples/video_replayer/python/CMakeLists.txt b/examples/video_replayer/python/CMakeLists.txt new file mode 100644 index 00000000..11c14319 --- /dev/null +++ b/examples/video_replayer/python/CMakeLists.txt @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Get relative folder path for the app +file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) + +# Copy video_replayer application file +add_custom_target(python_video_replayer ALL + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/video_replayer.py" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "video_replayer.py" + BYPRODUCTS "video_replayer.py" +) + +# Copy config file +add_custom_target(python_video_replayer_yaml + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/video_replayer.yaml" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "video_replayer.yaml" + BYPRODUCTS "video_replayer.yaml" +) + +add_dependencies(python_video_replayer python_video_replayer_yaml endoscopy_data) + +# Install the app +install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/video_replayer.py" + "${CMAKE_CURRENT_SOURCE_DIR}/video_replayer.yaml" + DESTINATION "${app_relative_dest_path}" + COMPONENT "holoscan-examples" +) + +# Testing +if(HOLOSCAN_BUILD_TESTS) + file(READ ${CMAKE_CURRENT_SOURCE_DIR}/video_replayer.yaml CONFIG_STRING) + string(REPLACE "count: 0" "count: 10" CONFIG_STRING ${CONFIG_STRING}) + set(CONFIG_FILE ${CMAKE_CURRENT_BINARY_DIR}/python_video_replayer_config.yaml) + file(WRITE ${CONFIG_FILE} ${CONFIG_STRING}) + + add_test(NAME EXAMPLE_PYTHON_VIDEO_REPLAYER_TEST + COMMAND python3 video_replayer.py python_video_replayer_config.yaml + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_PYTHON_VIDEO_REPLAYER_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Reach end of file or playback count reaches to the limit. Stop ticking." + ) +endif() diff --git a/examples/video_replayer/python/video_replayer.py b/examples/video_replayer/python/video_replayer.py new file mode 100644 index 00000000..d9618f9f --- /dev/null +++ b/examples/video_replayer/python/video_replayer.py @@ -0,0 +1,63 @@ +""" +SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" # noqa + +import os +import sys + +from holoscan.core import Application +from holoscan.operators import HolovizOp, VideoStreamReplayerOp + +sample_data_path = os.environ.get("HOLOSCAN_SAMPLE_DATA_PATH", "../data") + + +class VideoReplayerApp(Application): + """Example of an application that uses the operators defined above. + + This application has the following operators: + + - VideoStreamReplayerOp + - HolovizOp + + The VideoStreamReplayerOp reads a video file and sends the frames to the HolovizOp. + The HolovizOp displays the frames. + """ + + def compose(self): + video_dir = os.path.join(sample_data_path, "endoscopy", "video") + if not os.path.exists(video_dir): + raise ValueError(f"Could not find video data: {video_dir=}") + + # Define the replayer and holoviz operators + replayer = VideoStreamReplayerOp( + self, name="replayer", directory=video_dir, **self.kwargs("replayer") + ) + visualizer = HolovizOp(self, name="holoviz", **self.kwargs("holoviz")) + + # Define the workflow + self.add_flow(replayer, visualizer, {("output", "receivers")}) + + +if __name__ == "__main__": + + config_file = os.path.join(os.path.dirname(__file__), "video_replayer.yaml") + + if len(sys.argv) >= 2: + config_file = sys.argv[1] + + app = VideoReplayerApp() + app.config(config_file) + app.run() diff --git a/tests/data/emergent.yaml b/examples/video_replayer/python/video_replayer.yaml similarity index 55% rename from tests/data/emergent.yaml rename to examples/video_replayer/python/video_replayer.yaml index 25b0b360..bf64c32a 100644 --- a/tests/data/emergent.yaml +++ b/examples/video_replayer/python/video_replayer.yaml @@ -1,5 +1,5 @@ %YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,15 +14,19 @@ # See the License for the specific language governing permissions and # limitations under the License. --- -extensions: - - libgxf_std.so - - libgxf_cuda.so - - libgxf_multimedia.so - - libgxf_serialization.so - - libemergent_source.so +replayer: + #directory: "../data/endoscopy/video" + basename: "surgical_video" + frame_rate: 0 # as specified in timestamps + repeat: true # default: false + realtime: true # default: true + count: 0 # default: 0 (no frame count restriction) -emergent: - width: 4200 - height: 2160 - framerate: 240 - rdma: false +holoviz: + width: 854 + height: 480 + tensors: + - name: "" + type: color + opacity: 1.0 + priority: 0 diff --git a/examples/video_sources/README.md b/examples/video_sources/README.md deleted file mode 100644 index d25e1d6f..00000000 --- a/examples/video_sources/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# Video Sources - -Minimal examples using GXF YAML API to illustrate the usage of various video sources: -- `aja_capture`: uses the AJA capture card with [GPUDirect RDMA](https://docs.nvidia.com/cuda/gpudirect-rdma/index.html) to avoid copies to the CPU. The renderer (holoviz) leverages the videobuffer from CUDA to Vulkan to avoid copies to the CPU also. Requires to [set up the AJA hardware and drivers](https://docs.nvidia.com/clara-holoscan/sdk-user-guide/aja_setup.html). -- `v4l2_camera`: uses [Video4Linux](https://www.kernel.org/doc/html/v4.9/media/uapi/v4l/v4l2.html) as a source, to use with a V4L2 node such as a USB webcam (goes through the CPU). It uses CUDA/OpenGL interop to avoid copies to the CPU. -- `video_replayer`: loads a video from the disk, does some format conversions, and provides a basic visualization of tensors. - -### Requirements - -- `aja_capture`: follow the [setup instructions from the user guide](https://docs.nvidia.com/clara-holoscan/sdk-user-guide/aja_setup.html) to use the AJA capture card. -- `v4l2_camera`: if using a container, add `--device /dev/video0:/dev/video0` to the `docker run` command to make your USB cameras available to the V4L2 codelet in the container (automatically done by `./run launch`). Note that your container might not have permissions to open the video devices, run `sudo chmod 666 /dev/video*` to make them available. - -### Build instructions - -Built with the SDK, see instructions from the top level README. - -### Run instructions - -First, go in your `build` or `install` directory (automatically done by `./run launch`). - -Then, run the commands of your choice: - -```bash -./examples/video_sources/gxf/aja_capture -./examples/video_sources/gxf/v4l2_camera -./examples/video_sources/gxf/video_replayer -``` diff --git a/examples/video_sources/gxf/aja_capture.yaml b/examples/video_sources/gxf/aja_capture.yaml deleted file mode 100644 index 1bb16ab6..00000000 --- a/examples/video_sources/gxf/aja_capture.yaml +++ /dev/null @@ -1,68 +0,0 @@ -%YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -name: source -components: -- name: signal - type: nvidia::gxf::DoubleBufferTransmitter -- type: nvidia::gxf::DownstreamReceptiveSchedulingTerm - parameters: - transmitter: signal - min_size: 1 -- type: nvidia::holoscan::AJASource - parameters: - video_buffer_output: signal - width: 1920 - height: 1080 - rdma: true - enable_overlay: false - overlay_rdma: true -- type: nvidia::gxf::CountSchedulingTerm - parameters: - count: 300 ---- -name: sink -components: -- name: signal - type: nvidia::gxf::DoubleBufferReceiver -- type: nvidia::gxf::MessageAvailableSchedulingTerm - parameters: - receiver: signal - min_size: 1 -- name: boolean_scheduling_term - type: nvidia::gxf::BooleanSchedulingTerm -- type: nvidia::holoscan::Holoviz - parameters: - window_close_scheduling_term: boolean_scheduling_term - receivers: - - signal -- type: nvidia::gxf::CountSchedulingTerm - parameters: - count: 300 ---- -components: -- type: nvidia::gxf::Connection - parameters: - source: source/signal - target: sink/signal ---- -components: -- name: clock - type: nvidia::gxf::RealtimeClock -- type: nvidia::gxf::GreedyScheduler - parameters: - clock: clock - max_duration_ms: 1000000 diff --git a/examples/video_sources/gxf/video_replayer.yaml b/examples/video_sources/gxf/video_replayer.yaml deleted file mode 100644 index b9abbddb..00000000 --- a/examples/video_sources/gxf/video_replayer.yaml +++ /dev/null @@ -1,256 +0,0 @@ -%YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -name: replayer -components: - - name: output - type: nvidia::gxf::DoubleBufferTransmitter - - name: allocator - type: nvidia::gxf::UnboundedAllocator - - name: component_serializer - type: nvidia::gxf::StdComponentSerializer - parameters: - allocator: allocator - - name: entity_serializer - type: - nvidia::holoscan::stream_playback::VideoStreamSerializer - # You can set to 'nvidia::gxf::StdEntitySerializer' instead but it would cause warning messages when 'repeat' is true. - parameters: - component_serializers: [component_serializer] - - type: nvidia::holoscan::stream_playback::VideoStreamReplayer - parameters: - transmitter: output - entity_serializer: entity_serializer - boolean_scheduling_term: boolean_scheduling - directory: "../data/endoscopy/video" - basename: "surgical_video" - frame_rate: 0 # as specified in timestamps - repeat: true # default: false - realtime: true # default: true - count: 0 # default: 0 (no frame count restriction) - - name: boolean_scheduling - type: nvidia::gxf::BooleanSchedulingTerm - - type: nvidia::gxf::DownstreamReceptiveSchedulingTerm - parameters: - transmitter: output - min_size: 1 ---- -name: broadcast -components: - - name: input - type: nvidia::gxf::DoubleBufferReceiver - - name: output_1 - type: nvidia::gxf::DoubleBufferTransmitter - - name: output_2 - type: nvidia::gxf::DoubleBufferTransmitter - - type: nvidia::gxf::Broadcast - parameters: - source: input - - type: nvidia::gxf::MessageAvailableSchedulingTerm - parameters: - receiver: input - min_size: 1 - - type: nvidia::gxf::DownstreamReceptiveSchedulingTerm - parameters: - transmitter: output_1 - min_size: 1 - - type: nvidia::gxf::DownstreamReceptiveSchedulingTerm - parameters: - transmitter: output_2 - min_size: 1 ---- -name: recorder -components: - - name: input - type: nvidia::gxf::DoubleBufferReceiver - - name: allocator - type: nvidia::gxf::UnboundedAllocator - - name: component_serializer - type: nvidia::gxf::StdComponentSerializer - parameters: - allocator: allocator - - name: entity_serializer - type: nvidia::gxf::StdEntitySerializer - parameters: - component_serializers: [component_serializer] - - type: nvidia::gxf::EntityRecorder - parameters: - receiver: input - entity_serializer: entity_serializer - directory: "/tmp" - basename: "tensor" - - type: nvidia::gxf::MessageAvailableSchedulingTerm - parameters: - receiver: input - min_size: 1 ---- -name: format_converter -components: - - name: in_tensor - type: nvidia::gxf::DoubleBufferReceiver - - type: nvidia::gxf::MessageAvailableSchedulingTerm - parameters: - receiver: in_tensor - min_size: 1 - - name: out_tensor - type: nvidia::gxf::DoubleBufferTransmitter - - type: nvidia::gxf::DownstreamReceptiveSchedulingTerm - parameters: - transmitter: out_tensor - min_size: 1 - - name: pool - type: nvidia::gxf::BlockMemoryPool - parameters: - storage_type: 1 - block_size: 33177600 # 1920 * 1080 * 4 (channel) * 4 (bytes per pixel) - num_blocks: 3 - - type: nvidia::holoscan::formatconverter::FormatConverter - parameters: - in: in_tensor - out: out_tensor - out_tensor_name: source_video - out_dtype: "float32" - resize_width: 1920 - resize_height: 1080 - out_channel_order: [0,1,2] # can skip - pool: pool ---- -name: format_converter2 -components: - - name: in_tensor - type: nvidia::gxf::DoubleBufferReceiver - - type: nvidia::gxf::MessageAvailableSchedulingTerm - parameters: - receiver: in_tensor - min_size: 1 - - name: out_tensor - type: nvidia::gxf::DoubleBufferTransmitter - - type: nvidia::gxf::DownstreamReceptiveSchedulingTerm - parameters: - transmitter: out_tensor - min_size: 1 - - name: pool - type: nvidia::gxf::BlockMemoryPool - parameters: - storage_type: 1 - block_size: 33177600 # 1920 * 1080 * 4 (channel) * 4 (bytes per pixel) - num_blocks: 2 - - type: nvidia::holoscan::formatconverter::FormatConverter - parameters: - in: in_tensor - in_tensor_name: source_video - out: out_tensor - out_dtype: "uint8" - pool: pool ---- -name: format_converter3 -components: - - name: in_tensor - type: nvidia::gxf::DoubleBufferReceiver - - type: nvidia::gxf::MessageAvailableSchedulingTerm - parameters: - receiver: in_tensor - min_size: 1 - - name: out_tensor - type: nvidia::gxf::DoubleBufferTransmitter - - type: nvidia::gxf::DownstreamReceptiveSchedulingTerm - parameters: - transmitter: out_tensor - min_size: 1 - - name: pool - type: nvidia::gxf::BlockMemoryPool - parameters: - storage_type: 1 - block_size: 33177600 # 1920 * 1080 * 4 (channel) * 4 (bytes per pixel) - num_blocks: 2 - - type: nvidia::holoscan::formatconverter::FormatConverter - parameters: - in: in_tensor - out: out_tensor - out_dtype: "rgba8888" - pool: pool ---- -name: sink -components: - - name: tensor - type: nvidia::gxf::DoubleBufferReceiver - - type: nvidia::gxf::MessageAvailableSchedulingTerm - parameters: - receiver: tensor - min_size: 1 - - name: boolean_scheduling_term - type: nvidia::gxf::BooleanSchedulingTerm - - type: nvidia::holoscan::Holoviz - parameters: - window_close_scheduling_term: boolean_scheduling_term - receivers: - - tensor ---- -components: - - type: nvidia::gxf::Connection - parameters: - source: replayer/output - target: broadcast/input ---- -components: - - type: nvidia::gxf::Connection - parameters: - source: broadcast/output_1 - target: format_converter/in_tensor ---- -components: - - type: nvidia::gxf::Connection - parameters: - source: format_converter/out_tensor - target: format_converter2/in_tensor ---- -components: - - type: nvidia::gxf::Connection - parameters: - source: format_converter2/out_tensor - target: format_converter3/in_tensor ---- -components: - - type: nvidia::gxf::Connection - parameters: - source: format_converter3/out_tensor - target: sink/tensor ---- -components: - - type: nvidia::gxf::Connection - parameters: - source: broadcast/output_2 - target: recorder/input ---- -name: utils -components: - - name: rt_clock - type: nvidia::gxf::RealtimeClock - - type: nvidia::gxf::GreedyScheduler - parameters: - clock: rt_clock - max_duration_ms: 1000000 - - name: cuda_stream - type: nvidia::gxf::CudaStreamPool - parameters: - dev_id: 0 - stream_flags: 0 - stream_priority: 0 - reserved_size: 1 - max_size: 5 - - type: nvidia::gxf::JobStatistics - parameters: - clock: rt_clock diff --git a/examples/wrap_operator_as_gxf_extension/CMakeLists.txt b/examples/wrap_operator_as_gxf_extension/CMakeLists.txt new file mode 100644 index 00000000..9f67369d --- /dev/null +++ b/examples/wrap_operator_as_gxf_extension/CMakeLists.txt @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_subdirectory(gxf_extension) +add_subdirectory(gxf_app) +add_subdirectory(ping_rx_native_op) +add_subdirectory(ping_tx_native_op) + +install( + FILES README.md + DESTINATION "examples/wrap_operator_as_gxf_extension" + COMPONENT "holoscan-examples" +) diff --git a/examples/wrap_operator_as_gxf_extension/README.md b/examples/wrap_operator_as_gxf_extension/README.md new file mode 100644 index 00000000..b9ae0982 --- /dev/null +++ b/examples/wrap_operator_as_gxf_extension/README.md @@ -0,0 +1,28 @@ +# Wrap Operator As GXF Extension + +This example demonstrates how to wrap Holoscan SDK Native Operators as GXF codelets so +that they can be used in a gxf application. + +This example wraps two C++ native operators: + 1. a transmitter, which sends empty `gxf::Entity` messages to it's "out" port. + However, in general, this could be a tensor object. + 2. a receiver that receives `gxf::Entity` messages from it's "in" port. + In this example we ignore the empty message and simply print to terminal that we received a ping. + +## Run instructions + +* **using deb package install and NGC container**: + ```bash + cd /opt/nvidia/holoscan # for GXE to find GXF extensions + ./examples/wrap_operator_as_gxf_extension/gxf_app/test_operator_as_gxf_ext + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + ./examples/wrap_operator_as_gxf_extension/gxf_app/test_operator_as_gxf_ext + ``` +* **source (local env)**: + ```bash + cd ${BUILD_OR_INSTALL_DIR} # for GXE to find GXF extensions + ./examples/wrap_operator_as_gxf_extension/gxf_app/test_operator_as_gxf_ext + ``` diff --git a/examples/wrap_operator_as_gxf_extension/gxf_app/CMakeLists.min.txt b/examples/wrap_operator_as_gxf_extension/gxf_app/CMakeLists.min.txt new file mode 100644 index 00000000..ff4e4534 --- /dev/null +++ b/examples/wrap_operator_as_gxf_extension/gxf_app/CMakeLists.min.txt @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the \"License\"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.20) +project(holoscan_wrap_operator_as_gxf_app CXX) + +# Finds the package holoscan +find_package(holoscan REQUIRED CONFIG + PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install") + +# For create_gxe_application +include(GenerateGXEApp) + +create_gxe_application( + NAME test_operator_as_gxf_ext + YAML ping.yaml + EXTENSIONS + GXF::std + gxf_holoscan_wrapper + gxf_wrapped_ping_rx_native_op + gxf_wrapped_ping_tx_native_op + COMPONENT holoscan-examples +) + +# Testing +if(BUILD_TESTING) + add_test(NAME EXAMPLE_OPERATOR_AS_GXF_EXT_TEST + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test_operator_as_gxf_ext + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_OPERATOR_AS_GXF_EXT_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Number of pings received: 10") +endif() diff --git a/examples/wrap_operator_as_gxf_extension/gxf_app/CMakeLists.txt b/examples/wrap_operator_as_gxf_extension/gxf_app/CMakeLists.txt new file mode 100644 index 00000000..6842ac40 --- /dev/null +++ b/examples/wrap_operator_as_gxf_extension/gxf_app/CMakeLists.txt @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# For create_gxe_application +include(GenerateGXEApp) + +create_gxe_application( + NAME test_operator_as_gxf_ext + YAML ping.yaml + EXTENSIONS + GXF::std + gxf_holoscan_wrapper + gxf_wrapped_ping_rx_native_op + gxf_wrapped_ping_tx_native_op + COMPONENT holoscan-examples +) + +# Install following the relative folder path +file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) + +if(HOLOSCAN_INSTALL_EXAMPLE_SOURCE) +# Install the minimal CMakeLists.txt file +install(FILES CMakeLists.min.txt + RENAME "CMakeLists.txt" + DESTINATION "${app_relative_dest_path}" + COMPONENT holoscan-examples +) +endif() + +# Testing +if(HOLOSCAN_BUILD_TESTS) + add_test(NAME EXAMPLE_OPERATOR_AS_GXF_EXT_TEST + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test_operator_as_gxf_ext + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_OPERATOR_AS_GXF_EXT_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Number of pings received: 10") +endif() diff --git a/examples/wrap_operator_as_gxf_extension/gxf_app/ping.yaml b/examples/wrap_operator_as_gxf_extension/gxf_app/ping.yaml new file mode 100644 index 00000000..57b153ff --- /dev/null +++ b/examples/wrap_operator_as_gxf_extension/gxf_app/ping.yaml @@ -0,0 +1,57 @@ +%YAML 1.2 +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +name: tx +components: + # spec.output("out"); + - name: out + type: nvidia::gxf::DoubleBufferTransmitter + - type: nvidia::gxf::DownstreamReceptiveSchedulingTerm + parameters: + transmitter: out + min_size: 1 + - name: ping_tx_native_op + type: myexts::PingTxNativeOpCodelet + - name: count_condition + type: nvidia::gxf::CountSchedulingTerm + parameters: + count: 10 +--- +name: rx +components: + # spec.input("in"); + - name: in + type: nvidia::gxf::DoubleBufferReceiver + - type: nvidia::gxf::MessageAvailableSchedulingTerm + parameters: + receiver: in + min_size: 1 + - name: ping_rx_native_op + type: myexts::PingRxNativeOpCodelet +--- +components: + - type: nvidia::gxf::Connection + parameters: + source: tx/out + target: rx/in +--- +components: + - name: rt_clock + type: nvidia::gxf::RealtimeClock + - type: nvidia::gxf::GreedyScheduler + parameters: + clock: rt_clock + max_duration_ms: 1000000 diff --git a/examples/wrap_operator_as_gxf_extension/gxf_extension/CMakeLists.min.txt b/examples/wrap_operator_as_gxf_extension/gxf_extension/CMakeLists.min.txt new file mode 100644 index 00000000..97d7cb74 --- /dev/null +++ b/examples/wrap_operator_as_gxf_extension/gxf_extension/CMakeLists.min.txt @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the \"License\"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.20) +project(holoscan_wrap_operator_as_gxf_extension CXX) + +# Finds the package holoscan +find_package(holoscan REQUIRED CONFIG + PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install") + +include(WrapOperatorAsGXFExtension) +wrap_operator_as_gxf_extension( + OPERATOR_CLASS "myops::PingTxNativeOp" + OPERATOR_HEADER_INCLUDE "ping_tx_native_op/ping_tx_native_op.hpp" + OPERATOR_TARGET ping_tx_native_op + CODELET_ID_HASH1 "0x83a6aede926f4a44" + CODELET_ID_HASH2 "0xbcdf73a7008bdad9" + CODELET_NAME "PingTxNativeOpCodelet" + CODELET_NAMESPACE "myexts" + CODELET_DESCRIPTION "Ping Tx Native Operator codelet" + CODELET_TARGET_NAME "gxf_wrapped_ping_tx_native_op_lib" # optional, defaults to CODELET_NAME lowercase + CODELET_TARGET_PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + INCLUDE_DIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR}/.. # to find OPERATOR_HEADER_INCLUDE + EXTENSION_ID_HASH1 "0x2f3f69b27c2c4fd8" + EXTENSION_ID_HASH2 "0xb119237f5110572d" + EXTENSION_NAME "PingTxNativeOpExtension" + EXTENSION_DESCRIPTION "Ping Tx Native Operator extension" + EXTENSION_AUTHOR "NVIDIA" + EXTENSION_VERSION "0.5.0" + EXTENSION_LICENSE "Apache-2.0" + EXTENSION_TARGET_NAME "gxf_wrapped_ping_tx_native_op" # optional, defaults to EXTENSION_NAME lowercase + EXTENSION_TARGET_PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} +) + +wrap_operator_as_gxf_extension( + OPERATOR_CLASS "myops::PingRxNativeOp" + OPERATOR_HEADER_INCLUDE "ping_rx_native_op/ping_rx_native_op.hpp" + OPERATOR_TARGET ping_rx_native_op + CODELET_ID_HASH1 "0x3c9db42c37084788" + CODELET_ID_HASH2 "0x95f931de85a3c5dd" + CODELET_NAME "PingRxNativeOpCodelet" + CODELET_NAMESPACE "myexts" + CODELET_DESCRIPTION "Ping Rx Native Operator codelet" + CODELET_TARGET_NAME "gxf_wrapped_ping_rx_native_op_lib" # optional, defaults to CODELET_NAME lowercase + CODELET_TARGET_PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + INCLUDE_DIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR}/.. # to find OPERATOR_HEADER_INCLUDE + EXTENSION_ID_HASH1 "0x2e62c3eec4f04784" + EXTENSION_ID_HASH2 "0xaed183505e49dc73" + EXTENSION_NAME "PingRxNativeOpExtension" + EXTENSION_DESCRIPTION "Ping Rx Native Operator extension" + EXTENSION_AUTHOR "NVIDIA" + EXTENSION_VERSION "0.5.0" + EXTENSION_LICENSE "Apache-2.0" + EXTENSION_TARGET_NAME "gxf_wrapped_ping_rx_native_op" # optional, defaults to EXTENSION_NAME lowercase + EXTENSION_TARGET_PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} +) diff --git a/examples/wrap_operator_as_gxf_extension/gxf_extension/CMakeLists.txt b/examples/wrap_operator_as_gxf_extension/gxf_extension/CMakeLists.txt new file mode 100644 index 00000000..4b01e629 --- /dev/null +++ b/examples/wrap_operator_as_gxf_extension/gxf_extension/CMakeLists.txt @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include(WrapOperatorAsGXFExtension) +wrap_operator_as_gxf_extension( + OPERATOR_CLASS "myops::PingTxNativeOp" + OPERATOR_HEADER_INCLUDE "ping_tx_native_op/ping_tx_native_op.hpp" + OPERATOR_TARGET ping_tx_native_op + CODELET_ID_HASH1 "0x83a6aede926f4a44" + CODELET_ID_HASH2 "0xbcdf73a7008bdad9" + CODELET_NAME "PingTxNativeOpCodelet" + CODELET_NAMESPACE "myexts" + CODELET_DESCRIPTION "Ping Tx Native Operator codelet" + CODELET_TARGET_NAME "gxf_wrapped_ping_tx_native_op_lib" # optional, defaults to CODELET_NAME lowercase + CODELET_TARGET_PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + INCLUDE_DIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR}/.. # to find OPERATOR_HEADER_INCLUDE + EXTENSION_ID_HASH1 "0x2f3f69b27c2c4fd8" + EXTENSION_ID_HASH2 "0xb119237f5110572d" + EXTENSION_NAME "PingTxNativeOpExtension" + EXTENSION_DESCRIPTION "Ping Tx Native Operator extension" + EXTENSION_AUTHOR "NVIDIA" + EXTENSION_VERSION "0.5.0" + EXTENSION_LICENSE "Apache-2.0" + EXTENSION_TARGET_NAME "gxf_wrapped_ping_tx_native_op" # optional, defaults to EXTENSION_NAME lowercase + EXTENSION_TARGET_PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} +) + +wrap_operator_as_gxf_extension( + OPERATOR_CLASS "myops::PingRxNativeOp" + OPERATOR_HEADER_INCLUDE "ping_rx_native_op/ping_rx_native_op.hpp" + OPERATOR_TARGET ping_rx_native_op + CODELET_ID_HASH1 "0x3c9db42c37084788" + CODELET_ID_HASH2 "0x95f931de85a3c5dd" + CODELET_NAME "PingRxNativeOpCodelet" + CODELET_NAMESPACE "myexts" + CODELET_DESCRIPTION "Ping Rx Native Operator codelet" + CODELET_TARGET_NAME "gxf_wrapped_ping_rx_native_op_lib" # optional, defaults to CODELET_NAME lowercase + CODELET_TARGET_PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + INCLUDE_DIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR}/.. # to find OPERATOR_HEADER_INCLUDE + EXTENSION_ID_HASH1 "0x2e62c3eec4f04784" + EXTENSION_ID_HASH2 "0xaed183505e49dc73" + EXTENSION_NAME "PingRxNativeOpExtension" + EXTENSION_DESCRIPTION "Ping Rx Native Operator extension" + EXTENSION_AUTHOR "NVIDIA" + EXTENSION_VERSION "0.5.0" + EXTENSION_LICENSE "Apache-2.0" + EXTENSION_TARGET_NAME "gxf_wrapped_ping_rx_native_op" # optional, defaults to EXTENSION_NAME lowercase + EXTENSION_TARGET_PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} +) + + +## Installing ping_tx_native_op + +# Set RPATH to find extensions and libs +file(RELATIVE_PATH gxf_holoscan_wrapper_lib_relative_path + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/${HOLOSCAN_INSTALL_LIB_DIR}/gxf_extensions +) +set(ping_tx_native_op_relative_path ../ping_tx_native_op) +set_target_properties(gxf_wrapped_ping_tx_native_op_lib + PROPERTIES + INSTALL_RPATH "\$ORIGIN/${ping_tx_native_op_relative_path}:\$ORIGIN/${gxf_holoscan_wrapper_lib_relative_path}" +) +set_target_properties(gxf_wrapped_ping_tx_native_op + PROPERTIES + INSTALL_RPATH "\$ORIGIN:\$ORIGIN/${gxf_holoscan_wrapper_lib_relative_path}" +) + +# Install following the relative folder path +file(RELATIVE_PATH relative_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) +install(TARGETS gxf_wrapped_ping_tx_native_op gxf_wrapped_ping_tx_native_op_lib + DESTINATION ${relative_path} + COMPONENT "holoscan-examples" +) + + +## Installing ping_rx_native_op + +# Set RPATH to find extensions and libs +file(RELATIVE_PATH gxf_holoscan_wrapper_lib_relative_path + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/${HOLOSCAN_INSTALL_LIB_DIR}/gxf_extensions +) +set(ping_rx_native_op_relative_path ../ping_rx_native_op) +set_target_properties(gxf_wrapped_ping_rx_native_op_lib + PROPERTIES + INSTALL_RPATH "\$ORIGIN/${ping_rx_native_op_relative_path}:\$ORIGIN/${gxf_holoscan_wrapper_lib_relative_path}" +) +set_target_properties(gxf_wrapped_ping_rx_native_op + PROPERTIES + INSTALL_RPATH "\$ORIGIN:\$ORIGIN/${gxf_holoscan_wrapper_lib_relative_path}" +) + +# Install following the relative folder path +file(RELATIVE_PATH relative_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) +install(TARGETS gxf_wrapped_ping_rx_native_op gxf_wrapped_ping_rx_native_op_lib + DESTINATION "${relative_path}" + COMPONENT holoscan-examples +) + +if(HOLOSCAN_INSTALL_EXAMPLE_SOURCE) +# Install the minimal CMakeLists.txt file +install(FILES CMakeLists.min.txt + RENAME "CMakeLists.txt" + DESTINATION "${relative_path}" + COMPONENT holoscan-examples +) +endif() diff --git a/examples/native_operator/cpp/CMakeLists.min.txt b/examples/wrap_operator_as_gxf_extension/ping_rx_native_op/CMakeLists.min.txt similarity index 53% rename from examples/native_operator/cpp/CMakeLists.min.txt rename to examples/wrap_operator_as_gxf_extension/ping_rx_native_op/CMakeLists.min.txt index c1cc96fb..7e242003 100644 --- a/examples/native_operator/cpp/CMakeLists.min.txt +++ b/examples/wrap_operator_as_gxf_extension/ping_rx_native_op/CMakeLists.min.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the \"License\"); @@ -14,22 +14,26 @@ # limitations under the License. cmake_minimum_required(VERSION 3.20) -project(holoscan_native_operator CXX) +project(holoscan_wrap_operator_as_gxf_ping_rx_native_op CXX) # Finds the package holoscan -find_package(holoscan REQUIRED) +find_package(holoscan REQUIRED CONFIG + PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install") -add_executable(ping - ping.cpp +# Create library +add_library(ping_rx_native_op + ping_rx_native_op.cpp + ping_rx_native_op.hpp ) -target_link_libraries(ping - PRIVATE - holoscan::holoscan +set_target_properties(ping_rx_native_op PROPERTIES POSITION_INDEPENDENT_CODE ON) + +target_link_libraries(ping_rx_native_op + PUBLIC + holoscan::core ) -# Copy config file to the build tree -add_custom_target(ping_yaml - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_SOURCE_DIR}/app_config.yaml" ${CMAKE_CURRENT_BINARY_DIR} +set_target_properties(ping_rx_native_op + PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) -add_dependencies(ping ping_yaml) \ No newline at end of file diff --git a/examples/wrap_operator_as_gxf_extension/ping_rx_native_op/CMakeLists.txt b/examples/wrap_operator_as_gxf_extension/ping_rx_native_op/CMakeLists.txt new file mode 100644 index 00000000..f5288b52 --- /dev/null +++ b/examples/wrap_operator_as_gxf_extension/ping_rx_native_op/CMakeLists.txt @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Create library +add_library(ping_rx_native_op + ping_rx_native_op.cpp + ping_rx_native_op.hpp +) +target_link_libraries(ping_rx_native_op + PUBLIC + holoscan::core +) +set_target_properties(ping_rx_native_op + PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} +) + +# Install operator + +# Set the install RPATH based on the location of the Holoscan SDK libraries +file(RELATIVE_PATH install_lib_relative_path + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/${HOLOSCAN_INSTALL_LIB_DIR} +) +set_target_properties(ping_rx_native_op + PROPERTIES + INSTALL_RPATH "\$ORIGIN/${install_lib_relative_path}" +) + +# Install following the relative folder path +file(RELATIVE_PATH relative_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) + +if(HOLOSCAN_INSTALL_EXAMPLE_SOURCE) +# Install the source +install(FILES ping_rx_native_op.cpp ping_rx_native_op.hpp + DESTINATION "${relative_path}" + COMPONENT holoscan-examples +) + +# Install the minimal CMakeLists.txt file +install(FILES CMakeLists.min.txt + RENAME "CMakeLists.txt" + DESTINATION "${relative_path}" + COMPONENT holoscan-examples +) +endif() + +# Install the compiled operator +install(TARGETS ping_rx_native_op + DESTINATION "${relative_path}" + COMPONENT "holoscan-examples" +) diff --git a/examples/wrap_operator_as_gxf_extension/ping_rx_native_op/ping_rx_native_op.cpp b/examples/wrap_operator_as_gxf_extension/ping_rx_native_op/ping_rx_native_op.cpp new file mode 100644 index 00000000..e57882a7 --- /dev/null +++ b/examples/wrap_operator_as_gxf_extension/ping_rx_native_op/ping_rx_native_op.cpp @@ -0,0 +1,40 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ping_rx_native_op.hpp" + +#include + +using namespace holoscan; + +namespace myops { + +void PingRxNativeOp::setup(OperatorSpec& spec) { + HOLOSCAN_LOG_INFO("PingRxNativeOp::setup() called."); + spec.input("in"); +} + +void PingRxNativeOp::compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) { + HOLOSCAN_LOG_INFO("PingRxNativeOp::compute() called."); + + // The type of `in_message` is 'holoscan::gxf::Entity'. + auto in_message = op_input.receive("in"); + HOLOSCAN_LOG_INFO("Number of pings received: {}", count_++); +} + +} // namespace myops diff --git a/examples/wrap_operator_as_gxf_extension/ping_rx_native_op/ping_rx_native_op.hpp b/examples/wrap_operator_as_gxf_extension/ping_rx_native_op/ping_rx_native_op.hpp new file mode 100644 index 00000000..6d328e45 --- /dev/null +++ b/examples/wrap_operator_as_gxf_extension/ping_rx_native_op/ping_rx_native_op.hpp @@ -0,0 +1,42 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PING_RX_NATIVE_OP_HPP +#define PING_RX_NATIVE_OP_HPP + +#include "holoscan/holoscan.hpp" + +namespace myops { + +class PingRxNativeOp : public holoscan::Operator { + public: + HOLOSCAN_OPERATOR_FORWARD_ARGS(PingRxNativeOp) + + PingRxNativeOp() = default; + + void setup(holoscan::OperatorSpec& spec) override; + + void compute(holoscan::InputContext& op_input, holoscan::OutputContext& op_output, + holoscan::ExecutionContext& context) override; + + private: + int count_ = 1; +}; + +} // namespace myops + +#endif /* PING_RX_NATIVE_OP_HPP */ diff --git a/examples/basic_workflow/cpp/CMakeLists.min.txt b/examples/wrap_operator_as_gxf_extension/ping_tx_native_op/CMakeLists.min.txt similarity index 54% rename from examples/basic_workflow/cpp/CMakeLists.min.txt rename to examples/wrap_operator_as_gxf_extension/ping_tx_native_op/CMakeLists.min.txt index 6306bee7..9982f7fc 100644 --- a/examples/basic_workflow/cpp/CMakeLists.min.txt +++ b/examples/wrap_operator_as_gxf_extension/ping_tx_native_op/CMakeLists.min.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the \"License\"); @@ -14,22 +14,25 @@ # limitations under the License. cmake_minimum_required(VERSION 3.20) -project(holoscan_basic_workflow CXX) +project(holoscan_wrap_operator_as_gxf_extension CXX) # Finds the package holoscan -find_package(holoscan REQUIRED) +find_package(holoscan REQUIRED CONFIG + PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install") -add_executable(basic_workflow - basic_workflow.cpp +# Create library +add_library(ping_tx_native_op + ping_tx_native_op.cpp + ping_tx_native_op.hpp ) +set_target_properties(ping_tx_native_op PROPERTIES POSITION_INDEPENDENT_CODE ON) -target_link_libraries(basic_workflow - PRIVATE - holoscan::holoscan +target_link_libraries(ping_tx_native_op + PUBLIC + holoscan::core ) -# Copy config file to the build tree -add_custom_target(basic_workflow_yaml - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_SOURCE_DIR}/app_config.yaml" ${CMAKE_CURRENT_BINARY_DIR} +set_target_properties(ping_tx_native_op + PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) -add_dependencies(basic_workflow basic_workflow_yaml) \ No newline at end of file diff --git a/examples/wrap_operator_as_gxf_extension/ping_tx_native_op/CMakeLists.txt b/examples/wrap_operator_as_gxf_extension/ping_tx_native_op/CMakeLists.txt new file mode 100644 index 00000000..c47d81d2 --- /dev/null +++ b/examples/wrap_operator_as_gxf_extension/ping_tx_native_op/CMakeLists.txt @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Create library +add_library(ping_tx_native_op + ping_tx_native_op.cpp + ping_tx_native_op.hpp +) +target_link_libraries(ping_tx_native_op + PUBLIC + holoscan::core +) +set_target_properties(ping_tx_native_op + PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} +) + +# Install operator + +# Set the install RPATH based on the location of the Holoscan SDK libraries +file(RELATIVE_PATH install_lib_relative_path + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_SOURCE_DIR}/${HOLOSCAN_INSTALL_LIB_DIR} +) +set_target_properties(ping_tx_native_op + PROPERTIES + INSTALL_RPATH "\$ORIGIN/${install_lib_relative_path}" +) + +# Install following the relative folder path + +file(RELATIVE_PATH relative_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) + +if(HOLOSCAN_INSTALL_EXAMPLE_SOURCE) +# Install the source +install(FILES ping_tx_native_op.cpp ping_tx_native_op.hpp + DESTINATION "${relative_path}" + COMPONENT holoscan-examples +) + +# Install the minimal CMakeLists.txt file +install(FILES CMakeLists.min.txt + RENAME "CMakeLists.txt" + DESTINATION "${relative_path}" + COMPONENT holoscan-examples +) +endif() + +# Install the compiled operator +install(TARGETS ping_tx_native_op + DESTINATION "${relative_path}" + COMPONENT "holoscan-examples" +) diff --git a/examples/wrap_operator_as_gxf_extension/ping_tx_native_op/ping_tx_native_op.cpp b/examples/wrap_operator_as_gxf_extension/ping_tx_native_op/ping_tx_native_op.cpp new file mode 100644 index 00000000..62035a73 --- /dev/null +++ b/examples/wrap_operator_as_gxf_extension/ping_tx_native_op/ping_tx_native_op.cpp @@ -0,0 +1,41 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ping_tx_native_op.hpp" + +#include + +using namespace holoscan; + +namespace myops { + +void PingTxNativeOp::setup(OperatorSpec& spec) { + HOLOSCAN_LOG_INFO("PingTxNativeOp::setup() called."); + spec.output("out"); +} + +void PingTxNativeOp::compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) { + HOLOSCAN_LOG_INFO("PingTxNativeOp::compute() called."); + + // Create a new message (Entity) + auto out_message = gxf::Entity::New(&context); + // Send the empty message. + op_output.emit(out_message, "out"); +} + +} // namespace myops diff --git a/examples/wrap_operator_as_gxf_extension/ping_tx_native_op/ping_tx_native_op.hpp b/examples/wrap_operator_as_gxf_extension/ping_tx_native_op/ping_tx_native_op.hpp new file mode 100644 index 00000000..b54aca6b --- /dev/null +++ b/examples/wrap_operator_as_gxf_extension/ping_tx_native_op/ping_tx_native_op.hpp @@ -0,0 +1,39 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PING_TX_NATIVE_OP_HPP +#define PING_TX_NATIVE_OP_HPP + +#include "holoscan/holoscan.hpp" + +namespace myops { + +class PingTxNativeOp : public holoscan::Operator { + public: + HOLOSCAN_OPERATOR_FORWARD_ARGS(PingTxNativeOp) + + PingTxNativeOp() = default; + + void setup(holoscan::OperatorSpec& spec) override; + + void compute(holoscan::InputContext& op_input, holoscan::OutputContext& op_output, + holoscan::ExecutionContext& context) override; +}; + +} // namespace myops + +#endif /* PING_TX_NATIVE_OP_HPP */ diff --git a/gxf_extensions/CMakeLists.txt b/gxf_extensions/CMakeLists.txt index f5b885ee..7554c1f0 100644 --- a/gxf_extensions/CMakeLists.txt +++ b/gxf_extensions/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,6 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +# For install_gxf_extension +# TODO: move that function to its own CMake module/file +include(GenerateGXEApp) + list(APPEND CMAKE_INSTALL_RPATH_LIST \$ORIGIN # load lib${extension}_lib.so \$ORIGIN/.. # load 3rd-party dependency libs @@ -23,33 +27,9 @@ list(JOIN CMAKE_INSTALL_RPATH_LIST ":" CMAKE_INSTALL_RPATH) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib/gxf_extensions) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib/gxf_extensions) -add_subdirectory(aja) -add_subdirectory(custom_lstm_inference) -add_subdirectory(format_converter) - add_subdirectory(opengl) -add_subdirectory(probe) - -add_subdirectory(segmentation_postprocessor) -add_subdirectory(segmentation_visualizer) add_subdirectory(stream_playback) - add_subdirectory(tensor_rt) -add_subdirectory(tool_tracking_postprocessor) add_subdirectory(v4l2) - -add_subdirectory(visualizer_tool_tracking) - -add_subdirectory(holoviz) add_subdirectory(bayer_demosaic) -add_subdirectory(multiai_inference) -add_subdirectory(multiai_postprocessor) -add_subdirectory(visualizer_icardio) - -if(HOLOSCAN_BUILD_HI_SPEED_ENDO_APP) - add_subdirectory(emergent) -endif() - -add_subdirectory(mocks) - -add_subdirectory(sample) +add_subdirectory(gxf_holoscan_wrapper) diff --git a/gxf_extensions/README.md b/gxf_extensions/README.md index 457416ef..faefba95 100644 --- a/gxf_extensions/README.md +++ b/gxf_extensions/README.md @@ -2,22 +2,9 @@ See the User Guide for details regarding the extensions in GXF and Holoscan SDK, and for instructions to build your own extensions -- `aja`: support AJA capture card as source. It offers support for GPUDirect-RDMA on Quadro GPUs. The output is a VideoBuffer object. -- `bayer_demosaic`: calls to the NVIDIA CUDA based demosaicing algorithm to create RGB planes from Bayer format. -- `custom_lstm_inference`: provide LSTM (Long-Short Term Memory) stateful inference module using TensorRT -- `emergent`: support camera from Emergent Vision Technologies as the source. The datastream from this camera is transferred through Mellanox ConnectX NIC using Rivermax SDK. -- `format_converter`: provide common video or tensor operations in inference pipelines to change datatypes, resize images, reorder channels, and normalize and scale values. -- `holoviz`: Holoviz is a Vulkan API based visualizer to display video buffer and geometry from a tensor. -- `multiai_inference`: Multi AI inference supported by TRT and onnxruntime backend. -- `multiai_postprocessor`: Executes data processing on multiple tensors. Currently supports only one operation for icardio models. -- `visualizer_icardio`: Prepares results from plax chamber model (from icardio) for visualization. -- `opengl`: OpenGL Renderer(visualizer) to display a VideoBuffer, leveraging OpenGL/CUDA interop. -- `probe`: print tensor information -- `segmentation_postprocessor`: segmentation model postprocessing converting inference output to highest-probability class index, including support for sigmoid, softmax, and activations. -- `segmentation_visualizer`: OpenGL renderer that combines segmentation output overlaid on video input, using CUDA/OpenGL interop. -- `stream_playback`: provide video stream playback module to output video frames as a Tensor object. -- `tensor_rt` _(duplicate from GXF)_: Run inference with TensorRT -- `v4l2`: Video for Linux 2 source supporting USB cameras and other media inputs. The output is a VideoBuffer object. -- `visualizer_tool_tracking`: custom visualizer component that handles compositing, blending, and visualization of tool labels, tips, and masks given the output tensors of the custom_lstm_inference -- `mocks`: provide mock components such as VideoBufferMock. -- `sample`: provide sample ping_tx/ping_rx components. +- `bayer_demosaic`: includes the `nvidia::holoscan::BayerDemosaic` codelet. It performs color filter array (CFA) interpolation for 1-channel inputs of 8 or 16-bit unsigned integer and outputs an RGB or RGBA image. +- `gxf_holoscan_wrapper`: includes the `holoscan::gxf::OperatorWrapper` codelet. It is used as a utility base class to wrap a holoscan operator to interface with the GXF framework. +- `opengl_renderer`: includes the `nvidia::holoscan::OpenGLRenderer` codelet. It displays a VideoBuffer, leveraging OpenGL/CUDA interop. +- `tensor_rt`: includes the `nvidia::holoscan::TensorRtInference` codelet. It takes input tensors and feeds them into TensorRT for inference. +- `stream_playback`: includes the `nvidia::holoscan::stream_playback::VideoStreamSerializer` entity serializer to/from a Tensor Object. +- `v4l2_source`: includes the `nvidia::holoscan::V4L2Source` codelet. It uses V4L2 to get image frames from a USB cameras. The output is a VideoBuffer object. diff --git a/gxf_extensions/aja/CMakeLists.txt b/gxf_extensions/aja/CMakeLists.txt deleted file mode 100644 index 5f83dd73..00000000 --- a/gxf_extensions/aja/CMakeLists.txt +++ /dev/null @@ -1,39 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Create library -add_library(aja_source_lib SHARED - aja_source.cpp - aja_source.hpp -) -target_link_libraries(aja_source_lib - PUBLIC - AJA::ajantv2 - CUDA::cudart - CUDA::cuda_driver - GXF::multimedia - GXF::std - yaml-cpp -) - -# Create extension -add_library(aja_source SHARED - aja_ext.cpp -) -target_link_libraries(aja_source - PUBLIC aja_source_lib -) -# Install GXF extension as a component 'holoscan-gxf_extensions' -install_gxf_extension(aja_source) diff --git a/gxf_extensions/aja/aja_source.cpp b/gxf_extensions/aja/aja_source.cpp deleted file mode 100644 index fa6ee8cd..00000000 --- a/gxf_extensions/aja/aja_source.cpp +++ /dev/null @@ -1,523 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "aja_source.hpp" - -#include -#include - -#include -#include -#include - -#include "gxf/multimedia/video.hpp" - -template <> -struct YAML::convert { - static Node encode(const NTV2Channel& rhs) { - Node node; - int channel = static_cast(rhs); - std::stringstream ss; - ss << "NTV2_CHANNEL"; - ss << channel; - node.push_back(ss.str()); - YAML::Node value_node = node[0]; - return value_node; - } - - static bool decode(const Node& node, NTV2Channel& rhs) { - if (!node.IsScalar()) return false; - - const std::string prefix("NTV2_CHANNEL"); - auto value = node.Scalar(); - if (value.find(prefix) != 0) return false; - value = value.substr(prefix.length()); - - try { - size_t len; - const auto index = std::stoi(value, &len); - if (index < 1 || index > NTV2_MAX_NUM_CHANNELS || len != value.length()) { return false; } - rhs = static_cast(index - 1); - return true; - } catch (...) { return false; } - } -}; - -namespace nvidia { -namespace holoscan { - -AJASource::AJASource() : pixel_format_(kDefaultPixelFormat), current_buffer_(0), - current_hw_frame_(0), current_overlay_hw_frame_(0), - use_tsi_(false), is_kona_hdmi_(false) {} - -gxf_result_t AJASource::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - result &= registrar->parameter(video_buffer_output_, "video_buffer_output", "VideoBufferOutput", - "Output for the video buffer."); - result &= registrar->parameter(device_specifier_, "device", "Device", "Device specifier.", - std::string(kDefaultDevice)); - result &= - registrar->parameter(channel_, "channel", "Channel", "NTV2Channel to use.", kDefaultChannel); - result &= registrar->parameter(width_, "width", "Width", "Width of the stream.", kDefaultWidth); - result &= - registrar->parameter(height_, "height", "Height", "Height of the stream.", kDefaultHeight); - result &= registrar->parameter(framerate_, "framerate", "Framerate", "Framerate of the stream.", - kDefaultFramerate); - result &= registrar->parameter(use_rdma_, "rdma", "RDMA", "Enable RDMA.", kDefaultRDMA); - - result &= registrar->parameter(enable_overlay_, "enable_overlay", "EnableOverlay", - "Enable overlay.", kDefaultEnableOverlay); - result &= registrar->parameter(overlay_channel_, "overlay_channel", - "OverlayChannel", "NTV2Channel to use for overlay output.", - kDefaultOverlayChannel); - result &= registrar->parameter(overlay_rdma_, "overlay_rdma", "OverlayRDMA", - "Enable Overlay RDMA.", kDefaultOverlayRDMA); - result &= registrar->parameter(overlay_buffer_output_, "overlay_buffer_output", - "OverlayBufferOutput", "Output for an empty overlay buffer.", - gxf::Registrar::NoDefaultParameter(), - GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(overlay_buffer_input_, "overlay_buffer_input", - "OverlayBufferInput", "Input for a filled overlay buffer.", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - - return gxf::ToResultCode(result); -} - -AJAStatus AJASource::DetermineVideoFormat() { - if (width_ == 1920 && height_ == 1080 && framerate_ == 60) { - video_format_ = NTV2_FORMAT_1080p_6000_A; - } else if (width_ == 3840 && height_ == 2160 && framerate_ == 60) { - video_format_ = NTV2_FORMAT_3840x2160p_6000; - } else { - return AJA_STATUS_UNSUPPORTED; - } - - return AJA_STATUS_SUCCESS; -} - -AJAStatus AJASource::OpenDevice() { - // Get the requested device. - if (!CNTV2DeviceScanner::GetFirstDeviceFromArgument(device_specifier_, device_)) { - GXF_LOG_ERROR("Device %s not found.", device_specifier_.get().c_str()); - return AJA_STATUS_OPEN; - } - - // Check if the device is ready. - if (!device_.IsDeviceReady(false)) { - GXF_LOG_ERROR("Device %s not ready.", device_specifier_.get().c_str()); - return AJA_STATUS_INITIALIZE; - } - - // Get the device ID. - device_id_ = device_.GetDeviceID(); - - // Detect Kona HDMI device. - is_kona_hdmi_ = NTV2DeviceGetNumHDMIVideoInputs(device_id_) > 1; - - // Check if a TSI 4x format is needed. - if (is_kona_hdmi_) { use_tsi_ = GetNTV2VideoFormatTSI(&video_format_); } - - // Check device capabilities. - if (!NTV2DeviceCanDoVideoFormat(device_id_, video_format_)) { - GXF_LOG_ERROR("AJA device does not support requested video format."); - return AJA_STATUS_UNSUPPORTED; - } - if (!NTV2DeviceCanDoFrameBufferFormat(device_id_, pixel_format_)) { - GXF_LOG_ERROR("AJA device does not support requested pixel format."); - return AJA_STATUS_UNSUPPORTED; - } - if (!NTV2DeviceCanDoCapture(device_id_)) { - GXF_LOG_ERROR("AJA device cannot capture video."); - return AJA_STATUS_UNSUPPORTED; - } - if (!NTV2_IS_VALID_CHANNEL(channel_)) { - GXF_LOG_ERROR("Invalid AJA channel: %d", channel_); - return AJA_STATUS_UNSUPPORTED; - } - - // Check overlay capabilities. - if (enable_overlay_) { - if (!NTV2_IS_VALID_CHANNEL(overlay_channel_)) { - GXF_LOG_ERROR("Invalid overlay channel: %d", overlay_channel_); - return AJA_STATUS_UNSUPPORTED; - } - - if (NTV2DeviceGetNumVideoChannels(device_id_) < 2) { - GXF_LOG_ERROR("Insufficient number of video channels"); - return AJA_STATUS_UNSUPPORTED; - } - - if (NTV2DeviceGetNumFrameStores(device_id_) < 2) { - GXF_LOG_ERROR("Insufficient number of frame stores"); - return AJA_STATUS_UNSUPPORTED; - } - - if (NTV2DeviceGetNumMixers(device_id_) < 1) { - GXF_LOG_ERROR("Hardware mixing not supported"); - return AJA_STATUS_UNSUPPORTED; - } - - if (!NTV2DeviceHasBiDirectionalSDI(device_id_)) { - GXF_LOG_ERROR("BiDirectional SDI not supported"); - return AJA_STATUS_UNSUPPORTED; - } - } - - return AJA_STATUS_SUCCESS; -} - -AJAStatus AJASource::SetupVideo() { - NTV2InputSourceKinds input_kind = is_kona_hdmi_ ? NTV2_INPUTSOURCES_HDMI : NTV2_INPUTSOURCES_SDI; - NTV2InputSource input_src = ::NTV2ChannelToInputSource(channel_, input_kind); - NTV2Channel tsi_channel = static_cast(channel_ + 1); - - if (!IsRGBFormat(pixel_format_)) { - GXF_LOG_ERROR("YUV formats not yet supported"); - return AJA_STATUS_UNSUPPORTED; - } - - // Detect if the source is YUV or RGB (i.e. if CSC is required or not). - bool is_input_rgb(false); - if (input_kind == NTV2_INPUTSOURCES_HDMI) { - NTV2LHIHDMIColorSpace input_color; - device_.GetHDMIInputColor(input_color, channel_); - is_input_rgb = (input_color == NTV2_LHIHDMIColorSpaceRGB); - } - - // Setup the input routing. - device_.ClearRouting(); - device_.EnableChannel(channel_); - if (use_tsi_) { - device_.SetTsiFrameEnable(true, channel_); - device_.EnableChannel(tsi_channel); - } - device_.SetMode(channel_, NTV2_MODE_CAPTURE); - if (NTV2DeviceHasBiDirectionalSDI(device_id_) && NTV2_INPUT_SOURCE_IS_SDI(input_src)) { - device_.SetSDITransmitEnable(channel_, false); - } - device_.SetVideoFormat(video_format_, false, false, channel_); - device_.SetFrameBufferFormat(channel_, pixel_format_); - if (use_tsi_) { device_.SetFrameBufferFormat(tsi_channel, pixel_format_); } - device_.EnableInputInterrupt(channel_); - device_.SubscribeInputVerticalEvent(channel_); - - NTV2OutputXptID input_output_xpt = - GetInputSourceOutputXpt(input_src, /*DS2*/ false, is_input_rgb, /*Quadrant*/ 0); - NTV2InputXptID fb_input_xpt(GetFrameBufferInputXptFromChannel(channel_)); - if (use_tsi_) { - if (!is_input_rgb) { - if (NTV2DeviceGetNumCSCs(device_id_) < 4) { - GXF_LOG_ERROR("CSCs not available for TSI input."); - return AJA_STATUS_UNSUPPORTED; - } - device_.Connect(NTV2_XptFrameBuffer1Input, NTV2_Xpt425Mux1ARGB); - device_.Connect(NTV2_XptFrameBuffer1BInput, NTV2_Xpt425Mux1BRGB); - device_.Connect(NTV2_XptFrameBuffer2Input, NTV2_Xpt425Mux2ARGB); - device_.Connect(NTV2_XptFrameBuffer2BInput, NTV2_Xpt425Mux2BRGB); - device_.Connect(NTV2_Xpt425Mux1AInput, NTV2_XptCSC1VidRGB); - device_.Connect(NTV2_Xpt425Mux1BInput, NTV2_XptCSC2VidRGB); - device_.Connect(NTV2_Xpt425Mux2AInput, NTV2_XptCSC3VidRGB); - device_.Connect(NTV2_Xpt425Mux2BInput, NTV2_XptCSC4VidRGB); - device_.Connect(NTV2_XptCSC1VidInput, NTV2_XptHDMIIn1); - device_.Connect(NTV2_XptCSC2VidInput, NTV2_XptHDMIIn1Q2); - device_.Connect(NTV2_XptCSC3VidInput, NTV2_XptHDMIIn1Q3); - device_.Connect(NTV2_XptCSC4VidInput, NTV2_XptHDMIIn1Q4); - } else { - device_.Connect(NTV2_XptFrameBuffer1Input, NTV2_Xpt425Mux1ARGB); - device_.Connect(NTV2_XptFrameBuffer1BInput, NTV2_Xpt425Mux1BRGB); - device_.Connect(NTV2_XptFrameBuffer2Input, NTV2_Xpt425Mux2ARGB); - device_.Connect(NTV2_XptFrameBuffer2BInput, NTV2_Xpt425Mux2BRGB); - device_.Connect(NTV2_Xpt425Mux1AInput, NTV2_XptHDMIIn1RGB); - device_.Connect(NTV2_Xpt425Mux1BInput, NTV2_XptHDMIIn1Q2RGB); - device_.Connect(NTV2_Xpt425Mux2AInput, NTV2_XptHDMIIn1Q3RGB); - device_.Connect(NTV2_Xpt425Mux2BInput, NTV2_XptHDMIIn1Q4RGB); - } - } else if (!is_input_rgb) { - if (NTV2DeviceGetNumCSCs(device_id_) <= static_cast(channel_)) { - GXF_LOG_ERROR("No CSC available for NTV2_CHANNEL%d", channel_ + 1); - return AJA_STATUS_UNSUPPORTED; - } - NTV2InputXptID csc_input = GetCSCInputXptFromChannel(channel_); - NTV2OutputXptID csc_output = - GetCSCOutputXptFromChannel(channel_, /*inIsKey*/ false, /*inIsRGB*/ true); - device_.Connect(fb_input_xpt, csc_output); - device_.Connect(csc_input, input_output_xpt); - } else { - device_.Connect(fb_input_xpt, input_output_xpt); - } - - if (enable_overlay_) { - // Setup output channel. - device_.SetReference(NTV2_REFERENCE_INPUT1); - device_.SetMode(overlay_channel_, NTV2_MODE_DISPLAY); - device_.SetSDITransmitEnable(overlay_channel_, true); - device_.SetVideoFormat(video_format_, false, false, overlay_channel_); - device_.SetFrameBufferFormat(overlay_channel_, NTV2_FBF_ABGR); - - // Setup mixer controls. - device_.SetMixerFGInputControl(0, NTV2MIXERINPUTCONTROL_SHAPED); - device_.SetMixerBGInputControl(0, NTV2MIXERINPUTCONTROL_FULLRASTER); - device_.SetMixerCoefficient(0, 0x10000); - device_.SetMixerFGMatteEnabled(0, false); - device_.SetMixerBGMatteEnabled(0, false); - - // Setup routing (overlay frame to CSC, CSC and SDI input to mixer, mixer to SDI output). - NTV2OutputDestination output_dst = ::NTV2ChannelToOutputDestination(overlay_channel_); - device_.Connect(GetCSCInputXptFromChannel(overlay_channel_), - GetFrameBufferOutputXptFromChannel(overlay_channel_, true /*RGB*/)); - device_.Connect(NTV2_XptMixer1FGVidInput, - GetCSCOutputXptFromChannel(overlay_channel_, false /*Key*/)); - device_.Connect(NTV2_XptMixer1FGKeyInput, - GetCSCOutputXptFromChannel(overlay_channel_, true /*Key*/)); - device_.Connect(NTV2_XptMixer1BGVidInput, input_output_xpt); - device_.Connect(GetOutputDestInputXpt(output_dst), NTV2_XptMixer1VidYUV); - - // Set initial output frame (overlay uses HW frames 2 and 3). - current_overlay_hw_frame_ = 2; - device_.SetOutputFrame(overlay_channel_, current_overlay_hw_frame_); - } - - // Wait for a number of frames to acquire video signal. - current_hw_frame_ = 0; - device_.SetInputFrame(channel_, current_hw_frame_); - device_.WaitForInputVerticalInterrupt(channel_, kWarmupFrames); - - return AJA_STATUS_SUCCESS; -} - -bool AJASource::AllocateBuffers(std::vector& buffers, size_t num_buffers, - size_t buffer_size, bool rdma) { - buffers.resize(num_buffers); - for (auto& buf : buffers) { - if (rdma) { - cudaMalloc(&buf, buffer_size); - unsigned int syncFlag = 1; - if (cuPointerSetAttribute(&syncFlag, CU_POINTER_ATTRIBUTE_SYNC_MEMOPS, - reinterpret_cast(buf))) { - GXF_LOG_ERROR("Failed to set SYNC_MEMOPS CUDA attribute for RDMA"); - return false; - } - } else { - buf = malloc(buffer_size); - } - - if (!buf) { - GXF_LOG_ERROR("Failed to allocate buffer memory"); - return false; - } - - if (!device_.DMABufferLock(static_cast(buf), buffer_size, true, rdma)) { - GXF_LOG_ERROR("Failed to map buffer for DMA"); - return false; - } - } - - return true; -} - -void AJASource::FreeBuffers(std::vector& buffers, bool rdma) { - for (auto& buf : buffers) { - if (rdma) { - cudaFree(buf); - } else { - free(buf); - } - } - buffers.clear(); -} - -AJAStatus AJASource::SetupBuffers() { - auto size = GetVideoWriteSize(video_format_, pixel_format_); - - if (!AllocateBuffers(buffers_, kNumBuffers, size, use_rdma_)) { - return AJA_STATUS_INITIALIZE; - } - - if (enable_overlay_) { - if (!AllocateBuffers(overlay_buffers_, kNumBuffers, size, overlay_rdma_)) { - return AJA_STATUS_INITIALIZE; - } - } - - return AJA_STATUS_SUCCESS; -} - -gxf_result_t AJASource::start() { - GXF_LOG_INFO("AJA Source: Capturing from NTV2_CHANNEL%d", (channel_.get() + 1)); - GXF_LOG_INFO("AJA Source: RDMA is %s", use_rdma_ ? "enabled" : "disabled"); - if (enable_overlay_) { - GXF_LOG_INFO("AJA Source: Outputting overlay to NTV2_CHANNEL%d", (overlay_channel_.get() + 1)); - GXF_LOG_INFO("AJA Source: Overlay RDMA is %s", overlay_rdma_ ? "enabled" : "disabled"); - } else { - GXF_LOG_INFO("AJA Source: Overlay output is disabled"); - } - - AJAStatus status = DetermineVideoFormat(); - if (AJA_FAILURE(status)) { - GXF_LOG_ERROR("Video format could not be determined based on parameters."); - return GXF_FAILURE; - } - - status = OpenDevice(); - if (AJA_FAILURE(status)) { - GXF_LOG_ERROR("Failed to open device %s", device_specifier_.get().c_str()); - return GXF_FAILURE; - } - - status = SetupVideo(); - if (AJA_FAILURE(status)) { - GXF_LOG_ERROR("Failed to setup device %s", device_specifier_.get().c_str()); - return GXF_FAILURE; - } - - status = SetupBuffers(); - if (AJA_FAILURE(status)) { - GXF_LOG_ERROR("Failed to setup AJA buffers."); - return GXF_FAILURE; - } - - return GXF_SUCCESS; -} - -gxf_result_t AJASource::stop() { - device_.UnsubscribeInputVerticalEvent(channel_); - device_.DMABufferUnlockAll(); - - if (enable_overlay_) { - device_.SetMixerMode(0, NTV2MIXERMODE_FOREGROUND_OFF); - } - - FreeBuffers(buffers_, use_rdma_); - FreeBuffers(overlay_buffers_, overlay_rdma_); - - return GXF_SUCCESS; -} - -gxf_result_t AJASource::tick() { - // Update the overlay frame. - if (enable_overlay_ && overlay_buffer_input_.try_get()) { - const auto& overlay_buffer_input = overlay_buffer_input_.try_get().value()->receive(); - if (overlay_buffer_input) { - const auto& overlay_buffer = overlay_buffer_input.value().get(); - if (overlay_buffer) { - // Overlay uses HW frames 2 and 3. - current_overlay_hw_frame_ = ((current_overlay_hw_frame_ + 1) % 2) + 2; - - const auto& buffer = overlay_buffer.value(); - ULWord* ptr = reinterpret_cast(buffer->pointer()); - device_.DMAWriteFrame(current_overlay_hw_frame_, ptr, buffer->size()); - device_.SetOutputFrame(overlay_channel_, current_overlay_hw_frame_); - device_.SetMixerMode(0, NTV2MIXERMODE_MIX); - } - } - } - - // Update the next input frame and wait until it starts. - uint32_t next_hw_frame = (current_hw_frame_ + 1) % 2; - device_.SetInputFrame(channel_, next_hw_frame); - device_.WaitForInputVerticalInterrupt(channel_); - - // Read the last completed frame. - auto size = GetVideoWriteSize(video_format_, pixel_format_); - auto ptr = static_cast(buffers_[current_buffer_]); - device_.DMAReadFrame(current_hw_frame_, ptr, size); - - // Set the frame to read for the next tick. - current_hw_frame_ = next_hw_frame; - - // Common (output and overlay) buffer info - gxf::VideoTypeTraits video_type; - gxf::VideoFormatSize color_format; - auto color_planes = color_format.getDefaultColorPlanes(width_, height_); - gxf::VideoBufferInfo info{width_, height_, video_type.value, color_planes, - gxf::SurfaceLayout::GXF_SURFACE_LAYOUT_PITCH_LINEAR}; - - // Pass an empty overlay buffer downstream. - if (enable_overlay_ && overlay_buffer_output_.try_get()) { - auto overlay_output = gxf::Entity::New(context()); - if (!overlay_output) { - GXF_LOG_ERROR("Failed to allocate overlay output; terminating."); - return GXF_FAILURE; - } - - auto overlay_buffer = overlay_output.value().add(); - if (!overlay_buffer) { - GXF_LOG_ERROR("Failed to allocate overlay buffer; terminating."); - return GXF_FAILURE; - } - - auto overlay_storage_type = - overlay_rdma_ ? gxf::MemoryStorageType::kDevice : gxf::MemoryStorageType::kHost; - overlay_buffer.value()->wrapMemory(info, size, overlay_storage_type, - overlay_buffers_[current_buffer_], nullptr); - - auto overlay_result = overlay_buffer_output_.try_get().value()->publish( - std::move(overlay_output.value())); - if (GXF_SUCCESS != gxf::ToResultCode(overlay_result)) { - GXF_LOG_ERROR("Failed to publish overlay buffer; terminating."); - return GXF_FAILURE; - } - } - - // Pass the video output buffer downstream. - auto video_output = gxf::Entity::New(context()); - if (!video_output) { - GXF_LOG_ERROR("Failed to allocate video output; terminating."); - return GXF_FAILURE; - } - - auto video_buffer = video_output.value().add(); - if (!video_buffer) { - GXF_LOG_ERROR("Failed to allocate video buffer; terminating."); - return GXF_FAILURE; - } - - auto storage_type = use_rdma_ ? gxf::MemoryStorageType::kDevice : gxf::MemoryStorageType::kHost; - video_buffer.value()->wrapMemory(info, size, storage_type, buffers_[current_buffer_], nullptr); - - auto result = video_buffer_output_->publish(std::move(video_output.value())); - if (GXF_SUCCESS != gxf::ToResultCode(result)) { - GXF_LOG_ERROR("Failed to publish video buffer; terminating."); - return GXF_FAILURE; - } - - // Update the current buffer (index shared between video and overlay) - current_buffer_ = (current_buffer_ + 1) % kNumBuffers; - - return GXF_SUCCESS; -} - -bool AJASource::GetNTV2VideoFormatTSI(NTV2VideoFormat* format) { - switch (*format) { - case NTV2_FORMAT_3840x2160p_2400: - *format = NTV2_FORMAT_4x1920x1080p_2400; - return true; - case NTV2_FORMAT_3840x2160p_6000: - *format = NTV2_FORMAT_4x1920x1080p_6000; - return true; - case NTV2_FORMAT_4096x2160p_2400: - *format = NTV2_FORMAT_4x2048x1080p_2400; - return true; - case NTV2_FORMAT_4096x2160p_6000: - *format = NTV2_FORMAT_4x2048x1080p_6000; - return true; - default: - return false; - } -} - -} // namespace holoscan -} // namespace nvidia diff --git a/gxf_extensions/aja/aja_source.hpp b/gxf_extensions/aja/aja_source.hpp deleted file mode 100644 index 534c6420..00000000 --- a/gxf_extensions/aja/aja_source.hpp +++ /dev/null @@ -1,110 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_AJA_SOURCE_HPP_ -#define NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_AJA_SOURCE_HPP_ - -#include -#include -#include - -#include -#include - -#include "gxf/std/codelet.hpp" -#include "gxf/std/receiver.hpp" -#include "gxf/std/transmitter.hpp" - -namespace nvidia { -namespace holoscan { - -constexpr uint32_t kNumBuffers = 2; - -constexpr char kDefaultDevice[] = "0"; -constexpr NTV2Channel kDefaultChannel = NTV2_CHANNEL1; -constexpr uint32_t kDefaultWidth = 1920; -constexpr uint32_t kDefaultHeight = 1080; -constexpr uint32_t kDefaultFramerate = 60; -constexpr bool kDefaultRDMA = false; -constexpr NTV2PixelFormat kDefaultPixelFormat = NTV2_FBF_ABGR; -constexpr size_t kWarmupFrames = 5; -constexpr bool kDefaultEnableOverlay = false; -constexpr bool kDefaultOverlayRDMA = false; -constexpr NTV2Channel kDefaultOverlayChannel = NTV2_CHANNEL2; - -/* - * TODO - * - Make pixel format configurable (non-ABGR must be supported downstream) - */ - -/// @brief Video input codelet for use with AJA capture cards. -/// -/// Provides a codelet for supporting AJA capture card as a source. -/// It offers support for GPUDirect-RDMA on Quadro/NVIDIA RTX GPUs. -/// The output is a VideoBuffer object. -class AJASource : public gxf::Codelet { - public: - AJASource(); - - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - - gxf_result_t start() override; - gxf_result_t tick() override; - gxf_result_t stop() override; - - private: - AJAStatus DetermineVideoFormat(); - AJAStatus OpenDevice(); - AJAStatus SetupVideo(); - AJAStatus SetupBuffers(); - AJAStatus StartAutoCirculate(); - bool AllocateBuffers(std::vector& buffers, size_t num_buffers, - size_t buffer_size, bool rdma); - void FreeBuffers(std::vector& buffers, bool rdma); - bool GetNTV2VideoFormatTSI(NTV2VideoFormat* format); - - gxf::Parameter> video_buffer_output_; - gxf::Parameter device_specifier_; - gxf::Parameter channel_; - gxf::Parameter width_; - gxf::Parameter height_; - gxf::Parameter framerate_; - gxf::Parameter use_rdma_; - - gxf::Parameter enable_overlay_; - gxf::Parameter overlay_channel_; - gxf::Parameter overlay_rdma_; - gxf::Parameter> overlay_buffer_output_; - gxf::Parameter> overlay_buffer_input_; - - CNTV2Card device_; - NTV2DeviceID device_id_; - NTV2VideoFormat video_format_; - NTV2PixelFormat pixel_format_; - bool is_kona_hdmi_; - bool use_tsi_; - - std::vector buffers_; - std::vector overlay_buffers_; - uint8_t current_buffer_; - uint8_t current_hw_frame_; - uint8_t current_overlay_hw_frame_; -}; - -} // namespace holoscan -} // namespace nvidia - -#endif // NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_AJA_SOURCE_HPP_ diff --git a/gxf_extensions/bayer_demosaic/CMakeLists.txt b/gxf_extensions/bayer_demosaic/CMakeLists.txt index aedf12f7..0d4fcb05 100644 --- a/gxf_extensions/bayer_demosaic/CMakeLists.txt +++ b/gxf_extensions/bayer_demosaic/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,11 +15,11 @@ # create component library -add_library(bayer_demosaic_lib SHARED +add_library(gxf_bayer_demosaic_lib SHARED bayer_demosaic.cpp bayer_demosaic.hpp ) -target_link_libraries(bayer_demosaic_lib +target_link_libraries(gxf_bayer_demosaic_lib PUBLIC CUDA::cudart CUDA::nppicc @@ -30,13 +30,13 @@ target_link_libraries(bayer_demosaic_lib ) # Create extension -add_library(bayer_demosaic SHARED +add_library(gxf_bayer_demosaic SHARED bayer_demosaic_ext.cpp ) -target_link_libraries(bayer_demosaic +target_link_libraries(gxf_bayer_demosaic PUBLIC - bayer_demosaic_lib + gxf_bayer_demosaic_lib ) # Install GXF extension as a component 'holoscan-gxf_extensions' -install_gxf_extension(bayer_demosaic) +install_gxf_extension(gxf_bayer_demosaic) diff --git a/gxf_extensions/custom_lstm_inference/CMakeLists.txt b/gxf_extensions/custom_lstm_inference/CMakeLists.txt deleted file mode 100644 index 9cfaeeb8..00000000 --- a/gxf_extensions/custom_lstm_inference/CMakeLists.txt +++ /dev/null @@ -1,39 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Create library -add_library(custom_lstm_inference_lib SHARED - tensor_rt_inference.cpp - tensor_rt_inference.hpp -) -target_link_libraries(custom_lstm_inference_lib - PUBLIC - CUDA::cudart - GXF::cuda - GXF::std - # TensorRT::nvinfer_plugin - TensorRT::nvonnxparser - yaml-cpp -) - -# Create extension -add_library(custom_lstm_inference SHARED - lstm_tensor_rt_extension.cpp -) -target_link_libraries(custom_lstm_inference - PUBLIC custom_lstm_inference_lib -) -# Install GXF extension as a component 'holoscan-gxf_extensions' -install_gxf_extension(custom_lstm_inference) diff --git a/gxf_extensions/custom_lstm_inference/lstm_tensor_rt_extension.cpp b/gxf_extensions/custom_lstm_inference/lstm_tensor_rt_extension.cpp deleted file mode 100644 index a361be86..00000000 --- a/gxf_extensions/custom_lstm_inference/lstm_tensor_rt_extension.cpp +++ /dev/null @@ -1,28 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "gxf/std/extension_factory_helper.hpp" - -#include "tensor_rt_inference.hpp" - -GXF_EXT_FACTORY_BEGIN() -GXF_EXT_FACTORY_SET_INFO(0xef68668e8e1748f7, 0x88bd37f33565889d, "CustomLSTMInferenceExtension", - "TensorRT Custom LSTM Inference extension", "NVIDIA", "0.2.0", "LICENSE"); -GXF_EXT_FACTORY_ADD(0x207880b30f3f404a, 0xb3b12659a29f3e26, - nvidia::holoscan::custom_lstm_inference::TensorRtInference, - nvidia::gxf::Codelet, - "Codelet taking input tensors and feed them into TensorRT for LSTM inference."); -GXF_EXT_FACTORY_END() diff --git a/gxf_extensions/custom_lstm_inference/tensor_rt_inference.cpp b/gxf_extensions/custom_lstm_inference/tensor_rt_inference.cpp deleted file mode 100644 index da0c7d4c..00000000 --- a/gxf_extensions/custom_lstm_inference/tensor_rt_inference.cpp +++ /dev/null @@ -1,955 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "tensor_rt_inference.hpp" - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "gxf/cuda/cuda_stream_id.hpp" -#include "gxf/std/parameter_parser_std.hpp" -#include "gxf/std/tensor.hpp" -#include "gxf/std/timestamp.hpp" - -#define CUDA_TRY(stmt) \ - ({ \ - cudaError_t _holoscan_cuda_err = stmt; \ - if (cudaSuccess != _holoscan_cuda_err) { \ - GXF_LOG_ERROR("CUDA Runtime call %s in line %d of file %s failed with '%s' (%d).\n", #stmt, \ - __LINE__, __FILE__, cudaGetErrorString(_holoscan_cuda_err), \ - _holoscan_cuda_err); \ - } \ - _holoscan_cuda_err; \ - }) - -namespace nvidia { -namespace holoscan { -namespace custom_lstm_inference { -namespace { -// Checks whether a string ends with a certain string -inline bool EndsWith(const std::string& str, const std::string& suffix) { - return str.size() >= suffix.size() && - str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0; -} - -bool IsValidFile(const std::string& path) { - struct stat st; - if (stat(path.c_str(), &st) != 0) { return false; } - return static_cast(st.st_mode & S_IFREG); -} - -bool IsValidDirectory(const std::string& path) { - struct stat st; - if (stat(path.c_str(), &st) != 0) { return false; } - return static_cast(st.st_mode & S_IFDIR); -} - -bool ReadEntireBinaryFile(const std::string& file_path, std::vector& buffer) { - // Make sure we are opening a valid file. - if (!IsValidFile(file_path)) { return false; } - // Open the file in binary mode and seek to the end - std::ifstream file(file_path, std::ios::binary | std::ios::ate); - if (!file) { return false; } - // Get the size of the file and seek back to the beginning - const size_t size = file.tellg(); - file.seekg(0); - // Reserve enough space in the output buffer and read the file contents into it - buffer.resize(size); - const bool ret = static_cast(file.read(buffer.data(), size)); - file.close(); - return ret; -} - -// Formats gxf tensor shape specified by std::array for console spew -const std::string FormatDims(const std::array& dimensions, - const int32_t rank) { - std::stringbuf sbuf; - std::ostream stream(&sbuf); - stream << "["; - for (int i = 0; i < rank; ++i) { - if (i > 0) { stream << ", "; } - stream << dimensions[i]; - } - stream << "]"; - return sbuf.str(); -} - -// Formats gxf shape for console spew -const std::string FormatTensorShape(const gxf::Shape& shape) { - std::array dimensions; - for (uint32_t i = 0; i < shape.rank(); ++i) { dimensions[i] = shape.dimension(i); } - return FormatDims(dimensions, shape.rank()); -} - -// Converts TensorRT dimensions to Gxf Tensor dimensions (std::array) -std::array Dims2Dimensions(const nvinfer1::Dims& dims) { - std::array dimensions; - dimensions.fill(1); - for (int32_t i = 0; i < dims.nbDims; i++) { dimensions[i] = dims.d[i]; } - return dimensions; -} - -// Converts TensorRT data type to gxf::Tensor element type (gxf::PrimitiveType) -gxf::Expected NvInferDatatypeToTensorElementType(nvinfer1::DataType data_type) { - switch (data_type) { - case nvinfer1::DataType::kFLOAT: { - return gxf::PrimitiveType::kFloat32; - } - case nvinfer1::DataType::kINT8: { - return gxf::PrimitiveType::kInt8; - } - case nvinfer1::DataType::kINT32: { - return gxf::PrimitiveType::kInt32; - } - // case nvinfer1::DataType::kBOOL: - case nvinfer1::DataType::kHALF: - default: { - GXF_LOG_ERROR("Unsupported DataType %d", data_type); - return gxf::Unexpected{GXF_FAILURE}; - } - } -} - -// Writes engine plan to specified file path -gxf::Expected SerializeEnginePlan(const std::vector& plan, const std::string& path) { - // Write Plan To Disk - std::ofstream out_stream(path.c_str(), std::ofstream::binary); - if (!out_stream.is_open()) { - GXF_LOG_ERROR("Failed to create engine file %s.", path.c_str()); - return gxf::Unexpected{GXF_FAILURE}; - } - out_stream.write(plan.data(), plan.size()); - if (out_stream.bad()) { - GXF_LOG_ERROR("Failed to writing to engine file %s.", path.c_str()); - return gxf::Unexpected{GXF_FAILURE}; - } - out_stream.close(); - GXF_LOG_INFO("TensorRT engine serialized at %s", path.c_str()); - return gxf::Success; -} - -} // namespace - -// Logging interface for the TensorRT builder, engine and runtime, to redirect logging, -void TensorRTInferenceLogger::log(ILogger::Severity severity, const char* msg) throw() { - switch (severity) { - case Severity::kINTERNAL_ERROR: { - GXF_LOG_ERROR("TRT INTERNAL_ERROR: %s", msg); - break; - } - case Severity::kERROR: { - GXF_LOG_ERROR("TRT ERROR: %s", msg); - break; - } - case Severity::kWARNING: { - GXF_LOG_WARNING("TRT WARNING: %s", msg); - break; - } - case Severity::kINFO: { - GXF_LOG_DEBUG("TRT INFO: %s", msg); - break; - } - case Severity::kVERBOSE: { - if (verbose_) { GXF_LOG_DEBUG("TRT VERBOSE: %s", msg); } - break; - } - default: { - GXF_LOG_ERROR("TRT UNKNOWN SEVERITY ERROR: %s", msg); - break; - } - } -} - -void TensorRTInferenceLogger::setVerbose(bool verbose) { - verbose_ = verbose; -} - -gxf_result_t TensorRtInference::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - - result &= registrar->parameter(model_file_path_, "model_file_path", "Model File Path", - "Path to ONNX model to be loaded."); - result &= registrar->parameter( - engine_cache_dir_, "engine_cache_dir", "Engine Cache Directory", - "Path to a folder containing cached engine files to be serialized and loaded from."); - result &= registrar->parameter( - plugins_lib_namespace_, "plugins_lib_namespace", "Plugins Lib Namespace", - "Namespace used to register all the plugins in this library.", std::string("")); - result &= registrar->parameter(force_engine_update_, "force_engine_update", "Force Engine Update", - "Always update engine regard less of existing engine file. " - "Such conversion may take minutes. Default to false.", - false); - - result &= registrar->parameter(input_tensor_names_, "input_tensor_names", "Input Tensor Names", - "Names of input tensors in the order to be fed into the model."); - result &= registrar->parameter(input_binding_names_, "input_binding_names", "Input Binding Names", - "Names of input bindings as in the model in the same order of " - "what is provided in input_tensor_names."); - - result &= registrar->parameter( - input_state_tensor_names_, "input_state_tensor_names", "Input State Tensor Names", - "Names of input state tensors that are used internally by TensorRT.", - std::vector{}); - - result &= registrar->parameter(output_tensor_names_, "output_tensor_names", "Output Tensor Names", - "Names of output tensors in the order to be retrieved " - "from the model."); - result &= registrar->parameter( - output_state_tensor_names_, "output_state_tensor_names", "Output State Tensor Names", - "Names of output state tensors that are used internally by TensorRT.", - std::vector{}); - - result &= - registrar->parameter(output_binding_names_, "output_binding_names", "Output Binding Names", - "Names of output bindings in the model in the same " - "order of of what is provided in output_tensor_names."); - result &= registrar->parameter(pool_, "pool", "Pool", "Allocator instance for output tensors."); - result &= registrar->parameter(cuda_stream_pool_, "cuda_stream_pool", "Cuda Stream Pool", - "Instance of gxf::CudaStreamPool to allocate CUDA stream."); - - result &= registrar->parameter(max_workspace_size_, "max_workspace_size", "Max Workspace Size", - "Size of working space in bytes. Default to 64MB", 67108864l); - result &= - registrar->parameter(dla_core_, "dla_core", "DLA Core", - "DLA Core to use. Fallback to GPU is always enabled. " - "Default to use GPU only.", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(max_batch_size_, "max_batch_size", "Max Batch Size", - "Maximum possible batch size in case the first dimension is " - "dynamic and used as batch size.", - 1); - result &= registrar->parameter(enable_fp16_, "enable_fp16_", "Enable FP16 Mode", - "Enable inference with FP16 and FP32 fallback.", false); - - result &= registrar->parameter(verbose_, "verbose", "Verbose", - "Enable verbose logging on console. Default to false.", false); - result &= registrar->parameter(relaxed_dimension_check_, "relaxed_dimension_check", - "Relaxed Dimension Check", - "Ignore dimensions of 1 for input tensor dimension check.", true); - result &= - registrar->parameter(clock_, "clock", "Clock", "Instance of clock for publish time.", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - - result &= registrar->parameter(rx_, "rx", "RX", "List of receivers to take input tensors"); - result &= registrar->parameter(tx_, "tx", "TX", "Transmitter to publish output tensors"); - - return gxf::ToResultCode(result); -} - -gxf_result_t TensorRtInference::start() { - // Validates parameter - if (!EndsWith(model_file_path_.get(), ".onnx")) { - GXF_LOG_ERROR("Only supports ONNX model: %s.", model_file_path_.get().c_str()); - return GXF_FAILURE; - } - if (rx_.get().size() == 0) { - GXF_LOG_ERROR("At least one receiver is needed."); - return GXF_FAILURE; - } - - if (input_tensor_names_.get().size() != input_binding_names_.get().size()) { - GXF_LOG_ERROR("Mismatching number of input tensor names and bindings: %lu vs %lu.", - input_tensor_names_.get().size(), input_binding_names_.get().size()); - return GXF_FAILURE; - } - if (output_tensor_names_.get().size() != output_binding_names_.get().size()) { - GXF_LOG_ERROR("Mismatching number of output tensor names and bindings: %lu vs %lu.", - output_tensor_names_.get().size(), output_binding_names_.get().size()); - return GXF_FAILURE; - } - - // Check input and output state tensor names - state_tensor_count_ = input_state_tensor_names_.get().size(); - if (input_state_tensor_names_.get().size() != output_state_tensor_names_.get().size()) { - GXF_LOG_ERROR( - "Number of output state tensors %d does not match number of input state " - "tensors %d", - output_state_tensor_names_.get().size(), input_state_tensor_names_.get().size()); - return GXF_FAILURE; - } - - // Initializes TensorRT registered plugins - cuda_logger_.setVerbose(verbose_.get()); - const auto plugins_lib_namespace = plugins_lib_namespace_.try_get(); - if (plugins_lib_namespace && - !initLibNvInferPlugins(&cuda_logger_, plugins_lib_namespace.value().c_str())) { - // Tries to proceed to see if the model would work - GXF_LOG_WARNING("Could not initialize LibNvInferPlugins."); - } - - // Create the cuda stream if it does not yet exist. - // This is needed to populate cuda_stream_ before calling queryHostEngineCapability() - if (!cuda_stream_) { - auto maybe_stream = cuda_stream_pool_.get()->allocateStream(); - if (!maybe_stream) { - GXF_LOG_ERROR("Failed to allocate CUDA stream"); - return maybe_stream.error(); - } - cuda_stream_ = std::move(maybe_stream.value()); - } - - gxf::Expected maybe_host_engine_capability = queryHostEngineCapability(); - if (!maybe_host_engine_capability) { - GXF_LOG_ERROR("Failed to query host engine capability."); - return GXF_FAILURE; - } - - std::string host_engine_capability = maybe_host_engine_capability.value(); - GXF_LOG_INFO("Using Host Engine Capability: %s", host_engine_capability.c_str()); - - gxf::Expected maybe_engine_file_path = findEngineFilePath(host_engine_capability); - if (!maybe_engine_file_path) { - GXF_LOG_ERROR("Failed to find an engine file!"); - return GXF_FAILURE; - } - std::string engine_file_path = maybe_engine_file_path.value(); - engine_file_path_ = engine_file_path; - - if (force_engine_update_) { - // Deletes engine plan file if exists for forced update - std::remove(engine_file_path.c_str()); - if (std::ifstream(engine_file_path.c_str()).good()) { - GXF_LOG_ERROR("Failed to remove engine plan file %s for forced engine update.", - engine_file_path.c_str()); - return GXF_FAILURE; - } - } - - // Loads Cuda engine into std::vector plan or creates it if needed. - std::vector plan; - if (force_engine_update_ || !ReadEntireBinaryFile(engine_file_path, plan)) { - const char* warning_note = force_engine_update_ ? " (forced by config)" : ""; - GXF_LOG_WARNING( - "Rebuilding CUDA engine %s%s. " - "Note: this process may take up to several minutes.", - engine_file_path.c_str(), warning_note); - auto result = convertModelToEngine(); - if (!result) { - GXF_LOG_ERROR("Failed to create engine plan for model %s.", model_file_path_.get().c_str()); - return gxf::ToResultCode(result); - } - - // Skips loading file and uses in-memory engine plan directly. - plan = std::move(result.value()); - - // Tries to serializes the plan and proceeds anyway - if (!SerializeEnginePlan(plan, engine_file_path)) { - GXF_LOG_ERROR( - "Engine plan serialization failed. Proceeds with in-memory engine plan anyway."); - } - } - - // Creates inference runtime for the plan - NvInferHandle infer_runtime(nvinfer1::createInferRuntime(cuda_logger_)); - - // Deserialize the CUDA engine - if (verbose_.get()) { GXF_LOG_DEBUG("Creating inference runtime."); } - cuda_engine_.reset(infer_runtime->deserializeCudaEngine(plan.data(), plan.size(), NULL)); - - // Debug spews - if (verbose_.get()) { - GXF_LOG_DEBUG("Number of CUDA bindings: %d", cuda_engine_->getNbBindings()); - for (int i = 0; i < cuda_engine_->getNbBindings(); ++i) { - GXF_LOG_DEBUG("CUDA binding No.%d: name %s Format %s", i, cuda_engine_->getBindingName(i), - cuda_engine_->getBindingFormatDesc(i)); - } - } - - // Checks binding numbers against parameter - const uint64_t input_number = input_tensor_names_.get().size(); - const uint64_t output_number = output_tensor_names_.get().size(); - const int64_t total_bindings_number = input_number + output_number; - if (cuda_engine_->getNbBindings() != static_cast(total_bindings_number)) { - GXF_LOG_ERROR( - "Numbers of CUDA bindings mismatch: configured for %lu vs model requires %d. " - "Please check TensorRTInference codelet configuration.\n", - total_bindings_number, cuda_engine_->getNbBindings()); - return GXF_ARGUMENT_INVALID; - } - - // Creates cuda execution context - cuda_execution_ctx_.reset(cuda_engine_->createExecutionContext()); - - // Allocates CUDA buffer pointers for binding to be populated in tick() - cuda_buffers_.resize(input_tensor_names_.get().size() + output_tensor_names_.get().size(), - nullptr); - - // Initialize internal state tensors - internal_states_ = gxf::Entity::New(context()); - if (!internal_states_) { return gxf::ToResultCode(internal_states_); } - - // Keeps record of input bindings - binding_infos_.clear(); - for (uint64_t j = 0; j < input_number; ++j) { - const std::string& tensor_name = input_tensor_names_.get()[j]; - const std::string& binding_name = input_binding_names_.get()[j]; - - const int32_t binding_index = cuda_engine_->getBindingIndex(binding_name.c_str()); - if (binding_index == -1) { - GXF_LOG_ERROR("Failed to get binding index for input %s in model %s", binding_name.c_str(), - engine_file_path.c_str()); - return GXF_FAILURE; - } - - if (binding_index >= static_cast(cuda_buffers_.size())) { - GXF_LOG_ERROR("Binding index for input %s is out of range in model %s.", binding_name.c_str(), - engine_file_path.c_str()); - return GXF_FAILURE; - } - - // Checks element type - const auto maybe_element_type = - NvInferDatatypeToTensorElementType(cuda_engine_->getBindingDataType(binding_index)); - if (!maybe_element_type) { - GXF_LOG_ERROR("Unsupported element type for binding input %s on index %d. ", - binding_name.c_str(), binding_index); - return maybe_element_type.error(); - } - - // Keeps binding info - const auto& dims = cuda_engine_->getBindingDimensions(binding_index); - binding_infos_[tensor_name] = - BindingInfo{binding_index, static_cast(dims.nbDims), binding_name, - maybe_element_type.value(), Dims2Dimensions(dims)}; - - // Debug spew - if (verbose_.get()) { - GXF_LOG_DEBUG( - "Input Tensor %s:%s index %d Dimensions %s.", tensor_name.c_str(), binding_name.c_str(), - binding_index, - FormatDims(binding_infos_[tensor_name].dimensions, binding_infos_[tensor_name].rank) - .c_str()); - } - - // Create tensor for input states - if (std::find(input_state_tensor_names_.get().begin(), input_state_tensor_names_.get().end(), - tensor_name) != input_state_tensor_names_.get().end()) { - const BindingInfo& binding_info = binding_infos_[tensor_name]; - const gxf::Shape shape{Dims2Dimensions(dims), binding_info.rank}; - - const auto maybe_input_state_tensor = - internal_states_.value().add(tensor_name.c_str()); - if (!maybe_input_state_tensor) { - GXF_LOG_ERROR("Failed to create input state tensor %s.", tensor_name.c_str()); - return maybe_input_state_tensor.error(); - } - - const auto result = maybe_input_state_tensor.value()->reshapeCustom( - shape, binding_info.element_type, gxf::PrimitiveTypeSize(binding_info.element_type), - gxf::Unexpected{GXF_UNINITIALIZED_VALUE}, gxf::MemoryStorageType::kDevice, pool_); - if (!result) { - GXF_LOG_ERROR("Failed to allocate for input state tensor %s", tensor_name.c_str()); - return gxf::ToResultCode(result); - } - } - } - - // Keeps record of output bindings - for (uint64_t j = 0; j < output_number; ++j) { - const std::string& tensor_name = output_tensor_names_.get()[j]; - const std::string& binding_name = output_binding_names_.get()[j]; - - const int32_t binding_index = cuda_engine_->getBindingIndex(binding_name.c_str()); - if (binding_index == -1) { - GXF_LOG_ERROR("Failed to get binding index for output %s", binding_name.c_str()); - return GXF_FAILURE; - } - if (binding_index >= static_cast(cuda_buffers_.size())) { - GXF_LOG_ERROR("Binding index for output %s is out of range. %d >= %d", binding_name.c_str(), - binding_index, static_cast(cuda_buffers_.size())); - return GXF_FAILURE; - } - - // Checks element type - const auto maybe_element_type = - NvInferDatatypeToTensorElementType(cuda_engine_->getBindingDataType(binding_index)); - if (!maybe_element_type) { - GXF_LOG_ERROR("Unsupported element type for binding output %s on index %d. ", - binding_name.c_str(), binding_index); - return maybe_element_type.error(); - } - - // Keeps binding info - const auto& dims = cuda_engine_->getBindingDimensions(binding_index); - binding_infos_[tensor_name] = - BindingInfo{binding_index, static_cast(dims.nbDims), binding_name, - maybe_element_type.value(), Dims2Dimensions(dims)}; - cuda_buffers_[binding_index] = nullptr; // populate cuda_buffers dynamically, in tick() - - if (verbose_.get()) { - GXF_LOG_DEBUG( - "Output Tensor %s:%s (%d), Dimensions: %s.", tensor_name.c_str(), binding_name.c_str(), - binding_index, - FormatDims(binding_infos_[tensor_name].dimensions, binding_infos_[tensor_name].rank) - .c_str()); - } - } - - // Grabs CUDA stream and creates CUDA event for synchronization - if (!cuda_stream_) { - auto maybe_stream = cuda_stream_pool_.get()->allocateStream(); - if (!maybe_stream) { - GXF_LOG_ERROR("Failed to allocate CUDA stream"); - return maybe_stream.error(); - } - cuda_stream_ = std::move(maybe_stream.value()); - } - auto stream = cuda_stream_.get()->stream(); - if (!stream) { - GXF_LOG_ERROR("Failed to grab CUDA stream"); - return stream.error(); - } - cached_cuda_stream_ = stream.value(); - auto result = cudaEventCreate(&cuda_event_consumed_); - if (cudaSuccess != result) { - GXF_LOG_ERROR("Failed to create consumed CUDA event: %s", cudaGetErrorString(result)); - return GXF_FAILURE; - } - auto result2 = cudaEventCreate(&cuda_event_done_); - if (cudaSuccess != result2) { - GXF_LOG_ERROR("Failed to create done CUDA event: %s", cudaGetErrorString(result2)); - return GXF_FAILURE; - } - - return GXF_SUCCESS; -} - -gxf::Expected> TensorRtInference::convertModelToEngine() { - // Creates the engine Builder - NvInferHandle builder(nvinfer1::createInferBuilder(cuda_logger_)); - - // Builder Config provides options to the Builder - NvInferHandle builderConfig(builder->createBuilderConfig()); - builderConfig->setMaxWorkspaceSize(max_workspace_size_); - - // Sets DLA core if provided and always fall back to GPU - auto dla_core = dla_core_.try_get(); - if (dla_core) { - builderConfig->setDefaultDeviceType(nvinfer1::DeviceType::kDLA); - builderConfig->setFlag(nvinfer1::BuilderFlag::kGPU_FALLBACK); - builderConfig->setDLACore(dla_core.value()); - } - if (enable_fp16_.get()) { builderConfig->setFlag(nvinfer1::BuilderFlag::kFP16); } - - // Parses ONNX with explicit batch size for support of dynamic shapes/batch - NvInferHandle network(builder->createNetworkV2( - 1U << static_cast(nvinfer1::NetworkDefinitionCreationFlag::kEXPLICIT_BATCH))); - - NvInferHandle onnx_parser( - nvonnxparser::createParser(*network, cuda_logger_)); - if (!onnx_parser->parseFromFile(model_file_path_.get().c_str(), - static_cast(nvinfer1::ILogger::Severity::kWARNING))) { - GXF_LOG_ERROR("Failed to parse ONNX file %s", model_file_path_.get().c_str()); - return gxf::Unexpected{GXF_FAILURE}; - } - - // Provides optimization profile for dynamic size input bindings - nvinfer1::IOptimizationProfile* optimization_profile = builder->createOptimizationProfile(); - // Checks input dimensions and adds to optimization profile if needed - const int number_inputs = network->getNbInputs(); - for (int i = 0; i < number_inputs; ++i) { - auto* bind_tensor = network->getInput(i); - const char* bind_name = bind_tensor->getName(); - nvinfer1::Dims dims = bind_tensor->getDimensions(); - - // Validates binding info - if (dims.nbDims <= 0) { - GXF_LOG_ERROR("Invalid input tensor dimensions for binding %s", bind_name); - return gxf::Unexpected{GXF_ARGUMENT_INVALID}; - } - for (int j = 1; j < dims.nbDims; ++j) { - if (dims.d[j] <= 0) { - GXF_LOG_ERROR( - "Input binding %s requires dynamic size on dimension No.%d which is not supported", - bind_tensor->getName(), j); - return gxf::Unexpected{GXF_ARGUMENT_OUT_OF_RANGE}; - } - } - if (dims.d[0] == -1) { - // Only case with first dynamic dimension is supported and assumed to be batch size. - // Always optimizes for 1-batch. - dims.d[0] = 1; - optimization_profile->setDimensions(bind_name, nvinfer1::OptProfileSelector::kMIN, dims); - optimization_profile->setDimensions(bind_name, nvinfer1::OptProfileSelector::kOPT, dims); - dims.d[0] = max_batch_size_.get(); - if (max_batch_size_.get() <= 0) { - GXF_LOG_ERROR("Maximum batch size %d is invalid. Uses 1 instead.", max_batch_size_.get()); - dims.d[0] = 1; - } - optimization_profile->setDimensions(bind_name, nvinfer1::OptProfileSelector::kMAX, dims); - } - } - builderConfig->addOptimizationProfile(optimization_profile); - - // Creates TensorRT Engine Plan - NvInferHandle engine( - builder->buildEngineWithConfig(*network, *builderConfig)); - if (!engine) { - GXF_LOG_ERROR("Failed to build TensorRT engine from model %s.", model_file_path_.get().c_str()); - return gxf::Unexpected{GXF_FAILURE}; - } - - NvInferHandle model_stream(engine->serialize()); - if (!model_stream || model_stream->size() == 0 || model_stream->data() == nullptr) { - GXF_LOG_ERROR("Fail to serialize TensorRT Engine."); - return gxf::Unexpected{GXF_FAILURE}; - } - - // Prepares return value - std::vector result; - const char* data = static_cast(model_stream->data()); - result.resize(model_stream->size()); - std::copy(data, data + model_stream->size(), result.data()); - return result; -} - -gxf_result_t TensorRtInference::stop() { - cuda_execution_ctx_ = nullptr; - cuda_engine_ = nullptr; - cuda_buffers_.clear(); - - auto result = cudaEventDestroy(cuda_event_consumed_); - if (cudaSuccess != result) { - GXF_LOG_ERROR("Failed to destroy consumed CUDA event: %s", cudaGetErrorString(result)); - return GXF_FAILURE; - } - auto result2 = cudaEventDestroy(cuda_event_done_); - if (cudaSuccess != result2) { - GXF_LOG_ERROR("Failed to create done CUDA event: %s", cudaGetErrorString(result2)); - return GXF_FAILURE; - } - - // Release resources for the internal states (tensor map as entity). - // NOTE:: Without this, the following message can be seen at the end of the execution: - // "PANIC ./gxf/core/handle.hpp@174: Invalid Component Pointer." - internal_states_ = gxf::Unexpected{GXF_UNINITIALIZED_VALUE}; - - return GXF_SUCCESS; -} - -gxf_result_t TensorRtInference::tick() { - // Grabs latest messages from all receivers - std::vector messages; - messages.reserve(rx_.get().size()); - for (auto& rx : rx_.get()) { - gxf::Expected maybe_message = rx->receive(); - if (maybe_message) { messages.push_back(std::move(maybe_message.value())); } - } - if (messages.empty()) { - GXF_LOG_ERROR("No message available."); - return GXF_CONTRACT_MESSAGE_NOT_AVAILABLE; - } - // Tries to retrieve timestamp if clock present - gxf::Expected> maybe_input_timestamp = gxf::Unexpected{GXF_FAILURE}; - for (auto& msg : messages) { - maybe_input_timestamp = msg.get("timestamp"); - if (maybe_input_timestamp) { break; } - } - // Populates input tensors - for (uint32_t input_index = 0; input_index < input_tensor_names_.get().size(); ++input_index) { - const auto& tensor_name = input_tensor_names_.get()[input_index]; - - gxf::Expected> maybe_tensor = gxf::Unexpected{GXF_UNINITIALIZED_VALUE}; - if (std::find(input_state_tensor_names_.get().begin(), input_state_tensor_names_.get().end(), - tensor_name) == input_state_tensor_names_.get().end()) { - for (auto& msg : messages) { - maybe_tensor = msg.get(tensor_name.c_str()); - if (maybe_tensor) { break; } - } - if (!maybe_tensor) { - GXF_LOG_ERROR("Failed to retrieve Tensor %s", tensor_name.c_str()); - return GXF_FAILURE; - } - } else { - maybe_tensor = internal_states_.value().get(tensor_name.c_str()); - if (!maybe_tensor) { - GXF_LOG_ERROR("Failed to retrieve Tensor %s from internal states", tensor_name.c_str()); - return GXF_FAILURE; - } - } - - if (!maybe_tensor) { - GXF_LOG_ERROR("Failed to retrieve Tensor %s", tensor_name.c_str()); - return GXF_FAILURE; - } - - gxf::Tensor& input_tensor = *maybe_tensor.value(); - - // Validates input tensor against model bindings then binds and populates buffers - const auto& shape = input_tensor.shape(); - const auto& binding_info = binding_infos_[tensor_name]; - nvinfer1::Dims dims; - dims.nbDims = binding_info.rank; - for (int32_t i = 0; i < dims.nbDims; ++i) { dims.d[i] = binding_info.dimensions[i]; } - - // Checks input tensor element type - if (input_tensor.element_type() != binding_info.element_type) { - GXF_LOG_ERROR("Mismatching tensor element type required %d vs provided %d", - binding_info.element_type, input_tensor.element_type()); - return GXF_FAILURE; - } - - if (relaxed_dimension_check_.get()) { - // Relaxed dimension match. Ignore all 1s. Binding of -1 is considered as match. - const uint32_t shape_rank = shape.rank(); - uint32_t shape_rank_matched = 0; - uint32_t binding_rank_matched = 0; - bool matched = true; - for (uint32_t i = 0; i < gxf::Shape::kMaxRank * 2; ++i) { - if (shape_rank_matched >= shape_rank || binding_rank_matched >= binding_info.rank) { - break; - } - if (shape.dimension(shape_rank_matched) == 1) { - shape_rank_matched++; - continue; - } - if (binding_info.dimensions[binding_rank_matched] == 1) { - binding_rank_matched++; - continue; - } - if (binding_info.dimensions[binding_rank_matched] == -1) { - // Matches dimension - dims.d[binding_rank_matched] = shape.dimension(shape_rank_matched); - shape_rank_matched++; - binding_rank_matched++; - continue; - } - if (shape.dimension(shape_rank_matched) != binding_info.dimensions[binding_rank_matched]) { - matched = false; - break; - } - shape_rank_matched++; - binding_rank_matched++; - } - if (!matched || shape_rank_matched != shape_rank || - binding_rank_matched != binding_info.rank) { - GXF_LOG_ERROR( - "Input Tensor %s bound to %s:" - " dimensions does not meet model spec with relaxed matching. Expected: %s Real: %s", - tensor_name.c_str(), binding_info.binding_name.c_str(), - FormatDims(binding_info.dimensions, binding_info.rank).c_str(), - FormatTensorShape(shape).c_str()); - return GXF_FAILURE; - } - } else { - // Strict dimension match. All dimensions must match. Binding of -1 is considered as match. - if (shape.rank() != binding_info.rank) { - GXF_LOG_ERROR("Tensor %s bound to %s has mismatching rank %d (%d required)", - tensor_name.c_str(), binding_info.binding_name.c_str(), shape.rank(), - binding_info.rank); - return GXF_FAILURE; - } - for (uint32_t i = 0; i < binding_info.rank; i++) { - if (binding_info.dimensions[i] == -1) { dims.d[i] = shape.dimension(i); } - if (shape.dimension(i) != binding_info.dimensions[i] && binding_info.dimensions[i] != -1) { - GXF_LOG_ERROR("Tensor %s bound to %s has mismatching dimension %d:%d (%d required)", - tensor_name.c_str(), binding_info.binding_name.c_str(), i, - shape.dimension(i), binding_info.dimensions[i]); - return GXF_FAILURE; - } - } - } - - // Updates the latest dimension of input tensor - if (!cuda_execution_ctx_->setBindingDimensions(binding_info.index, dims)) { - GXF_LOG_ERROR("Failed to update input binding %s dimensions.", - binding_info.binding_name.c_str()); - return GXF_FAILURE; - } - - // Binds input tensor buffer - cuda_buffers_[binding_info.index] = input_tensor.pointer(); - } - - // Creates result message entity - gxf::Expected maybe_result_message = gxf::Entity::New(context()); - if (!maybe_result_message) { return gxf::ToResultCode(maybe_result_message); } - - // Creates tensors for output - for (const auto& tensor_name : output_tensor_names_.get()) { - auto maybe_result_tensor = maybe_result_message.value().add(tensor_name.c_str()); - if (!maybe_result_tensor) { - GXF_LOG_ERROR("Failed to create output tensor %s", tensor_name.c_str()); - return gxf::ToResultCode(maybe_result_tensor); - } - - // Queries binding dimension from context and allocates tensor accordingly - const auto& binding_info = binding_infos_[tensor_name]; - const auto binding_dims = cuda_engine_->getBindingDimensions(binding_info.index); - gxf::Shape shape{Dims2Dimensions(binding_dims), binding_info.rank}; - - auto result = maybe_result_tensor.value()->reshapeCustom( - shape, binding_info.element_type, gxf::PrimitiveTypeSize(binding_info.element_type), - gxf::Unexpected{GXF_UNINITIALIZED_VALUE}, gxf::MemoryStorageType::kDevice, pool_); - if (!result) { - GXF_LOG_ERROR("Failed to allocate for output tensor %s", tensor_name.c_str()); - return gxf::ToResultCode(result); - } - - // Allocates gpu buffer for output tensors - cuda_buffers_[binding_info.index] = maybe_result_tensor.value()->pointer(); - } - - auto maybe_stream_id = maybe_result_message.value().add("TensorRTCuStream"); - if (!maybe_stream_id) { - GXF_LOG_ERROR("failed to add TensorRTCuStream"); - return gxf::ToResultCode(maybe_stream_id); - } - maybe_stream_id.value()->stream_cid = cuda_stream_.cid(); - GXF_ASSERT(maybe_stream_id.value()->stream_cid != kNullUid, "Internal error: stream_cid is null"); - - // Runs inference on specified CUDA stream - if (!cuda_execution_ctx_->enqueueV2(cuda_buffers_.data(), cached_cuda_stream_, - &cuda_event_consumed_)) { - GXF_LOG_ERROR("TensorRT task enqueue for engine %s failed.", engine_file_path_.c_str()); - return GXF_FAILURE; - } - // Blocks until job is done - auto result = cudaEventRecord(cuda_event_done_, cached_cuda_stream_); - if (cudaSuccess != result) { - GXF_LOG_ERROR("Failed to queue event for task done with engine %s: %s", - engine_file_path_.c_str(), cudaGetErrorString(result)); - return GXF_FAILURE; - } - auto result2 = cudaEventSynchronize(cuda_event_done_); - if (cudaSuccess != result2) { - GXF_LOG_ERROR("Failed to synchronize for task done with engine %s: %s", - engine_file_path_.c_str(), cudaGetErrorString(result2)); - return GXF_FAILURE; - } - - // Copy output state tensor to the input state tensor of the next stage - for (uint32_t i = 0; i < state_tensor_count_; ++i) { - // Get output state tensor - const auto out_state_tensor = maybe_result_message.value() - .get(output_state_tensor_names_.get()[i].c_str()) - .value(); - if (!out_state_tensor) { - GXF_LOG_ERROR("Failed to get output tensor %s", output_state_tensor_names_.get()[i].c_str()); - return GXF_FAILURE; - } - const auto out_tensor_ptr = out_state_tensor->pointer(); - if (!out_tensor_ptr) { - GXF_LOG_ERROR("Output tensor %s has null pointer", - output_state_tensor_names_.get()[i].c_str()); - return GXF_FAILURE; - } - - // Get input state tensor - const auto in_state_tensor = internal_states_.value() - .get(input_state_tensor_names_.get()[i].c_str()) - .value(); - if (!in_state_tensor) { - GXF_LOG_ERROR("Failed to get output tensor %s", output_state_tensor_names_.get()[i].c_str()); - return GXF_FAILURE; - } - const auto in_state_tensor_ptr = in_state_tensor->pointer(); - if (!in_state_tensor_ptr) { - GXF_LOG_ERROR("Input state tensor %s has null pointer", - input_state_tensor_names_.get()[i].c_str()); - return GXF_FAILURE; - } - - const size_t state_tensor_size = out_state_tensor->size(); - if (state_tensor_size != in_state_tensor->size()) { - GXF_LOG_ERROR( - "Output state tensor %s (size: %d) does not match input state " - "tensor %s (size: %d)", - output_state_tensor_names_.get()[i].c_str(), state_tensor_size, - input_state_tensor_names_.get()[i].c_str(), in_state_tensor->size()); - return GXF_FAILURE; - } - - // Copy output state tensor to input state tensor - cudaError_t cuda_status = CUDA_TRY(cudaMemcpy(in_state_tensor_ptr, out_tensor_ptr, - state_tensor_size, cudaMemcpyDeviceToDevice)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to copy output tensor %s to input tensor %s: %s", - output_state_tensor_names_.get()[0].c_str(), - input_state_tensor_names_.get()[0].c_str(), cudaGetErrorString(cuda_status)); - return GXF_FAILURE; - } - } - - // Publishes result with acqtime - if (maybe_input_timestamp) { // if input timestamp is present, use it's acqtime - return gxf::ToResultCode( - tx_->publish(maybe_result_message.value(), maybe_input_timestamp.value()->acqtime)); - } else { // else simply use 0 as acqtime - return gxf::ToResultCode(tx_->publish(maybe_result_message.value(), 0)); - } -} - -static std::string replaceChar(const std::string& string, char match, char replacement) { - std::string result = string; - std::replace(result.begin(), result.end(), match, replacement); - return result; -} - -gxf::Expected TensorRtInference::queryHostEngineCapability() const { - char* env_var = std::getenv("GXF_TENSORRT_HOST_ENGINE_CAPABILITY"); - if (env_var != nullptr) { - GXF_LOG_INFO("Using GXF_TENSORRT_HOST_ENGINE_CAPABILITY overwrite: %s", env_var); - return std::string(env_var); - } - cudaDeviceProp device_prop = {0}; - cudaError_t status = cudaGetDeviceProperties(&device_prop, cuda_stream_.get()->dev_id()); - if (status != cudaSuccess) { - GXF_LOG_ERROR("Failed to get cuda device properties with errorcode: %d", status); - return gxf::Unexpected{}; - } - std::string device_name = device_prop.name; - device_name = replaceChar(device_name, ' ', '-'); - std::stringstream ss; - // TensorRT builds an engine file per device that changes based on the number of SMs available. - // This returns a string that should be a unique mapping per device and SM configuration. - ss << device_name << "_c" << device_prop.major << device_prop.minor << "_n" - << device_prop.multiProcessorCount; - return ss.str(); -} - -gxf::Expected TensorRtInference::findEngineFilePath( - const std::string& host_engine_capability) const { - std::string engine_file_path; - if (!IsValidDirectory(engine_cache_dir_.get())) { - GXF_LOG_ERROR( - "Engine cache directory '%s' does not exist! Please create a valid cache directory.", - engine_cache_dir_.get().c_str()); - return gxf::Unexpected{}; - } - engine_file_path = engine_cache_dir_.get() + "/" + host_engine_capability + ".engine"; - GXF_LOG_INFO("Loading engine cache dir file: %s", engine_file_path.c_str()); - if (engine_file_path.empty()) { - GXF_LOG_ERROR("Engine file path not specified!"); - return gxf::Unexpected{}; - } - return engine_file_path; -} - -} // namespace custom_lstm_inference -} // namespace holoscan -} // namespace nvidia diff --git a/gxf_extensions/custom_lstm_inference/tensor_rt_inference.hpp b/gxf_extensions/custom_lstm_inference/tensor_rt_inference.hpp deleted file mode 100644 index 03898ba2..00000000 --- a/gxf_extensions/custom_lstm_inference/tensor_rt_inference.hpp +++ /dev/null @@ -1,140 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_CUSTOM_LSTM_INFERENCE_TENSOR_RT_INFERENCE_HPP_ -#define NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_CUSTOM_LSTM_INFERENCE_TENSOR_RT_INFERENCE_HPP_ - -#include -#include - -#include -#include -#include -#include -#include - -#include "gxf/core/entity.hpp" -#include "gxf/core/gxf.h" -#include "gxf/core/parameter.hpp" -#include "gxf/cuda/cuda_stream.hpp" -#include "gxf/cuda/cuda_stream_pool.hpp" -#include "gxf/std/allocator.hpp" -#include "gxf/std/clock.hpp" -#include "gxf/std/codelet.hpp" -#include "gxf/std/receiver.hpp" -#include "gxf/std/tensor.hpp" -#include "gxf/std/transmitter.hpp" - -namespace nvidia::holoscan::custom_lstm_inference { - -// Logger for TensorRT to redirect logging into gxf console spew. -class TensorRTInferenceLogger : public nvinfer1::ILogger { - public: - void log(ILogger::Severity severity, const char* msg) throw() override; - // Sets verbose flag for logging - void setVerbose(bool verbose); - - private: - bool verbose_; -}; - -/// @brief Loads ONNX model, takes input tensors and run inference against them with TensorRT. -/// -/// It takes input from all receivers provided and try to locate Tensor component with specified -/// name on them one by one. The first occurrence would be used. Only takes gpu memory tensor. -/// Supports dynamic batch as first dimension. -/// The codelet has an engine cache directory that can be pre-populated to reduce start time. -/// If the engine cache directory has no pre-existing engine file for an architecture, it will -/// generate this dynamically. -/// Requires gxf::CudaStream to run load on specific CUDA stream. -class TensorRtInference : public gxf::Codelet { - public: - gxf_result_t start() override; - gxf_result_t tick() override; - gxf_result_t stop() override; - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - - private: - // Helper to return a string for the TRT engine capability. - gxf::Expected queryHostEngineCapability() const; - // Helper to search for the engine file path. - gxf::Expected findEngineFilePath(const std::string& host_engine_capability) const; - - // Helper deleter to call destroy while destroying the cuda objects - template - struct DeleteFunctor { - inline void operator()(void* ptr) { reinterpret_cast(ptr)->destroy(); } - }; - // unique_ptr using custom Delete Functor above - template - using NvInferHandle = std::unique_ptr>; - - // To cache binding info for tensors - typedef struct { - int32_t index; - uint32_t rank; - std::string binding_name; - gxf::PrimitiveType element_type; - std::array dimensions; - } BindingInfo; - std::unordered_map binding_infos_; - - // Converts loaded model to engine plan - gxf::Expected> convertModelToEngine(); - - gxf::Parameter model_file_path_; - gxf::Parameter engine_cache_dir_; - gxf::Parameter plugins_lib_namespace_; - gxf::Parameter force_engine_update_; - gxf::Parameter> input_tensor_names_; - gxf::Parameter> input_state_tensor_names_; - gxf::Parameter> input_binding_names_; - gxf::Parameter> output_tensor_names_; - gxf::Parameter> output_state_tensor_names_; - gxf::Parameter> output_binding_names_; - gxf::Parameter> pool_; - gxf::Parameter> cuda_stream_pool_; - gxf::Parameter max_workspace_size_; - gxf::Parameter dla_core_; - gxf::Parameter max_batch_size_; - gxf::Parameter enable_fp16_; - gxf::Parameter relaxed_dimension_check_; - gxf::Parameter verbose_; - gxf::Parameter> clock_; - - gxf::Parameter>> rx_; - gxf::Parameter> tx_; - - // Logger instance for TensorRT - TensorRTInferenceLogger cuda_logger_; - - NvInferHandle cuda_execution_ctx_; - NvInferHandle cuda_engine_; - - gxf::Handle cuda_stream_; - std::vector cuda_buffers_; - cudaStream_t cached_cuda_stream_; - cudaEvent_t cuda_event_consumed_; - cudaEvent_t cuda_event_done_; - - uint32_t state_tensor_count_ = 0; - gxf::Expected internal_states_ = gxf::Unexpected{GXF_UNINITIALIZED_VALUE}; - std::string engine_file_path_; -}; - -} // namespace nvidia::holoscan::custom_lstm_inference - -#endif // NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_CUSTOM_LSTM_INFERENCE_TENSOR_RT_INFERENCE_HPP_ diff --git a/gxf_extensions/emergent/CMakeLists.txt b/gxf_extensions/emergent/CMakeLists.txt deleted file mode 100644 index 097ebf65..00000000 --- a/gxf_extensions/emergent/CMakeLists.txt +++ /dev/null @@ -1,51 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Create library -add_library(emergent_source_lib SHARED - emergent_source.cpp - emergent_source.hpp -) -target_link_libraries(emergent_source_lib - PUBLIC - CUDA::cudart - CUDA::cuda_driver - GXF::multimedia - GXF::std - yaml-cpp - EmergentCamera -) -target_include_directories(emergent_source_lib - SYSTEM - BEFORE - PUBLIC - "/opt/EVT/eSDK/include" -) -target_link_directories(emergent_source_lib - BEFORE - PUBLIC - "/opt/EVT/eSDK/lib/" -) - -# Create extension -add_library(emergent_source SHARED - emergent_ext.cpp -) -target_link_libraries(emergent_source - PUBLIC emergent_source_lib -) - -# Install GXF extension as a component 'holoscan-gxf_extensions' -install_gxf_extension(emergent_source) diff --git a/gxf_extensions/emergent/emergent_ext.cpp b/gxf_extensions/emergent/emergent_ext.cpp deleted file mode 100644 index 02dbb9de..00000000 --- a/gxf_extensions/emergent/emergent_ext.cpp +++ /dev/null @@ -1,26 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "gxf/std/extension_factory_helper.hpp" - -#include "emergent_source.hpp" - -GXF_EXT_FACTORY_BEGIN() -GXF_EXT_FACTORY_SET_INFO(0xa874c830207a47e7, 0x9270c6997e5ff8dc, "Emergent", "Emergent Extension", - "NVIDIA", "1.0.0", "LICENSE"); -GXF_EXT_FACTORY_ADD(0xcaf7eb335da24a28, 0xbdcd3f3319a356db, nvidia::holoscan::EmergentSource, - nvidia::gxf::Codelet, "Emergent Source Codelet"); -GXF_EXT_FACTORY_END() diff --git a/gxf_extensions/emergent/emergent_source.cpp b/gxf_extensions/emergent/emergent_source.cpp deleted file mode 100644 index 212c5551..00000000 --- a/gxf_extensions/emergent/emergent_source.cpp +++ /dev/null @@ -1,302 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include "gxf/core/handle.hpp" -#include "gxf/multimedia/video.hpp" -#include "emergent_source.hpp" - - -namespace nvidia { -namespace holoscan { - -EmergentSource::EmergentSource() {} - -gxf_result_t EmergentSource::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - result &= registrar->parameter( - signal_, "signal", "Output", "Output channel"); - result &= registrar->parameter( - width_, "width", "Width", "Width of the stream", kDefaultWidth); - result &= registrar->parameter( - height_, "height", "Height", "Height of the stream", kDefaultHeight); - result &= registrar->parameter( - framerate_, "framerate", "Framerate", - "Framerate of the stream.", kDefaultFramerate); - result &= registrar->parameter( - use_rdma_, "rdma", "RDMA", "Enable RDMA.", kDefaultRDMA); - return gxf::ToResultCode(result); -} - -EVT_ERROR EmergentSource::OpenEVTCamera() { - struct GigEVisionDeviceInfo deviceInfo[kMaxCameras]; - unsigned int count, camera_index; - EVT_ERROR err = EVT_SUCCESS; - - // Find all cameras in system. - unsigned int listcam_buf_size = kMaxCameras; - EVT_ListDevices(deviceInfo, &listcam_buf_size, &count); - if (count == 0) { - GXF_LOG_ERROR("No cameras found.\n"); - return EVT_ERROR_ENODEV; - } - - // Find and use first EVT camera. - for (camera_index = 0; camera_index < kMaxCameras; camera_index++) { - char* EVT_models[] = { "HS", "HT", "HR", "HB", "LR", "LB", "HZ" }; - int EVT_models_count = sizeof(EVT_models) / sizeof(EVT_models[0]); - bool is_EVT_camera = false; - for (int i = 0; i < EVT_models_count; i++) { - if (strncmp(deviceInfo[camera_index].modelName, EVT_models[i], 2) == 0) { - is_EVT_camera = true; - break; // it is an EVT camera - } - } - if (is_EVT_camera) { - break; // Use the first EVT camera. - } - if (camera_index == (kMaxCameras - 1)) { - GXF_LOG_ERROR("No EVT cameras found.\n"); - return EVT_ERROR_ENODEV; - } - } - - // Open the camera with GPU 0 if use_rdma_ == true. - if (use_rdma_) { - camera_.gpuDirectDeviceId = 0; - } - err = EVT_CameraOpen(&camera_, &deviceInfo[camera_index]); - if (err != EVT_SUCCESS) { - GXF_LOG_ERROR("Error while opening camera.\n"); - return err; - } - return err; -} - -EVT_ERROR EmergentSource::CheckCameraCapabilities() { - EVT_ERROR err = EVT_SUCCESS; - unsigned int height_max, width_max; - unsigned int frame_rate_max, frame_rate_min; - - // Check resolution - EVT_CameraGetUInt32ParamMax(&camera_, "Height", &height_max); - EVT_CameraGetUInt32ParamMax(&camera_, "Width", &width_max); - if ((width_ == 0U) || (width_ > width_max) || - (height_ == 0U) || (height_ > height_max)) { - GXF_LOG_ERROR("Given resolution is not supported. Supported max" - " resolution is (%u, %u)\n", width_max, height_max); - return EVT_ERROR_INVAL; - } - EVT_CameraSetUInt32Param(&camera_, "Width", width_); - EVT_CameraSetUInt32Param(&camera_, "Height", height_); - - // Check Framerate - EVT_CameraGetUInt32ParamMax(&camera_, "FrameRate", &frame_rate_max); - EVT_CameraGetUInt32ParamMin(&camera_, "FrameRate", &frame_rate_min); - if ((framerate_ > frame_rate_max) || (framerate_ < frame_rate_min)) { - GXF_LOG_ERROR("Given framerate is not supported. Supported framrate" - " range is [%u, %u]\n", frame_rate_min, frame_rate_max); - return EVT_ERROR_INVAL; - } - EVT_CameraSetUInt32Param(&camera_, "FrameRate", framerate_); - - return err; -} - -void EmergentSource::SetDefaultConfiguration() { - unsigned int width_max, height_max, param_val_max; - unsigned long enum_buffer_size_return = 0; - const unsigned long enum_buffer_size = 1000; - const size_t kEnum_buffer_size = enum_buffer_size; - char enum_buffer[kEnum_buffer_size]; - char* next_token; - - // Order is important as param max/mins get updated. - EVT_CameraGetEnumParamRange(&camera_, "PixelFormat", enum_buffer, - enum_buffer_size, &enum_buffer_size_return); - char* enum_member = strtok_s(enum_buffer, ",", &next_token); - EVT_CameraSetEnumParam(&camera_, "PixelFormat", enum_member); - - EVT_CameraSetUInt32Param(&camera_, "FrameRate", 30); - - EVT_CameraSetUInt32Param(&camera_, "OffsetX", 0); - EVT_CameraSetUInt32Param(&camera_, "OffsetY", 0); - - EVT_CameraGetUInt32ParamMax(&camera_, "Width", &width_max); - EVT_CameraSetUInt32Param(&camera_, "Width", width_max); - - EVT_CameraGetUInt32ParamMax(&camera_, "Height", &height_max); - EVT_CameraSetUInt32Param(&camera_, "Height", height_max); - - EVT_CameraSetEnumParam(&camera_, "AcquisitionMode", "Continuous"); - EVT_CameraSetUInt32Param(&camera_, "AcquisitionFrameCount", 1); - EVT_CameraSetEnumParam(&camera_, "TriggerSelector", "AcquisitionStart"); - EVT_CameraSetEnumParam(&camera_, "TriggerMode", "Off"); - EVT_CameraSetEnumParam(&camera_, "TriggerSource", "Software"); - EVT_CameraSetEnumParam(&camera_, "BufferMode", "Off"); - EVT_CameraSetUInt32Param(&camera_, "BufferNum", 0); - - EVT_CameraGetUInt32ParamMax(&camera_, "GevSCPSPacketSize", ¶m_val_max); - EVT_CameraSetUInt32Param(&camera_, "GevSCPSPacketSize", param_val_max); - - EVT_CameraSetUInt32Param(&camera_, "Exposure", 3072); - - EVT_CameraSetUInt32Param(&camera_, "Gain", 4095); - EVT_CameraSetUInt32Param(&camera_, "Offset", 0); - - EVT_CameraSetBoolParam(&camera_, "LUTEnable", false); - EVT_CameraSetBoolParam(&camera_, "AutoGain", false); - - EVT_CameraSetUInt32Param(&camera_, "WB_R_GAIN_Value", 256); - EVT_CameraSetUInt32Param(&camera_, "WB_GR_GAIN_Value", 166); - EVT_CameraSetUInt32Param(&camera_, "WB_GB_GAIN_Value", 162); - EVT_CameraSetUInt32Param(&camera_, "WB_B_GAIN_Value", 272); -} - -gxf_result_t EmergentSource::start() { - GXF_LOG_INFO("Emergent Source: RDMA is %s", use_rdma_ ? "enabled" : "disabled"); - - EVT_ERROR err = EVT_SUCCESS; - - // Open EVT camera in system. - if (OpenEVTCamera() != EVT_SUCCESS) { - GXF_LOG_ERROR("No EVT camera found.\n"); - return GXF_FAILURE; - } - - SetDefaultConfiguration(); - - // Check Camera Capabilities - if (CheckCameraCapabilities() != EVT_SUCCESS) { - GXF_LOG_ERROR("EVT Camera does not support requested format.\n"); - return GXF_FAILURE; - } - - // Prepare for streaming. - err = EVT_CameraOpenStream(&camera_); - if (err != EVT_SUCCESS) { - GXF_LOG_ERROR("EVT_CameraOpenStream failed. Error: %d\n", err); - EVT_CameraClose(&camera_); - GXF_LOG_ERROR("Camera Closed\n"); - return GXF_FAILURE; - } - - // Allocate buffers - for (unsigned int frame_count = 0U; frame_count < FRAMES_BUFFERS; frame_count++) { - evt_frame_[frame_count].size_x = width_; - evt_frame_[frame_count].size_y = height_; - // TODO: Add the option for providing different pixel_type - evt_frame_[frame_count].pixel_type = kDefaultPixelFormat; - - err = EVT_AllocateFrameBuffer(&camera_, &evt_frame_[frame_count], EVT_FRAME_BUFFER_ZERO_COPY); - if (err != EVT_SUCCESS) { - GXF_LOG_ERROR("EVT_AllocateFrameBuffer Error!\n"); - return GXF_FAILURE; - } - - err = EVT_CameraQueueFrame(&camera_, &evt_frame_[frame_count]); - if (err != EVT_SUCCESS) { - GXF_LOG_ERROR("EVT_CameraQueueFrame Error!\n"); - return GXF_FAILURE; - } - } - - // Start streaming - err = EVT_CameraExecuteCommand(&camera_, "AcquisitionStart"); - if (err != EVT_SUCCESS) { - GXF_LOG_ERROR("Acquisition start failed. Error %d\n", err); - return GXF_FAILURE; - } - return GXF_SUCCESS; -} - -gxf_result_t EmergentSource::tick() { - EVT_ERROR err = EVT_SUCCESS; - - err = EVT_CameraGetFrame(&camera_, &evt_frame_recv_, EVT_INFINITE); - if (err != EVT_SUCCESS) { - GXF_LOG_ERROR("Failed to get frame. Error %d\n", err); - return GXF_FAILURE; - } - - auto message = gxf::Entity::New(context()); - if (!message) { - GXF_LOG_ERROR("Failed to allocate message.\n"); - return GXF_FAILURE; - } - - auto buffer = message.value().add(); - if (!buffer) { - GXF_LOG_ERROR("Failed to allocate video buffer.\nr"); - return GXF_FAILURE; - } - - gxf::VideoTypeTraits video_type; - gxf::VideoFormatSize color_format; - auto color_planes = color_format.getDefaultColorPlanes(width_, height_); - gxf::VideoBufferInfo info{width_, height_, video_type.value, color_planes, - gxf::SurfaceLayout::GXF_SURFACE_LAYOUT_PITCH_LINEAR}; - auto storage_type = use_rdma_ ? gxf::MemoryStorageType::kDevice : gxf::MemoryStorageType::kHost; - buffer.value()->wrapMemory(info, evt_frame_recv_.bufferSize, storage_type, - evt_frame_recv_.imagePtr, nullptr); - - signal_->publish(std::move(message.value())); - - err = EVT_CameraQueueFrame(&camera_, &evt_frame_recv_); // Re-queue. - if (err != EVT_SUCCESS) { - GXF_LOG_ERROR("Failed to queue the frame.\n"); - return GXF_FAILURE; - } - - return gxf::ToResultCode(message); -} - -gxf_result_t EmergentSource::stop() { - EVT_ERROR err = EVT_SUCCESS; - - // Tell camera to stop streaming - err = EVT_CameraExecuteCommand(&camera_, "AcquisitionStop"); - if (err != EVT_SUCCESS) { - GXF_LOG_ERROR("EVT_CameraExecuteCommand failed. Error: %d\n", err); - return GXF_FAILURE; - } - - // Release frame buffers - for (unsigned int frame_count = 0U; frame_count < kNumBuffers; frame_count++) { - if (EVT_ReleaseFrameBuffer(&camera_, &evt_frame_[frame_count]) != EVT_SUCCESS) { - GXF_LOG_ERROR("Failed to release buffers.\n"); - return GXF_FAILURE; - } - } - - // Host side tear down for stream. - if (EVT_CameraCloseStream(&camera_) != EVT_SUCCESS) { - GXF_LOG_ERROR("Failed to close camera successfully.\n"); - return GXF_FAILURE; - } - - if (EVT_CameraClose(&camera_) != EVT_SUCCESS) { - GXF_LOG_ERROR("Failed to close camera successfully.\n"); - return GXF_FAILURE; - } - - return GXF_SUCCESS; -} - -} // namespace holoscan -} // namespace nvidia diff --git a/gxf_extensions/emergent/emergent_source.hpp b/gxf_extensions/emergent/emergent_source.hpp deleted file mode 100644 index a7694fa2..00000000 --- a/gxf_extensions/emergent/emergent_source.hpp +++ /dev/null @@ -1,83 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_EMERGENT_SOURCE_HPP_ -#define NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_EMERGENT_SOURCE_HPP_ - -#include -#include -#include -#include -#include - -#include -#include - -#include "gxf/std/codelet.hpp" -#include "gxf/std/transmitter.hpp" - -#define FRAMES_BUFFERS 256 - -using namespace Emergent; - -namespace nvidia { -namespace holoscan { - -constexpr uint32_t kMaxCameras = 10; -constexpr uint32_t kNumBuffers = 10; - -constexpr uint32_t kDefaultWidth = 4200; -constexpr uint32_t kDefaultHeight = 2160; -constexpr uint32_t kDefaultFramerate = 240; -constexpr bool kDefaultRDMA = false; -constexpr PIXEL_FORMAT kDefaultPixelFormat = GVSP_PIX_BAYGB8; - -/// @brief Video input codelet for use with Emergent cameras using ConnectX-6 -/// -/// Provides a codelet for supporting Emergent camera as a source. -/// It offers support for GPUDirect-RDMA on Quadro/NVIDIA RTX GPUs. -/// The output is a VideoBuffer object. - -class EmergentSource : public gxf::Codelet { - public: - EmergentSource(); - - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - - gxf_result_t start() override; - gxf_result_t tick() override; - gxf_result_t stop() override; - - private: - EVT_ERROR CheckCameraCapabilities(); - EVT_ERROR OpenEVTCamera(); - void SetDefaultConfiguration(); - - gxf::Parameter> signal_; - gxf::Parameter width_; - gxf::Parameter height_; - gxf::Parameter framerate_; - gxf::Parameter use_rdma_; - - CEmergentCamera camera_; - CEmergentFrame evt_frame_[FRAMES_BUFFERS]; - CEmergentFrame evt_frame_recv_; -}; - -} // namespace holoscan -} // namespace nvidia - -#endif // NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_EMERGENT_SOURCE_HPP_ diff --git a/gxf_extensions/format_converter/CMakeLists.txt b/gxf_extensions/format_converter/CMakeLists.txt deleted file mode 100644 index 621457e1..00000000 --- a/gxf_extensions/format_converter/CMakeLists.txt +++ /dev/null @@ -1,39 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Create library -add_library(format_converter_lib SHARED - format_converter.cpp - format_converter.hpp -) -target_link_libraries(format_converter_lib - PUBLIC - CUDA::cudart - CUDA::nppidei - CUDA::nppig - GXF::multimedia - GXF::std - yaml-cpp -) - -# Create extension -add_library(format_converter SHARED - format_converter_ext.cpp -) -target_link_libraries(format_converter - PUBLIC format_converter_lib -) -# Install GXF extension as a component 'holoscan-gxf_extensions' -install_gxf_extension(format_converter) diff --git a/gxf_extensions/format_converter/format_converter.cpp b/gxf_extensions/format_converter/format_converter.cpp deleted file mode 100644 index 7e6e4cbd..00000000 --- a/gxf_extensions/format_converter/format_converter.cpp +++ /dev/null @@ -1,659 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "format_converter.hpp" - -#include -#include - -#include -#include -#include - -#include "gxf/core/entity.hpp" -#include "gxf/core/expected.hpp" -#include "gxf/multimedia/video.hpp" -#include "gxf/std/tensor.hpp" - -#define CUDA_TRY(stmt) \ - ({ \ - cudaError_t _holoscan_cuda_err = stmt; \ - if (cudaSuccess != _holoscan_cuda_err) { \ - GXF_LOG_ERROR("CUDA Runtime call %s in line %d of file %s failed with '%s' (%d).\n", #stmt, \ - __LINE__, __FILE__, cudaGetErrorString(_holoscan_cuda_err), \ - _holoscan_cuda_err); \ - } \ - _holoscan_cuda_err; \ - }) - -namespace nvidia::holoscan::formatconverter { - -static FormatDType toFormatDType(const std::string& str) { - if (str == "rgb888") { - return FormatDType::kRGB888; - } else if (str == "uint8") { - return FormatDType::kUnsigned8; - } else if (str == "float32") { - return FormatDType::kFloat32; - } else if (str == "rgba8888") { - return FormatDType::kRGBA8888; - } else { - return FormatDType::kUnknown; - } -} - -static constexpr FormatConversionType getFormatConversionType(FormatDType from, FormatDType to) { - if (from != FormatDType::kUnknown && to != FormatDType::kUnknown && from == to) { - return FormatConversionType::kNone; - } else if (from == FormatDType::kUnsigned8 && to == FormatDType::kFloat32) { - return FormatConversionType::kUnsigned8ToFloat32; - } else if (from == FormatDType::kFloat32 && to == FormatDType::kUnsigned8) { - return FormatConversionType::kFloat32ToUnsigned8; - } else if (from == FormatDType::kUnsigned8 && to == FormatDType::kRGBA8888) { - return FormatConversionType::kRGB888ToRGBA8888; - } else if (from == FormatDType::kRGBA8888 && to == FormatDType::kUnsigned8) { - return FormatConversionType::kRGBA8888ToRGB888; - } else if (from == FormatDType::kRGBA8888 && to == FormatDType::kFloat32) { - return FormatConversionType::kRGBA8888ToFloat32; - } else { - return FormatConversionType::kUnknown; - } -} - -static constexpr FormatDType normalizeFormatDType(FormatDType dtype) { - switch (dtype) { - case FormatDType::kRGB888: - return FormatDType::kUnsigned8; - default: - return dtype; - } -} - -static constexpr gxf::PrimitiveType primitiveTypeFromFormatDType(FormatDType dtype) { - switch (dtype) { - case FormatDType::kRGB888: - case FormatDType::kRGBA8888: - case FormatDType::kUnsigned8: - return gxf::PrimitiveType::kUnsigned8; - case FormatDType::kFloat32: - return gxf::PrimitiveType::kFloat32; - default: - return gxf::PrimitiveType::kCustom; - } -} - -static constexpr FormatDType FormatDTypeFromPrimitiveType(gxf::PrimitiveType type) { - switch (type) { - case gxf::PrimitiveType::kUnsigned8: - return FormatDType::kUnsigned8; - case gxf::PrimitiveType::kFloat32: - return FormatDType::kFloat32; - default: - return FormatDType::kUnknown; - } -} - -static gxf_result_t verifyFormatDTypeChannels(FormatDType dtype, int channel_count) { - switch (dtype) { - case FormatDType::kRGB888: - if (channel_count != 3) { - GXF_LOG_ERROR("Invalid channel count for RGB888 %d != 3\n", channel_count); - return GXF_FAILURE; - } - break; - case FormatDType::kRGBA8888: - if (channel_count != 4) { - GXF_LOG_ERROR("Invalid channel count for RGBA8888 %d != 4\n", channel_count); - return GXF_FAILURE; - } - break; - default: - break; - } - return GXF_SUCCESS; -} - -gxf_result_t FormatConverter::start() { - out_dtype_ = toFormatDType(out_dtype_str_.get()); - if (out_dtype_ == FormatDType::kUnknown) { - GXF_LOG_ERROR("Unsupported output format dtype: %s\n", out_dtype_str_.get().c_str()); - return GXF_FAILURE; - } - out_primitive_type_ = primitiveTypeFromFormatDType(out_dtype_); - - if (out_primitive_type_ == gxf::PrimitiveType::kCustom) { - GXF_LOG_ERROR("Unsupported output format dtype: %s\n", out_dtype_str_.get().c_str()); - return GXF_FAILURE; - } - - if (!in_dtype_str_.get().empty()) { - in_dtype_ = toFormatDType(in_dtype_str_.get()); - if (in_dtype_ == FormatDType::kUnknown) { - GXF_LOG_ERROR("Unsupported input format dtype: %s\n", in_dtype_str_.get().c_str()); - return GXF_FAILURE; - } - format_conversion_type_ = - getFormatConversionType(normalizeFormatDType(in_dtype_), normalizeFormatDType(out_dtype_)); - in_primitive_type_ = primitiveTypeFromFormatDType(in_dtype_); - } - - switch (resize_mode_) { - case 0: - resize_mode_.set(NPPI_INTER_CUBIC); - break; - case 1: // NPPI_INTER_NN - case 2: // NPPI_INTER_LINEAR - case 4: // NPPI_INTER_CUBIC - case 5: // NPPI_INTER_CUBIC2P_BSPLINE - case 6: // NPPI_INTER_CUBIC2P_CATMULLROM - case 7: // NPPI_INTER_CUBIC2P_B05C03 - case 8: // NPPI_INTER_SUPER - case 16: // NPPI_INTER_LANCZOS - case 17: // NPPI_INTER_LANCZOS3_ADVANCED - case static_cast(0x8000000): // NPPI_SMOOTH_EDGE - break; - default: - GXF_LOG_ERROR("Unsupported resize mode: %d\n", resize_mode_.get()); - return GXF_FAILURE; - } - - return GXF_SUCCESS; -} - -gxf_result_t FormatConverter::stop() { - resize_buffer_.freeBuffer(); - channel_buffer_.freeBuffer(); - device_scratch_buffer_.freeBuffer(); - return GXF_SUCCESS; -} - -gxf_result_t FormatConverter::tick() { - // Process input message - const auto in_message = in_->receive(); - if (!in_message || in_message.value().is_null()) { return GXF_CONTRACT_MESSAGE_NOT_AVAILABLE; } - - gxf::Shape out_shape{0, 0, 0}; - void* in_tensor_data = nullptr; - gxf::PrimitiveType in_primitive_type = gxf::PrimitiveType::kCustom; - gxf::MemoryStorageType in_memory_storage_type = gxf::MemoryStorageType::kHost; - int32_t rows = 0; - int32_t columns = 0; - int16_t in_channels = 0; - int16_t out_channels = 0; - - // Get tensor attached to the message - auto maybe_video = in_message.value().get(); - gxf::Handle in_tensor; - if (maybe_video) { - // Convert VideoBuffer to Tensor - auto frame = maybe_video.value(); - - // NOTE: VideoBuffer::moveToTensor() converts VideoBuffer instance to the Tensor instance - // with an unexpected shape: [width, height] or [width, height, num_planes]. - // And, if we use moveToTensor() to convert VideoBuffer to Tensor, we may lose the the original - // video buffer when the VideoBuffer instance is used in other places. For that reason, we - // directly access internal data of VideoBuffer instance to access Tensor data. - const auto& buffer_info = frame->video_frame_info(); - switch (buffer_info.color_format) { - case gxf::VideoFormat::GXF_VIDEO_FORMAT_RGBA: - in_primitive_type = gxf::PrimitiveType::kUnsigned8; - break; - default: - GXF_LOG_ERROR("Unsupported input format: %d\n", buffer_info.color_format); - return GXF_FAILURE; - } - - // Get needed information from the tensor - in_memory_storage_type = frame->storage_type(); - out_shape = gxf::Shape{static_cast(buffer_info.height), - static_cast(buffer_info.width), 4}; - in_tensor_data = frame->pointer(); - rows = buffer_info.height; - columns = buffer_info.width; - in_channels = 4; // RGBA - out_channels = in_channels; - - // If the buffer is in host memory, copy it to a device (GPU) buffer - // as needed for the NPP resize/convert operations. - if (in_memory_storage_type == gxf::MemoryStorageType::kHost) { - size_t buffer_size = rows * columns * in_channels; - if (buffer_size > device_scratch_buffer_.size()) { - device_scratch_buffer_.resize(pool_, buffer_size, gxf::MemoryStorageType::kDevice); - if (!device_scratch_buffer_.pointer()) { - GXF_LOG_ERROR("Failed to allocate device scratch buffer (%d bytes)", buffer_size); - return GXF_FAILURE; - } - } - CUDA_TRY(cudaMemcpy(device_scratch_buffer_.pointer(), frame->pointer(), buffer_size, - cudaMemcpyHostToDevice)); - in_tensor_data = device_scratch_buffer_.pointer(); - in_memory_storage_type = gxf::MemoryStorageType::kDevice; - } - } else { - const auto maybe_tensor = in_message.value().get(in_tensor_name_.get().c_str()); - if (!maybe_tensor) { - GXF_LOG_ERROR("Tensor '%s' not found in message.\n", in_tensor_name_.get().c_str()); - return GXF_FAILURE; - } - in_tensor = maybe_tensor.value(); - - // Get needed information from the tensor - out_shape = in_tensor->shape(); - in_tensor_data = in_tensor->pointer(); - in_primitive_type = in_tensor->element_type(); - in_memory_storage_type = in_tensor->storage_type(); - rows = in_tensor->shape().dimension(0); - columns = in_tensor->shape().dimension(1); - in_channels = in_tensor->shape().dimension(2); - out_channels = in_channels; - } - - if (in_memory_storage_type != gxf::MemoryStorageType::kDevice) { - GXF_LOG_ERROR("Tensor('%s') or VideoBuffer is not allocated on device.\n", - in_tensor_name_.get().c_str()); - return GXF_MEMORY_INVALID_STORAGE_MODE; - } - - if (in_dtype_ == FormatDType::kUnknown) { - in_primitive_type_ = in_primitive_type; - in_dtype_ = FormatDTypeFromPrimitiveType(in_primitive_type_); - format_conversion_type_ = - getFormatConversionType(normalizeFormatDType(in_dtype_), normalizeFormatDType(out_dtype_)); - } - - // Check if input tensor is consistent over all the frames - if (in_primitive_type != in_primitive_type_) { - GXF_LOG_ERROR("Input tensor element type is inconsistent over all the frames.\n"); - return GXF_FAILURE; - } - - // Check if the input/output tensor is compatible with the format conversion - if (format_conversion_type_ == FormatConversionType::kUnknown) { - GXF_LOG_ERROR("Unsupported format conversion: %s (%" PRIu32 ") -> %s\n", - in_dtype_str_.get().c_str(), static_cast(in_dtype_), - out_dtype_str_.get().c_str()); - return GXF_FAILURE; - } - - // Check that if the format requires a specific number of channels they are consistent. - // Some formats (e.g. float32) are agnostic to channel count, while others (e.g. RGB888) have a - // specific channel count. - if (GXF_SUCCESS != verifyFormatDTypeChannels(in_dtype_, in_channels)) { - GXF_LOG_ERROR("Failed to verify the channels for the expected input dtype [%d]: %d.", in_dtype_, - in_channels); - return GXF_FAILURE; - } - - // Resize the input image before converting data type - if (resize_width_ > 0 && resize_height_ > 0) { - auto resize_result = resizeImage(in_tensor_data, rows, columns, in_channels, in_primitive_type, - resize_width_, resize_height_); - if (!resize_result) { - GXF_LOG_ERROR("Failed to resize image.\n"); - return resize_result.error(); - } - - // Update the tensor pointer and shape - out_shape = gxf::Shape{resize_height_, resize_width_, in_channels}; - in_tensor_data = resize_result.value(); - rows = resize_height_; - columns = resize_width_; - } - - // Create output message - const uint32_t dst_typesize = gxf::PrimitiveTypeSize(out_primitive_type_); - - // Adjust output shape if the conversion involves the change in the channel dimension - switch (format_conversion_type_) { - case FormatConversionType::kRGB888ToRGBA8888: { - out_channels = 4; - out_shape = gxf::Shape{out_shape.dimension(0), out_shape.dimension(1), out_channels}; - break; - } - case FormatConversionType::kRGBA8888ToRGB888: - case FormatConversionType::kRGBA8888ToFloat32: { - out_channels = 3; - out_shape = gxf::Shape{out_shape.dimension(0), out_shape.dimension(1), out_channels}; - break; - } - default: - break; - } - - // Check that if the format requires a specific number of channels they are consistent. - // Some formats (e.g. float32) are agnostic to channel count, while others (e.g. RGB888) have a - // specific channel count. - if (GXF_SUCCESS != verifyFormatDTypeChannels(out_dtype_, out_channels)) { - GXF_LOG_ERROR("Failed to verify the channels for the expected output dtype [%d]: %d.", - out_dtype_, out_channels); - return GXF_FAILURE; - } - - gxf::Expected out_message = CreateTensorMap( - context(), pool_, - {{out_tensor_name_.get(), gxf::MemoryStorageType::kDevice, out_shape, out_primitive_type_, 0, - gxf::ComputeTrivialStrides(out_shape, dst_typesize)}}); - - if (!out_message) return out_message.error(); - const auto out_tensor = out_message.value().get(); - if (!out_tensor) { return out_tensor.error(); } - - // Set tensor to constant using NPP - if (in_channels == 3 || in_channels == 4) { - gxf_result_t convert_result = convertTensorFormat(in_tensor_data, out_tensor.value()->pointer(), - rows, columns, in_channels, out_channels); - - if (convert_result != GXF_SUCCESS) { - GXF_LOG_ERROR("Failed to convert tensor format (conversion type:%d)", - static_cast(format_conversion_type_)); - return GXF_FAILURE; - } - } else { - GXF_LOG_ERROR("Only support 3 or 4 channel input tensor"); - return GXF_NOT_IMPLEMENTED; - } - - const auto result = out_->publish(out_message.value()); - - // Publish output message - return gxf::ToResultCode(result); -} - -gxf::Expected FormatConverter::resizeImage(const void* in_tensor_data, const int32_t rows, - const int32_t columns, const int16_t channels, - const gxf::PrimitiveType primitive_type, - const int32_t resize_width, - const int32_t resize_height) { - if (resize_buffer_.size() == 0) { - uint64_t buffer_size = resize_width * resize_height * channels; - resize_buffer_.resize(pool_, buffer_size, gxf::MemoryStorageType::kDevice); - } - - const auto converted_tensor_ptr = resize_buffer_.pointer(); - if (converted_tensor_ptr == nullptr) { - GXF_LOG_ERROR("Failed to allocate memory for the resizing image"); - return gxf::ExpectedOrCode(GXF_FAILURE, nullptr); - } - - // Resize image - NppStatus status = NPP_ERROR; - const NppiSize src_size = {static_cast(columns), static_cast(rows)}; - const NppiRect src_roi = {0, 0, static_cast(columns), static_cast(rows)}; - const NppiSize dst_size = {static_cast(resize_width), static_cast(resize_height)}; - const NppiRect dst_roi = {0, 0, static_cast(resize_width), static_cast(resize_height)}; - - switch (channels) { - case 3: - switch (primitive_type) { - case gxf::PrimitiveType::kUnsigned8: - status = nppiResize_8u_C3R(static_cast(in_tensor_data), columns * channels, - src_size, src_roi, converted_tensor_ptr, - resize_width * channels, dst_size, dst_roi, resize_mode_); - break; - default: - GXF_LOG_ERROR("Unsupported input primitive type for resizing image"); - return gxf::ExpectedOrCode(GXF_FAILURE, nullptr); - } - break; - case 4: - switch (primitive_type) { - case gxf::PrimitiveType::kUnsigned8: - status = nppiResize_8u_C4R(static_cast(in_tensor_data), columns * channels, - src_size, src_roi, converted_tensor_ptr, - resize_width * channels, dst_size, dst_roi, resize_mode_); - break; - default: - GXF_LOG_ERROR("Unsupported input primitive type for resizing image"); - return gxf::ExpectedOrCode(GXF_FAILURE, nullptr); - } - break; - default: - GXF_LOG_ERROR("Unsupported input primitive type for resizing image (%d, %d)", channels, - static_cast(primitive_type)); - return gxf::ExpectedOrCode(GXF_FAILURE, nullptr); - break; - } - - if (status != NPP_SUCCESS) { return gxf::ExpectedOrCode(GXF_FAILURE, nullptr); } - - return gxf::ExpectedOrCode(GXF_SUCCESS, converted_tensor_ptr); -} - -gxf_result_t FormatConverter::convertTensorFormat(const void* in_tensor_data, void* out_tensor_data, - const int32_t rows, const int32_t columns, - const int16_t in_channels, - const int16_t out_channels) { - const uint32_t src_typesize = gxf::PrimitiveTypeSize(in_primitive_type_); - const uint32_t dst_typesize = gxf::PrimitiveTypeSize(out_primitive_type_); - - const int32_t src_step = columns * in_channels * src_typesize; - const int32_t dst_step = columns * out_channels * dst_typesize; - - const auto& out_channel_order = out_channel_order_.get(); - - NppStatus status = NPP_ERROR; - const NppiSize roi = {static_cast(columns), static_cast(rows)}; - - switch (format_conversion_type_) { - case FormatConversionType::kNone: { - const auto in_tensor_ptr = static_cast(in_tensor_data); - auto out_tensor_ptr = static_cast(out_tensor_data); - - cudaError_t cuda_status = CUDA_TRY( - cudaMemcpy(out_tensor_ptr, in_tensor_ptr, src_step * rows, cudaMemcpyDeviceToDevice)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to copy GPU data to GPU memory."); - return GXF_FAILURE; - } - status = NPP_SUCCESS; - break; - } - case FormatConversionType::kUnsigned8ToFloat32: { - const auto in_tensor_ptr = static_cast(in_tensor_data); - const auto out_tensor_ptr = static_cast(out_tensor_data); - status = nppiScale_8u32f_C3R(in_tensor_ptr, src_step, out_tensor_ptr, dst_step, roi, - scale_min_, scale_max_); - break; - } - case FormatConversionType::kFloat32ToUnsigned8: { - const auto in_tensor_ptr = static_cast(in_tensor_data); - const auto out_tensor_ptr = static_cast(out_tensor_data); - status = nppiScale_32f8u_C3R(in_tensor_ptr, src_step, out_tensor_ptr, dst_step, roi, - scale_min_, scale_max_); - break; - } - case FormatConversionType::kRGB888ToRGBA8888: { - const auto in_tensor_ptr = static_cast(in_tensor_data); - const auto out_tensor_ptr = static_cast(out_tensor_data); - // Convert RGB888 to RGBA8888 (3 channels -> 4 channels, uint8_t) - int dst_order[4]{0, 1, 2, 3}; - if (!out_channel_order.empty()) { - if (out_channel_order.size() != 4) { - GXF_LOG_ERROR("Invalid channel order for RGBA8888"); - return GXF_FAILURE; - } - for (int i = 0; i < 4; i++) { dst_order[i] = out_channel_order[i]; } - } - status = nppiSwapChannels_8u_C3C4R(in_tensor_ptr, src_step, out_tensor_ptr, - out_channels * dst_typesize * columns, roi, dst_order, - alpha_value_.get()); - break; - } - case FormatConversionType::kRGBA8888ToRGB888: { - const auto in_tensor_ptr = static_cast(in_tensor_data); - const auto out_tensor_ptr = static_cast(out_tensor_data); - // Convert RGBA8888 to RGB888 (4 channels -> 3 channels, uint8_t) - int dst_order[3]{0, 1, 2}; - if (!out_channel_order.empty()) { - if (out_channel_order.size() != 3) { - GXF_LOG_ERROR("Invalid channel order for RGB888"); - return GXF_FAILURE; - } - for (int i = 0; i < 3; i++) { dst_order[i] = out_channel_order[i]; } - } - status = nppiSwapChannels_8u_C4C3R(in_tensor_ptr, src_step, out_tensor_ptr, - out_channels * dst_typesize * columns, roi, dst_order); - break; - } - case FormatConversionType::kRGBA8888ToFloat32: { - const auto in_tensor_ptr = static_cast(in_tensor_data); - const auto out_tensor_ptr = static_cast(out_tensor_data); - - if (channel_buffer_.size() == 0) { - uint64_t buffer_size = rows * columns * 3; // 4 channels -> 3 channels - channel_buffer_.resize(pool_, buffer_size, gxf::MemoryStorageType::kDevice); - } - - const auto converted_tensor_ptr = channel_buffer_.pointer(); - if (converted_tensor_ptr == nullptr) { - GXF_LOG_ERROR("Failed to allocate memory for the channel conversion"); - return GXF_FAILURE; - } - - int dst_order[3]{0, 1, 2}; - if (!out_channel_order.empty()) { - if (out_channel_order.size() != 3) { - GXF_LOG_ERROR("Invalid channel order for RGB888"); - return GXF_FAILURE; - } - for (int i = 0; i < 3; i++) { dst_order[i] = out_channel_order[i]; } - } - - status = nppiSwapChannels_8u_C4C3R(in_tensor_ptr, src_step, converted_tensor_ptr, - out_channels * src_typesize * columns, roi, dst_order); - - if (status == NPP_SUCCESS) { - const int32_t new_src_step = columns * out_channels * src_typesize; - status = nppiScale_8u32f_C3R(converted_tensor_ptr, new_src_step, out_tensor_ptr, dst_step, - roi, scale_min_, scale_max_); - } else { - GXF_LOG_ERROR("Failed to convert channel order (NPP error code: %d)", status); - return GXF_FAILURE; - } - break; - } - default: - GXF_LOG_ERROR("Unsupported format conversion: %s (%" PRIu32 ") -> %s\n", - in_dtype_str_.get().c_str(), static_cast(in_dtype_), - out_dtype_str_.get().c_str()); - } - - // Reorder channels in the output tensor (inplace) if needed. - switch (format_conversion_type_) { - case FormatConversionType::kNone: - case FormatConversionType::kUnsigned8ToFloat32: - case FormatConversionType::kFloat32ToUnsigned8: { - if (!out_channel_order.empty()) { - switch (out_channels) { - case 3: { - int dst_order[3]{0, 1, 2}; - if (out_channel_order.size() != 3) { - GXF_LOG_ERROR("Invalid channel order for %s", out_dtype_str_.get().c_str()); - return GXF_FAILURE; - } - for (int i = 0; i < 3; i++) { dst_order[i] = out_channel_order[i]; } - switch (out_primitive_type_) { - case gxf::PrimitiveType::kUnsigned8: { - auto out_tensor_ptr = static_cast(out_tensor_data); - status = nppiSwapChannels_8u_C3IR(out_tensor_ptr, dst_step, roi, dst_order); - break; - } - case gxf::PrimitiveType::kFloat32: { - auto out_tensor_ptr = static_cast(out_tensor_data); - status = nppiSwapChannels_32f_C3IR(out_tensor_ptr, dst_step, roi, dst_order); - break; - } - default: - GXF_LOG_ERROR("Unsupported output data type for reordering channels: %s", - out_dtype_str_.get().c_str()); - } - break; - } - case 4: { - int dst_order[4]{0, 1, 2, 3}; - if (out_channel_order.size() != 4) { - GXF_LOG_ERROR("Invalid channel order for %s", out_dtype_str_.get().c_str()); - return GXF_FAILURE; - } - for (int i = 0; i < 4; i++) { dst_order[i] = out_channel_order[i]; } - switch (out_primitive_type_) { - case gxf::PrimitiveType::kUnsigned8: { - auto out_tensor_ptr = static_cast(out_tensor_data); - status = nppiSwapChannels_8u_C4IR(out_tensor_ptr, dst_step, roi, dst_order); - break; - } - case gxf::PrimitiveType::kFloat32: { - auto out_tensor_ptr = static_cast(out_tensor_data); - status = nppiSwapChannels_32f_C4IR(out_tensor_ptr, dst_step, roi, dst_order); - break; - } - default: - GXF_LOG_ERROR("Unsupported output data type for reordering channels: %s\n", - out_dtype_str_.get().c_str()); - } - break; - } - } - if (status != NPP_SUCCESS) { - GXF_LOG_ERROR("Failed to convert channel order"); - return GXF_FAILURE; - } - } - } - default: - break; - } - - if (status != NPP_SUCCESS) { return GXF_FAILURE; } - - return GXF_SUCCESS; -} - -gxf_result_t FormatConverter::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - result &= registrar->parameter(in_, "in", "Input", "Input channel."); - result &= registrar->parameter(in_tensor_name_, "in_tensor_name", "InputTensorName", - "Name of the input tensor.", std::string("")); - result &= registrar->parameter(in_dtype_str_, "in_dtype", "InputDataType", "Source data type.", - std::string("")); - result &= registrar->parameter(out_, "out", "Output", "Output channel."); - result &= registrar->parameter(out_tensor_name_, "out_tensor_name", "OutputTensorName", - "Name of the output tensor.", std::string("")); - result &= - registrar->parameter(out_dtype_str_, "out_dtype", "OutputDataType", "Destination data type."); - result &= registrar->parameter(scale_min_, "scale_min", "Scale min", - "Minimum value of the scale.", 0.f); - result &= registrar->parameter(scale_max_, "scale_max", "Scale max", - "Maximum value of the scale.", 1.f); - result &= registrar->parameter(alpha_value_, "alpha_value", "Alpha value", - "Alpha value that can be used to fill the alpha channel when " - "converting RGB888 to RGBA8888.", - static_cast(255)); - result &= registrar->parameter(resize_width_, "resize_width", "Resize width", - "Width for resize. No actions if this value is zero.", 0); - result &= registrar->parameter(resize_height_, "resize_height", "Resize height", - "Height for resize. No actions if this value is zero.", 0); - result &= registrar->parameter(resize_mode_, "resize_mode", "Resize mode", - "Mode for resize. 4 (NPPI_INTER_CUBIC) if this value is zero.", 0); - result &= registrar->parameter( - out_channel_order_, "out_channel_order", "Output channel order", - "Host memory integer array describing how channel values are permutated.", - std::vector{}); - result &= registrar->parameter(pool_, "pool", "Pool", "Pool to allocate the output message."); - return gxf::ToResultCode(result); -} - -} // namespace nvidia::holoscan::formatconverter diff --git a/gxf_extensions/format_converter/format_converter.hpp b/gxf_extensions/format_converter/format_converter.hpp deleted file mode 100644 index b21d40c9..00000000 --- a/gxf_extensions/format_converter/format_converter.hpp +++ /dev/null @@ -1,92 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_FORMAT_CONVERTER_FORMAT_CONVERTER_HPP_ -#define NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_FORMAT_CONVERTER_FORMAT_CONVERTER_HPP_ - -#include -#include -#include - -#include "gxf/std/allocator.hpp" -#include "gxf/std/codelet.hpp" -#include "gxf/std/receiver.hpp" -#include "gxf/std/tensor.hpp" -#include "gxf/std/transmitter.hpp" - -namespace nvidia::holoscan::formatconverter { - -enum class FormatDType { kUnknown, kRGB888, kRGBA8888, kUnsigned8, kFloat32 }; - -enum class FormatConversionType { - kUnknown, - kNone, - kUnsigned8ToFloat32, - kFloat32ToUnsigned8, - kRGB888ToRGBA8888, - kRGBA8888ToRGB888, - kRGBA8888ToFloat32, -}; - -/// @brief Helper codelet for common tensor operations in inference pipelines. -/// -/// Provides a codelet that provides common video or tensor operations in inference pipelines to -/// change datatypes, resize images, reorder channels, and normalize and scale values. -class FormatConverter : public gxf::Codelet { - public: - gxf_result_t start() override; - gxf_result_t tick() override; - gxf_result_t stop() override; - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - - gxf::Expected resizeImage(const void* in_tensor_data, const int32_t rows, - const int32_t columns, const int16_t channels, - const gxf::PrimitiveType primitive_type, - const int32_t resize_width, const int32_t resize_height); - gxf_result_t convertTensorFormat(const void* in_tensor_data, void* out_tensor_data, - const int32_t rows, const int32_t columns, - const int16_t in_channels, const int16_t out_channels); - - private: - gxf::Parameter> in_; - gxf::Parameter in_tensor_name_; - gxf::Parameter in_dtype_str_; - gxf::Parameter> out_; - gxf::Parameter out_tensor_name_; - gxf::Parameter out_dtype_str_; - gxf::Parameter scale_min_; - gxf::Parameter scale_max_; - gxf::Parameter alpha_value_; - gxf::Parameter resize_width_; - gxf::Parameter resize_height_; - gxf::Parameter resize_mode_; - gxf::Parameter> out_channel_order_; - gxf::Parameter> pool_; - - FormatDType in_dtype_ = FormatDType::kUnknown; - FormatDType out_dtype_ = FormatDType::kUnknown; - gxf::PrimitiveType in_primitive_type_ = gxf::PrimitiveType::kCustom; - gxf::PrimitiveType out_primitive_type_ = gxf::PrimitiveType::kCustom; - FormatConversionType format_conversion_type_ = FormatConversionType::kUnknown; - - gxf::MemoryBuffer resize_buffer_; - gxf::MemoryBuffer channel_buffer_; - gxf::MemoryBuffer device_scratch_buffer_; -}; - -} // namespace nvidia::holoscan::formatconverter - -#endif // NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_FORMAT_CONVERTER_FORMAT_CONVERTER_HPP_ diff --git a/gxf_extensions/gxf_holoscan_wrapper/CMakeLists.txt b/gxf_extensions/gxf_holoscan_wrapper/CMakeLists.txt new file mode 100644 index 00000000..ad0a3782 --- /dev/null +++ b/gxf_extensions/gxf_holoscan_wrapper/CMakeLists.txt @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Create library +add_library(gxf_holoscan_wrapper_lib SHARED + operator_wrapper.cpp + operator_wrapper.hpp + operator_wrapper_fragment.cpp + operator_wrapper_fragment.hpp +) +target_link_libraries(gxf_holoscan_wrapper_lib + PUBLIC + holoscan::core +) + +# Create extension +add_library(gxf_holoscan_wrapper SHARED + gxf_holoscan_wrapper_ext.cpp +) +target_link_libraries(gxf_holoscan_wrapper + PUBLIC gxf_holoscan_wrapper_lib +) + +# Add include directories +target_include_directories(gxf_holoscan_wrapper_lib + PUBLIC + # Include headers from the parent directory (./gxf_extensions) + # so that the headers can be included as + # TODO: need to find better way to do this + $ + $ +) + +# Install the header files +install(FILES + operator_wrapper.hpp + operator_wrapper_fragment.hpp + # Install headers to the directory 'gxf_holoscan_wrapper' in the include directory + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/gxf_holoscan_wrapper + COMPONENT holoscan-gxf_extensions +) + +# Install GXF extension as a component 'holoscan-gxf_extensions' +install_gxf_extension(gxf_holoscan_wrapper) diff --git a/gxf_extensions/gxf_holoscan_wrapper/gxf_holoscan_wrapper_ext.cpp b/gxf_extensions/gxf_holoscan_wrapper/gxf_holoscan_wrapper_ext.cpp new file mode 100644 index 00000000..0d44e11c --- /dev/null +++ b/gxf_extensions/gxf_holoscan_wrapper/gxf_holoscan_wrapper_ext.cpp @@ -0,0 +1,39 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "gxf/std/extension_factory_helper.hpp" + +#include "holoscan/core/domain/tensor.hpp" +#include "holoscan/core/gxf/gxf_tensor.hpp" +#include "holoscan/core/message.hpp" +#include "operator_wrapper.hpp" + +GXF_EXT_FACTORY_BEGIN() +GXF_EXT_FACTORY_SET_INFO(0x12d01b4ee06f49ef, 0x93c4961834347385, "HoloscanWrapperExtension", + "Holoscan Wrapper extension", "NVIDIA", "0.5.0", "LICENSE"); + +// Register types/components that are used by Holoscan +GXF_EXT_FACTORY_ADD_0(0x61510ca06aa9493b, 0x8a777d0bf87476b7, holoscan::Message, + "Holoscan message type"); +GXF_EXT_FACTORY_ADD(0xa02945eaf20e418c, 0x8e6992b68672ce40, holoscan::gxf::GXFTensor, + nvidia::gxf::Tensor, "Holoscan's GXF Tensor type"); +GXF_EXT_FACTORY_ADD_0(0xa5eb0ed57d7f4aa2, 0xb5865ccca0ef955c, holoscan::Tensor, + "Holoscan's Tensor type"); + +// Register the wrapper codelet +GXF_EXT_FACTORY_ADD(0x04f99794e01b4bd1, 0xb42653a2e6d07347, holoscan::gxf::OperatorWrapper, + nvidia::gxf::Codelet, "Codelet for wrapping Holoscan Operator"); +GXF_EXT_FACTORY_END() diff --git a/gxf_extensions/gxf_holoscan_wrapper/operator_wrapper.cpp b/gxf_extensions/gxf_holoscan_wrapper/operator_wrapper.cpp new file mode 100644 index 00000000..ab5dd8b3 --- /dev/null +++ b/gxf_extensions/gxf_holoscan_wrapper/operator_wrapper.cpp @@ -0,0 +1,317 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "operator_wrapper.hpp" + +#include "gxf/cuda/cuda_stream_pool.hpp" +#include "gxf/std/scheduling_term.hpp" +#include "gxf/std/unbounded_allocator.hpp" +// #include "gxf/std/block_memory_pool.hpp" // TODO: uncomment when available in GXF SDK package + +#include "holoscan/core/common.hpp" +#include "holoscan/core/gxf/gxf_execution_context.hpp" +#include "holoscan/core/gxf/gxf_utils.hpp" +#include "holoscan/core/io_context.hpp" +#include "holoscan/core/resources/gxf/block_memory_pool.hpp" +#include "holoscan/core/resources/gxf/cuda_stream_pool.hpp" +#include "holoscan/core/resources/gxf/unbounded_allocator.hpp" + +namespace holoscan::gxf { + +// Whether the log level has been set from the environment. +static bool g_operator_wrapper_env_log_level_set = false; + +OperatorWrapper::OperatorWrapper() : nvidia::gxf::Codelet() { + // Check if the environment variable for log level has been set. + if (!g_operator_wrapper_env_log_level_set) { + g_operator_wrapper_env_log_level_set = true; + holoscan::load_env_log_level(); + } +} + +gxf_result_t OperatorWrapper::initialize() { + HOLOSCAN_LOG_TRACE("OperatorWrapper::initialize()"); + if (!op_) { + HOLOSCAN_LOG_ERROR("OperatorWrapper::initialize() - op_ is null"); + return GXF_FAILURE; + } + if (!fragment_.executor().context()) { + // Set the current GXF context to the fragment's executor. + fragment_.executor().context(context()); + // Set the executor (GXFExecutor)'s 'op_eid_' to this codelet's eid + // so that the operator is initialized under this codelet's eid. + fragment_.gxf_executor().op_eid(eid()); + // Set the executor (GXFExecutor)'s 'op_cid_' to this codelet's cid + // so that the operator is initialized with this codelet's cid. + fragment_.gxf_executor().op_cid(cid()); + + // Set the operator's name to the typename of the operator. + // (it doesn't affect anything) + op_->name(holoscan_typename()); + // Set the operator's fragment to the fragment we created. + op_->fragment(&fragment_); + op_->spec()->fragment(&fragment_); // lazy fragment setup + + // Set parameters. + for (auto& gxf_param : parameters_) { + const auto& arg_type = gxf_param.arg_type; + const auto param_ptr = gxf_param.param_ptr; + const auto& arg = gxf_param.param.try_get(); + if (arg) { + auto& param_gxf = const_cast(arg.value()); + + // Handle cases where the argument is a handle such as a condition or a resource. + if (arg_type.element_type() == ArgElementType::kCondition) { + // Create a condition object based on the handle name. + HOLOSCAN_LOG_TRACE(" adding condition: {}", param_ptr->key()); + + // Parse string from node + std::string tag; + try { + tag = param_gxf.as(); + } catch (...) { + std::stringstream ss; + ss << param_gxf; + HOLOSCAN_LOG_ERROR("Could not parse parameter {} from {}", param_ptr->key(), ss.str()); + continue; + } + + gxf_uid_t condition_cid = find_component_handle( + context(), cid(), param_ptr->key().c_str(), tag, ""); + + gxf_tid_t condition_tid{}; + gxf_result_t code = GxfComponentType(context(), condition_cid, &condition_tid); + + gxf_tid_t boolean_condition_tid{}; + GxfComponentTypeId( + context(), "nvidia::gxf::BooleanSchedulingTerm", &boolean_condition_tid); + + if (condition_tid == boolean_condition_tid) { + nvidia::gxf::BooleanSchedulingTerm* boolean_condition_ptr = nullptr; + code = GxfComponentPointer(context(), + condition_cid, + condition_tid, + reinterpret_cast(&boolean_condition_ptr)); + + if (boolean_condition_ptr) { + auto condition = + std::make_shared(tag, boolean_condition_ptr); + op_->add_arg(Arg(param_ptr->key()) = condition); + } + } else { + HOLOSCAN_LOG_ERROR("Unsupported condition type for the handle: {}", tag); + } + } else if (arg_type.element_type() == ArgElementType::kResource) { + // Create a resource object based on the handle name. + HOLOSCAN_LOG_TRACE(" adding resource: {}", param_ptr->key()); + + // Parse string from node + std::string tag; + try { + tag = param_gxf.as(); + } catch (...) { + std::stringstream ss; + ss << param_gxf; + HOLOSCAN_LOG_ERROR("Could not parse parameter {} from {}", param_ptr->key(), ss.str()); + continue; + } + + gxf_uid_t resource_cid = find_component_handle( + context(), cid(), param_ptr->key().c_str(), tag, ""); + + gxf_tid_t resource_tid{}; + gxf_result_t code = GxfComponentType(context(), resource_cid, &resource_tid); + + gxf_tid_t unbounded_allocator_tid{}; + GxfComponentTypeId( + context(), "nvidia::gxf::UnboundedAllocator", &unbounded_allocator_tid); + + gxf_tid_t block_memory_pool_tid{}; + GxfComponentTypeId(context(), "nvidia::gxf::BlockMemoryPool", &block_memory_pool_tid); + + gxf_tid_t cuda_stream_pool_tid{}; + GxfComponentTypeId(context(), "nvidia::gxf::CudaStreamPool", &cuda_stream_pool_tid); + + if (resource_tid == unbounded_allocator_tid) { + nvidia::gxf::UnboundedAllocator* unbounded_allocator_ptr = nullptr; + code = GxfComponentPointer(context(), + resource_cid, + resource_tid, + reinterpret_cast(&unbounded_allocator_ptr)); + + if (unbounded_allocator_ptr) { + auto resource = + std::make_shared(tag, unbounded_allocator_ptr); + op_->add_arg(Arg(param_ptr->key()) = resource); + } + } else if (resource_tid == block_memory_pool_tid) { + nvidia::gxf::BlockMemoryPool* block_memory_pool_ptr = nullptr; + code = GxfComponentPointer(context(), + resource_cid, + resource_tid, + reinterpret_cast(&block_memory_pool_ptr)); + + if (block_memory_pool_ptr) { + auto resource = + std::make_shared(tag, block_memory_pool_ptr); + op_->add_arg(Arg(param_ptr->key()) = resource); + } + } else if (resource_tid == cuda_stream_pool_tid) { + nvidia::gxf::CudaStreamPool* cuda_stream_pool_ptr = nullptr; + code = GxfComponentPointer(context(), + resource_cid, + resource_tid, + reinterpret_cast(&cuda_stream_pool_ptr)); + + if (cuda_stream_pool_ptr) { + auto resource = std::make_shared(tag, cuda_stream_pool_ptr); + op_->add_arg(Arg(param_ptr->key()) = resource); + } + } else { + HOLOSCAN_LOG_ERROR("Unsupported resource type for the handle: {}", tag); + } + } else if (arg_type.element_type() == ArgElementType::kIOSpec && + arg_type.container_type() == ArgContainerType::kVector) { + // Create IOSpec objects based on the receivers name + HOLOSCAN_LOG_TRACE(" adding receivers: {}", param_ptr->key()); + + // Parse string from node + std::vector tags; + try { + tags = param_gxf.as>(); + } catch (...) { + std::stringstream ss; + ss << param_gxf; + HOLOSCAN_LOG_ERROR("Could not parse parameter {} from {}", param_ptr->key(), ss.str()); + continue; + } + + // Get the receivers parameter pointer. + auto receivers_param_ptr = + reinterpret_cast>*>(param_ptr); + // Set the default value. + receivers_param_ptr->set_default_value(); + // Get the vector of IOSpec pointers. + std::vector& iospec_vector = receivers_param_ptr->get(); + iospec_vector.reserve(tags.size()); + for (auto& tag : tags) { + HOLOSCAN_LOG_TRACE(" creating new input port: {}", tag); + // Create and add the new input port to the vector. + auto& input_port = op_->spec()->input(tag); + iospec_vector.push_back(&input_port); + } + } else { + // Add argument to the operator (as YAML::Node object). + op_->add_arg(Arg(param_ptr->key()) = param_gxf); + } + } + } + + // Initialize the operator. + op_->initialize(); + } + + return GXF_SUCCESS; +} +gxf_result_t OperatorWrapper::deinitialize() { + HOLOSCAN_LOG_TRACE("OperatorWrapper::deinitialize()"); + return GXF_SUCCESS; +} + +gxf_result_t OperatorWrapper::registerInterface(nvidia::gxf::Registrar* registrar) { + HOLOSCAN_LOG_TRACE("OperatorWrapper::registerInterface()"); + nvidia::gxf::Expected result; + if (!op_) { + HOLOSCAN_LOG_ERROR("OperatorWrapper::registerInterface() - op_ is null"); + return GXF_FAILURE; + } + + // This method (registerInterface()) is called before initialize() multiple times. + if (!op_->spec()) { + // Setup the operator. + auto spec = std::make_shared(nullptr); + op_->setup(*spec.get()); + op_->spec(spec); + + // Initialize the list of GXFParameter objects + for (auto& param : op_->spec()->params()) { + HOLOSCAN_LOG_TRACE(" adding param: {}", param.first); + // Cast the storage pointer to a Parameter pointer, to access metadata. + // (Accessing the value is illegal, because the type is unknown.) + auto storage_ptr = static_cast*>(param.second.storage_ptr()); + if (!storage_ptr) { + HOLOSCAN_LOG_ERROR("OperatorWrapper::registerInterface() - storage_ptr is null"); + return GXF_FAILURE; + } + parameters_.emplace_back(GXFParameter{{}, param.second.arg_type(), storage_ptr}); + } + } + + // Register the operator's parameters. + for (auto& gxf_param : parameters_) { + const auto param_ptr = gxf_param.param_ptr; + HOLOSCAN_LOG_TRACE(" registering param: {}", param_ptr->key()); + // TODO(gbae): support parameter flags + result &= registrar->parameter(gxf_param.param, + param_ptr->key().c_str(), + param_ptr->headline().c_str(), + param_ptr->description().c_str(), + nvidia::gxf::Unexpected(), + GXF_PARAMETER_FLAGS_OPTIONAL); + } + + return nvidia::gxf::ToResultCode(result); +} + +gxf_result_t OperatorWrapper::start() { + HOLOSCAN_LOG_TRACE("OperatorWrapper::start()"); + if (op_ == nullptr) { + HOLOSCAN_LOG_ERROR("OperatorWrapper::start() - Operator is not set"); + return GXF_FAILURE; + } + op_->start(); + return GXF_SUCCESS; +} + +gxf_result_t OperatorWrapper::tick() { + HOLOSCAN_LOG_TRACE("OperatorWrapper::tick()"); + if (!op_) { + HOLOSCAN_LOG_ERROR("OperatorWrapper::tick() - Operator is not set"); + return GXF_FAILURE; + } + + HOLOSCAN_LOG_TRACE("Calling operator: {}", op_->name()); + + GXFExecutionContext exec_context(context(), op_.get()); + InputContext* op_input = exec_context.input(); + OutputContext* op_output = exec_context.output(); + op_->compute(*op_input, *op_output, exec_context); + + return GXF_SUCCESS; +} + +gxf_result_t OperatorWrapper::stop() { + HOLOSCAN_LOG_TRACE("OperatorWrapper::stop()"); + if (!op_) { + HOLOSCAN_LOG_ERROR("OperatorWrapper::stop() - Operator is not set"); + return GXF_FAILURE; + } + op_->stop(); + return GXF_SUCCESS; +} + +} // namespace holoscan::gxf diff --git a/gxf_extensions/gxf_holoscan_wrapper/operator_wrapper.hpp b/gxf_extensions/gxf_holoscan_wrapper/operator_wrapper.hpp new file mode 100644 index 00000000..20eb5896 --- /dev/null +++ b/gxf_extensions/gxf_holoscan_wrapper/operator_wrapper.hpp @@ -0,0 +1,73 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef GXF_HOLOSCAN_WRAPPER_OPERATOR_WRAPPER_HPP +#define GXF_HOLOSCAN_WRAPPER_OPERATOR_WRAPPER_HPP + +#include +#include + +#include "holoscan/core/operator.hpp" +#include "holoscan/core/parameter.hpp" +#include "operator_wrapper_fragment.hpp" + +#include "gxf/std/codelet.hpp" +#include "gxf/std/parameter_parser_std.hpp" + +namespace holoscan::gxf { + +/** + * @brief Class to wrap an Operator to interface with the GXF framework. + */ +class OperatorWrapper : public nvidia::gxf::Codelet { + public: + OperatorWrapper(); + virtual ~OperatorWrapper() = default; + + /// Get the type name of the Operator. + virtual const char* holoscan_typename() const = 0; + + /// Create and initialize the Operator. + gxf_result_t initialize() override; + /// Destroy the Operator and free resources. + gxf_result_t deinitialize() override; + + /// Register the Operator's parameters with the GXF framework. + gxf_result_t registerInterface(nvidia::gxf::Registrar* registrar) override; + + /// Delegate to the Operator's start() method. + gxf_result_t start() override; + /// Delegate to the Operator's compute() method. + gxf_result_t tick() override; + /// Delegate to the Operator's stop() method. + gxf_result_t stop() override; + + struct GXFParameter { + nvidia::gxf::Parameter param; ///< The GXF parameter. + holoscan::ArgType arg_type; ///< The type of the parameter (in Holoscan) + holoscan::Parameter* param_ptr; ///< The pointer to the parameter (in Holoscan) + }; + + protected: + std::shared_ptr op_; ///< The Operator to wrap. + OperatorWrapperFragment fragment_; ///< The fragment to use for the Operator. + std::list parameters_; ///< The parameters to use for the GXF Codelet. +}; + +} // namespace holoscan::gxf + +#endif /* GXF_HOLOSCAN_WRAPPER_OPERATOR_WRAPPER_HPP */ diff --git a/gxf_extensions/gxf_holoscan_wrapper/operator_wrapper_fragment.cpp b/gxf_extensions/gxf_holoscan_wrapper/operator_wrapper_fragment.cpp new file mode 100644 index 00000000..0b276db5 --- /dev/null +++ b/gxf_extensions/gxf_holoscan_wrapper/operator_wrapper_fragment.cpp @@ -0,0 +1,30 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "operator_wrapper_fragment.hpp" + +#include "holoscan/core/executors/gxf/gxf_executor.hpp" + +namespace holoscan::gxf { + +OperatorWrapperFragment::OperatorWrapperFragment() : Fragment() { + // Create a GXFExecutor without creating a GXF Context + // This is because the GXF Context is already created by the GXF executor (gxe). + executor_ = make_executor(this, false); +} + +} // namespace holoscan::gxf diff --git a/gxf_extensions/gxf_holoscan_wrapper/operator_wrapper_fragment.hpp b/gxf_extensions/gxf_holoscan_wrapper/operator_wrapper_fragment.hpp new file mode 100644 index 00000000..e8104f0f --- /dev/null +++ b/gxf_extensions/gxf_holoscan_wrapper/operator_wrapper_fragment.hpp @@ -0,0 +1,42 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef GXF_HOLOSCAN_WRAPPER_OPERATOR_WRAPPER_FRAGMENT_HPP +#define GXF_HOLOSCAN_WRAPPER_OPERATOR_WRAPPER_FRAGMENT_HPP + +#include "holoscan/core/executors/gxf/gxf_executor.hpp" +#include "holoscan/core/fragment.hpp" + +namespace holoscan::gxf { + +/** + * @brief Class to wrap an Operator's Fragment to interface with the GXF framework. + * + * This class is used to create Operator instances for OperatorWrapper objects. + */ +class OperatorWrapperFragment : public holoscan::Fragment { + public: + OperatorWrapperFragment(); + + GXFExecutor& gxf_executor() { + return static_cast(executor()); + } +}; + +} // namespace holoscan::gxf + +#endif /* GXF_HOLOSCAN_WRAPPER_OPERATOR_WRAPPER_FRAGMENT_HPP */ diff --git a/gxf_extensions/holoviz/holoviz.cpp b/gxf_extensions/holoviz/holoviz.cpp deleted file mode 100644 index 9882ead6..00000000 --- a/gxf_extensions/holoviz/holoviz.cpp +++ /dev/null @@ -1,981 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#include "holoviz.hpp" - -#include - -#include -#include -#include - -#include "gxf/cuda/cuda_stream.hpp" -#include "gxf/cuda/cuda_stream_id.hpp" -#include "gxf/multimedia/video.hpp" -#include "gxf/std/tensor.hpp" - -#include "holoviz/holoviz.hpp" - -#define CUDA_TRY(stmt) \ - ({ \ - cudaError_t _holoscan_cuda_err = stmt; \ - if (cudaSuccess != _holoscan_cuda_err) { \ - GXF_LOG_ERROR("CUDA Runtime call %s in line %d of file %s failed with '%s' (%d).\n", \ - #stmt, \ - __LINE__, \ - __FILE__, \ - cudaGetErrorString(_holoscan_cuda_err), \ - _holoscan_cuda_err); \ - } \ - _holoscan_cuda_err; \ - }) - -namespace viz = holoscan::viz; - -namespace { - -/** - * Check is the message contains a cuda stream, if yes, set the Holoviz cuda stream with that - * stream. - * - * @param context current GXF context - * @param message GXF message to check for a cuda stream - */ -void setCudaStreamFromMessage(gxf_context_t context, const nvidia::gxf::Entity& message) { - // check if the message contains a Cuda stream - const auto maybe_cuda_stream_id = message.get(); - if (maybe_cuda_stream_id) { - const auto maybe_cuda_stream_handle = nvidia::gxf::Handle::Create( - context, maybe_cuda_stream_id.value()->stream_cid); - if (maybe_cuda_stream_handle) { - const auto cuda_stream = maybe_cuda_stream_handle.value(); - if (cuda_stream) { viz::SetCudaStream(cuda_stream->stream().value()); } - } - } -} - -/// Buffer information, can be initialized either with a tensor or a video buffer -struct BufferInfo { - /** - * Initialize with tensor - * - * @returns error code - */ - gxf_result_t init(const nvidia::gxf::Handle& tensor) { - rank = tensor->rank(); - shape = tensor->shape(); - element_type = tensor->element_type(); - name = tensor.name(); - buffer_ptr = tensor->pointer(); - storage_type = tensor->storage_type(); - bytes_size = tensor->bytes_size(); - for (uint32_t i = 0; i < rank; ++i) { stride[i] = tensor->stride(i); } - - return GXF_SUCCESS; - } - - /** - * Initialize with video buffer - * - * @returns error code - */ - gxf_result_t init(const nvidia::gxf::Handle& video) { - // NOTE: VideoBuffer::moveToTensor() converts VideoBuffer instance to the Tensor instance - // with an unexpected shape: [width, height] or [width, height, num_planes]. - // And, if we use moveToTensor() to convert VideoBuffer to Tensor, we may lose the original - // video buffer when the VideoBuffer instance is used in other places. For that reason, we - // directly access internal data of VideoBuffer instance to access Tensor data. - const auto& buffer_info = video->video_frame_info(); - - int32_t channels; - switch (buffer_info.color_format) { - case nvidia::gxf::VideoFormat::GXF_VIDEO_FORMAT_GRAY: - element_type = nvidia::gxf::PrimitiveType::kUnsigned8; - channels = 1; - break; - case nvidia::gxf::VideoFormat::GXF_VIDEO_FORMAT_GRAY16: - element_type = nvidia::gxf::PrimitiveType::kUnsigned16; - channels = 1; - break; - case nvidia::gxf::VideoFormat::GXF_VIDEO_FORMAT_GRAY32: - element_type = nvidia::gxf::PrimitiveType::kUnsigned32; - channels = 1; - break; - case nvidia::gxf::VideoFormat::GXF_VIDEO_FORMAT_RGB: - element_type = nvidia::gxf::PrimitiveType::kUnsigned8; - channels = 3; - break; - case nvidia::gxf::VideoFormat::GXF_VIDEO_FORMAT_RGBA: - element_type = nvidia::gxf::PrimitiveType::kUnsigned8; - channels = 4; - break; - default: - GXF_LOG_ERROR("Unsupported input format: %" PRId64 "\n", - static_cast(buffer_info.color_format)); - return GXF_FAILURE; - } - - rank = 3; - shape = nvidia::gxf::Shape{static_cast(buffer_info.height), - static_cast(buffer_info.width), - channels}; - name = video.name(); - buffer_ptr = video->pointer(); - storage_type = video->storage_type(); - bytes_size = video->size(); - stride[0] = buffer_info.color_planes[0].stride; - stride[1] = channels; - stride[2] = PrimitiveTypeSize(element_type); - - return GXF_SUCCESS; - } - - uint32_t rank; - nvidia::gxf::Shape shape; - nvidia::gxf::PrimitiveType element_type; - std::string name; - const nvidia::byte* buffer_ptr; - nvidia::gxf::MemoryStorageType storage_type; - uint64_t bytes_size; - nvidia::gxf::Tensor::stride_array_t stride; -}; - -/** - * Get the Holoviz image format for a given buffer. - * - * @param buffer_info buffer info - * @return Holoviz image format - */ -nvidia::gxf::Expected getImageFormatFromTensor(const BufferInfo& buffer_info) { - if (buffer_info.rank != 3) { - GXF_LOG_ERROR("Invalid tensor rank count, expected 3, got %u", buffer_info.rank); - return nvidia::gxf::Unexpected{GXF_INVALID_DATA_FORMAT}; - } - - struct Format { - nvidia::gxf::PrimitiveType type_; - int32_t channels_; - viz::ImageFormat format_; - }; - constexpr Format kGFXToHolovizFormats[] = { - {nvidia::gxf::PrimitiveType::kUnsigned8, 1, viz::ImageFormat::R8_UINT}, - {nvidia::gxf::PrimitiveType::kUnsigned16, 1, viz::ImageFormat::R16_UINT}, - {nvidia::gxf::PrimitiveType::kUnsigned32, 1, viz::ImageFormat::R32_UINT}, - {nvidia::gxf::PrimitiveType::kFloat32, 1, viz::ImageFormat::R32_SFLOAT}, - {nvidia::gxf::PrimitiveType::kUnsigned8, 3, viz::ImageFormat::R8G8B8_UNORM}, - {nvidia::gxf::PrimitiveType::kUnsigned8, 4, viz::ImageFormat::R8G8B8A8_UNORM}, - {nvidia::gxf::PrimitiveType::kUnsigned16, 4, viz::ImageFormat::R16G16B16A16_UNORM}, - {nvidia::gxf::PrimitiveType::kFloat32, 4, viz::ImageFormat::R32G32B32A32_SFLOAT}}; - - viz::ImageFormat image_format = (viz::ImageFormat)-1; - for (auto&& format : kGFXToHolovizFormats) { - if ((format.type_ == buffer_info.element_type) && - (format.channels_ == buffer_info.shape.dimension(2))) { - image_format = format.format_; - break; - } - } - if (image_format == (viz::ImageFormat)-1) { - GXF_LOG_ERROR("Element type %d and channel count %d not supported", - static_cast(buffer_info.element_type), - buffer_info.shape.dimension(3)); - return nvidia::gxf::Unexpected{GXF_INVALID_DATA_FORMAT}; - } - - return image_format; -} - -/// table to convert input type to string -static const std::array, 11> - kInputTypeToStr{{{nvidia::holoscan::Holoviz::InputType::UNKNOWN, "unknown"}, - {nvidia::holoscan::Holoviz::InputType::COLOR, "color"}, - {nvidia::holoscan::Holoviz::InputType::COLOR_LUT, "color_lut"}, - {nvidia::holoscan::Holoviz::InputType::POINTS, "points"}, - {nvidia::holoscan::Holoviz::InputType::LINES, "lines"}, - {nvidia::holoscan::Holoviz::InputType::LINE_STRIP, "line_strip"}, - {nvidia::holoscan::Holoviz::InputType::TRIANGLES, "triangles"}, - {nvidia::holoscan::Holoviz::InputType::CROSSES, "crosses"}, - {nvidia::holoscan::Holoviz::InputType::RECTANGLES, "rectangles"}, - {nvidia::holoscan::Holoviz::InputType::OVALS, "ovals"}, - {nvidia::holoscan::Holoviz::InputType::TEXT, "text"}}}; - -/** - * Convert a string to a input type enum - * - * @param string input type string - * @return input type enum - */ -static nvidia::gxf::Expected inputTypeFromString( - const std::string& string) { - const auto it = std::find_if(std::cbegin(kInputTypeToStr), - std::cend(kInputTypeToStr), - [&string](const auto& v) { return v.second == string; }); - if (it != std::cend(kInputTypeToStr)) { return it->first; } - - GXF_LOG_ERROR("Unsupported tensor type '%s'", string.c_str()); - return nvidia::gxf::Unexpected{GXF_FAILURE}; -} - -/** - * Convert a input type enum to a string - * - * @param input_type input type enum - * @return input type string - */ -static std::string inputTypeToString(nvidia::holoscan::Holoviz::InputType input_type) { - const auto it = std::find_if(std::cbegin(kInputTypeToStr), - std::cend(kInputTypeToStr), - [&input_type](const auto& v) { return v.first == input_type; }); - if (it != std::cend(kInputTypeToStr)) { return it->second; } - - return "invalid"; -} - -/** - * Try to detect the input type enum for given buffer properties. - * - * @param buffer_info buffer info - * @param has_lut true if the user specified a LUT - * - * @return input type enum - */ -nvidia::gxf::Expected detectInputType( - const BufferInfo& buffer_info, bool has_lut) { - // auto detect type - if (buffer_info.rank == 3) { - if ((buffer_info.shape.dimension(2) == 2) && (buffer_info.shape.dimension(0) == 1) && - (buffer_info.element_type == nvidia::gxf::PrimitiveType::kFloat32)) { - // array of 2D coordinates, draw crosses - return nvidia::holoscan::Holoviz::InputType::CROSSES; - } else if ((buffer_info.shape.dimension(2) == 1) && has_lut) { - // color image with lookup table - return nvidia::holoscan::Holoviz::InputType::COLOR_LUT; - } else if ((buffer_info.shape.dimension(2) == 3) || (buffer_info.shape.dimension(2) == 4)) { - // color image (RGB or RGBA) - return nvidia::holoscan::Holoviz::InputType::COLOR; - } else { - GXF_LOG_ERROR("Can't auto detect type of input %s", buffer_info.name.c_str()); - } - } - return nvidia::gxf::Unexpected{GXF_FAILURE}; -} - -/** - * Log the input spec - * - * @param input_specs input spec to log - */ -void logInputSpec(const std::vector& input_specs) { - std::stringstream ss; - ss << "Input spec:" << std::endl; - for (auto&& input_spec : input_specs) { - ss << "- name: '" << input_spec.tensor_name_ << "'" << std::endl; - ss << " type: '" << inputTypeToString(input_spec.type_) << "'" << std::endl; - ss << " opacity: " << input_spec.opacity_ << std::endl; - ss << " priority: " << input_spec.priority_ << std::endl; - if ((input_spec.type_ == nvidia::holoscan::Holoviz::InputType::POINTS) || - (input_spec.type_ == nvidia::holoscan::Holoviz::InputType::LINES) || - (input_spec.type_ == nvidia::holoscan::Holoviz::InputType::LINE_STRIP) || - (input_spec.type_ == nvidia::holoscan::Holoviz::InputType::TRIANGLES) || - (input_spec.type_ == nvidia::holoscan::Holoviz::InputType::CROSSES) || - (input_spec.type_ == nvidia::holoscan::Holoviz::InputType::RECTANGLES) || - (input_spec.type_ == nvidia::holoscan::Holoviz::InputType::OVALS) || - (input_spec.type_ == nvidia::holoscan::Holoviz::InputType::TEXT)) { - ss << " color: ["; - for (auto it = input_spec.color_.cbegin(); it < input_spec.color_.cend(); ++it) { - ss << *it; - if (it + 1 != input_spec.color_.cend()) { ss << ", "; } - } - ss << "]" << std::endl; - ss << " line_width: " << input_spec.line_width_ << std::endl; - ss << " point_size: " << input_spec.point_size_ << std::endl; - ss << " text: ["; - for (auto it = input_spec.text_.cbegin(); it < input_spec.text_.cend(); ++it) { - ss << *it; - if (it + 1 != input_spec.text_.cend()) { ss << ", "; } - } - ss << "]" << std::endl; - } - } - GXF_LOG_INFO(ss.str().c_str()); -} - -} // namespace - -/** - * Custom YAML parser for InputSpec class - */ -template <> -struct YAML::convert { - static Node encode(const nvidia::holoscan::Holoviz::InputSpec& input_spec) { - Node node; - node["type"] = inputTypeToString(input_spec.type_); - node["name"] = input_spec.tensor_name_; - node["opacity"] = std::to_string(input_spec.opacity_); - node["priority"] = std::to_string(input_spec.priority_); - node["color"] = input_spec.color_; - node["line_width"] = std::to_string(input_spec.line_width_); - node["point_size"] = std::to_string(input_spec.point_size_); - node["text"] = input_spec.text_; - return node; - } - - static bool decode(const Node& node, nvidia::holoscan::Holoviz::InputSpec& input_spec) { - if (!node.IsMap()) { - GXF_LOG_ERROR("InputSpec: expected a map"); - return false; - } - - // YAML is using exceptions, catch them - try { - const auto maybe_input_type = inputTypeFromString(node["type"].as()); - if (!maybe_input_type) { return false; } - - input_spec.tensor_name_ = node["name"].as(); - input_spec.type_ = maybe_input_type.value(); - input_spec.opacity_ = node["opacity"].as(input_spec.opacity_); - input_spec.priority_ = node["priority"].as(input_spec.priority_); - input_spec.color_ = node["color"].as>(input_spec.color_); - input_spec.line_width_ = node["line_width"].as(input_spec.line_width_); - input_spec.point_size_ = node["point_size"].as(input_spec.point_size_); - input_spec.text_ = node["text"].as>(input_spec.text_); - - return true; - } catch (const std::exception& e) { - GXF_LOG_ERROR(e.what()); - return false; - } - } -}; - -namespace nvidia::holoscan { - -constexpr uint32_t kDefaultWidth = 1920; -constexpr uint32_t kDefaultHeight = 1080; -constexpr uint32_t kDefaultFramerate = 60; -const std::string kDefaultWindowTitle = "Holoviz"; // NOLINT -const std::string kDefaultDisplayName = "DP-0"; // NOLINT -constexpr bool kDefaultExclusiveDisplay = false; -constexpr bool kDefaultFullscreen = false; -constexpr bool kDefaultHeadless = false; - -gxf_result_t Holoviz::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - result &= - registrar->parameter(receivers_, "receivers", "Input Receivers", "List of input receivers."); - - result &= registrar->parameter(render_buffer_input_, - "render_buffer_input", - "RenderBufferInput", - "Input for an empty render buffer.", - gxf::Registrar::NoDefaultParameter(), - GXF_PARAMETER_FLAGS_OPTIONAL); - result &= - registrar->parameter(render_buffer_output_, - "render_buffer_output", - "RenderBufferOutput", - "Output for a filled render buffer. If an input render buffer is " - "specified it is using that one, otherwise it allocates a new buffer.", - gxf::Registrar::NoDefaultParameter(), - GXF_PARAMETER_FLAGS_OPTIONAL); - - result &= registrar->parameter( - tensors_, - "tensors", - "Input Tensors", - "List of input tensors. 'name' is required, 'type' is optional (unknown, color, color_lut, " - "points, lines, line_strip, triangles, crosses, rectangles, ovals, text).", - std::vector()); - - result &= registrar->parameter(color_lut_, - "color_lut", - "ColorLUT", - "Color lookup table for tensors of type 'color_lut'", - std::vector>()); - - result &= registrar->parameter( - window_title_, "window_title", "Window title", "Title on window canvas", kDefaultWindowTitle); - result &= registrar->parameter(display_name_, - "display_name", - "Display name", - "In exclusive mode, name of display to use as shown with xrandr.", - kDefaultDisplayName); - result &= registrar->parameter( - width_, - "width", - "Width", - "Window width or display resolution width if in exclusive or fullscreen mode.", - kDefaultWidth); - result &= registrar->parameter( - height_, - "height", - "Height", - "Window height or display resolution height if in exclusive or fullscreen mode.", - kDefaultHeight); - result &= registrar->parameter(framerate_, - "framerate", - "Framerate", - "Display framerate if in exclusive mode.", - kDefaultFramerate); - result &= registrar->parameter(use_exclusive_display_, - "use_exclusive_display", - "Use exclusive display", - "Enable exclusive display", - kDefaultExclusiveDisplay); - result &= registrar->parameter(fullscreen_, - "fullscreen", - "Use fullscreen window", - "Enable fullscreen window", - kDefaultFullscreen); - result &= registrar->parameter(headless_, - "headless", - "Headless", - "Enable headless mode. No window is opened, the render buffer is " - "output to `render_buffer_output`", - kDefaultHeadless); - result &= registrar->parameter( - window_close_scheduling_term_, - "window_close_scheduling_term", - "WindowCloseSchedulingTerm", - "BooleanSchedulingTerm to stop the codelet from ticking when the window is closed.", - gxf::Handle()); - - result &= registrar->parameter(allocator_, - "allocator", - "Allocator", - "Allocator used to allocate render buffer output.", - gxf::Registrar::NoDefaultParameter(), - GXF_PARAMETER_FLAGS_OPTIONAL); - - return gxf::ToResultCode(result); -} - -gxf_result_t Holoviz::start() { - try { - // initialize Holoviz - viz::InitFlags init_flags = viz::InitFlags::NONE; - if (fullscreen_ && headless_) { - GXF_LOG_ERROR("Headless and fullscreen are mutually exclusive."); - return GXF_FAILURE; - } - if (fullscreen_) { init_flags = viz::InitFlags::FULLSCREEN; } - if (headless_) { init_flags = viz::InitFlags::HEADLESS; } - - if (use_exclusive_display_) { - viz::Init(display_name_.get().c_str(), width_, height_, framerate_, init_flags); - } else { - viz::Init(width_, height_, window_title_.get().c_str(), init_flags); - } - - // get the color lookup table - const auto& color_lut = color_lut_.get(); - lut_.reserve(color_lut.size() * 4); - for (auto&& color : color_lut) { - if (color.size() != 4) { - GXF_LOG_ERROR("Expected four components in color lookup table element, but got %zu", - color.size()); - return GXF_FAILURE; - } - lut_.insert(lut_.end(), color.begin(), color.end()); - } - - if (window_close_scheduling_term_.get()) { - const auto result = window_close_scheduling_term_->enable_tick(); - if (!result) { - GXF_LOG_ERROR("Failed to enable entity execution using '%s'", - window_close_scheduling_term_->name()); - return gxf::ToResultCode(result); - } - } - - // Copy the user defined input spec list to the internal input spec list. If there is no user - // defined input spec it will be generated from the first messages received. - if (!tensors_.get().empty()) { - input_spec_.reserve(tensors_.get().size()); - input_spec_.insert(input_spec_.begin(), tensors_.get().begin(), tensors_.get().end()); - } - } catch (const std::exception& e) { - GXF_LOG_ERROR(e.what()); - return GXF_FAILURE; - } - - return GXF_SUCCESS; -} - -gxf_result_t Holoviz::stop() { - try { - viz::Shutdown(); - } catch (const std::exception& e) { - GXF_LOG_ERROR(e.what()); - return GXF_FAILURE; - } - return GXF_SUCCESS; -} - -gxf_result_t Holoviz::tick() { - try { - // Grabs the messages from all receivers - std::vector messages; - messages.reserve(receivers_.get().size()); - for (auto& rx : receivers_.get()) { - gxf::Expected maybe_message = rx->receive(); - if (maybe_message) { messages.push_back(std::move(maybe_message.value())); } - } - if (messages.empty()) { - GXF_LOG_ERROR("No message available."); - return GXF_CONTRACT_MESSAGE_NOT_AVAILABLE; - } - - if (window_close_scheduling_term_.get()) { - // check if the window had been closed, if yes, stop the execution - if (viz::WindowShouldClose()) { - const auto result = window_close_scheduling_term_->disable_tick(); - if (!result) { - GXF_LOG_ERROR("Failed to enable entity execution using '%s'", - window_close_scheduling_term_->name()); - return gxf::ToResultCode(result); - } - return GXF_SUCCESS; - } - } - - // nothing to do if minimized - if (viz::WindowIsMinimized()) { return GXF_SUCCESS; } - - // if user provided it, we have a input spec which had been copied at start(). - // else we build the input spec automatically by inspecting the tensors/videobuffers of all - // messages - if (input_spec_.empty()) { - // get all tensors and video buffers of all messages and build the input spec - for (auto&& message : messages) { - const auto tensors = message.findAll(); - for (auto&& tensor : tensors.value()) { - BufferInfo buffer_info; - if (buffer_info.init(tensor.value()) != GXF_FAILURE) { - // try to detect the input type, if we can't detect it, ignore the tensor - const auto maybe_input_type = detectInputType(buffer_info, !lut_.empty()); - if (maybe_input_type) { - input_spec_.emplace_back(tensor->name(), maybe_input_type.value()); - } - } - } - const auto video_buffers = message.findAll(); - for (auto&& video_buffer : video_buffers.value()) { - BufferInfo buffer_info; - if (buffer_info.init(video_buffer.value()) != GXF_FAILURE) { - // try to detect the input type, if we can't detect it, ignore the tensor - const auto maybe_input_type = detectInputType(buffer_info, !lut_.empty()); - if (maybe_input_type) { - input_spec_.emplace_back(video_buffer->name(), maybe_input_type.value()); - } - } - } - } - } - - // begin visualization - viz::Begin(); - - // get the tensors attached to the messages by the tensor names defined by the input spec and - // display them - for (auto& input_spec : input_spec_) { - gxf::Expected> maybe_input_tensor = - gxf::Unexpected{GXF_UNINITIALIZED_VALUE}; - gxf::Expected> maybe_input_video = - gxf::Unexpected{GXF_UNINITIALIZED_VALUE}; - auto message = messages.begin(); - while (message != messages.end()) { - maybe_input_tensor = message->get(input_spec.tensor_name_.c_str()); - if (maybe_input_tensor) { - // pick the first one with that name - break; - } - // check for video if no tensor found - maybe_input_video = message->get(input_spec.tensor_name_.c_str()); - if (maybe_input_video) { // pick the first one with that name - break; - } - ++message; - } - if (message == messages.end()) { - GXF_LOG_ERROR("Failed to retrieve input '%s'", input_spec.tensor_name_.c_str()); - return GXF_FAILURE; - } - - BufferInfo buffer_info; - gxf_result_t result; - if (maybe_input_tensor) { - result = buffer_info.init(maybe_input_tensor.value()); - } else { - result = buffer_info.init(maybe_input_video.value()); - } - if (result != GXF_SUCCESS) { - GXF_LOG_ERROR("Unsupported buffer format tensor/video buffer '%s'", - input_spec.tensor_name_.c_str()); - return result; - } - - // if the input type is unknown it now can be detected using the image properties - if (input_spec.type_ == InputType::UNKNOWN) { - const auto maybe_input_type = detectInputType(buffer_info, !lut_.empty()); - if (!maybe_input_type) { return gxf::ToResultCode(maybe_input_type); } - input_spec.type_ = maybe_input_type.value(); - } - - switch (input_spec.type_) { - case InputType::COLOR: - case InputType::COLOR_LUT: { - // 2D color image - - // sanity checks - if (buffer_info.rank != 3) { - GXF_LOG_ERROR("Expected rank 3 for tensor '%s', type '%s', but got %u", - buffer_info.name.c_str(), - inputTypeToString(input_spec.type_).c_str(), - buffer_info.rank); - return GXF_FAILURE; - } - - /// @todo this is assuming HWC, should either auto-detect (if possible) or make user - /// configurable - const auto height = buffer_info.shape.dimension(0); - const auto width = buffer_info.shape.dimension(1); - const auto channels = buffer_info.shape.dimension(2); - - if (input_spec.type_ == InputType::COLOR_LUT) { - if (channels != 1) { - GXF_LOG_ERROR( - "Expected one channel for tensor '%s' when using lookup table, but got %d", - buffer_info.name.c_str(), - channels); - return GXF_FAILURE; - } - if (lut_.empty()) { - GXF_LOG_ERROR( - "Type of tensor '%s' is '%s', but a color lookup table has not been specified", - buffer_info.name.c_str(), - inputTypeToString(input_spec.type_).c_str()); - return GXF_FAILURE; - } - } - - auto maybe_image_format = getImageFormatFromTensor(buffer_info); - if (!maybe_image_format) { return gxf::ToResultCode(maybe_image_format); } - const viz::ImageFormat image_format = maybe_image_format.value(); - - // start a image layer - viz::BeginImageLayer(); - viz::LayerPriority(input_spec.priority_); - viz::LayerOpacity(input_spec.opacity_); - - if (input_spec.type_ == InputType::COLOR_LUT) { - viz::LUT(lut_.size() / 4, - viz::ImageFormat::R32G32B32A32_SFLOAT, - lut_.size() * sizeof(float), - lut_.data()); - } - - if (buffer_info.storage_type == gxf::MemoryStorageType::kDevice) { - setCudaStreamFromMessage(context(), *message); - - // if it's the device convert to `CUDeviceptr` - const auto cu_buffer_ptr = reinterpret_cast(buffer_info.buffer_ptr); - viz::ImageCudaDevice(width, height, image_format, cu_buffer_ptr); - } else { - // convert to void * if using the system/host - const auto host_buffer_ptr = reinterpret_cast(buffer_info.buffer_ptr); - viz::ImageHost(width, height, image_format, host_buffer_ptr); - } - viz::EndLayer(); - } break; - - case InputType::POINTS: - case InputType::LINES: - case InputType::LINE_STRIP: - case InputType::TRIANGLES: - case InputType::CROSSES: - case InputType::RECTANGLES: - case InputType::OVALS: - case InputType::TEXT: { - // geometry layer - - // get pointer to tensor buffer - std::vector host_buffer; - if (buffer_info.storage_type == gxf::MemoryStorageType::kDevice) { - host_buffer.resize(buffer_info.bytes_size); - - CUDA_TRY(cudaMemcpy(static_cast(host_buffer.data()), - static_cast(buffer_info.buffer_ptr), - buffer_info.bytes_size, - cudaMemcpyDeviceToHost)); - - buffer_info.buffer_ptr = host_buffer.data(); - } - - // start a geometry layer - viz::BeginGeometryLayer(); - viz::LayerPriority(input_spec.priority_); - viz::LayerOpacity(input_spec.opacity_); - std::array color{1.f, 1.f, 1.f, 1.f}; - for (size_t index = 0; index < std::min(input_spec.color_.size(), color.size()); - ++index) { - color[index] = input_spec.color_[index]; - } - viz::Color(color[0], color[1], color[2], color[3]); - - /// @todo this is assuming NHW, should either auto-detect (if possible) or make user - /// configurable - const auto coordinates = buffer_info.shape.dimension(1); - const auto components = buffer_info.shape.dimension(2); - - if (input_spec.type_ == InputType::TEXT) { - // text is defined by the top left coordinate and the size (x, y, s) per string, text - // strings are define by InputSpec::text_ - if ((components < 2) || (components > 3)) { - GXF_LOG_ERROR("Expected two or three values per text, but got '%d'", components); - return GXF_FAILURE; - } - const float* src_coord = reinterpret_cast(buffer_info.buffer_ptr); - for (int32_t index = 0; index < coordinates; ++index) { - viz::Text( - src_coord[0], - src_coord[1], - (components == 3) ? src_coord[2] : 0.05f, - input_spec.text_[std::min(index, (int32_t)input_spec.text_.size() - 1)].c_str()); - src_coord += components; - } - } else { - viz::LineWidth(input_spec.line_width_); - - std::vector coords; - viz::PrimitiveTopology topology; - uint32_t primitive_count; - uint32_t coordinate_count; - uint32_t values_per_coordinate; - std::vector default_coord; - if (input_spec.type_ == InputType::POINTS) { - // point primitives, one coordinate (x, y) per primitive - if (components != 2) { - GXF_LOG_ERROR("Expected two values per point, but got '%d'", components); - return GXF_FAILURE; - } - - viz::PointSize(input_spec.point_size_); - - topology = viz::PrimitiveTopology::POINT_LIST; - primitive_count = coordinates; - coordinate_count = primitive_count; - values_per_coordinate = 2; - default_coord = {0.f, 0.f}; - } else if (input_spec.type_ == InputType::LINES) { - // line primitives, two coordinates (x0, y0) and (x1, y1) per primitive - if (components != 2) { - GXF_LOG_ERROR("Expected two values per line vertex, but got '%d'", components); - return GXF_FAILURE; - } - topology = viz::PrimitiveTopology::LINE_LIST; - primitive_count = coordinates / 2; - coordinate_count = primitive_count * 2; - values_per_coordinate = 2; - default_coord = {0.f, 0.f}; - } else if (input_spec.type_ == InputType::LINE_STRIP) { - // line primitives, two coordinates (x0, y0) and (x1, y1) per primitive - if (components != 2) { - GXF_LOG_ERROR("Expected two values per line strip vertex, but got '%d'", - components); - return GXF_FAILURE; - } - topology = viz::PrimitiveTopology::LINE_STRIP; - primitive_count = coordinates - 1; - coordinate_count = coordinates; - values_per_coordinate = 2; - default_coord = {0.f, 0.f}; - } else if (input_spec.type_ == InputType::TRIANGLES) { - // triangle primitive, three coordinates (x0, y0), (x1, y1) and (x2, y2) per primitive - if (components != 2) { - GXF_LOG_ERROR("Expected two values per triangle vertex, but got '%d'", components); - return GXF_FAILURE; - } - topology = viz::PrimitiveTopology::TRIANGLE_LIST; - primitive_count = coordinates / 3; - coordinate_count = primitive_count * 3; - values_per_coordinate = 2; - default_coord = {0.f, 0.f}; - } else if (input_spec.type_ == InputType::CROSSES) { - // cross primitive, a cross is defined by the center coordinate and the size (xi, yi, - // si) - if ((components < 2) || (components > 3)) { - GXF_LOG_ERROR("Expected two or three values per cross, but got '%d'", components); - return GXF_FAILURE; - } - - topology = viz::PrimitiveTopology::CROSS_LIST; - primitive_count = coordinates; - coordinate_count = primitive_count; - values_per_coordinate = 3; - default_coord = {0.f, 0.f, 0.05f}; - } else if (input_spec.type_ == InputType::RECTANGLES) { - // axis aligned rectangle primitive, each rectangle is defined by two coordinates (xi, - // yi) and (xi+1, yi+1) - if (components != 2) { - GXF_LOG_ERROR("Expected two values per rectangle vertex, but got '%d'", components); - return GXF_FAILURE; - } - topology = viz::PrimitiveTopology::RECTANGLE_LIST; - primitive_count = coordinates / 2; - coordinate_count = primitive_count * 2; - values_per_coordinate = 2; - default_coord = {0.f, 0.f}; - } else if (input_spec.type_ == InputType::OVALS) { - // oval primitive, an oval primitive is defined by the center coordinate and the axis - // sizes (xi, yi, sxi, syi) - if ((components < 2) || (components > 4)) { - GXF_LOG_ERROR("Expected two, three or four values per oval, but got '%d'", - components); - return GXF_FAILURE; - } - topology = viz::PrimitiveTopology::OVAL_LIST; - primitive_count = coordinates; - coordinate_count = primitive_count; - values_per_coordinate = 4; - default_coord = {0.f, 0.f, 0.05f, 0.05f}; - } else { - GXF_LOG_ERROR("Unhandled tensor type '%s'", - inputTypeToString(input_spec.type_).c_str()); - return GXF_FAILURE; - } - - // copy coordinates - const float* src_coord = reinterpret_cast(buffer_info.buffer_ptr); - coords.reserve(coordinate_count * values_per_coordinate); - for (int32_t index = 0; index < coordinate_count; ++index) { - int32_t component_index = 0; - // copy from source array - while (component_index < std::min(components, int32_t(values_per_coordinate))) { - coords.push_back(src_coord[component_index]); - ++component_index; - } - // fill from default array - while (component_index < values_per_coordinate) { - coords.push_back(default_coord[component_index]); - ++component_index; - } - src_coord += buffer_info.stride[1] / sizeof(float); - } - - if (primitive_count) { - viz::Primitive(topology, primitive_count, coords.size(), coords.data()); - } - } - - viz::EndLayer(); - } break; - default: - GXF_LOG_ERROR("Unhandled input type '%s'", inputTypeToString(input_spec.type_).c_str()); - return GXF_FAILURE; - } - } - - viz::End(); - - // check if the render buffer should be output - if (render_buffer_output_.try_get()) { - auto entity = gxf::Entity::New(context()); - if (!entity) { - GXF_LOG_ERROR("Failed to allocate message for the render buffer output."); - return GXF_FAILURE; - } - - auto video_buffer = entity.value().add("render_buffer_output"); - if (!video_buffer) { - GXF_LOG_ERROR("Failed to allocate the video buffer for the render buffer output."); - return GXF_FAILURE; - } - - // check if there is a input buffer given to copy the output into - if (render_buffer_input_.try_get()) { - const auto& render_buffer_input = render_buffer_input_.try_get().value()->receive(); - if (!render_buffer_input) { - GXF_LOG_ERROR("No message available at 'render_buffer_input'."); - return GXF_FAILURE; - } - - // Get the empty input buffer - const auto& video_buffer_in = render_buffer_input.value().get(); - if (!video_buffer_in) { - GXF_LOG_ERROR("No video buffer attached to message on 'render_buffer_input'."); - return GXF_FAILURE; - } - - const gxf::VideoBufferInfo info = video_buffer_in.value()->video_frame_info(); - - if ((info.color_format != gxf::VideoFormat::GXF_VIDEO_FORMAT_RGBA)) { - GXF_LOG_ERROR("Invalid render buffer input, expected an RGBA buffer."); - return GXF_FAILURE; - } - - video_buffer.value()->wrapMemory(info, - video_buffer_in.value()->size(), - video_buffer_in.value()->storage_type(), - video_buffer_in.value()->pointer(), - nullptr); - } else { - // if there is no input buffer given, allocate one - if (!allocator_.try_get()) { - GXF_LOG_ERROR("No render buffer input specified and no allocator set."); - return GXF_FAILURE; - } - - video_buffer.value()->resize( - width_, - height_, - gxf::SurfaceLayout::GXF_SURFACE_LAYOUT_BLOCK_LINEAR, - gxf::MemoryStorageType::kDevice, - allocator_.try_get().value()); - if (!video_buffer.value()->pointer()) { - GXF_LOG_ERROR("Failed to allocate render output buffer."); - return GXF_FAILURE; - } - } - - // read the framebuffer - viz::ReadFramebuffer(viz::ImageFormat::R8G8B8A8_UNORM, - video_buffer.value()->size(), - reinterpret_cast(video_buffer.value()->pointer())); - - // Output the filled render buffer object - const auto result = - render_buffer_output_.try_get().value()->publish(std::move(entity.value())); - if (GXF_SUCCESS != gxf::ToResultCode(result)) { - GXF_LOG_ERROR("Failed to publish render output buffer"); - return GXF_FAILURE; - } - } - - // print the input spec on first tick to let the user know what had been detected - if (isFirstTick()) { logInputSpec(input_spec_); - } - } catch (const std::exception& e) { - GXF_LOG_ERROR(e.what()); - return GXF_FAILURE; - } - - return GXF_SUCCESS; -} - -} // namespace nvidia::holoscan diff --git a/gxf_extensions/holoviz/holoviz.hpp b/gxf_extensions/holoviz/holoviz.hpp deleted file mode 100644 index 5c42d5b8..00000000 --- a/gxf_extensions/holoviz/holoviz.hpp +++ /dev/null @@ -1,118 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef GXF_EXTENSIONS_HOLOVIZ_HOLOVIZ_HPP -#define GXF_EXTENSIONS_HOLOVIZ_HOLOVIZ_HPP - -#include -#include -#include -#include - -#include "gxf/core/handle.hpp" -#include "gxf/std/allocator.hpp" -#include "gxf/std/codelet.hpp" -#include "gxf/std/receiver.hpp" -#include "gxf/std/scheduling_terms.hpp" - -namespace nvidia::holoscan { - -class Holoviz : public gxf::Codelet { - public: - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - - gxf_result_t start() override; - gxf_result_t tick() override; - gxf_result_t stop() override; - - /** - * Input type. - * - * All geometric primitives expect a 1d array of coordinates. Coordinates range from 0.0 (left, - * top) to 1.0 (right, bottom). - */ - enum class InputType { - UNKNOWN, ///< unknown type, the operator tries to guess the type by inspecting the tensor - COLOR, ///< RGB or RGBA color 2d image - COLOR_LUT, ///< single channel 2d image, color is looked up - POINTS, ///< point primitives, one coordinate (x, y) per primitive - LINES, ///< line primitives, two coordinates (x0, y0) and (x1, y1) per primitive - LINE_STRIP, ///< line strip primitive, a line primitive i is defined by each coordinate (xi, - ///< yi) and the following (xi+1, yi+1) - TRIANGLES, ///< triangle primitive, three coordinates (x0, y0), (x1, y1) and (x2, y2) per - ///< primitive - CROSSES, ///< cross primitive, a cross is defined by the center coordinate and the size (xi, - ///< yi, si) - RECTANGLES, ///< axis aligned rectangle primitive, each rectangle is defined by two coordinates - ///< (xi, yi) and (xi+1, yi+1) - OVALS, ///< oval primitive, an oval primitive is defined by the center coordinate and the axis - ///< sizes (xi, yi, sxi, syi) - TEXT ///< text is defined by the top left coordinate and the size (x, y, s) per string, text - ///< strings are define by InputSpec::text_ - }; - - /** - * Input specification - */ - struct InputSpec { - InputSpec() = default; - InputSpec(const std::string tensor_name, InputType type) - : tensor_name_(tensor_name), type_(type) {} - - std::string tensor_name_; ///< name of the tensor containing the input data - InputType type_; ///< input type - float opacity_ = 1.f; ///< layer opacity, 1.0 is fully opaque, 0.0 is fully transparent - int32_t priority_ = - 0; ///< layer priority, determines the render order, layers with higher priority values are - ///< rendered on top of layers with lower priority values - std::vector color_{1.f, 1.f, 1.f, 1.f}; ///< color of rendered geometry - float line_width_ = 1.f; ///< line width for geometry made of lines - float point_size_ = 1.f; ///< point size for geometry made of points - std::vector text_; ///< array of text strings, used when type_ is text. - }; - - private: - // parameters - gxf::Parameter>> receivers_; - - gxf::Parameter> render_buffer_input_; - gxf::Parameter> render_buffer_output_; - - gxf::Parameter> tensors_; - - gxf::Parameter>> color_lut_; - - gxf::Parameter window_title_; - gxf::Parameter display_name_; - gxf::Parameter width_; - gxf::Parameter height_; - gxf::Parameter framerate_; - gxf::Parameter use_exclusive_display_; - gxf::Parameter fullscreen_; - gxf::Parameter headless_; - gxf::Parameter> window_close_scheduling_term_; - - gxf::Parameter> allocator_; - - // internal state - std::vector lut_; - std::vector input_spec_; -}; - -} // namespace nvidia::holoscan - -#endif /* GXF_EXTENSIONS_HOLOVIZ_HOLOVIZ_HPP */ diff --git a/gxf_extensions/mocks/CMakeLists.txt b/gxf_extensions/mocks/CMakeLists.txt deleted file mode 100644 index 3fe925ce..00000000 --- a/gxf_extensions/mocks/CMakeLists.txt +++ /dev/null @@ -1,41 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Create library -add_library(video_buffer_mock_lib SHARED - video_buffer_mock.cpp - video_buffer_mock.hpp -) -target_link_libraries(video_buffer_mock_lib - PUBLIC - CUDA::cudart - CUDA::nppidei - GXF::core - GXF::multimedia - GXF::std - yaml-cpp -) - -# Create extension -add_library(mocks SHARED - mocks_ext.cpp -) -target_link_libraries(mocks - PUBLIC video_buffer_mock_lib -) - -# Install GXF extension as a component 'holoscan-gxf_extensions' -install_gxf_extension(mocks) -install_gxf_extension(video_buffer_mock_lib) # not following '{name}_lib' so install explicitly diff --git a/gxf_extensions/mocks/mocks_ext.cpp b/gxf_extensions/mocks/mocks_ext.cpp deleted file mode 100644 index 12853daf..00000000 --- a/gxf_extensions/mocks/mocks_ext.cpp +++ /dev/null @@ -1,27 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "gxf/std/extension_factory_helper.hpp" - -#include "video_buffer_mock.hpp" - -GXF_EXT_FACTORY_BEGIN() -GXF_EXT_FACTORY_SET_INFO(0xa494ec3e9f114704, 0xa0d181b3f1700d2e, "TestMockExtension", - "Holoscan Test Mock extension", "NVIDIA", "0.2.0", "LICENSE"); -GXF_EXT_FACTORY_ADD(0xa7d54d962b244b49, 0x94e4d60c0bb39903, - nvidia::holoscan::mocks::VideoBufferMock, - nvidia::gxf::Codelet, "VideoBuffer Mock codelet."); -GXF_EXT_FACTORY_END() diff --git a/gxf_extensions/mocks/video_buffer_mock.cpp b/gxf_extensions/mocks/video_buffer_mock.cpp deleted file mode 100644 index 948a6e50..00000000 --- a/gxf_extensions/mocks/video_buffer_mock.cpp +++ /dev/null @@ -1,133 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "video_buffer_mock.hpp" - -#include -#include - -#include - -#include "gxf/core/entity.hpp" -#include "gxf/core/expected.hpp" -#include "gxf/multimedia/video.hpp" -#include "gxf/std/tensor.hpp" - - -#define CUDA_TRY(stmt) \ - ({ \ - cudaError_t _holoscan_cuda_err = stmt; \ - if (cudaSuccess != _holoscan_cuda_err) { \ - GXF_LOG_ERROR("CUDA Runtime call %s in line %d of file %s failed with '%s' (%d).\n", #stmt, \ - __LINE__, __FILE__, cudaGetErrorString(_holoscan_cuda_err), \ - _holoscan_cuda_err); \ - } \ - _holoscan_cuda_err; \ - }) - -namespace nvidia { -namespace holoscan { -namespace mocks { - -constexpr int32_t DEFAULT_SRC_WIDTH = 640; -constexpr int32_t DEFAULT_SRC_HEIGHT = 480; -constexpr int16_t DEFAULT_SRC_CHANNELS = 3; -constexpr uint8_t DEFAULT_SRC_BYTES_PER_PIXEL = 1; - -gxf_result_t VideoBufferMock::start() { - return GXF_SUCCESS; -} - -gxf_result_t VideoBufferMock::stop() { - buffer_.freeBuffer(); - return GXF_SUCCESS; -} - -gxf_result_t VideoBufferMock::tick() { - const int32_t rows = in_height_; - const int32_t columns = in_width_; - const int16_t channels = in_channels_; - const uint8_t bytes_per_pixel = in_bytes_per_pixel_; - - if (buffer_.size() == 0) { - buffer_.resize(pool_, columns * rows * sizeof(float), gxf::MemoryStorageType::kDevice); - } - - // Pass the frame downstream. - auto message = gxf::Entity::New(context()); - if (!message) { - GXF_LOG_ERROR("Failed to allocate message"); - return GXF_FAILURE; - } - - auto buffer = message.value().add(); - if (!buffer) { - GXF_LOG_ERROR("Failed to allocate video buffer"); - return GXF_FAILURE; - } - - gxf::VideoTypeTraits video_type; - gxf::VideoFormatSize color_format; - auto color_planes = color_format.getDefaultColorPlanes(columns, rows); - gxf::VideoBufferInfo info{static_cast(columns), static_cast(rows), - video_type.value, color_planes, - gxf::SurfaceLayout::GXF_SURFACE_LAYOUT_PITCH_LINEAR}; - - const auto buffer_ptr = buffer_.pointer(); - const int32_t row_step = channels * bytes_per_pixel * columns; - - nppiSet_8u_C4R(std::array{255, 0, 0, 255}.data(), buffer_ptr, row_step, - NppiSize{static_cast(columns), static_cast(rows / 3)}); - nppiSet_8u_C4R(std::array{0, 255, 0, 255}.data(), buffer_ptr + (row_step * rows / 3), - row_step, NppiSize{static_cast(columns), static_cast(rows / 3)}); - nppiSet_8u_C4R(std::array{0, 0, 255, 255}.data(), - buffer_ptr + (row_step * rows * 2 / 3), row_step, - NppiSize{static_cast(columns), static_cast(rows / 3)}); - - auto storage_type = gxf::MemoryStorageType::kDevice; - buffer.value()->wrapMemory(info, buffer_.size(), storage_type, buffer_.pointer(), nullptr); - - out_->publish(std::move(message.value())); - - printf("count: %" PRIu64 "\n", ++count_); - - return gxf::ToResultCode(message); -} - -gxf_result_t VideoBufferMock::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - result &= registrar->parameter(in_width_, "in_width", "SourceWidth", "Width of the image.", - DEFAULT_SRC_WIDTH); - result &= registrar->parameter(in_height_, "in_height", "SourceHeight", "Height of the image.", - DEFAULT_SRC_HEIGHT); - result &= registrar->parameter(in_channels_, "in_channels", "SourceChannels", - "Number of channels.", DEFAULT_SRC_CHANNELS); - result &= registrar->parameter(in_bytes_per_pixel_, - "in_bytes_per_pixel", - "InputBytesPerPixel", - "Number of bytes per pixel of the image.", - DEFAULT_SRC_BYTES_PER_PIXEL); - result &= registrar->parameter(out_tensor_name_, "out_tensor_name", "OutputTensorName", - "Name of the output tensor.", std::string("")); - - result &= registrar->parameter(out_, "out", "Output", "Output channel."); - result &= registrar->parameter(pool_, "pool", "Pool", "Pool to allocate the output message."); - return gxf::ToResultCode(result); -} - -} // namespace mocks -} // namespace holoscan -} // namespace nvidia diff --git a/gxf_extensions/mocks/video_buffer_mock.hpp b/gxf_extensions/mocks/video_buffer_mock.hpp deleted file mode 100644 index 486992fa..00000000 --- a/gxf_extensions/mocks/video_buffer_mock.hpp +++ /dev/null @@ -1,60 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef MOCKS_VIDEO_BUFFER_MOCK_HPP -#define MOCKS_VIDEO_BUFFER_MOCK_HPP - -#include -#include -#include - -#include "gxf/std/allocator.hpp" -#include "gxf/std/codelet.hpp" -#include "gxf/std/memory_buffer.hpp" -#include "gxf/std/transmitter.hpp" - -namespace nvidia { -namespace holoscan { -namespace mocks { - -/// @brief Mock video input codelet for testing. -class VideoBufferMock : public gxf::Codelet { - public: - gxf_result_t start() override; - gxf_result_t tick() override; - gxf_result_t stop() override; - - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - - private: - gxf::Parameter in_width_; - gxf::Parameter in_height_; - gxf::Parameter in_channels_; - gxf::Parameter in_bytes_per_pixel_; - gxf::Parameter> out_; - gxf::Parameter out_tensor_name_; - gxf::Parameter> pool_; - - // Tick counter - gxf::MemoryBuffer buffer_; - uint64_t count_ = 0; -}; - -} // namespace mocks -} // namespace holoscan -} // namespace nvidia - -#endif /* MOCKS_VIDEO_BUFFER_MOCK_HPP */ diff --git a/gxf_extensions/multiai_inference/CMakeLists.txt b/gxf_extensions/multiai_inference/CMakeLists.txt deleted file mode 100644 index a6a3b107..00000000 --- a/gxf_extensions/multiai_inference/CMakeLists.txt +++ /dev/null @@ -1,42 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -add_library(multiai_inference_lib SHARED - multiai_inference.cpp - multiai_inference.hpp -) - -target_include_directories(multiai_inference_lib - PUBLIC - ${CMAKE_SOURCE_DIR}/modules/holoinfer/src/include) - -target_link_libraries(multiai_inference_lib - PUBLIC - CUDA::cudart - GXF::cuda - GXF::std - GXF::multimedia - yaml-cpp - holoinfer -) - -add_library(multiai_inference SHARED - multiai_inference_extension.cpp -) -target_link_libraries(multiai_inference - PUBLIC multiai_inference_lib -) - -install_gxf_extension(multiai_inference) diff --git a/gxf_extensions/multiai_inference/multiai_inference.cpp b/gxf_extensions/multiai_inference/multiai_inference.cpp deleted file mode 100644 index 0548a909..00000000 --- a/gxf_extensions/multiai_inference/multiai_inference.cpp +++ /dev/null @@ -1,174 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "multiai_inference.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace nvidia { -namespace holoscan { -namespace multiai { - -gxf_result_t MultiAIInference::start() { - try { - // Check for the validity of parameters from configuration - auto status = HoloInfer::multiai_inference_validity_check(model_path_map_.get(), - pre_processor_map_.get(), - inference_map_.get(), - in_tensor_names_.get(), - out_tensor_names_.get()); - if (status.get_code() != HoloInfer::holoinfer_code::H_SUCCESS) { - return HoloInfer::report_error(module_, - "Parameter Validation failed: " + status.get_message()); - } - - // Create multiai specification structure - multiai_specs_ = std::make_shared(backend_.get(), - model_path_map_.get(), - inference_map_.get(), - is_engine_path_.get(), - infer_on_cpu_.get(), - parallel_inference_.get(), - enable_fp16_.get(), - input_on_cuda_.get(), - output_on_cuda_.get()); - - // Create holoscan inference context - holoscan_infer_context_ = std::make_unique(); - - // Set and transfer inference specification to inference context - // Multi AI specifications are updated with memory allocations - status = holoscan_infer_context_->set_inference_params(multiai_specs_); - if (status.get_code() != HoloInfer::holoinfer_code::H_SUCCESS) { - return HoloInfer::report_error(module_, "Start, Parameters setup, " + status.get_message()); - } - } catch (const std::bad_alloc& b_) { - return HoloInfer::report_error(module_, - "Start, Memory allocation, Message: " + std::string(b_.what())); - } catch (...) { return HoloInfer::report_error(module_, "Start, Unknown exception"); } - - return GXF_SUCCESS; -} - -gxf_result_t MultiAIInference::stop() { - holoscan_infer_context_.reset(); - return GXF_SUCCESS; -} - -gxf_result_t MultiAIInference::tick() { - try { - // Extract relevant data from input GXF Receivers, and update multiai specifications - gxf_result_t stat = HoloInfer::multiai_get_data_per_model(receivers_.get(), - in_tensor_names_.get(), - multiai_specs_->data_per_tensor_, - dims_per_tensor_, - input_on_cuda_.get(), - module_); - - if (stat != GXF_SUCCESS) { return HoloInfer::report_error(module_, "Tick, Data extraction"); } - - auto status = HoloInfer::map_data_to_model_from_tensor(pre_processor_map_.get(), - multiai_specs_->data_per_model_, - multiai_specs_->data_per_tensor_); - if (status.get_code() != HoloInfer::holoinfer_code::H_SUCCESS) { - return HoloInfer::report_error(module_, "Tick, Data mapping, " + status.get_message()); - } - - // Execute inference and populate output buffer in multiai specifications - status = holoscan_infer_context_->execute_inference(multiai_specs_->data_per_model_, - multiai_specs_->output_per_model_); - if (status.get_code() != HoloInfer::holoinfer_code::H_SUCCESS) { - return HoloInfer::report_error(module_, "Tick, Inference execution, " + status.get_message()); - } - GXF_LOG_DEBUG("%s", status.get_message().c_str()); - - // Get output dimensions - auto model_out_dims_map = holoscan_infer_context_->get_output_dimensions(); - - auto cont = context(); - - // Transmit output buffers via a single GXF transmitter - stat = HoloInfer::multiai_transmit_data_per_model(cont, - inference_map_.get(), - multiai_specs_->output_per_model_, - transmitter_.get(), - out_tensor_names_.get(), - model_out_dims_map, - output_on_cuda_.get(), - transmit_on_cuda_.get(), - data_type_, - module_, - allocator_.get()); - if (stat != GXF_SUCCESS) { return HoloInfer::report_error(module_, "Tick, Data Transmission"); } - } catch (const std::runtime_error& r_) { - return HoloInfer::report_error(module_, - "Tick, Inference execution, Message->" + std::string(r_.what())); - } catch (...) { return HoloInfer::report_error(module_, "Tick, unknown exception"); } - return GXF_SUCCESS; -} - -gxf_result_t MultiAIInference::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - - result &= registrar->parameter(backend_, "backend", "Supported backend"); - - result &= registrar->parameter(model_path_map_, - "model_path_map", - "Model Keyword with File Path", - "Path to ONNX model to be loaded."); - result &= registrar->parameter(pre_processor_map_, - "pre_processor_map", - "Pre processor setting per model", - "Pre processed data to model map."); - result &= registrar->parameter( - inference_map_, "inference_map", "Inferred tensor per model", "Tensor to model map."); - result &= registrar->parameter( - in_tensor_names_, "in_tensor_names", "Input Tensors", "Input tensors", {std::string("")}); - result &= registrar->parameter( - out_tensor_names_, "out_tensor_names", "Output Tensors", "Output tensors", {std::string("")}); - result &= registrar->parameter(allocator_, "allocator", "Allocator", "Output Allocator"); - result &= registrar->parameter( - is_engine_path_, "is_engine_path", "Input file paths are trt engine files", "", false); - result &= - registrar->parameter(infer_on_cpu_, "infer_on_cpu", "Inference on CPU", "Use CPU.", false); - result &= - registrar->parameter(enable_fp16_, "enable_fp16", "use FP16 engine", "Use FP16.", false); - result &= registrar->parameter( - input_on_cuda_, "input_on_cuda", "Input for inference on cuda", "", true); - result &= - registrar->parameter(output_on_cuda_, "output_on_cuda", "Inferred output on cuda", "", true); - result &= registrar->parameter( - transmit_on_cuda_, "transmit_on_cuda", "Transmit message on cuda", "", true); - - result &= registrar->parameter(parallel_inference_, "parallel_inference", "Parallel inference"); - - result &= registrar->parameter( - receivers_, "receivers", "Receivers", "List of receivers to take input tensors"); - result &= registrar->parameter(transmitter_, "transmitter", "Transmitter", "Transmitter"); - - return gxf::ToResultCode(result); -} - -} // namespace multiai -} // namespace holoscan -} // namespace nvidia diff --git a/gxf_extensions/multiai_inference/multiai_inference.hpp b/gxf_extensions/multiai_inference/multiai_inference.hpp deleted file mode 100644 index 1e906bf6..00000000 --- a/gxf_extensions/multiai_inference/multiai_inference.hpp +++ /dev/null @@ -1,135 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef NVIDIA_GXF_EXTENSIONS_MULTIAI_INFERENCE_HPP_ -#define NVIDIA_GXF_EXTENSIONS_MULTIAI_INFERENCE_HPP_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "gxf/core/entity.hpp" -#include "gxf/core/gxf.h" -#include "gxf/core/parameter.hpp" -#include "gxf/cuda/cuda_stream.hpp" -#include "gxf/cuda/cuda_stream_id.hpp" -#include "gxf/cuda/cuda_stream_pool.hpp" -#include "gxf/multimedia/video.hpp" -#include "gxf/std/allocator.hpp" -#include "gxf/std/clock.hpp" -#include "gxf/std/codelet.hpp" -#include "gxf/std/parameter_parser_std.hpp" -#include "gxf/std/receiver.hpp" -#include "gxf/std/tensor.hpp" -#include "gxf/std/timestamp.hpp" -#include "gxf/std/transmitter.hpp" - -#include -#include - -namespace HoloInfer = holoscan::inference; - -namespace nvidia { -namespace holoscan { -namespace multiai { -/** - * @brief Generic Multi AI inference codelet to perform multi model inference. - */ -class MultiAIInference : public gxf::Codelet { - public: - gxf_result_t start() override; - gxf_result_t tick() override; - gxf_result_t stop() override; - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - - private: - /// Map with key as model name and value as model file path - gxf::Parameter model_path_map_; - - /// Map with key as model name and value as vector of input tensor names - gxf::Parameter pre_processor_map_; - - /// Map with key as model name and value as inferred tensor name - gxf::Parameter inference_map_; - - /// Flag to show if input model path mapping is for cached trt engine files - gxf::Parameter is_engine_path_; - - /// Input tensor names - gxf::Parameter > in_tensor_names_; - - /// Output tensor names - gxf::Parameter > out_tensor_names_; - - /// Memory allocator - gxf::Parameter > allocator_; - - /// Flag to enable inference on CPU (only supported by onnxruntime). - /// Default is False. - gxf::Parameter infer_on_cpu_; - - /// Flag to enable parallel inference. Default is True. - gxf::Parameter parallel_inference_; - - /// Flag showing if trt engine file conversion will use FP16. Default is False. - gxf::Parameter enable_fp16_; - - /// Backend to do inference on. Supported values: "trt", "onnxrt". - gxf::Parameter backend_; - - /// Vector of input receivers. Multiple receivers supported. - gxf::Parameter receivers_; - - /// Output transmitter. Single transmitter supported. - gxf::Parameter transmitter_; - - /// Flag showing if input buffers are on CUDA. Default is True. - gxf::Parameter input_on_cuda_; - - /// Flag showing if output buffers are on CUDA. Default is True. - gxf::Parameter output_on_cuda_; - - /// Flag showing if data transmission is on CUDA. Default is True. - gxf::Parameter transmit_on_cuda_; - - /// Pointer to inference context. - std::unique_ptr holoscan_infer_context_; - - /// Pointer to multi ai inference specifications - std::shared_ptr multiai_specs_; - - /// Data type of message to be transmitted. Supported value: float32 - gxf::PrimitiveType data_type_ = gxf::PrimitiveType::kFloat32; - - /// Map holding dimensions per model. Key is model name and value is a vector with - /// dimensions. - std::map > dims_per_tensor_; - - /// Codelet Identifier, used in reporting. - const std::string module_{"Multi AI Inference Codelet"}; -}; - -} // namespace multiai -} // namespace holoscan -} // namespace nvidia - -#endif // NVIDIA_GXF_EXTENSIONS_MULTIAI_INFERENCE_HPP_ diff --git a/gxf_extensions/multiai_inference/multiai_inference_extension.cpp b/gxf_extensions/multiai_inference/multiai_inference_extension.cpp deleted file mode 100644 index de098522..00000000 --- a/gxf_extensions/multiai_inference/multiai_inference_extension.cpp +++ /dev/null @@ -1,35 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include - -#include "gxf/core/gxf.h" -#include "gxf/std/extension_factory_helper.hpp" - -#include "multiai_inference.hpp" - -extern "C" { - -GXF_EXT_FACTORY_BEGIN() -GXF_EXT_FACTORY_SET_INFO(0xa5b5234fe9004901, 0x9e098b37d539ecbd, "MultiAIExtension", - "MultiAI with Holoscan Inference", "NVIDIA", "1.0.0", "LICENSE"); - -GXF_EXT_FACTORY_ADD(0x43c2567eb5a442f7, 0x9a1781764a01822e, - nvidia::holoscan::multiai::MultiAIInference, nvidia::gxf::Codelet, - "Multi AI Codelet."); - -GXF_EXT_FACTORY_END() -} diff --git a/gxf_extensions/multiai_postprocessor/CMakeLists.txt b/gxf_extensions/multiai_postprocessor/CMakeLists.txt deleted file mode 100644 index 619d6731..00000000 --- a/gxf_extensions/multiai_postprocessor/CMakeLists.txt +++ /dev/null @@ -1,43 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -add_library(multiai_postprocessor_lib SHARED - multiai_postprocessor.cpp - multiai_postprocessor.hpp -) - -target_link_libraries(multiai_postprocessor_lib - PUBLIC - CUDA::cudart - GXF::cuda - GXF::std - GXF::multimedia - yaml-cpp - holoinfer -) - -target_include_directories(multiai_postprocessor_lib - PUBLIC - ${CMAKE_SOURCE_DIR}/modules/holoinfer/src/include) - -# Create extension -add_library(multiai_postprocessor SHARED - multiai_postprocessor_extension.cpp -) -target_link_libraries(multiai_postprocessor - PUBLIC multiai_postprocessor_lib -) - -install_gxf_extension(multiai_postprocessor) diff --git a/gxf_extensions/multiai_postprocessor/multiai_postprocessor.cpp b/gxf_extensions/multiai_postprocessor/multiai_postprocessor.cpp deleted file mode 100644 index 02007066..00000000 --- a/gxf_extensions/multiai_postprocessor/multiai_postprocessor.cpp +++ /dev/null @@ -1,135 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "multiai_postprocessor.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace nvidia { -namespace holoscan { -namespace multiai { - -gxf_result_t MultiAIPostprocessor::start() { - try { - // Check for the validity of parameters from configuration - if (input_on_cuda_.get() || output_on_cuda_.get() || transmit_on_cuda_.get()) { - return HoloInfer::report_error(module_, - "CUDA based data not supported in Multi AI post processor"); - } - auto status = HoloInfer::multiai_processor_validity_check( - processed_map_.get(), in_tensor_names_.get(), out_tensor_names_.get()); - if (status.get_code() != HoloInfer::holoinfer_code::H_SUCCESS) { - return HoloInfer::report_error(module_, - "Parameter Validation failed: " + status.get_message()); - } - - // Create holoscan processing context - holoscan_postprocess_context_ = std::make_unique(); - } catch (const std::bad_alloc& b_) { - return HoloInfer::report_error(module_, - "Start, Memory allocation, Message: " + std::string(b_.what())); - } catch (...) { return HoloInfer::report_error(module_, "Start, Unknown exception"); } - - // Initialize holoscan processing context - auto status = holoscan_postprocess_context_->initialize(process_operations_.get()); - if (status.get_code() != HoloInfer::holoinfer_code::H_SUCCESS) { - return HoloInfer::report_error(module_, "Start, Out data setup"); - } - - return GXF_SUCCESS; -} - -gxf_result_t MultiAIPostprocessor::stop() { - return GXF_SUCCESS; -} - -gxf_result_t MultiAIPostprocessor::tick() { - try { - // Extract relevant data from input GXF Receivers, and update data per model - gxf_result_t stat = HoloInfer::multiai_get_data_per_model(receivers_.get(), - in_tensor_names_.get(), - data_per_tensor_, - dims_per_tensor_, - input_on_cuda_.get(), - module_); - if (stat != GXF_SUCCESS) { return HoloInfer::report_error(module_, "Tick, Data extraction"); } - - // Execute processing - auto status = holoscan_postprocess_context_->process( - process_operations_.get(), processed_map_.get(), data_per_tensor_, dims_per_tensor_); - if (status.get_code() != HoloInfer::holoinfer_code::H_SUCCESS) { - return HoloInfer::report_error(module_, "Tick, post_process"); - } - - // Get processed data and dimensions (currently only on host) - auto processed_data = holoscan_postprocess_context_->get_processed_data(); - auto processed_dims = holoscan_postprocess_context_->get_processed_data_dims(); - - auto cont = context(); - // Transmit output buffers via a single GXF transmitter - stat = HoloInfer::multiai_transmit_data_per_model(cont, - processed_map_.get(), - processed_data, - transmitter_.get(), - out_tensor_names_.get(), - processed_dims, - output_on_cuda_.get(), - transmit_on_cuda_.get(), - gxf::PrimitiveType::kFloat32, - module_, - allocator_.get()); - if (stat != GXF_SUCCESS) { return HoloInfer::report_error(module_, "Tick, Data Transmission"); } - } catch (const std::runtime_error& r_) { - return HoloInfer::report_error(module_, "Tick, Message->" + std::string(r_.what())); - } catch (...) { return HoloInfer::report_error(module_, "Tick, unknown exception"); } - return GXF_SUCCESS; -} - -gxf_result_t MultiAIPostprocessor::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - result &= registrar->parameter(process_operations_, - "process_operations", - "Operations per tensor", - "Operations in sequence on tensors."); - result &= registrar->parameter(processed_map_, "processed_map", "Processed tensor map"); - result &= registrar->parameter( - in_tensor_names_, "in_tensor_names", "Input Tensors", "Input tensors", {std::string("")}); - result &= registrar->parameter( - out_tensor_names_, "out_tensor_names", "Output Tensors", "Output tensors", {std::string("")}); - result &= registrar->parameter( - input_on_cuda_, "input_on_cuda", "Input for processing on cuda", "", false); - result &= registrar->parameter( - output_on_cuda_, "output_on_cuda", "Processed output on cuda", "", false); - result &= registrar->parameter( - transmit_on_cuda_, "transmit_on_cuda", "Transmit message on cuda", "", false); - result &= registrar->parameter(allocator_, "allocator", "Allocator", "Output Allocator"); - result &= registrar->parameter( - receivers_, "receivers", "Receivers", "List of receivers to take input tensors"); - result &= registrar->parameter(transmitter_, "transmitter", "Transmitter", "Transmitter"); - - return gxf::ToResultCode(result); -} - -} // namespace multiai -} // namespace holoscan -} // namespace nvidia diff --git a/gxf_extensions/multiai_postprocessor/multiai_postprocessor.hpp b/gxf_extensions/multiai_postprocessor/multiai_postprocessor.hpp deleted file mode 100644 index 0c07cb70..00000000 --- a/gxf_extensions/multiai_postprocessor/multiai_postprocessor.hpp +++ /dev/null @@ -1,116 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef NVIDIA_GXF_EXTENSIONS_MULTIAI_POSTPROCESSOR_HPP_ -#define NVIDIA_GXF_EXTENSIONS_MULTIAI_POSTPROCESSOR_HPP_ - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "gxf/core/entity.hpp" -#include "gxf/core/gxf.h" -#include "gxf/core/parameter.hpp" -#include "gxf/cuda/cuda_stream.hpp" -#include "gxf/cuda/cuda_stream_id.hpp" -#include "gxf/cuda/cuda_stream_pool.hpp" -#include "gxf/multimedia/video.hpp" -#include "gxf/std/allocator.hpp" -#include "gxf/std/clock.hpp" -#include "gxf/std/codelet.hpp" -#include "gxf/std/parameter_parser_std.hpp" -#include "gxf/std/receiver.hpp" -#include "gxf/std/tensor.hpp" -#include "gxf/std/timestamp.hpp" -#include "gxf/std/transmitter.hpp" - -#include -#include - -namespace HoloInfer = holoscan::inference; - -namespace nvidia { -namespace holoscan { -namespace multiai { -/** - * @brief Generic Multi AI postprocessor codelet to perform multiple operations on multiple tensors. - */ -class MultiAIPostprocessor : public gxf::Codelet { - public: - gxf_result_t start() override; - gxf_result_t tick() override; - gxf_result_t stop() override; - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - - private: - /// Map with key as tensor name and value as vector of supported operations. - /// Supported operations: "max_per_channel_scaled" - gxf::Parameter process_operations_; - - /// Map with key as input tensor name and value as processed tensor name - gxf::Parameter processed_map_; - - /// Vector of input tensor names - gxf::Parameter > in_tensor_names_; - - /// Vector of output tensor names - gxf::Parameter > out_tensor_names_; - - /// Memory allocator - gxf::Parameter > allocator_; - - /// Vector of input receivers. Multiple receivers supported. - gxf::Parameter receivers_; - - /// Output transmitter. Single transmitter supported. - gxf::Parameter transmitter_; - - /// Flag showing if input buffers are on CUDA. Default is False. - /// Supported value: False - gxf::Parameter input_on_cuda_; - - /// Flag showing if output buffers are on CUDA. Default is False. - /// Supported value: False - gxf::Parameter output_on_cuda_; - - /// Flag showing if data transmission on CUDA. Default is False. - /// Supported value: False - gxf::Parameter transmit_on_cuda_; - - /// Pointer to Data Processor context. - std::unique_ptr holoscan_postprocess_context_; - - /// Map holding data per input tensor. - HoloInfer::DataMap data_per_tensor_; - - /// Map holding dimensions per model. Key is model name and value is a vector with - /// dimensions. - std::map > dims_per_tensor_; - - /// Codelet Identifier, used in reporting. - const std::string module_{"Multi AI Postprocessor Codelet"}; -}; -} // namespace multiai -} // namespace holoscan -} // namespace nvidia - -#endif // NVIDIA_GXF_EXTENSIONS_MULTIAI_POSTPROCESSOR_HPP_ diff --git a/gxf_extensions/multiai_postprocessor/multiai_postprocessor_extension.cpp b/gxf_extensions/multiai_postprocessor/multiai_postprocessor_extension.cpp deleted file mode 100644 index 8f7962c9..00000000 --- a/gxf_extensions/multiai_postprocessor/multiai_postprocessor_extension.cpp +++ /dev/null @@ -1,37 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "multiai_postprocessor.hpp" - -#include - -#include "gxf/core/gxf.h" -#include "gxf/std/extension_factory_helper.hpp" - -extern "C" { - -GXF_EXT_FACTORY_BEGIN() - -GXF_EXT_FACTORY_SET_INFO(0xaa47a2a6d6b9471c, 0xbe5292af3cbb59f8, "MultiAIPostprocessorExtension", - "MultiAI Postprocessor", "NVIDIA", "1.0.0", "LICENSE"); - -GXF_EXT_FACTORY_ADD(0xa569e128ee09448a, 0xb6e08107f0edd2ea, - nvidia::holoscan::multiai::MultiAIPostprocessor, nvidia::gxf::Codelet, - "Multi AI Postprocessor Codelet."); - -GXF_EXT_FACTORY_END() -} diff --git a/gxf_extensions/probe/CMakeLists.txt b/gxf_extensions/probe/CMakeLists.txt deleted file mode 100644 index 45dd7d00..00000000 --- a/gxf_extensions/probe/CMakeLists.txt +++ /dev/null @@ -1,36 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Create library -add_library(tensor_probe_lib SHARED - tensor_probe.cpp - tensor_probe.hpp -) -target_link_libraries(tensor_probe_lib - PUBLIC - CUDA::cudart - GXF::std - yaml-cpp -) - -# Create extension -add_library(tensor_probe SHARED - tensor_probe_ext.cpp -) -target_link_libraries(tensor_probe - PUBLIC tensor_probe_lib -) -# Install GXF extension as a component 'holoscan-gxf_extensions' -install_gxf_extension(tensor_probe) diff --git a/gxf_extensions/probe/tensor_probe.cpp b/gxf_extensions/probe/tensor_probe.cpp deleted file mode 100644 index 09b8a349..00000000 --- a/gxf_extensions/probe/tensor_probe.cpp +++ /dev/null @@ -1,89 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "tensor_probe.hpp" - -#include -#include - -#include "gxf/core/handle.hpp" -#include "gxf/std/tensor.hpp" - -namespace nvidia { -namespace holoscan { -namespace probe { - -gxf_result_t TensorProbe::start() { - return GXF_SUCCESS; -} - -gxf_result_t TensorProbe::stop() { - return GXF_SUCCESS; -} - -gxf_result_t TensorProbe::tick() { - gxf::Expected maybe_message = rx_->receive(); - if (!maybe_message) { - GXF_LOG_ERROR("Message not available."); - return maybe_message.error(); - } - - auto tensor_metas = maybe_message.value().findAll(); - GXF_LOG_INFO("Getting tensors"); - - for (const auto& tensor_meta : tensor_metas.value()) { - GXF_LOG_INFO( - "Tensor name: %s (name length %d)", tensor_meta->name(), strlen(tensor_meta->name())); - - gxf::Expected> maybe_tensor = - maybe_message.value().get(tensor_meta->name()); - if (!maybe_tensor) { - GXF_LOG_ERROR("Tensor %s not available.", tensor_meta->name()); - return maybe_tensor.error(); - } - - const auto& tensor = maybe_tensor.value(); - auto shape = tensor->shape(); - // Prints out dimension - { - std::stringbuf sbuf; - std::ostream stream(&sbuf); - stream << "["; - for (uint32_t i = 0; i < shape.rank(); ++i) { - if (i > 0) { stream << ", "; } - stream << shape.dimension(i); - } - stream << "]"; - - GXF_LOG_INFO("Input tensor: %s, Dimension: %s", tensor_meta->name(), sbuf.str().c_str()); - } - - // Print element type - GXF_LOG_INFO("Input tensor: %s, Element Type: %d", tensor_meta->name(), tensor->element_type()); - } - - return GXF_SUCCESS; -} - -gxf_result_t TensorProbe::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - result &= registrar->parameter(rx_, "rx", "RX", "Receiver of tensor message."); - return gxf::ToResultCode(result); -} - -} // namespace probe -} // namespace holoscan -} // namespace nvidia diff --git a/gxf_extensions/probe/tensor_probe.hpp b/gxf_extensions/probe/tensor_probe.hpp deleted file mode 100644 index ad1c2672..00000000 --- a/gxf_extensions/probe/tensor_probe.hpp +++ /dev/null @@ -1,52 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_TENSOR_PROBE_HPP_ -#define NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_TENSOR_PROBE_HPP_ - -#include -#include -#include - -#include "gxf/std/codelet.hpp" -#include "gxf/std/receiver.hpp" -#include "gxf/std/transmitter.hpp" - -namespace nvidia { -namespace holoscan { -namespace probe { - -/// @brief Introspection codlet for observing tensor shapes and printing to standard output. -/// -/// This codelet can be helpful for debugging information like tensor shapes and sizes sent between -/// codelets. -class TensorProbe : public gxf::Codelet { - public: - gxf_result_t start() override; - gxf_result_t tick() override; - gxf_result_t stop() override; - - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - - private: - gxf::Parameter> rx_; -}; - -} // namespace probe -} // namespace holoscan -} // namespace nvidia - -#endif // NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_TENSOR_PROBE_HPP_ diff --git a/gxf_extensions/probe/tensor_probe_ext.cpp b/gxf_extensions/probe/tensor_probe_ext.cpp deleted file mode 100644 index 22f92d51..00000000 --- a/gxf_extensions/probe/tensor_probe_ext.cpp +++ /dev/null @@ -1,26 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "gxf/std/extension_factory_helper.hpp" - -#include "tensor_probe.hpp" - -GXF_EXT_FACTORY_BEGIN() -GXF_EXT_FACTORY_SET_INFO(0xada0b1069d4c445a, 0xb2e045c0b816499c, "TensorProbeExtension", - "Probe extension", "NVIDIA", "1.0.0", "LICENSE"); -GXF_EXT_FACTORY_ADD(0x8be4de857921ddd9, 0x182106d326154aeb, nvidia::holoscan::probe::TensorProbe, - nvidia::gxf::Codelet, "Tensor name probe passthrough codelet."); -GXF_EXT_FACTORY_END() diff --git a/gxf_extensions/sample/CMakeLists.txt b/gxf_extensions/sample/CMakeLists.txt deleted file mode 100644 index 6ab0355a..00000000 --- a/gxf_extensions/sample/CMakeLists.txt +++ /dev/null @@ -1,52 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Create library -add_library(ping_tx SHARED - ping_tx.hpp - ping_tx.cpp -) -target_link_libraries(ping_tx - PUBLIC - GXF::std - yaml-cpp -) - -# Create library -add_library(ping_rx SHARED - ping_rx.hpp - ping_rx.cpp -) -target_link_libraries(ping_rx - PUBLIC - GXF::std - yaml-cpp -) - -# Create extension -add_library(holoscan_sample SHARED - sample.cpp -) -target_link_libraries(holoscan_sample - PUBLIC - GXF::std - ping_tx - ping_rx - yaml-cpp -) -# Install GXF extension as a component 'holoscan-gxf_extensions' -install_gxf_extension(holoscan_sample) -install_gxf_extension(ping_tx) # not following '{name}_lib' so install explicitly -install_gxf_extension(ping_rx) # not following '{name}_lib' so install explicitly diff --git a/gxf_extensions/sample/ping_rx.hpp b/gxf_extensions/sample/ping_rx.hpp deleted file mode 100644 index 1d7ca04b..00000000 --- a/gxf_extensions/sample/ping_rx.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef SAMPLE_PING_RX_HPP -#define SAMPLE_PING_RX_HPP - -#include "gxf/std/codelet.hpp" -#include "gxf/std/receiver.hpp" - -namespace nvidia { -namespace holoscan { -namespace sample { -// Sample codelet implementation to receive an entity -class PingRx : public gxf::Codelet { - public: - virtual ~PingRx() = default; - - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - gxf_result_t start() override { return GXF_SUCCESS; } - gxf_result_t tick() override; - gxf_result_t stop() override { return GXF_SUCCESS; } - - private: - gxf::Parameter> signal_; - int count = 1; -}; - -} // namespace sample -} // namespace holoscan -} // namespace nvidia - -#endif /* SAMPLE_PING_RX_HPP */ diff --git a/gxf_extensions/sample/ping_tx.cpp b/gxf_extensions/sample/ping_tx.cpp deleted file mode 100644 index c2ab0535..00000000 --- a/gxf_extensions/sample/ping_tx.cpp +++ /dev/null @@ -1,65 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "ping_tx.hpp" - -namespace nvidia { -namespace holoscan { -namespace sample { - -gxf_result_t PingTx::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - result &= registrar->parameter(signal_, "signal"); - result &= registrar->parameter(clock_, - "clock", - "Clock", - "Clock for Timestamp", - gxf::Registrar::NoDefaultParameter(), - GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(signal_vector_, "signal_vector"); - result &= registrar->parameter(signal_data_, "signal_data", "Signal data", "Signal data", {}); - result &= registrar->parameter(pool_, "pool", "Pool", "Allocator instance for output tensors."); - return gxf::ToResultCode(result); -} - -gxf_result_t PingTx::tick() { - auto message = gxf::Entity::New(context()); - if (!message) { - GXF_LOG_ERROR("Failure creating message entity."); - return message.error(); - } - auto maybe_clock = clock_.try_get(); - int64_t now; - if (maybe_clock) { - now = maybe_clock.value()->timestamp(); - } else { - now = 0; - } - GXF_LOG_INFO("Signal vector size: %llu", signal_vector_.get().size()); - - signal_->publish(message.value(), now); - GXF_LOG_INFO("Message Sent: %d", this->count); - GXF_LOG_INFO("Message data size: %llu [0] = %d", - this->signal_data_.get().size(), - this->signal_data_.get()[0]); - this->count = this->count + 1; - return ToResultCode(message); -} - -} // namespace sample -} // namespace holoscan -} // namespace nvidia diff --git a/gxf_extensions/sample/ping_tx.hpp b/gxf_extensions/sample/ping_tx.hpp deleted file mode 100644 index 0f2b30df..00000000 --- a/gxf_extensions/sample/ping_tx.hpp +++ /dev/null @@ -1,58 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef SAMPLE_PING_TX_HPP -#define SAMPLE_PING_TX_HPP - -#include -#include -#include - -#include "gxf/std/allocator.hpp" -#include "gxf/std/clock.hpp" -#include "gxf/std/codelet.hpp" -#include "gxf/std/parameter_parser_std.hpp" -#include "gxf/std/transmitter.hpp" - -namespace nvidia { -namespace holoscan { -namespace sample { - -// Sample codelet implementation to send an entity -class PingTx : public gxf::Codelet { - public: - virtual ~PingTx() = default; - - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - gxf_result_t start() override { return GXF_SUCCESS; } - gxf_result_t tick() override; - gxf_result_t stop() override { return GXF_SUCCESS; } - - private: - gxf::Parameter> signal_; - gxf::Parameter> clock_; - gxf::Parameter>> signal_vector_; - gxf::Parameter> signal_data_; - gxf::Parameter> pool_; - int count = 1; -}; - -} // namespace sample -} // namespace holoscan -} // namespace nvidia - -#endif /* SAMPLE_PING_TX_HPP */ diff --git a/gxf_extensions/sample/sample.cpp b/gxf_extensions/sample/sample.cpp deleted file mode 100644 index 4ccbd6cb..00000000 --- a/gxf_extensions/sample/sample.cpp +++ /dev/null @@ -1,33 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "gxf/std/extension_factory_helper.hpp" - -#include "ping_rx.hpp" -#include "ping_tx.hpp" - -GXF_EXT_FACTORY_BEGIN() -GXF_EXT_FACTORY_SET_INFO(0x3e9f4558bcc140bc, 0xa919ab36801265eb, "Holoscan SampleExtension", - "Sample extension to demonstrate the use of GXF features", "NVIDIA", - "1.0.0", "LICENSE"); - -GXF_EXT_FACTORY_ADD(0xb52ab9e5a33341a2, 0xb72574f35d77463e, nvidia::holoscan::sample::PingTx, - nvidia::gxf::Codelet, "Sends an entity"); -GXF_EXT_FACTORY_ADD(0x2d3a1965f0534de1, 0xbd713bccd5c85c4f, nvidia::holoscan::sample::PingRx, - nvidia::gxf::Codelet, "Receives an entity"); - -GXF_EXT_FACTORY_END() diff --git a/gxf_extensions/segmentation_postprocessor/CMakeLists.txt b/gxf_extensions/segmentation_postprocessor/CMakeLists.txt deleted file mode 100644 index fce02097..00000000 --- a/gxf_extensions/segmentation_postprocessor/CMakeLists.txt +++ /dev/null @@ -1,50 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Mark CUDA files -# Note: needed because we can't use .cu or .cuh with the -# bazel build system. We could switch to using those instead -# if we only support CMake -set_source_files_properties( - segmentation_postprocessor.cu.cpp - segmentation_postprocessor.cu.hpp - PROPERTIES - LANGUAGE CUDA -) - -# Create library -add_library(segmentation_postprocessor_lib SHARED - segmentation_postprocessor.cpp - segmentation_postprocessor.cu.cpp - segmentation_postprocessor.cu.hpp - segmentation_postprocessor.hpp -) -target_link_libraries(segmentation_postprocessor_lib - PUBLIC - CUDA::cuda_driver - GXF::multimedia - GXF::std - yaml-cpp -) - -# Create extension -add_library(segmentation_postprocessor SHARED - segmentation_postprocessor_ext.cpp -) -target_link_libraries(segmentation_postprocessor - PUBLIC segmentation_postprocessor_lib -) -# Install GXF extension as a component 'holoscan-gxf_extensions' -install_gxf_extension(segmentation_postprocessor) diff --git a/gxf_extensions/segmentation_postprocessor/segmentation_postprocessor.cpp b/gxf_extensions/segmentation_postprocessor/segmentation_postprocessor.cpp deleted file mode 100644 index b3e296ad..00000000 --- a/gxf_extensions/segmentation_postprocessor/segmentation_postprocessor.cpp +++ /dev/null @@ -1,168 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "segmentation_postprocessor.hpp" - -#include -#include - -#include "gxf/multimedia/video.hpp" -#include "gxf/std/tensor.hpp" - -#define CUDA_TRY(stmt) \ - ({ \ - cudaError_t _holoscan_cuda_err = stmt; \ - if (cudaSuccess != _holoscan_cuda_err) { \ - GXF_LOG_ERROR("CUDA Runtime call %s in line %d of file %s failed with '%s' (%d).", #stmt, \ - __LINE__, __FILE__, cudaGetErrorString(_holoscan_cuda_err), \ - _holoscan_cuda_err); \ - } \ - _holoscan_cuda_err; \ - }) - -namespace nvidia { -namespace holoscan { -namespace segmentation_postprocessor { - -gxf_result_t Postprocessor::start() { - const std::string& network_output_type_name = network_output_type_.get(); - if (network_output_type_name == "sigmoid") { - network_output_type_value_ = NetworkOutputType::kSigmoid; - } else if (network_output_type_name == "softmax") { - network_output_type_value_ = NetworkOutputType::kSoftmax; - } else { - GXF_LOG_ERROR("Unsupported network type %s", network_output_type_name.c_str()); - return GXF_FAILURE; - } - const std::string& data_format_name = data_format_.get(); - if (data_format_name == "nchw") { - data_format_value_ = DataFormat::kNCHW; - } else if (data_format_name == "hwc") { - data_format_value_ = DataFormat::kHWC; - } else if (data_format_name == "nhwc") { - data_format_value_ = DataFormat::kNHWC; - } else { - GXF_LOG_ERROR("Unsupported format type %s", data_format_name.c_str()); - return GXF_FAILURE; - } - return GXF_SUCCESS; -} - -gxf_result_t Postprocessor::tick() { - // Process input message - const auto in_message = in_->receive(); - if (!in_message || in_message.value().is_null()) { return GXF_CONTRACT_MESSAGE_NOT_AVAILABLE; } - - // Get tensor attached to the message - auto maybe_tensor = in_message.value().get(in_tensor_name_.get().c_str()); - if (!maybe_tensor) { - maybe_tensor = in_message.value().get(); - if (!maybe_tensor) { - GXF_LOG_ERROR("Tensor '%s' not found in message.", in_tensor_name_.get().c_str()); - return GXF_FAILURE; - } - } - gxf::Handle in_tensor = maybe_tensor.value(); - - Shape shape = {}; - switch (data_format_value_) { - case DataFormat::kHWC: { - shape.height = in_tensor->shape().dimension(0); - shape.width = in_tensor->shape().dimension(1); - shape.channels = in_tensor->shape().dimension(2); - } break; - case DataFormat::kNCHW: { - shape.channels = in_tensor->shape().dimension(1); - shape.height = in_tensor->shape().dimension(2); - shape.width = in_tensor->shape().dimension(3); - } break; - case DataFormat::kNHWC: { - shape.height = in_tensor->shape().dimension(1); - shape.width = in_tensor->shape().dimension(2); - shape.channels = in_tensor->shape().dimension(3); - } break; - } - - if (shape.channels > kMaxChannelCount) { - GXF_LOG_ERROR("Input channel count larger than allowed: %d > %d", shape.channels, - kMaxChannelCount); - return GXF_FAILURE; - } - - auto out_message = gxf::Entity::New(context()); - if (!out_message) { - GXF_LOG_ERROR("Failed to allocate message"); - return GXF_FAILURE; - } - - auto out_tensor = out_message.value().add("out_tensor"); - if (!out_tensor) { - GXF_LOG_ERROR("Failed to allocate output tensor"); - return GXF_FAILURE; - } - - // Allocate and convert output buffer on the device. - gxf::Shape output_shape{shape.height, shape.width, 1}; - out_tensor.value()->reshape(output_shape, gxf::MemoryStorageType::kDevice, allocator_); - if (!out_tensor.value()->pointer()) { - GXF_LOG_ERROR("Failed to allocate output tensor buffer."); - return GXF_FAILURE; - } - - gxf::Expected in_tensor_data = in_tensor->data(); - if (!in_tensor_data) { - GXF_LOG_ERROR("Failed to get in tensor data!"); - return GXF_FAILURE; - } - gxf::Expected out_tensor_data = out_tensor.value()->data(); - if (!out_tensor_data) { - GXF_LOG_ERROR("Failed to get out tensor data!"); - return GXF_FAILURE; - } - - cuda_postprocess(network_output_type_value_, data_format_value_, shape, in_tensor_data.value(), - out_tensor_data.value()); - - const auto result = out_->publish(std::move(out_message.value())); - if (!result) { - GXF_LOG_ERROR("Failed to publish output!"); - return GXF_FAILURE; - } - - return GXF_SUCCESS; -} - -gxf_result_t Postprocessor::stop() { - return GXF_SUCCESS; -} - -gxf_result_t Postprocessor::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - result &= registrar->parameter(in_, "in", "Input", "Input channel."); - result &= registrar->parameter(in_tensor_name_, "in_tensor_name", "InputTensorName", - "Name of the input tensor.", std::string("")); - result &= registrar->parameter(network_output_type_, "network_output_type", "NetworkOutputType", - "Network output type.", std::string("softmax")); - result &= registrar->parameter(out_, "out", "Output", "Output channel."); - result &= registrar->parameter(data_format_, "data_format", "DataFormat", - "Data format of network output", std::string("hwc")); - result &= registrar->parameter(allocator_, "allocator", "Allocator", "Output Allocator"); - return gxf::ToResultCode(result); -} - -} // namespace segmentation_postprocessor -} // namespace holoscan -} // namespace nvidia diff --git a/gxf_extensions/segmentation_postprocessor/segmentation_postprocessor.hpp b/gxf_extensions/segmentation_postprocessor/segmentation_postprocessor.hpp deleted file mode 100644 index 11d0ceae..00000000 --- a/gxf_extensions/segmentation_postprocessor/segmentation_postprocessor.hpp +++ /dev/null @@ -1,62 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_SEGMENTATION_POSTPROCESSOR_HPP_ -#define NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_SEGMENTATION_POSTPROCESSOR_HPP_ - -#include - -#include "gxf/std/allocator.hpp" -#include "gxf/std/codelet.hpp" -#include "gxf/std/receiver.hpp" -#include "gxf/std/transmitter.hpp" - -#include "segmentation_postprocessor.cu.hpp" - -namespace nvidia { -namespace holoscan { -namespace segmentation_postprocessor { - -/// @brief Segmentation model postproceessing Codelet converting inference output to class index. -/// -/// This Codelet performs segmentation model postprocessing in CUDA. -/// It takes in the output of inference, either with the final softmax layer (multiclass) or sigmoid -/// (2-class), and emits a Tensor that contains the highest probability class index. -/// The class index can then be consumed downstream for visualization or other purposes. -/// The inference output currently supported are either HWC or NCHW. -class Postprocessor : public gxf::Codelet { - public: - gxf_result_t start() override; - gxf_result_t tick() override; - gxf_result_t stop() override; - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - - private: - NetworkOutputType network_output_type_value_; - DataFormat data_format_value_; - - gxf::Parameter> in_; - gxf::Parameter in_tensor_name_; - gxf::Parameter network_output_type_; - gxf::Parameter data_format_; - gxf::Parameter> out_; - gxf::Parameter> allocator_; -}; -} // namespace segmentation_postprocessor -} // namespace holoscan -} // namespace nvidia - -#endif // NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_SEGMENTATION_POSTPROCESSOR_HPP_ diff --git a/gxf_extensions/segmentation_postprocessor/segmentation_postprocessor_ext.cpp b/gxf_extensions/segmentation_postprocessor/segmentation_postprocessor_ext.cpp deleted file mode 100644 index 809077ed..00000000 --- a/gxf_extensions/segmentation_postprocessor/segmentation_postprocessor_ext.cpp +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "gxf/std/extension_factory_helper.hpp" - -#include "segmentation_postprocessor.hpp" - -GXF_EXT_FACTORY_BEGIN() -GXF_EXT_FACTORY_SET_INFO(0x1486ad73a0f3496f, 0xa88ab90d35658f9a, - "SegmentationPostprocessorExtension", - "Holoscan Segmentation Model Postprocessing extension", "NVIDIA", "0.2.0", - "LICENSE"); -GXF_EXT_FACTORY_ADD(0x9676cb6784e048e1, 0xaf6ed81223dda5aa, - nvidia::holoscan::segmentation_postprocessor::Postprocessor, - nvidia::gxf::Codelet, "OpenGL Segmentation Postprocessor codelet."); -GXF_EXT_FACTORY_END() diff --git a/gxf_extensions/segmentation_visualizer/CMakeLists.txt b/gxf_extensions/segmentation_visualizer/CMakeLists.txt deleted file mode 100644 index fdd168e1..00000000 --- a/gxf_extensions/segmentation_visualizer/CMakeLists.txt +++ /dev/null @@ -1,49 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Create library -add_library(segmentation_visualizer_lib SHARED - segmentation_visualizer.cpp - segmentation_visualizer.hpp -) -target_link_libraries(segmentation_visualizer_lib - PUBLIC - CUDA::cudart - glad::glad - glfw - GXF::multimedia - GXF::std - # OpenGL::GL # Using glad + patch instead for GL/gl.h - yaml-cpp -) - -# Create extension -add_library(segmentation_visualizer SHARED - segmentation_visualizer_ext.cpp -) -target_link_libraries(segmentation_visualizer - PUBLIC segmentation_visualizer_lib -) - -# Copy resources -file(COPY glsl DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) - -# Install GXF extension as a component 'holoscan-gxf_extensions' -install_gxf_extension(segmentation_visualizer) -file(RELATIVE_PATH _relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) -install(DIRECTORY glsl - DESTINATION ${_relative_dest_path} - COMPONENT "holoscan-gxf_extensions" -) diff --git a/gxf_extensions/segmentation_visualizer/glsl/segmentation_mask.frag b/gxf_extensions/segmentation_visualizer/glsl/segmentation_mask.frag deleted file mode 100644 index cdd569f9..00000000 --- a/gxf_extensions/segmentation_visualizer/glsl/segmentation_mask.frag +++ /dev/null @@ -1,44 +0,0 @@ -#version 460 - -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -in vec2 tex_coord; - -// Note: MAX_LUT_COLORS should match the value used by the host code using this shader. -const int MAX_LUT_COLORS = 64; -layout(location = 0) uniform uint lut_count; -layout(location = 1) uniform vec4 lut_colors[MAX_LUT_COLORS]; - -layout(binding = 0) uniform sampler2D image_texture; -layout(binding = 1) uniform usampler2D class_texure; - -out vec4 out_color; - -void main() { - uint class_index = texture(class_texure, tex_coord).r; - // Set maximum index to last index that can be used as an "other" class. - class_index = min(class_index, lut_count - 1); - vec4 class_color = lut_colors[class_index]; - vec4 image_color = texture(image_texture, tex_coord); - - vec4 dst = vec4(0.0); - dst = (1.0 - image_color.a) * dst + image_color.a * image_color; - dst = (1.0 - class_color.a) * dst + class_color.a * class_color; - dst.a = 1.0; - out_color = dst; -}; diff --git a/gxf_extensions/segmentation_visualizer/segmentation_visualizer.cpp b/gxf_extensions/segmentation_visualizer/segmentation_visualizer.cpp deleted file mode 100644 index 70d417a2..00000000 --- a/gxf_extensions/segmentation_visualizer/segmentation_visualizer.cpp +++ /dev/null @@ -1,417 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "segmentation_visualizer.hpp" - -#include -#include - -#include -#include -#include -#include - -#include "gxf/multimedia/video.hpp" -#include "gxf/std/tensor.hpp" - -#define CUDA_TRY(stmt) \ - ({ \ - cudaError_t _holoscan_cuda_err = stmt; \ - if (cudaSuccess != _holoscan_cuda_err) { \ - GXF_LOG_ERROR("CUDA Runtime call %s in line %d of file %s failed with '%s' (%d).\n", #stmt, \ - __LINE__, __FILE__, cudaGetErrorString(_holoscan_cuda_err), \ - _holoscan_cuda_err); \ - } \ - _holoscan_cuda_err; \ - }) - -#define CUDA_TRY_OR_RETURN_FAILURE(stmt) \ - ({ \ - cudaError_t _holoscan_cuda_err = stmt; \ - if (cudaSuccess != _holoscan_cuda_err) { \ - GXF_LOG_ERROR("CUDA Runtime call %s in line %d of file %s failed with '%s' (%d).\n", #stmt, \ - __LINE__, __FILE__, cudaGetErrorString(_holoscan_cuda_err), \ - _holoscan_cuda_err); \ - return GXF_FAILURE; \ - } \ - }) - -namespace nvidia { -namespace holoscan { -namespace segmentation_visualizer { - -static const float kVertices[8] = {1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, -1.0f}; - -static void glfwPrintErrorCallback(int error, const char* msg) { - std::cerr << " [" << error << "] " << msg << "\n"; -} - -// process all input: query GLFW whether relevant keys are pressed/released this frame and react -// accordingly -// ---------------------------------------------------------------------------------------------- -static void glfwProcessInput(GLFWwindow* window) { - if (glfwGetKey(window, GLFW_KEY_ESCAPE) == GLFW_PRESS) glfwSetWindowShouldClose(window, true); -} - -// whenever the window size changed (by OS or user resize) this callback function executes -// --------------------------------------------------------------------------------------------- -static void glfwFramebufferSizeCallback(GLFWwindow* window, int width, int height) { - glViewport(0, 0, width, height); -} - -static gxf::Expected readFile(const std::string& path) { - std::ifstream istream(path); - if (istream.fail()) { - GXF_LOG_WARNING("Failed to find file: '%s'", path.c_str()); - return gxf::Unexpected{GXF_FAILURE}; - } - std::stringstream sstream; - sstream << istream.rdbuf(); - return sstream.str(); -} - -gxf_result_t Visualizer::start() { - window_ = nullptr; - - glfwSetErrorCallback(glfwPrintErrorCallback); - - // Create window - // ------------- - // initialize and configure - if (!glfwInit()) { - GXF_LOG_ERROR("Failed to initialize GLFW"); - glfwTerminate(); - return GXF_FAILURE; - } - - glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 4); - glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 6); - glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE); - window_ = glfwCreateWindow(image_width_, image_height_, "GXF Segmentation Visualizer", nullptr, - nullptr); - if (window_ == nullptr) { - GXF_LOG_ERROR("Failed to create GLFW window"); - glfwTerminate(); - return GXF_FAILURE; - } - glfwMakeContextCurrent(window_); - glfwSetFramebufferSizeCallback(window_, glfwFramebufferSizeCallback); - - // Load all OpenGL function pointers - GLADloadproc gl_loader = reinterpret_cast(glfwGetProcAddress); - if (!gladLoadGLLoader(gl_loader)) { - GXF_LOG_ERROR("Failed to initialize GLAD"); - return GXF_FAILURE; - } - - // Create shaders - // -------------- - // Compile the vertex shader - GLuint vertex_shader = glCreateShader(GL_VERTEX_SHADER); - // Attach the shader source code to the shader object and compile the shader: - gxf::Expected maybe_vertex_shader_source = - readFile("gxf_extensions/segmentation_visualizer/glsl/segmentation_mask.vert"); - if (!maybe_vertex_shader_source) { - GXF_LOG_ERROR("Could not open vertex shader source!"); - return GXF_FAILURE; - } - const char* vertex_shader_sources[] = {maybe_vertex_shader_source->c_str()}; - glShaderSource(vertex_shader, 1, vertex_shader_sources, nullptr); - glCompileShader(vertex_shader); - int success = -1; - glGetShaderiv(vertex_shader, GL_COMPILE_STATUS, &success); - if (!success) { - char info_log[512] = {0}; - glGetShaderInfoLog(vertex_shader, sizeof(info_log), nullptr, info_log); - GXF_LOG_ERROR("ERROR::SHADER::VERTEX::COMPILATION_FAILED\n%s", info_log); - return GXF_FAILURE; - } - - // Compile fragment shader - gxf::Expected maybe_fragment_shader_source = - readFile("gxf_extensions/segmentation_visualizer/glsl/segmentation_mask.frag"); - if (!maybe_fragment_shader_source) { - GXF_LOG_ERROR("Could not open fragment shader source!"); - return GXF_FAILURE; - } - GLuint fragment_shader = glCreateShader(GL_FRAGMENT_SHADER); - const char* fragment_shader_sources[] = {maybe_fragment_shader_source->c_str()}; - glShaderSource(fragment_shader, 1, fragment_shader_sources, nullptr); - glCompileShader(fragment_shader); - glGetShaderiv(fragment_shader, GL_COMPILE_STATUS, &success); - if (!success) { - char info_log[512] = {0}; - glGetShaderInfoLog(fragment_shader, sizeof(info_log), nullptr, info_log); - GXF_LOG_ERROR("ERROR::SHADER::FRAGMENT::COMPILATION_FAILED\n%s", info_log); - return GXF_FAILURE; - } - - // Create shader program object to link multiple shaders - // ----------------------------------------------------- - GLuint shader_program = glCreateProgram(); - - // Attach the previously compiled shaders to the program object and then link them - glAttachShader(shader_program, vertex_shader); - glAttachShader(shader_program, fragment_shader); - glLinkProgram(shader_program); - glGetProgramiv(shader_program, GL_LINK_STATUS, &success); - if (!success) { - char info_log[512] = {0}; - glGetProgramInfoLog(shader_program, 512, nullptr, info_log); - GXF_LOG_ERROR("ERROR::SHADER::PROGRAM::LINK_FAILED\n%s", info_log); - return GXF_FAILURE; - } - - // Activate the program. Every shader and rendering call after the activation will now use this - // program object (and thus the shaders) - glUseProgram(shader_program); - - // This is the maximum number of lookup colors used in our GLSL shader. It should be consistent - // with glsl/segmentation_mask.frag - static const uint32_t MAX_LUT_COLORS = 64; - const auto& class_color_lut = class_color_lut_.get(); - if (class_color_lut.size() >= MAX_LUT_COLORS) { - GXF_LOG_ERROR("Too many colors in the class_color_lut %d > %d", class_color_lut.size(), - MAX_LUT_COLORS); - return GXF_FAILURE; - } - glUniform1ui(0, class_color_lut.size()); - for (size_t i = 0; i < class_color_lut.size(); ++i) { - if (class_color_lut[i].size() != 4) { - GXF_LOG_ERROR("Not enough color components in class_color_lut[%d].size() %d != 4", i, - class_color_lut[i].size()); - return GXF_FAILURE; - } - glUniform4fv(1 + i, 1, class_color_lut[i].data()); - } - - // Delete the shader objects once we've linked them into the program object - glDeleteShader(vertex_shader); - glDeleteShader(fragment_shader); - - // Setup the vertex array. - GLuint vao, vbo; - glGenVertexArrays(1, &vao); - glBindVertexArray(vao); - glGenBuffers(1, &vbo); - glBindBuffer(GL_ARRAY_BUFFER, vbo); - glBufferData(GL_ARRAY_BUFFER, sizeof(kVertices), kVertices, GL_STATIC_DRAW); - glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 0, 0); - glEnableVertexAttribArray(0); - - // load and create a texture - glGenTextures(2, textures_); - cuda_resources_.resize(2, nullptr); - - glActiveTexture(GL_TEXTURE0); - glBindTexture(GL_TEXTURE_2D, textures_[0]); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - - // Set alignment requirement to 1 so that the tensor with any width can work. - glPixelStorei(GL_UNPACK_ALIGNMENT, 1); - - const size_t bytes_per_pixel = 4; - image_buffer_.resize(image_width_ * image_height_ * bytes_per_pixel, 0); - glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, image_width_, image_height_, 0, GL_RGBA, - GL_UNSIGNED_BYTE, image_buffer_.data()); - - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, textures_[1]); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); - glPixelStorei(GL_UNPACK_ALIGNMENT, 1); - - class_index_buffer_.resize(class_index_width_ * class_index_height_, 0); - glTexImage2D(GL_TEXTURE_2D, 0, GL_R8UI, class_index_width_, class_index_height_, 0, - GL_RED_INTEGER, GL_UNSIGNED_BYTE, class_index_buffer_.data()); - CUDA_TRY_OR_RETURN_FAILURE(cudaGraphicsGLRegisterImage( - &cuda_resources_[0], textures_[0], GL_TEXTURE_2D, cudaGraphicsMapFlagsWriteDiscard)); - CUDA_TRY_OR_RETURN_FAILURE(cudaGraphicsGLRegisterImage( - &cuda_resources_[1], textures_[1], GL_TEXTURE_2D, cudaGraphicsMapFlagsWriteDiscard)); - - window_close_scheduling_term_->enable_tick(); - - return GXF_SUCCESS; -} - -gxf_result_t Visualizer::unregisterCudaResources() { - // Unregister all cuda resources and empty the vector. - bool success = true; - for (cudaGraphicsResource* resource : cuda_resources_) { - if (resource == nullptr) { continue; } - auto result = CUDA_TRY(cudaGraphicsUnregisterResource(resource)); - if (result != cudaSuccess) { success = false; } - } - cuda_resources_.clear(); - return success ? GXF_SUCCESS : GXF_FAILURE; -} - -gxf_result_t Visualizer::stop() { - gxf_result_t cuda_result = unregisterCudaResources(); - - // Terminate GLFW regardless of cuda result. - if (window_ != nullptr) { - glfwDestroyWindow(window_); - window_ = nullptr; - } - glfwTerminate(); - - return cuda_result; -} - -gxf_result_t Visualizer::tick() { - glfwProcessInput(window_); - if (glfwWindowShouldClose(window_)) { - window_close_scheduling_term_->disable_tick(); - return GXF_SUCCESS; - } - - // Grabs latest messages from all receivers - const auto image_message = image_in_->receive(); - if (!image_message || image_message.value().is_null()) { - return GXF_CONTRACT_MESSAGE_NOT_AVAILABLE; - } - - // Get tensor attached to the message - gxf::Expected> image_tensor = image_message.value().get(); - if (!image_tensor) { - GXF_LOG_ERROR("Could not get input tensor data from message"); - return GXF_FAILURE; - } - if (image_tensor.value()->storage_type() != gxf::MemoryStorageType::kDevice) { - GXF_LOG_ERROR("Expecting image tensor to be allocated on the device"); - return GXF_MEMORY_INVALID_STORAGE_MODE; - } - - gxf::Expected image_tensor_data = image_tensor.value()->data(); - if (!image_tensor_data) { - GXF_LOG_ERROR("Could not get image tensor data"); - return GXF_FAILURE; - } - - const gxf::Shape image_shape = image_tensor.value()->shape(); - const int32_t image_height = image_shape.dimension(0); - const int32_t image_width = image_shape.dimension(1); - - if (image_height != image_height_ || image_width != image_width_) { - GXF_LOG_ERROR("Received Tensor has a different shape (%d, %d). Expected (%d, %d)", image_height, - image_width, image_height_.get(), image_width_.get()); - return GXF_FAILURE; - } - - const auto class_index_message = class_index_in_->receive(); - if (!class_index_message || class_index_message.value().is_null()) { - return GXF_CONTRACT_MESSAGE_NOT_AVAILABLE; - } - - // Get tensor attached to the message - gxf::Expected> class_index_tensor = - class_index_message.value().get(); - if (!class_index_tensor) { - GXF_LOG_ERROR("Could not get input class index data from message"); - return GXF_FAILURE; - } - if (class_index_tensor.value()->storage_type() != gxf::MemoryStorageType::kDevice) { - return GXF_MEMORY_INVALID_STORAGE_MODE; - } - - gxf::Expected class_index_tensor_data = - class_index_tensor.value()->data(); - if (!class_index_tensor_data) { - GXF_LOG_ERROR("Could not get input tensor data"); - return GXF_FAILURE; - } - - const gxf::Shape class_index_shape = class_index_tensor.value()->shape(); - const int32_t class_index_height = class_index_shape.dimension(0); - const int32_t class_index_width = class_index_shape.dimension(1); - - if (class_index_width != class_index_width_ || class_index_height != class_index_height_) { - GXF_LOG_ERROR("Received Tensor has a different shape (%d, %d). Expected (%d, %d)", - class_index_height, class_index_width, class_index_height_.get(), - class_index_width_.get()); - return GXF_FAILURE; - } - - glActiveTexture(GL_TEXTURE0); - glBindTexture(GL_TEXTURE_2D, textures_[0]); - CUDA_TRY_OR_RETURN_FAILURE(cudaGraphicsMapResources(1, &cuda_resources_[0], 0)); - cudaArray* image_opengl_cuda_ptr = nullptr; - CUDA_TRY_OR_RETURN_FAILURE( - cudaGraphicsSubResourceGetMappedArray(&image_opengl_cuda_ptr, cuda_resources_[0], 0, 0)); - const size_t image_bytes_per_pixel = 4; - const size_t image_pitch_width = image_width * image_bytes_per_pixel; - CUDA_TRY_OR_RETURN_FAILURE( - cudaMemcpy2DToArray(image_opengl_cuda_ptr, 0, 0, image_tensor_data.value(), image_pitch_width, - image_pitch_width, image_height, cudaMemcpyDeviceToDevice)); - CUDA_TRY_OR_RETURN_FAILURE(cudaGraphicsUnmapResources(1, &cuda_resources_[0], 0)); - - glActiveTexture(GL_TEXTURE1); - glBindTexture(GL_TEXTURE_2D, textures_[1]); - CUDA_TRY_OR_RETURN_FAILURE(cudaGraphicsMapResources(1, &cuda_resources_[1], 0)); - cudaArray* class_index_opengl_cuda_ptr = nullptr; - CUDA_TRY_OR_RETURN_FAILURE(cudaGraphicsSubResourceGetMappedArray(&class_index_opengl_cuda_ptr, - cuda_resources_[1], 0, 0)); - const size_t class_index_bytes_per_pixel = 1; - const size_t class_index_pitch_width = class_index_width * class_index_bytes_per_pixel; - CUDA_TRY_OR_RETURN_FAILURE(cudaMemcpy2DToArray( - class_index_opengl_cuda_ptr, 0, 0, class_index_tensor_data.value(), class_index_pitch_width, - class_index_pitch_width, class_index_height, cudaMemcpyDeviceToDevice)); - CUDA_TRY_OR_RETURN_FAILURE(cudaGraphicsUnmapResources(1, &cuda_resources_[1], 0)); - - glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); - - // swap buffers and poll IO events (keys pressed/released, mouse moved etc.) - // ------------------------------------------------------------------------------- - glfwSwapBuffers(window_); - glfwPollEvents(); - - return GXF_SUCCESS; -} - -gxf_result_t Visualizer::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - - result &= registrar->parameter(image_in_, "image_in", "Input", "Tensor input"); - result &= registrar->parameter(image_width_, "image_width", "ImageWidth", - "Width of the input image.", 1920); - result &= registrar->parameter(image_height_, "image_height", "ImageHeight", - "Height of the input image.", 1080); - - result &= registrar->parameter(class_index_in_, "class_index_in", "Input", "Tensor input"); - result &= registrar->parameter(class_index_width_, "class_index_width", "ClassIndexWidth", - "Width of the segmentation class index tensor.", 1920); - result &= registrar->parameter(class_index_height_, "class_index_height", "ClassIndexHeight", - "Height of the segmentation class index tensor.", 1080); - - result &= registrar->parameter(class_color_lut_, "class_color_lut", "ClassColorLUT", - "Overlay Image Segmentation Class Colormap"); - - result &= registrar->parameter( - window_close_scheduling_term_, "window_close_scheduling_term", "WindowCloseSchedulingTerm", - "BooleanSchedulingTerm to stop the codelet from ticking after all messages are published."); - - return gxf::ToResultCode(result); -} - -} // namespace segmentation_visualizer -} // namespace holoscan -} // namespace nvidia diff --git a/gxf_extensions/segmentation_visualizer/segmentation_visualizer.hpp b/gxf_extensions/segmentation_visualizer/segmentation_visualizer.hpp deleted file mode 100644 index 6f939b5c..00000000 --- a/gxf_extensions/segmentation_visualizer/segmentation_visualizer.hpp +++ /dev/null @@ -1,81 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_SEGMENTATION_VISUALIZER_HPP_ -#define NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_SEGMENTATION_VISUALIZER_HPP_ - -// clang-format off -#define GLFW_INCLUDE_NONE 1 -#include -#include // NOLINT(build/include_order) -// clang-format on - -#include -#include - -#include "gxf/std/codelet.hpp" -#include "gxf/std/receiver.hpp" -#include "gxf/std/scheduling_terms.hpp" - -// Forward declare from cuda_gl_interopt.h -struct cudaGraphicsResource; - -namespace nvidia { -namespace holoscan { -namespace segmentation_visualizer { - -/// @brief OpenGL Visualization Codelet that combines segmentation output overlaid on video input. -/// -/// This Codelet performs visualization of a segmentation model alpha blended with the original -// video input. -/// The window is displayed with GLFW and uses CUDA/OpenGL interopt propagate CUDA Tensors from the -/// CUDA context to OpenGL. -/// The final class color is rendered using a GLSL fragment shader to remap the class index to an -/// RGBA LUT. -/// The LUT can be configured inside the YAML for common class colors. In some cases the alpha -/// component may be desired to be 0 (fully transparent) to prevent drawing over segmentation -/// classes that are uninteresting. -class Visualizer : public gxf::Codelet { - public: - gxf_result_t start() override; - gxf_result_t tick() override; - gxf_result_t stop() override; - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - - private: - gxf_result_t unregisterCudaResources(); - - GLFWwindow* window_; - std::vector image_buffer_; - std::vector class_index_buffer_; - GLuint textures_[2]; - std::vector cuda_resources_; - - gxf::Parameter> class_index_in_; - gxf::Parameter class_index_width_; - gxf::Parameter class_index_height_; - gxf::Parameter> image_in_; - gxf::Parameter image_width_; - gxf::Parameter image_height_; - gxf::Parameter>> class_color_lut_; - gxf::Parameter> window_close_scheduling_term_; -}; - -} // namespace segmentation_visualizer -} // namespace holoscan -} // namespace nvidia - -#endif // NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_SEGMENTATION_VISUALIZER_HPP_ diff --git a/gxf_extensions/segmentation_visualizer/segmentation_visualizer_ext.cpp b/gxf_extensions/segmentation_visualizer/segmentation_visualizer_ext.cpp deleted file mode 100644 index 12b67c94..00000000 --- a/gxf_extensions/segmentation_visualizer/segmentation_visualizer_ext.cpp +++ /dev/null @@ -1,28 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "gxf/std/extension_factory_helper.hpp" - -#include "segmentation_visualizer.hpp" - -GXF_EXT_FACTORY_BEGIN() -GXF_EXT_FACTORY_SET_INFO(0xf52289414bec4f59, 0xb02f1b532c3f19a4, "SegmentationVisualizerExtension", - "Holoscan Segmentation Visualizer extension", "NVIDIA", "0.2.0", - "LICENSE"); -GXF_EXT_FACTORY_ADD(0xe048718b636f40ba, 0x993c15d1d530b56e, - nvidia::holoscan::segmentation_visualizer::Visualizer, nvidia::gxf::Codelet, - "OpenGL Segmentation Vizualizer codelet."); -GXF_EXT_FACTORY_END() diff --git a/gxf_extensions/stream_playback/CMakeLists.txt b/gxf_extensions/stream_playback/CMakeLists.txt index e626dc11..481e35ec 100644 --- a/gxf_extensions/stream_playback/CMakeLists.txt +++ b/gxf_extensions/stream_playback/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,27 +14,23 @@ # limitations under the License. # Create library -add_library(stream_playback_lib SHARED - video_stream_replayer.cpp - video_stream_replayer.hpp +add_library(gxf_stream_playback_lib SHARED video_stream_serializer.cpp video_stream_serializer.hpp ) -target_link_libraries(stream_playback_lib +target_link_libraries(gxf_stream_playback_lib PUBLIC - CUDA::cudart GXF::serialization - GXF::std yaml-cpp ) # Create extension -add_library(stream_playback SHARED +add_library(gxf_stream_playback SHARED stream_playback_ext.cpp ) -target_link_libraries(stream_playback +target_link_libraries(gxf_stream_playback PUBLIC - stream_playback_lib + gxf_stream_playback_lib ) # Install GXF extension as a component 'holoscan-gxf_extensions' -install_gxf_extension(stream_playback) +install_gxf_extension(gxf_stream_playback) diff --git a/gxf_extensions/stream_playback/stream_playback_ext.cpp b/gxf_extensions/stream_playback/stream_playback_ext.cpp index a8109d62..ea11e832 100644 --- a/gxf_extensions/stream_playback/stream_playback_ext.cpp +++ b/gxf_extensions/stream_playback/stream_playback_ext.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,15 +16,11 @@ */ #include "gxf/std/extension_factory_helper.hpp" -#include "video_stream_replayer.hpp" #include "video_stream_serializer.hpp" GXF_EXT_FACTORY_BEGIN() GXF_EXT_FACTORY_SET_INFO(0xe6c168715f3f428d, 0x96cd24dce2f42f46, "StreamPlaybackExtension", "Holoscan StreamPlayback extension", "NVIDIA", "0.2.0", "LICENSE"); -GXF_EXT_FACTORY_ADD(0xb4d16b16b7484e52, 0xaccb8c8989bfac91, - nvidia::holoscan::stream_playback::VideoStreamReplayer, nvidia::gxf::Codelet, - "VideoStreamReplayer codelet."); GXF_EXT_FACTORY_ADD(0x7ee08fcc84c94245, 0xa415022b42f4ef39, nvidia::holoscan::stream_playback::VideoStreamSerializer, nvidia::gxf::EntitySerializer, "VideoStreamSerializer component."); diff --git a/gxf_extensions/stream_playback/video_stream_replayer.cpp b/gxf_extensions/stream_playback/video_stream_replayer.cpp deleted file mode 100644 index 16dcc291..00000000 --- a/gxf_extensions/stream_playback/video_stream_replayer.cpp +++ /dev/null @@ -1,215 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "video_stream_replayer.hpp" - -#include -#include -#include -#include - -namespace nvidia { -namespace holoscan { -namespace stream_playback { - -gxf_result_t VideoStreamReplayer::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - result &= registrar->parameter(transmitter_, "transmitter", "Entity transmitter", - "Transmitter channel for replaying entities"); - result &= registrar->parameter(entity_serializer_, "entity_serializer", "Entity serializer", - "Serializer for serializing entities"); - result &= registrar->parameter( - boolean_scheduling_term_, "boolean_scheduling_term", "BooleanSchedulingTerm", - "BooleanSchedulingTerm to stop the codelet from ticking after all messages are published."); - result &= registrar->parameter(directory_, "directory", "Directory path", - "Directory path for storing files"); - result &= registrar->parameter( - basename_, "basename", "Base file name", "User specified file name without extension", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(batch_size_, "batch_size", "Batch Size", - "Number of entities to read and publish for one tick", 1UL); - result &= registrar->parameter( - ignore_corrupted_entities_, "ignore_corrupted_entities", "Ignore Corrupted Entities", - "If an entity could not be deserialized, it is ignored by default; otherwise a failure is " - "generated.", - true); - result &= registrar->parameter( - frame_rate_, "frame_rate", "Frame rate", - "Frame rate to replay. If zero value is specified, it follows timings in timestamps.", 0.f); - result &= registrar->parameter( - realtime_, "realtime", "Realtime playback", - "Playback video in realtime, based on frame_rate or timestamps (default: true).", true); - result &= registrar->parameter(repeat_, "repeat", "RepeatVideo", - "Repeat video stream (default: false)", false); - result &= registrar->parameter( - count_, "count", "Number of frame counts to playback", - "Number of frame counts to playback. If zero value is specified, it is ignored. If the count " - "is less than the number of frames in the video, it would be finished early.", - 0UL); - - return gxf::ToResultCode(result); -} - -gxf_result_t VideoStreamReplayer::initialize() { - // Create path by appending component name to directory path if basename is not provided - std::string path = directory_.get() + '/'; - if (const auto& basename = basename_.try_get()) { - path += basename.value(); - } else { - path += name(); - } - - // Filenames for index and data - const std::string index_filename = path + gxf::FileStream::kIndexFileExtension; - const std::string entity_filename = path + gxf::FileStream::kBinaryFileExtension; - - // Open index file stream as read-only - index_file_stream_ = gxf::FileStream(index_filename, ""); - gxf::Expected result = index_file_stream_.open(); - if (!result) { - GXF_LOG_WARNING("Could not open index file: %s", index_filename.c_str()); - return gxf::ToResultCode(result); - } - - // Open entity file stream as read-only - entity_file_stream_ = gxf::FileStream(entity_filename, ""); - result = entity_file_stream_.open(); - if (!result) { - GXF_LOG_WARNING("Could not open entity file: %s", entity_filename.c_str()); - return gxf::ToResultCode(result); - } - - boolean_scheduling_term_->enable_tick(); - - playback_index_ = 0; - playback_count_ = 0; - index_start_timestamp_ = 0; - index_last_timestamp_ = 0; - index_timestamp_duration_ = 0; - index_frame_count_ = 1; - playback_start_timestamp_ = 0; - - return GXF_SUCCESS; -} - -gxf_result_t VideoStreamReplayer::deinitialize() { - // Close binary file stream - gxf::Expected result = entity_file_stream_.close(); - if (!result) { return gxf::ToResultCode(result); } - - // Close index file stream - result = index_file_stream_.close(); - if (!result) { return gxf::ToResultCode(result); } - - return GXF_SUCCESS; -} - -gxf_result_t VideoStreamReplayer::tick() { - for (size_t i = 0; i < batch_size_; i++) { - // Read entity index from index file - // Break if index not found and clear stream errors - gxf::EntityIndex index; - gxf::Expected size = index_file_stream_.readTrivialType(&index); - if (!size) { - if (repeat_) { - // Rewind index stream - index_file_stream_.clear(); - if (!index_file_stream_.setReadOffset(0)) { GXF_LOG_ERROR("Could not rewind index file"); } - size = index_file_stream_.readTrivialType(&index); - - // Rewind entity stream - entity_file_stream_.clear(); - entity_file_stream_.setReadOffset(0); - if (!entity_file_stream_.setReadOffset(0)) { - GXF_LOG_ERROR("Could not rewind entity file"); - } - - // Initialize the frame index - playback_index_ = 0; - } - } - if ((!size && !repeat_) || (count_ > 0 && playback_count_ >= count_)) { - GXF_LOG_INFO("Reach end of file or playback count reaches to the limit. Stop ticking."); - boolean_scheduling_term_->disable_tick(); - index_file_stream_.clear(); - break; - } - - // Read entity from binary file - gxf::Expected entity = - entity_serializer_->deserializeEntity(context(), &entity_file_stream_); - if (!entity) { - if (ignore_corrupted_entities_) { - continue; - } else { - return gxf::ToResultCode(entity); - } - } - - int64_t time_to_delay = 0; - - if (playback_count_ == 0) { - playback_start_timestamp_ = std::chrono::system_clock::now().time_since_epoch().count(); - index_start_timestamp_ = index.log_time; - } - // Update last timestamp - if (index.log_time > index_last_timestamp_) { - index_last_timestamp_ = index.log_time; - index_frame_count_ = playback_count_ + 1; - index_timestamp_duration_ = index_last_timestamp_ - index_start_timestamp_; - } - - // Delay if realtime is specified - if (realtime_) { - // Calculate the delay time based on frame rate or timestamps. - uint64_t current_timestamp = std::chrono::system_clock::now().time_since_epoch().count(); - int64_t time_delta = static_cast(current_timestamp - playback_start_timestamp_); - if (frame_rate_ > 0.f) { - time_to_delay = - static_cast(1000000000 / frame_rate_) * playback_count_ - time_delta; - } else { - // Get timestamp from entity - uint64_t timestamp = index.log_time; - time_to_delay = static_cast((timestamp - index_start_timestamp_) + - index_timestamp_duration_ * - (playback_count_ / index_frame_count_)) - - time_delta; - } - if (time_to_delay < 0 && (playback_count_ % index_frame_count_ != 0)) { - GXF_LOG_INFO("Playing video stream is lagging behind (count: %" PRIu64 ", delay: %" PRId64 - " ns)", - playback_count_, time_to_delay); - } - } - - if (time_to_delay > 0) { std::this_thread::sleep_for(std::chrono::nanoseconds(time_to_delay)); } - - // Publish entity - gxf::Expected result = transmitter_->publish(entity.value()); - - // Increment frame counter and index - ++playback_count_; - ++playback_index_; - - if (!result) { return gxf::ToResultCode(result); } - } - - return GXF_SUCCESS; -} - -} // namespace stream_playback -} // namespace holoscan -} // namespace nvidia diff --git a/gxf_extensions/stream_playback/video_stream_replayer.hpp b/gxf_extensions/stream_playback/video_stream_replayer.hpp deleted file mode 100644 index db4558fa..00000000 --- a/gxf_extensions/stream_playback/video_stream_replayer.hpp +++ /dev/null @@ -1,79 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_STREAM_PLAYBACK_VIDEO_STREAM_REPLAYER_HPP_ -#define NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_STREAM_PLAYBACK_VIDEO_STREAM_REPLAYER_HPP_ - -#include - -#include "gxf/serialization/entity_serializer.hpp" -#include "gxf/serialization/file_stream.hpp" -#include "gxf/std/codelet.hpp" -#include "gxf/std/scheduling_terms.hpp" -#include "gxf/std/transmitter.hpp" - -namespace nvidia { -namespace holoscan { -namespace stream_playback { - -/// @brief Replays entities by reading and deserializing from a file. -/// -/// The file is processed sequentially and a single entity is published per tick. -/// This supports realtime, faster than realtime, or slower than realtime playback of prerecorded -/// data. -/// The input data can optionally be repeated to loop forever or only for a specified count. -class VideoStreamReplayer : public gxf::Codelet { - public: - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - gxf_result_t initialize() override; - gxf_result_t deinitialize() override; - - gxf_result_t start() override { return GXF_SUCCESS; } - gxf_result_t tick() override; - gxf_result_t stop() override { return GXF_SUCCESS; } - - private: - gxf::Parameter> transmitter_; - gxf::Parameter> entity_serializer_; - gxf::Parameter> boolean_scheduling_term_; - gxf::Parameter directory_; - gxf::Parameter basename_; - gxf::Parameter batch_size_; - gxf::Parameter ignore_corrupted_entities_; - gxf::Parameter frame_rate_; - gxf::Parameter realtime_; - gxf::Parameter repeat_; - gxf::Parameter count_; - - // File stream for entities - gxf::FileStream entity_file_stream_; - // File stream for index - gxf::FileStream index_file_stream_; - - uint64_t playback_index_ = 0; - uint64_t playback_count_ = 0; - uint64_t index_start_timestamp_ = 0; - uint64_t index_last_timestamp_ = 0; - uint64_t index_timestamp_duration_ = 0; - uint64_t index_frame_count_ = 1; - uint64_t playback_start_timestamp_ = 0; -}; - -} // namespace stream_playback -} // namespace holoscan -} // namespace nvidia - -#endif // NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_STREAM_PLAYBACK_VIDEO_STREAM_REPLAYER_HPP_ diff --git a/gxf_extensions/stream_playback/video_stream_serializer.cpp b/gxf_extensions/stream_playback/video_stream_serializer.cpp index c685aced..e0649ee8 100644 --- a/gxf_extensions/stream_playback/video_stream_serializer.cpp +++ b/gxf_extensions/stream_playback/video_stream_serializer.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -135,8 +135,6 @@ gxf::Expected VideoStreamSerializer::deserialize_entity_header_abi( .and_then([&]() { return DeserializeEntityHeader(endpoint); }) .map([&](EntityHeader entity_header) { if (entity_header.sequence_number != incoming_sequence_number_) { - GXF_LOG_WARNING("Got message %zu but expected message %zu", - entity_header.sequence_number, incoming_sequence_number_); incoming_sequence_number_ = entity_header.sequence_number; } incoming_sequence_number_++; diff --git a/gxf_extensions/tensor_rt/CMakeLists.txt b/gxf_extensions/tensor_rt/CMakeLists.txt index 30ad0d54..dbfef58d 100644 --- a/gxf_extensions/tensor_rt/CMakeLists.txt +++ b/gxf_extensions/tensor_rt/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,11 +14,11 @@ # limitations under the License. # Create library -add_library(tensor_rt_inference_lib SHARED +add_library(gxf_tensor_rt_lib SHARED tensor_rt_inference.cpp tensor_rt_inference.hpp ) -target_link_libraries(tensor_rt_inference_lib +target_link_libraries(gxf_tensor_rt_lib PUBLIC CUDA::cudart GXF::cuda @@ -28,13 +28,12 @@ target_link_libraries(tensor_rt_inference_lib ) # Create extension -add_library(tensor_rt SHARED +add_library(gxf_tensor_rt SHARED tensor_rt_extension.cpp ) -target_link_libraries(tensor_rt - PUBLIC tensor_rt_inference_lib +target_link_libraries(gxf_tensor_rt + PUBLIC gxf_tensor_rt_lib ) # Install GXF extension as a component 'holoscan-gxf_extensions' -install_gxf_extension(tensor_rt) -install_gxf_extension(tensor_rt_inference_lib) # not following '{name}_lib' so install explicitly +install_gxf_extension(gxf_tensor_rt) diff --git a/gxf_extensions/tensor_rt/tensor_rt_extension.cpp b/gxf_extensions/tensor_rt/tensor_rt_extension.cpp index c011dce2..2b56ec73 100644 --- a/gxf_extensions/tensor_rt/tensor_rt_extension.cpp +++ b/gxf_extensions/tensor_rt/tensor_rt_extension.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/gxf_extensions/tensor_rt/tensor_rt_inference.cpp b/gxf_extensions/tensor_rt/tensor_rt_inference.cpp index 91e12806..3157b45c 100644 --- a/gxf_extensions/tensor_rt/tensor_rt_inference.cpp +++ b/gxf_extensions/tensor_rt/tensor_rt_inference.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -180,62 +180,92 @@ void TensorRTInferenceLogger::setVerbose(bool verbose) { gxf_result_t TensorRtInference::registerInterface(gxf::Registrar* registrar) { gxf::Expected result; - result &= registrar->parameter(model_file_path_, "model_file_path", "Model File Path", - "Path to ONNX model to be loaded."); result &= registrar->parameter( - engine_cache_dir_, "engine_cache_dir", "Engine Cache Directory", - "Path to a folder containing cached engine files to be serialized and loaded from."); + model_file_path_, "model_file_path", "Model File Path", "Path to ONNX model to be loaded."); result &= registrar->parameter( - plugins_lib_namespace_, "plugins_lib_namespace", "Plugins Lib Namespace", - "Namespace used to register all the plugins in this library.", std::string("")); - result &= registrar->parameter(force_engine_update_, "force_engine_update", "Force Engine Update", + engine_cache_dir_, + "engine_cache_dir", + "Engine Cache Directory", + "Path to a folder containing cached engine files to be serialized and loaded from."); + result &= registrar->parameter(plugins_lib_namespace_, + "plugins_lib_namespace", + "Plugins Lib Namespace", + "Namespace used to register all the plugins in this library.", + std::string("")); + result &= registrar->parameter(force_engine_update_, + "force_engine_update", + "Force Engine Update", "Always update engine regard less of existing engine file. " "Such conversion may take minutes. Default to false.", false); - result &= registrar->parameter(input_tensor_names_, "input_tensor_names", "Input Tensor Names", + result &= registrar->parameter(input_tensor_names_, + "input_tensor_names", + "Input Tensor Names", "Names of input tensors in the order to be fed into the model."); - result &= registrar->parameter(input_binding_names_, "input_binding_names", "Input Binding Names", + result &= registrar->parameter(input_binding_names_, + "input_binding_names", + "Input Binding Names", "Names of input bindings as in the model in the same order of " "what is provided in input_tensor_names."); - result &= registrar->parameter(output_tensor_names_, "output_tensor_names", "Output Tensor Names", + result &= registrar->parameter(output_tensor_names_, + "output_tensor_names", + "Output Tensor Names", "Names of output tensors in the order to be retrieved " "from the model."); - result &= - registrar->parameter(output_binding_names_, "output_binding_names", "Output Binding Names", - "Names of output bindings in the model in the same " - "order of of what is provided in output_tensor_names."); + result &= registrar->parameter(output_binding_names_, + "output_binding_names", + "Output Binding Names", + "Names of output bindings in the model in the same " + "order of of what is provided in output_tensor_names."); result &= registrar->parameter(pool_, "pool", "Pool", "Allocator instance for output tensors."); - result &= registrar->parameter(cuda_stream_pool_, "cuda_stream_pool", "Cuda Stream Pool", - "Instance of gxf::CudaStreamPool to allocate CUDA stream."); - - result &= registrar->parameter(max_workspace_size_, "max_workspace_size", "Max Workspace Size", - "Size of working space in bytes. Default to 64MB", 67108864l); - result &= - registrar->parameter(dla_core_, "dla_core", "DLA Core", - "DLA Core to use. Fallback to GPU is always enabled. " - "Default to use GPU only.", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(max_batch_size_, "max_batch_size", "Max Batch Size", + + result &= registrar->parameter(max_workspace_size_, + "max_workspace_size", + "Max Workspace Size", + "Size of working space in bytes. Default to 64MB", + 67108864l); + result &= registrar->parameter(dla_core_, + "dla_core", + "DLA Core", + "DLA Core to use. Fallback to GPU is always enabled. " + "Default to use GPU only.", + gxf::Registrar::NoDefaultParameter(), + GXF_PARAMETER_FLAGS_OPTIONAL); + result &= registrar->parameter(max_batch_size_, + "max_batch_size", + "Max Batch Size", "Maximum possible batch size in case the first dimension is " "dynamic and used as batch size.", 1); - result &= registrar->parameter(enable_fp16_, "enable_fp16_", "Enable FP16 Mode", - "Enable inference with FP16 and FP32 fallback.", false); - - result &= registrar->parameter(verbose_, "verbose", "Verbose", - "Enable verbose logging on console. Default to false.", false); - result &= registrar->parameter(relaxed_dimension_check_, "relaxed_dimension_check", + result &= registrar->parameter(enable_fp16_, + "enable_fp16_", + "Enable FP16 Mode", + "Enable inference with FP16 and FP32 fallback.", + false); + result &= registrar->parameter(verbose_, + "verbose", + "Verbose", + "Enable verbose logging on console. Default to false.", + false); + result &= registrar->parameter(relaxed_dimension_check_, + "relaxed_dimension_check", "Relaxed Dimension Check", - "Ignore dimensions of 1 for input tensor dimension check.", true); - result &= - registrar->parameter(clock_, "clock", "Clock", "Instance of clock for publish time.", - gxf::Registrar::NoDefaultParameter(), GXF_PARAMETER_FLAGS_OPTIONAL); + "Ignore dimensions of 1 for input tensor dimension check.", + true); + result &= registrar->parameter(clock_, + "clock", + "Clock", + "Instance of clock for publish time.", + gxf::Registrar::NoDefaultParameter(), + GXF_PARAMETER_FLAGS_OPTIONAL); result &= registrar->parameter(rx_, "rx", "RX", "List of receivers to take input tensors"); result &= registrar->parameter(tx_, "tx", "TX", "Transmitter to publish output tensors"); + result &= cuda_stream_handler_.registerInterface(registrar, true); + return gxf::ToResultCode(result); } @@ -251,12 +281,14 @@ gxf_result_t TensorRtInference::start() { } if (input_tensor_names_.get().size() != input_binding_names_.get().size()) { GXF_LOG_ERROR("Mismatching number of input tensor names and bindings: %lu vs %lu.", - input_tensor_names_.get().size(), input_binding_names_.get().size()); + input_tensor_names_.get().size(), + input_binding_names_.get().size()); return GXF_FAILURE; } if (output_tensor_names_.get().size() != output_binding_names_.get().size()) { GXF_LOG_ERROR("Mismatching number of output tensor names and bindings: %lu vs %lu.", - output_tensor_names_.get().size(), output_binding_names_.get().size()); + output_tensor_names_.get().size(), + output_binding_names_.get().size()); return GXF_FAILURE; } @@ -269,18 +301,8 @@ gxf_result_t TensorRtInference::start() { GXF_LOG_WARNING("Could not initialize LibNvInferPlugins."); } - // Create the cuda stream if it does not yet exist. - // This is needed to populate cuda_stream_ before calling queryHostEngineCapability() - if (!cuda_stream_) { - auto maybe_stream = cuda_stream_pool_.get()->allocateStream(); - if (!maybe_stream) { - GXF_LOG_ERROR("Failed to allocate CUDA stream"); - return maybe_stream.error(); - } - cuda_stream_ = std::move(maybe_stream.value()); - } - - gxf::Expected maybe_host_engine_capability = queryHostEngineCapability(); + gxf::Expected maybe_host_engine_capability = + queryHostEngineCapability(cuda_stream_handler_.getStreamHandle()->dev_id()); if (!maybe_host_engine_capability) { GXF_LOG_ERROR("Failed to query host engine capability."); return GXF_FAILURE; @@ -314,7 +336,8 @@ gxf_result_t TensorRtInference::start() { GXF_LOG_WARNING( "Rebuilding CUDA engine %s%s. " "Note: this process may take up to several minutes.", - engine_file_path.c_str(), warning_note); + engine_file_path.c_str(), + warning_note); auto result = convertModelToEngine(); if (!result) { GXF_LOG_ERROR("Failed to create engine plan for model %s.", model_file_path_.get().c_str()); @@ -336,13 +359,15 @@ gxf_result_t TensorRtInference::start() { // Deserialize the CUDA engine if (verbose_.get()) { GXF_LOG_DEBUG("Creating inference runtime."); } - cuda_engine_.reset(infer_runtime->deserializeCudaEngine(plan.data(), plan.size(), NULL)); + cuda_engine_.reset(infer_runtime->deserializeCudaEngine(plan.data(), plan.size())); // Debug spews if (verbose_.get()) { GXF_LOG_DEBUG("Number of CUDA bindings: %d", cuda_engine_->getNbBindings()); for (int i = 0; i < cuda_engine_->getNbBindings(); ++i) { - GXF_LOG_DEBUG("CUDA binding No.%d: name %s Format %s", i, cuda_engine_->getBindingName(i), + GXF_LOG_DEBUG("CUDA binding No.%d: name %s Format %s", + i, + cuda_engine_->getBindingName(i), cuda_engine_->getBindingFormatDesc(i)); } } @@ -355,7 +380,8 @@ gxf_result_t TensorRtInference::start() { GXF_LOG_ERROR( "Numbers of CUDA bindings mismatch: configured for %lu vs model requires %d. " "Please check TensorRTInference codelet configuration.\n", - total_bindings_number, cuda_engine_->getNbBindings()); + total_bindings_number, + cuda_engine_->getNbBindings()); return GXF_ARGUMENT_INVALID; } @@ -374,13 +400,15 @@ gxf_result_t TensorRtInference::start() { const int32_t binding_index = cuda_engine_->getBindingIndex(binding_name.c_str()); if (binding_index == -1) { - GXF_LOG_ERROR("Failed to get binding index for input %s in model %s", binding_name.c_str(), + GXF_LOG_ERROR("Failed to get binding index for input %s in model %s", + binding_name.c_str(), engine_file_path_.c_str()); return GXF_FAILURE; } if (binding_index >= static_cast(cuda_buffers_.size())) { - GXF_LOG_ERROR("Binding index for input %s is out of range in model %s.", binding_name.c_str(), + GXF_LOG_ERROR("Binding index for input %s is out of range in model %s.", + binding_name.c_str(), engine_file_path.c_str()); return GXF_FAILURE; } @@ -390,20 +418,25 @@ gxf_result_t TensorRtInference::start() { NvInferDatatypeToTensorElementType(cuda_engine_->getBindingDataType(binding_index)); if (!maybe_element_type) { GXF_LOG_ERROR("Unsupported element type for binding input %s on index %d. ", - binding_name.c_str(), binding_index); + binding_name.c_str(), + binding_index); return maybe_element_type.error(); } // Keeps binding info const auto& dims = cuda_engine_->getBindingDimensions(binding_index); - binding_infos_[tensor_name] = - BindingInfo{binding_index, static_cast(dims.nbDims), binding_name, - maybe_element_type.value(), Dims2Dimensions(dims)}; + binding_infos_[tensor_name] = BindingInfo{binding_index, + static_cast(dims.nbDims), + binding_name, + maybe_element_type.value(), + Dims2Dimensions(dims)}; // Debug spew if (verbose_.get()) { GXF_LOG_DEBUG( - "Input Tensor %s:%s index %d Dimensions %s.", tensor_name.c_str(), binding_name.c_str(), + "Input Tensor %s:%s index %d Dimensions %s.", + tensor_name.c_str(), + binding_name.c_str(), binding_index, FormatDims(binding_infos_[tensor_name].dimensions, binding_infos_[tensor_name].rank) .c_str()); @@ -430,52 +463,31 @@ gxf_result_t TensorRtInference::start() { NvInferDatatypeToTensorElementType(cuda_engine_->getBindingDataType(binding_index)); if (!maybe_element_type) { GXF_LOG_ERROR("Unsupported element type for binding output %s on index %d. ", - binding_name.c_str(), binding_index); + binding_name.c_str(), + binding_index); return maybe_element_type.error(); } // Keeps binding info const auto& dims = cuda_engine_->getBindingDimensions(binding_index); - binding_infos_[tensor_name] = - BindingInfo{binding_index, static_cast(dims.nbDims), binding_name, - maybe_element_type.value(), Dims2Dimensions(dims)}; + binding_infos_[tensor_name] = BindingInfo{binding_index, + static_cast(dims.nbDims), + binding_name, + maybe_element_type.value(), + Dims2Dimensions(dims)}; cuda_buffers_[binding_index] = nullptr; // populate cuda_buffers dynamically, in tick() if (verbose_.get()) { GXF_LOG_DEBUG( - "Output Tensor %s:%s (%d), Dimensions: %s.", tensor_name.c_str(), binding_name.c_str(), + "Output Tensor %s:%s (%d), Dimensions: %s.", + tensor_name.c_str(), + binding_name.c_str(), binding_index, FormatDims(binding_infos_[tensor_name].dimensions, binding_infos_[tensor_name].rank) .c_str()); } } - // Grabs CUDA stream and creates CUDA event for synchronization - if (!cuda_stream_) { - auto maybe_stream = cuda_stream_pool_.get()->allocateStream(); - if (!maybe_stream) { - GXF_LOG_ERROR("Failed to allocate CUDA stream"); - return maybe_stream.error(); - } - cuda_stream_ = std::move(maybe_stream.value()); - } - auto stream = cuda_stream_.get()->stream(); - if (!stream) { - GXF_LOG_ERROR("Failed to grab CUDA stream"); - return stream.error(); - } - cached_cuda_stream_ = stream.value(); - auto result = cudaEventCreate(&cuda_event_consumed_); - if (cudaSuccess != result) { - GXF_LOG_ERROR("Failed to create consumed CUDA event: %s", cudaGetErrorString(result)); - return GXF_FAILURE; - } - auto result2 = cudaEventCreate(&cuda_event_done_); - if (cudaSuccess != result2) { - GXF_LOG_ERROR("Failed to create done CUDA event: %s", cudaGetErrorString(result2)); - return GXF_FAILURE; - } - return GXF_SUCCESS; } @@ -526,7 +538,8 @@ gxf::Expected> TensorRtInference::convertModelToEngine() { if (dims.d[j] <= 0) { GXF_LOG_ERROR( "Input binding %s requires dynamic size on dimension No.%d which is not supported", - bind_tensor->getName(), j); + bind_tensor->getName(), + j); return gxf::Unexpected{GXF_ARGUMENT_OUT_OF_RANGE}; } } @@ -547,14 +560,8 @@ gxf::Expected> TensorRtInference::convertModelToEngine() { builderConfig->addOptimizationProfile(optimization_profile); // Creates TensorRT Engine Plan - NvInferHandle engine( - builder->buildEngineWithConfig(*network, *builderConfig)); - if (!engine) { - GXF_LOG_ERROR("Failed to build TensorRT engine from model %s.", model_file_path_.get().c_str()); - return gxf::Unexpected{GXF_FAILURE}; - } - - NvInferHandle model_stream(engine->serialize()); + NvInferHandle model_stream(builder->buildSerializedNetwork( + *network, *builderConfig)); if (!model_stream || model_stream->size() == 0 || model_stream->data() == nullptr) { GXF_LOG_ERROR("Fail to serialize TensorRT Engine."); return gxf::Unexpected{GXF_FAILURE}; @@ -573,17 +580,6 @@ gxf_result_t TensorRtInference::stop() { cuda_engine_ = nullptr; cuda_buffers_.clear(); - auto result = cudaEventDestroy(cuda_event_consumed_); - if (cudaSuccess != result) { - GXF_LOG_ERROR("Failed to destroy consumed CUDA event: %s", cudaGetErrorString(result)); - return GXF_FAILURE; - } - auto result2 = cudaEventDestroy(cuda_event_done_); - if (cudaSuccess != result2) { - GXF_LOG_ERROR("Failed to create done CUDA event: %s", cudaGetErrorString(result2)); - return GXF_FAILURE; - } - return GXF_SUCCESS; } @@ -599,12 +595,20 @@ gxf_result_t TensorRtInference::tick() { GXF_LOG_ERROR("No message available."); return GXF_CONTRACT_MESSAGE_NOT_AVAILABLE; } + + // sync with the CUDA streams from the input messages + gxf_result_t stream_handler_result = cuda_stream_handler_.fromMessages(context(), messages); + if (stream_handler_result != GXF_SUCCESS) { + GXF_LOG_ERROR("Failed to get the CUDA stream from incoming messages"); + return stream_handler_result; + } // Tries to retrieve timestamp if clock present gxf::Expected> maybe_input_timestamp = gxf::Unexpected{GXF_FAILURE}; for (auto& msg : messages) { maybe_input_timestamp = msg.get("timestamp"); if (maybe_input_timestamp) { break; } } + // Populates input tensors for (const auto& tensor_name : input_tensor_names_.get()) { gxf::Expected> maybe_tensor = gxf::Unexpected{GXF_UNINITIALIZED_VALUE}; @@ -627,7 +631,8 @@ gxf_result_t TensorRtInference::tick() { // Checks input tensor element type if (maybe_tensor.value()->element_type() != binding_info.element_type) { GXF_LOG_ERROR("Mismatching tensor element type required %d vs provided %d", - binding_info.element_type, maybe_tensor.value()->element_type()); + binding_info.element_type, + maybe_tensor.value()->element_type()); return GXF_FAILURE; } @@ -668,7 +673,8 @@ gxf_result_t TensorRtInference::tick() { GXF_LOG_ERROR( "Input Tensor %s bound to %s:" " dimensions does not meet model spec with relaxed matching. Expected: %s Real: %s", - tensor_name.c_str(), binding_info.binding_name.c_str(), + tensor_name.c_str(), + binding_info.binding_name.c_str(), FormatDims(binding_info.dimensions, binding_info.rank).c_str(), FormatTensorShape(shape).c_str()); return GXF_FAILURE; @@ -677,7 +683,9 @@ gxf_result_t TensorRtInference::tick() { // Strict dimension match. All dimensions must match. Binding of -1 is considered as match. if (shape.rank() != binding_info.rank) { GXF_LOG_ERROR("Tensor %s bound to %s has mismatching rank %d (%d required)", - tensor_name.c_str(), binding_info.binding_name.c_str(), shape.rank(), + tensor_name.c_str(), + binding_info.binding_name.c_str(), + shape.rank(), binding_info.rank); return GXF_FAILURE; } @@ -685,8 +693,11 @@ gxf_result_t TensorRtInference::tick() { if (binding_info.dimensions[i] == -1) { dims.d[i] = shape.dimension(i); } if (shape.dimension(i) != binding_info.dimensions[i] && binding_info.dimensions[i] != -1) { GXF_LOG_ERROR("Tensor %s bound to %s has mismatching dimension %d:%d (%d required)", - tensor_name.c_str(), binding_info.binding_name.c_str(), i, - shape.dimension(i), binding_info.dimensions[i]); + tensor_name.c_str(), + binding_info.binding_name.c_str(), + i, + shape.dimension(i), + binding_info.dimensions[i]); return GXF_FAILURE; } } @@ -721,8 +732,12 @@ gxf_result_t TensorRtInference::tick() { gxf::Shape shape{Dims2Dimensions(binding_dims), binding_info.rank}; auto result = maybe_result_tensor.value()->reshapeCustom( - shape, binding_info.element_type, gxf::PrimitiveTypeSize(binding_info.element_type), - gxf::Unexpected{GXF_UNINITIALIZED_VALUE}, gxf::MemoryStorageType::kDevice, pool_); + shape, + binding_info.element_type, + gxf::PrimitiveTypeSize(binding_info.element_type), + gxf::Unexpected{GXF_UNINITIALIZED_VALUE}, + gxf::MemoryStorageType::kDevice, + pool_); if (!result) { GXF_LOG_ERROR("Failed to allocate for output tensor %s", tensor_name.c_str()); return gxf::ToResultCode(result); @@ -732,32 +747,18 @@ gxf_result_t TensorRtInference::tick() { cuda_buffers_[binding_info.index] = maybe_result_tensor.value()->pointer(); } - auto maybe_stream_id = maybe_result_message.value().add("TensorRTCuStream"); - if (!maybe_stream_id) { - GXF_LOG_ERROR("failed to add TensorRTCuStream"); - return gxf::ToResultCode(maybe_stream_id); - } - maybe_stream_id.value()->stream_cid = cuda_stream_.cid(); - GXF_ASSERT(maybe_stream_id.value()->stream_cid != kNullUid, "Internal error: stream_cid is null"); - // Runs inference on specified CUDA stream - if (!cuda_execution_ctx_->enqueueV2(cuda_buffers_.data(), cached_cuda_stream_, - &cuda_event_consumed_)) { + if (!cuda_execution_ctx_->enqueueV2( + cuda_buffers_.data(), cuda_stream_handler_.getCudaStream(), nullptr)) { GXF_LOG_ERROR("TensorRT task enqueue for engine %s failed.", engine_file_path_.c_str()); return GXF_FAILURE; } - // Blocks until job is done - auto result = cudaEventRecord(cuda_event_done_, cached_cuda_stream_); - if (cudaSuccess != result) { - GXF_LOG_ERROR("Failed to queue event for task done with engine %s: %s", - engine_file_path_.c_str(), cudaGetErrorString(result)); - return GXF_FAILURE; - } - auto result2 = cudaEventSynchronize(cuda_event_done_); - if (cudaSuccess != result2) { - GXF_LOG_ERROR("Failed to synchronize for task done with engine %s: %s", - engine_file_path_.c_str(), cudaGetErrorString(result2)); - return GXF_FAILURE; + + // pass the CUDA stream to the output message + stream_handler_result = cuda_stream_handler_.toMessage(maybe_result_message); + if (stream_handler_result != GXF_SUCCESS) { + GXF_LOG_ERROR("Failed to add the CUDA stream to the outgoing messages"); + return stream_handler_result; } // Publishes result with acqtime @@ -775,14 +776,14 @@ static std::string replaceChar(const std::string& string, char match, char repla return result; } -gxf::Expected TensorRtInference::queryHostEngineCapability() const { +gxf::Expected TensorRtInference::queryHostEngineCapability(int dev_id) const { char* env_var = std::getenv("GXF_TENSORRT_HOST_ENGINE_CAPABILITY"); if (env_var != nullptr) { GXF_LOG_INFO("Using GXF_TENSORRT_HOST_ENGINE_CAPABILITY overwrite: %s", env_var); return std::string(env_var); } cudaDeviceProp device_prop = {0}; - cudaError_t status = cudaGetDeviceProperties(&device_prop, cuda_stream_.get()->dev_id()); + cudaError_t status = cudaGetDeviceProperties(&device_prop, dev_id); if (status != cudaSuccess) { GXF_LOG_ERROR("Failed to get cuda device properties with errorcode: %d", status); return gxf::Unexpected{}; diff --git a/gxf_extensions/tensor_rt/tensor_rt_inference.hpp b/gxf_extensions/tensor_rt/tensor_rt_inference.hpp index 82a9c388..6a0401b4 100644 --- a/gxf_extensions/tensor_rt/tensor_rt_inference.hpp +++ b/gxf_extensions/tensor_rt/tensor_rt_inference.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,8 +14,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef NVIDIA_GXF_EXTENSIONS_TENSOR_RT_TENSOR_RT_INFERENCE_HPP_ -#define NVIDIA_GXF_EXTENSIONS_TENSOR_RT_TENSOR_RT_INFERENCE_HPP_ +#ifndef GXF_EXTENSIONS_TENSOR_RT_TENSOR_RT_INFERENCE_HPP +#define GXF_EXTENSIONS_TENSOR_RT_TENSOR_RT_INFERENCE_HPP #include #include @@ -38,6 +38,8 @@ #include "gxf/std/tensor.hpp" #include "gxf/std/transmitter.hpp" +#include "../utils/cuda_stream_handler.hpp" + namespace nvidia { namespace gxf { @@ -70,14 +72,14 @@ class TensorRtInference : public gxf::Codelet { private: // Helper to return a string for the TRT engine capability. - gxf::Expected queryHostEngineCapability() const; + gxf::Expected queryHostEngineCapability(int dev_id) const; // Helper to search for the engine file path. gxf::Expected findEngineFilePath(const std::string& host_engine_capability) const; // Helper deleter to call destroy while destroying the cuda objects template struct DeleteFunctor { - inline void operator()(void* ptr) { reinterpret_cast(ptr)->destroy(); } + inline void operator()(void* ptr) { delete reinterpret_cast(ptr); } }; // unique_ptr using custom Delete Functor above template @@ -105,7 +107,6 @@ class TensorRtInference : public gxf::Codelet { gxf::Parameter> output_tensor_names_; gxf::Parameter> output_binding_names_; gxf::Parameter> pool_; - gxf::Parameter> cuda_stream_pool_; gxf::Parameter max_workspace_size_; gxf::Parameter dla_core_; gxf::Parameter max_batch_size_; @@ -123,15 +124,13 @@ class TensorRtInference : public gxf::Codelet { NvInferHandle cuda_execution_ctx_; NvInferHandle cuda_engine_; - gxf::Handle cuda_stream_; std::vector cuda_buffers_; - cudaStream_t cached_cuda_stream_; - cudaEvent_t cuda_event_consumed_; - cudaEvent_t cuda_event_done_; std::string engine_file_path_; + + holoscan::CudaStreamHandler cuda_stream_handler_; }; } // namespace gxf } // namespace nvidia -#endif // NVIDIA_GXF_EXTENSIONS_TENSOR_RT_TENSOR_RT_INFERENCE_HPP_ +#endif /* GXF_EXTENSIONS_TENSOR_RT_TENSOR_RT_INFERENCE_HPP */ diff --git a/gxf_extensions/tool_tracking_postprocessor/CMakeLists.txt b/gxf_extensions/tool_tracking_postprocessor/CMakeLists.txt deleted file mode 100644 index e7009e63..00000000 --- a/gxf_extensions/tool_tracking_postprocessor/CMakeLists.txt +++ /dev/null @@ -1,50 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Mark CUDA files -# Note: needed because we can't use .cu or .cuh with the -# bazel build system. We could switch to using those instead -# if we only support CMake -set_source_files_properties( - tool_tracking_postprocessor.cu.cpp - tool_tracking_postprocessor.cu.hpp - PROPERTIES - LANGUAGE CUDA -) - -# Create library -add_library(tool_tracking_postprocessor_lib SHARED - tool_tracking_postprocessor.cpp - tool_tracking_postprocessor.cu.cpp - tool_tracking_postprocessor.cu.hpp - tool_tracking_postprocessor.hpp -) -target_link_libraries(tool_tracking_postprocessor_lib - PUBLIC - CUDA::cuda_driver - GXF::multimedia - GXF::std - yaml-cpp -) - -# Create extension -add_library(tool_tracking_postprocessor SHARED - tool_tracking_postprocessor_ext.cpp -) -target_link_libraries(tool_tracking_postprocessor - PUBLIC tool_tracking_postprocessor_lib -) -# Install GXF extension as a component 'holoscan-gxf_extensions' -install_gxf_extension(tool_tracking_postprocessor) diff --git a/gxf_extensions/tool_tracking_postprocessor/tool_tracking_postprocessor.cpp b/gxf_extensions/tool_tracking_postprocessor/tool_tracking_postprocessor.cpp deleted file mode 100644 index 0d44061b..00000000 --- a/gxf_extensions/tool_tracking_postprocessor/tool_tracking_postprocessor.cpp +++ /dev/null @@ -1,188 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "tool_tracking_postprocessor.hpp" - -#include -#include - -#include "gxf/std/tensor.hpp" - -#define CUDA_TRY(stmt) \ - ({ \ - cudaError_t _holoscan_cuda_err = stmt; \ - if (cudaSuccess != _holoscan_cuda_err) { \ - GXF_LOG_ERROR("CUDA Runtime call %s in line %d of file %s failed with '%s' (%d).", #stmt, \ - __LINE__, __FILE__, cudaGetErrorString(_holoscan_cuda_err), \ - _holoscan_cuda_err); \ - } \ - _holoscan_cuda_err; \ - }) - -namespace nvidia { -namespace holoscan { -namespace tool_tracking_postprocessor { - -constexpr float DEFAULT_MIN_PROB = 0.5f; -// 12 qualitative classes color scheme from colorbrewer2 -static const std::vector> DEFAULT_COLORS = { - {0.12f, 0.47f, 0.71f}, {0.20f, 0.63f, 0.17f}, {0.89f, 0.10f, 0.11f}, {1.00f, 0.50f, 0.00f}, - {0.42f, 0.24f, 0.60f}, {0.69f, 0.35f, 0.16f}, {0.65f, 0.81f, 0.89f}, {0.70f, 0.87f, 0.54f}, - {0.98f, 0.60f, 0.60f}, {0.99f, 0.75f, 0.44f}, {0.79f, 0.70f, 0.84f}, {1.00f, 1.00f, 0.60f}}; - -gxf_result_t Postprocessor::start() { - return GXF_SUCCESS; -} - -gxf_result_t Postprocessor::tick() { - // Process input message - const auto in_message = in_->receive(); - if (!in_message || in_message.value().is_null()) { return GXF_CONTRACT_MESSAGE_NOT_AVAILABLE; } - - // Get tensors attached to the message - auto maybe_tensor = in_message.value().get("probs"); - if (!maybe_tensor) { - GXF_LOG_ERROR("Tensor 'probs' not found in message."); - return GXF_FAILURE; - } - const gxf::Handle probs_tensor = maybe_tensor.value(); - std::vector probs(probs_tensor->size() / sizeof(float)); - CUDA_TRY(cudaMemcpy(probs.data(), probs_tensor->pointer(), probs_tensor->size(), - cudaMemcpyDeviceToHost)); - - maybe_tensor = in_message.value().get("scaled_coords"); - if (!maybe_tensor) { - GXF_LOG_ERROR("Tensor 'scaled_coords' not found in message."); - return GXF_FAILURE; - } - const gxf::Handle scaled_coords_tensor = maybe_tensor.value(); - std::vector scaled_coords(scaled_coords_tensor->size() / sizeof(float)); - CUDA_TRY(cudaMemcpy(scaled_coords.data(), scaled_coords_tensor->pointer(), - scaled_coords_tensor->size(), cudaMemcpyDeviceToHost)); - - maybe_tensor = in_message.value().get("binary_masks"); - if (!maybe_tensor) { - GXF_LOG_ERROR("Tensor 'binary_masks' not found in message."); - return GXF_FAILURE; - } - const gxf::Handle binary_masks_tensor = maybe_tensor.value(); - - auto out_message = gxf::Entity::New(context()); - if (!out_message) { - GXF_LOG_ERROR("Failed to allocate output message"); - return GXF_FAILURE; - } - - // filter coordinates based on probability - std::vector visible_classes; - { - std::vector filtered_scaled_coords; - for (size_t index = 0; index < probs.size(); ++index) { - if (probs[index] > min_prob_) { - filtered_scaled_coords.push_back(scaled_coords[index * 2]); - filtered_scaled_coords.push_back(scaled_coords[index * 2 + 1]); - visible_classes.push_back(index); - } else { - filtered_scaled_coords.push_back(-1.f); - filtered_scaled_coords.push_back(-1.f); - } - } - - const auto out_tensor = out_message.value().add("scaled_coords"); - if (!out_tensor) { - GXF_LOG_ERROR("Failed to allocate output tensor 'scaled_coords'"); - return GXF_FAILURE; - } - - const gxf::Shape output_shape{1, int32_t(filtered_scaled_coords.size() / 2), 2}; - out_tensor.value()->reshape(output_shape, gxf::MemoryStorageType::kHost, - host_allocator_); - if (!out_tensor.value()->pointer()) { - GXF_LOG_ERROR("Failed to allocate output tensor buffer for tensor 'scaled_coords'."); - return GXF_FAILURE; - } - memcpy(out_tensor.value()->data().value(), filtered_scaled_coords.data(), - filtered_scaled_coords.size() * sizeof(float)); - } - - // filter binary mask - { - const auto out_tensor = out_message.value().add("mask"); - if (!out_tensor) { - GXF_LOG_ERROR("Failed to allocate output tensor 'mask'"); - return GXF_FAILURE; - } - - const gxf::Shape output_shape{binary_masks_tensor->shape().dimension(2), - binary_masks_tensor->shape().dimension(3), 4}; - out_tensor.value()->reshape(output_shape, gxf::MemoryStorageType::kDevice, - device_allocator_); - if (!out_tensor.value()->pointer()) { - GXF_LOG_ERROR("Failed to allocate output tensor buffer for tensor 'mask'."); - return GXF_FAILURE; - } - - float* const out_data = out_tensor.value()->data().value(); - const size_t layer_size = output_shape.dimension(0) * output_shape.dimension(1); - bool first = true; - for (auto& index : visible_classes) { - const auto& img_color = - overlay_img_colors_.get()[std::min(index, uint32_t(overlay_img_colors_.get().size()))]; - const std::array color{{img_color[0], img_color[1], img_color[2]}}; - cuda_postprocess(output_shape.dimension(0), output_shape.dimension(1), color, first, - binary_masks_tensor->data().value() + index * layer_size, - reinterpret_cast(out_data)); - first = false; - } - } - - const auto result = out_->publish(std::move(out_message.value())); - if (!result) { - GXF_LOG_ERROR("Failed to publish output!"); - return GXF_FAILURE; - } - - return GXF_SUCCESS; -} - -gxf_result_t Postprocessor::stop() { - return GXF_SUCCESS; -} - -gxf_result_t Postprocessor::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - result &= registrar->parameter(in_, "in", "Input", "Input channel."); - result &= registrar->parameter(out_, "out", "Output", "Output channel."); - - result &= registrar->parameter(min_prob_, "min_prob", "Minimum probability", - "Minimum probability.", - DEFAULT_MIN_PROB); - - result &= registrar->parameter( - overlay_img_colors_, "overlay_img_colors", "Overlay Image Layer Colors", - "Color of the image overlays, a list of RGB values with components between 0 and 1", - DEFAULT_COLORS); - - result &= - registrar->parameter(host_allocator_, "host_allocator", "Allocator", "Output Allocator"); - result &= - registrar->parameter(device_allocator_, "device_allocator", "Allocator", "Output Allocator"); - return gxf::ToResultCode(result); -} - -} // namespace tool_tracking_postprocessor -} // namespace holoscan -} // namespace nvidia diff --git a/gxf_extensions/tool_tracking_postprocessor/tool_tracking_postprocessor.cu.cpp b/gxf_extensions/tool_tracking_postprocessor/tool_tracking_postprocessor.cu.cpp deleted file mode 100644 index 29d8cc2a..00000000 --- a/gxf_extensions/tool_tracking_postprocessor/tool_tracking_postprocessor.cu.cpp +++ /dev/null @@ -1,62 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "tool_tracking_postprocessor.cu.hpp" - -namespace nvidia { -namespace holoscan { -namespace tool_tracking_postprocessor { - -__global__ void postprocessing_kernel(uint32_t width, uint32_t height, const float3 color, - bool first, const float* input, float4* output) { - const uint32_t x = blockIdx.x * blockDim.x + threadIdx.x; - const uint32_t y = blockIdx.y * blockDim.y + threadIdx.y; - - if ((x >= width) || (y >= height)) { return; } - - float value = input[y * width + x]; - - const float minV = 0.3f; - const float maxV = 0.99f; - const float range = maxV - minV; - value = min(max(value, minV), maxV); - value -= minV; - value /= range; - value *= 0.7f; - - const float4 dst = first ? make_float4(0.f, 0.f, 0.f, 0.f) : output[y * width + x]; - output[y * width + x] = make_float4( - (1.0f - value) * dst.x + color.x * value, (1.0f - value) * dst.y + color.y * value, - (1.0f - value) * dst.z + color.z * value, (1.0f - value) * dst.w + 1.f * value); -} - -uint16_t ceil_div(uint16_t numerator, uint16_t denominator) { - uint32_t accumulator = numerator + denominator - 1; - return accumulator / denominator; -} - -void cuda_postprocess(uint32_t width, uint32_t height, - const std::array& color, bool first, const float* input, - float4* output) { - const dim3 block(32, 32, 1); - const dim3 grid(ceil_div(width, block.x), ceil_div(height, block.y), 1); - postprocessing_kernel<<>>(width, height, make_float3(color[0], color[1], color[2]), - first, input, output); -} - -} // namespace tool_tracking_postprocessor -} // namespace holoscan -} // namespace nvidia diff --git a/gxf_extensions/tool_tracking_postprocessor/tool_tracking_postprocessor.hpp b/gxf_extensions/tool_tracking_postprocessor/tool_tracking_postprocessor.hpp deleted file mode 100644 index 1b3ceda6..00000000 --- a/gxf_extensions/tool_tracking_postprocessor/tool_tracking_postprocessor.hpp +++ /dev/null @@ -1,58 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_TOOL_TRACKING_POSTPROCESSOR_HPP_ -#define NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_TOOL_TRACKING_POSTPROCESSOR_HPP_ - -#include -#include - -#include "gxf/std/allocator.hpp" -#include "gxf/std/codelet.hpp" -#include "gxf/std/receiver.hpp" -#include "gxf/std/transmitter.hpp" - -#include "tool_tracking_postprocessor.cu.hpp" - -namespace nvidia { -namespace holoscan { -namespace tool_tracking_postprocessor { - -/// @brief tool tracking model postproceessing Codelet converting inference output. -/// -/// This Codelet performs tool tracking model postprocessing in CUDA. -class Postprocessor : public gxf::Codelet { - public: - gxf_result_t start() override; - gxf_result_t tick() override; - gxf_result_t stop() override; - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - - private: - gxf::Parameter> in_; - gxf::Parameter> out_; - - gxf::Parameter min_prob_; - gxf::Parameter>> overlay_img_colors_; - - gxf::Parameter> host_allocator_; - gxf::Parameter> device_allocator_; -}; -} // namespace tool_tracking_postprocessor -} // namespace holoscan -} // namespace nvidia - -#endif // NVIDIA_CLARA_HOLOSCAN_GXF_EXTENSIONS_TOOL_TRACKING_POSTPROCESSOR_HPP_ diff --git a/gxf_extensions/tool_tracking_postprocessor/tool_tracking_postprocessor_ext.cpp b/gxf_extensions/tool_tracking_postprocessor/tool_tracking_postprocessor_ext.cpp deleted file mode 100644 index a6070256..00000000 --- a/gxf_extensions/tool_tracking_postprocessor/tool_tracking_postprocessor_ext.cpp +++ /dev/null @@ -1,29 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "gxf/std/extension_factory_helper.hpp" - -#include "tool_tracking_postprocessor.hpp" - -GXF_EXT_FACTORY_BEGIN() -GXF_EXT_FACTORY_SET_INFO(0xfbc6d815a2b54e7c, 0x8ced71e2f635a506, - "ToolTrackingPostprocessorExtension", - "Holoscan Tool Tracking Model Postprocessing extension", "NVIDIA", "0.4.0", - "LICENSE"); -GXF_EXT_FACTORY_ADD(0xb07a675dec0c43a4, 0x8ce6e472cd90a9a1, - nvidia::holoscan::tool_tracking_postprocessor::Postprocessor, - nvidia::gxf::Codelet, "Tool Tracking Model Postprocessor codelet."); -GXF_EXT_FACTORY_END() diff --git a/gxf_extensions/utils/cuda_stream_handler.hpp b/gxf_extensions/utils/cuda_stream_handler.hpp new file mode 100644 index 00000000..1473fbca --- /dev/null +++ b/gxf_extensions/utils/cuda_stream_handler.hpp @@ -0,0 +1,267 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef GXF_EXTENSIONS_UTILS_CUDA_STREAM_HANDLER_HPP +#define GXF_EXTENSIONS_UTILS_CUDA_STREAM_HANDLER_HPP + +#include +#include + +#include "gxf/cuda/cuda_stream.hpp" +#include "gxf/cuda/cuda_stream_id.hpp" +#include "gxf/cuda/cuda_stream_pool.hpp" + +namespace nvidia { +namespace holoscan { + +/** + * This class handles usage of CUDA streams for operators. + * + * When using CUDA operations the default stream '0' synchronizes with all other streams in the same + * context, see + * https://docs.nvidia.com/cuda/cuda-runtime-api/stream-sync-behavior.html#stream-sync-behavior. + * This can reduce performance. The CudaStreamHandler class manages streams across operators and + * makes sure that CUDA operations are properly chained. + * + * Usage: + * - add an instance of CudaStreamHandler to your operator + * - call CudaStreamHandler::registerInterface() from the operator registerInterface() function + * - in the tick() function call CudaStreamHandler::fromMessage(), this will get the CUDA stream + * from the message of the previous operator. When the operator receives multiple messages, then + * call CudaStreamHandler::fromMessages(). This will synchronize with multiple streams. + * - when executing CUDA functions CudaStreamHandler::get() to get the CUDA stream which should + * be used by your CUDA function + * - before publishing the output message(s) of your operator call CudaStreamHandler::toMessage() on + * each message. This will add the CUDA stream used by the CUDA functions in your operator to + * the output message. + */ +class CudaStreamHandler { + public: + /** + * @brief Destroy the CudaStreamHandler object + */ + ~CudaStreamHandler() { + for (auto&& event : cuda_events_) { + const cudaError_t result = cudaEventDestroy(event); + if (cudaSuccess != result) { + GXF_LOG_ERROR("Failed to destroy CUDA event: %s", cudaGetErrorString(result)); + } + } + cuda_events_.clear(); + } + + /** + * @brief Register the parameters used by this class. + * + * @param registrar + * @param required if set then it's required that the CUDA stream pool is specified + * @return gxf::Expected + */ + gxf::Expected registerInterface(gxf::Registrar* registrar, bool required = false) { + return registrar->parameter(cuda_stream_pool_, + "cuda_stream_pool", + "CUDA Stream Pool", + "Instance of gxf::CudaStreamPool.", + gxf::Registrar::NoDefaultParameter(), + required ? GXF_PARAMETER_FLAGS_NONE : GXF_PARAMETER_FLAGS_OPTIONAL); + } + + /** + * Get the CUDA stream for the operation from the incoming message + * + * @param context + * @param message + * @return gxf_result_t + */ + gxf_result_t fromMessage(gxf_context_t context, + const nvidia::gxf::Expected& message) { + // if the message contains a stream use this + const auto maybe_cuda_stream_id = message.value().get(); + if (maybe_cuda_stream_id) { + const auto maybe_cuda_stream_handle = + gxf::Handle::Create(context, maybe_cuda_stream_id.value()->stream_cid); + if (maybe_cuda_stream_handle) { + message_cuda_stream_handle_ = maybe_cuda_stream_handle.value(); + } + } else { + // if no stream had been found, allocate a stream and use that + gxf_result_t result = allocateInternalStream(); + if (result != GXF_SUCCESS) { return result; } + message_cuda_stream_handle_ = cuda_stream_handle_; + } + return GXF_SUCCESS; + } + + /** + * Get the CUDA stream for the operation from the incoming messages + * + * @param context + * @param messages + * @return gxf_result_t + */ + gxf_result_t fromMessages(gxf_context_t context, + const std::vector& messages) { + const gxf_result_t result = allocateInternalStream(); + if (result != GXF_SUCCESS) { return result; } + + if (!cuda_stream_handle_) { + // if no CUDA stream can be allocated because no stream pool is set, then don't sync + // with incoming streams. CUDA operations of this operator will use the default stream + // which sync with all other streams by default. + return GXF_SUCCESS; + } + + // iterate through all messages and use events to chain incoming streams with the internal + // stream + auto event_it = cuda_events_.begin(); + for (auto& msg : messages) { + const auto maybe_cuda_stream_id = msg.get(); + if (maybe_cuda_stream_id) { + const auto maybe_cuda_stream_handle = + gxf::Handle::Create(context, maybe_cuda_stream_id.value()->stream_cid); + if (maybe_cuda_stream_handle) { + const cudaStream_t cuda_stream = maybe_cuda_stream_handle.value()->stream().value(); + cudaError_t result; + + // allocate a new event if needed + if (event_it == cuda_events_.end()) { + cudaEvent_t cuda_event; + result = cudaEventCreateWithFlags(&cuda_event, cudaEventDisableTiming); + if (cudaSuccess != result) { + GXF_LOG_ERROR("Failed to create input CUDA event: %s", cudaGetErrorString(result)); + return GXF_FAILURE; + } + cuda_events_.push_back(cuda_event); + event_it = cuda_events_.end(); + --event_it; + } + + result = cudaEventRecord(*event_it, cuda_stream); + if (cudaSuccess != result) { + GXF_LOG_ERROR("Failed to record event for message stream: %s", + cudaGetErrorString(result)); + return GXF_FAILURE; + } + result = cudaStreamWaitEvent(cuda_stream_handle_->stream().value(), *event_it); + if (cudaSuccess != result) { + GXF_LOG_ERROR("Failed to record wait on message event: %s", cudaGetErrorString(result)); + return GXF_FAILURE; + } + ++event_it; + } + } + } + message_cuda_stream_handle_ = cuda_stream_handle_; + return GXF_SUCCESS; + } + + /** + * @brief Add the used CUDA stream to the outgoing message + * + * @param message + * @return gxf_result_t + */ + gxf_result_t toMessage(nvidia::gxf::Expected& message) { + if (message_cuda_stream_handle_) { + const auto maybe_stream_id = message.value().add(); + if (!maybe_stream_id) { + GXF_LOG_ERROR("Failed to add CUDA stream id to output message."); + return gxf::ToResultCode(maybe_stream_id); + } + maybe_stream_id.value()->stream_cid = message_cuda_stream_handle_.cid(); + } + return GXF_SUCCESS; + } + + /** + * Get the CUDA stream handle which should be used for CUDA commands + * + * @return gxf::Handle + */ + gxf::Handle getStreamHandle() { + // If there is a message stream handle, return this + if (message_cuda_stream_handle_) { return message_cuda_stream_handle_; } + + // else allocate an internal CUDA stream and return it + allocateInternalStream(); + return cuda_stream_handle_; + } + + /** + * Get the CUDA stream which should be used for CUDA commands. + * + * If no message stream is set and no stream can be allocated, return the default stream. + * + * @return cudaStream_t + */ + cudaStream_t getCudaStream() { + const gxf::Handle cuda_stream_handle = getStreamHandle(); + if (cuda_stream_handle) { return cuda_stream_handle->stream().value(); } + if (!default_stream_warning_) { + default_stream_warning_ = true; + GXF_LOG_WARNING( + "Parameter `cuda_stream_pool` is not set, using the default CUDA stream for CUDA " + "operations."); + } + return cudaStreamDefault; + } + + private: + /** + * Allocate the internal CUDA stream + * + * @return gxf_result_t + */ + gxf_result_t allocateInternalStream() { + // Create the CUDA stream if it does not yet exist. + if (!cuda_stream_handle_) { + const auto cuda_stream_pool = cuda_stream_pool_.try_get(); + if (cuda_stream_pool) { + // allocate a stream + auto maybe_stream = cuda_stream_pool.value()->allocateStream(); + if (!maybe_stream) { + GXF_LOG_ERROR("Failed to allocate CUDA stream"); + return gxf::ToResultCode(maybe_stream); + } + cuda_stream_handle_ = std::move(maybe_stream.value()); + } + } + return GXF_SUCCESS; + } + + /// CUDA stream pool used to allocate the internal CUDA stream + gxf::Parameter> cuda_stream_pool_; + + /// If the CUDA stream pool is not set and we can't use the incoming CUDA stream, issue + /// a warning once. + bool default_stream_warning_ = false; + + /// Array of CUDA events used to synchronize the internal CUDA stream with multiple incoming + /// streams + std::vector cuda_events_; + + /// The CUDA stream which is attached to the incoming message + gxf::Handle message_cuda_stream_handle_; + + /// Allocated internal CUDA stream handle + gxf::Handle cuda_stream_handle_; +}; + +} // namespace holoscan +} // namespace nvidia + +#endif /* GXF_EXTENSIONS_UTILS_CUDA_STREAM_HANDLER_HPP */ diff --git a/gxf_extensions/visualizer_icardio/CMakeLists.txt b/gxf_extensions/visualizer_icardio/CMakeLists.txt deleted file mode 100644 index f998121d..00000000 --- a/gxf_extensions/visualizer_icardio/CMakeLists.txt +++ /dev/null @@ -1,42 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -add_library(visualizer_icardio_lib SHARED - visualizer_icardio.cpp - visualizer_icardio.hpp -) - -target_link_libraries(visualizer_icardio_lib - PUBLIC - CUDA::cudart - GXF::cuda - GXF::std - GXF::multimedia - yaml-cpp - holoinfer -) - -target_include_directories(visualizer_icardio_lib - PUBLIC - ${CMAKE_SOURCE_DIR}/modules/holoinfer/src/include) - -add_library(visualizer_icardio SHARED - visualizer_icardio_extension.cpp -) -target_link_libraries(visualizer_icardio - PUBLIC visualizer_icardio_lib -) - -install_gxf_extension(visualizer_icardio) diff --git a/gxf_extensions/visualizer_icardio/visualizer_icardio.cpp b/gxf_extensions/visualizer_icardio/visualizer_icardio.cpp deleted file mode 100644 index 34f52988..00000000 --- a/gxf_extensions/visualizer_icardio/visualizer_icardio.cpp +++ /dev/null @@ -1,169 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "visualizer_icardio.hpp" - -namespace nvidia { -namespace holoscan { -namespace multiai { - -gxf_result_t VisualizerICardio::start() { - if (input_on_cuda_.get()) { - return HoloInfer::report_error(module_, "CUDA based data not supported in iCardio Visualizer"); - } - - std::vector logo_dim = tensor_to_shape_.at("logo"); - size_t logo_size = - std::accumulate(logo_dim.begin(), logo_dim.end(), 1, std::multiplies()); - logo_image_.assign(logo_size, 0); - - std::ifstream file_logo(path_to_logo_file_); - - if (!file_logo) { - GXF_LOG_WARNING("Logo file not found, Ignored."); - } else { - std::istream_iterator start(file_logo), end; - std::vector data_logo(start, end); - - logo_image_ = std::move(data_logo); - } - return GXF_SUCCESS; -} - -gxf_result_t VisualizerICardio::stop() { - return GXF_SUCCESS; -} - -gxf_result_t VisualizerICardio::tick() { - try { - gxf_result_t stat = HoloInfer::multiai_get_data_per_model(receivers_.get(), - in_tensor_names_.get(), - data_per_tensor, - tensor_size_map_, - input_on_cuda_.get(), - module_); - if (stat != GXF_SUCCESS) { return HoloInfer::report_error(module_, "Tick, Data extraction"); } - - if (tensor_size_map_.find(pc_tensor_name_) == tensor_size_map_.end()) { - HoloInfer::report_error(module_, "Dimension not found for tensor " + pc_tensor_name_); - } - if (data_per_tensor.find(pc_tensor_name_) == data_per_tensor.end()) { - HoloInfer::report_error(module_, "Data not found for tensor " + pc_tensor_name_); - } - auto coords = data_per_tensor.at(pc_tensor_name_)->host_buffer.data(); - auto datasize = tensor_size_map_[pc_tensor_name_]; - - if (transmitters_.get().size() > 0) { - for (unsigned int a = 0; a < transmitters_.get().size(); ++a) { - auto out_message = gxf::Entity::New(context()); - if (!out_message) { - return HoloInfer::report_error(module_, "Tick, Out message allocation"); - } - std::string current_tensor_name{out_tensor_names_.get()[a]}; - auto out_tensor = out_message.value().add(current_tensor_name.c_str()); - if (!out_tensor) { return HoloInfer::report_error(module_, "Tick, Out tensor allocation"); } - if (tensor_to_shape_.find(current_tensor_name) == tensor_to_shape_.end()) { - return HoloInfer::report_error( - module_, "Tick, Output Tensor shape mapping not found for " + current_tensor_name); - } - std::vector shape_dim = tensor_to_shape_.at(current_tensor_name); - gxf::Shape output_shape{shape_dim[0], shape_dim[1], shape_dim[2]}; - - if (current_tensor_name.compare("logo") == 0) { - out_tensor.value()->reshape( - output_shape, gxf::MemoryStorageType::kHost, allocator_); - if (!out_tensor.value()->pointer()) { - return HoloInfer::report_error(module_, "Tick, Out tensor buffer allocation"); - } - gxf::Expected out_tensor_data = out_tensor.value()->data(); - if (!out_tensor_data) { - return HoloInfer::report_error(module_, "Tick, Getting out tensor data"); - } - uint8_t* out_tensor_buffer = out_tensor_data.value(); - - for (unsigned int h = 0; h < shape_dim[0] * shape_dim[1] * shape_dim[2]; ++h) { - out_tensor_buffer[h] = uint8_t(logo_image_.data()[h]); - } - } else { - out_tensor.value()->reshape( - output_shape, gxf::MemoryStorageType::kHost, allocator_); - if (!out_tensor.value()->pointer()) { - return HoloInfer::report_error(module_, "Tick, Out tensor buffer allocation"); - } - gxf::Expected out_tensor_data = out_tensor.value()->data(); - if (!out_tensor_data) { - return HoloInfer::report_error(module_, "Tick, Getting out tensor data"); - } - float* out_tensor_buffer = out_tensor_data.value(); - - int property_size = shape_dim[2]; - - if (property_size <= 3) { - for (unsigned int i = 1; i < datasize[datasize.size() - 1] / 2; ++i) { - unsigned int index = (i - 1) * property_size; - out_tensor_buffer[index] = coords[2 * i + 1]; - out_tensor_buffer[index + 1] = coords[2 * i]; - - if (property_size == 3) { // keypoint - out_tensor_buffer[index + 2] = 0.01; - } - } - } else { - if (tensor_to_index_.find(current_tensor_name) == tensor_to_index_.end()) { - return HoloInfer::report_error(module_, "Tick, tensor to index mapping failed"); - } - int index_coord = tensor_to_index_.at(current_tensor_name); - if (index_coord >= 1 && index_coord <= 5) { - out_tensor_buffer[0] = coords[2 * index_coord + 1]; - out_tensor_buffer[1] = coords[2 * index_coord]; - out_tensor_buffer[2] = 0.04; - out_tensor_buffer[3] = 0.02; - } else { - return HoloInfer::report_error(module_, "Tick, invalid coordinate from tensor"); - } - } - } - const auto result = transmitters_.get()[a]->publish(std::move(out_message.value())); - if (!result) { return HoloInfer::report_error(module_, "Tick, Publishing output"); } - } - } - } catch (const std::runtime_error& r_) { - return HoloInfer::report_error(module_, "Tick, Message->" + std::string(r_.what())); - } catch (...) { return HoloInfer::report_error(module_, "Tick, unknown exception"); } - return GXF_SUCCESS; -} - -gxf_result_t VisualizerICardio::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - - result &= registrar->parameter( - in_tensor_names_, "in_tensor_names", "Input Tensors", "Input tensors", {std::string("")}); - result &= registrar->parameter( - out_tensor_names_, "out_tensor_names", "Output Tensors", "Output tensors", {std::string("")}); - result &= registrar->parameter(allocator_, "allocator", "Allocator", "Output Allocator"); - result &= registrar->parameter( - input_on_cuda_, "input_on_cuda", "Input for processing on cuda", "", false); - result &= registrar->parameter( - receivers_, "receivers", "Receivers", "List of receivers to take input tensors"); - result &= - registrar->parameter(transmitters_, "transmitters", "Transmitters", "List of transmitters"); - - return gxf::ToResultCode(result); -} - -} // namespace multiai -} // namespace holoscan -} // namespace nvidia diff --git a/gxf_extensions/visualizer_icardio/visualizer_icardio.hpp b/gxf_extensions/visualizer_icardio/visualizer_icardio.hpp deleted file mode 100644 index 95a47f5e..00000000 --- a/gxf_extensions/visualizer_icardio/visualizer_icardio.hpp +++ /dev/null @@ -1,75 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef NVIDIA_GXF_EXTENSIONS_VISUALIZER_ICARDIO_HPP_ -#define NVIDIA_GXF_EXTENSIONS_VISUALIZER_ICARDIO_HPP_ - -#include -#include -#include -#include - -#include - -namespace HoloInfer = holoscan::inference; - -namespace nvidia { -namespace holoscan { -namespace multiai { - -class VisualizerICardio : public gxf::Codelet { - public: - gxf_result_t start() override; - gxf_result_t tick() override; - gxf_result_t stop() override; - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - - private: - gxf::Parameter> in_tensor_names_; - gxf::Parameter> out_tensor_names_; - gxf::Parameter> allocator_; - - gxf::Parameter receivers_; - gxf::Parameter transmitters_; - - HoloInfer::DataMap data_per_tensor; - std::map> tensor_size_map_; - - const std::string module_{"Visualizer icardio Codelet"}; - const std::string pc_tensor_name_{"plax_chamber_processed"}; - - gxf::Parameter input_on_cuda_; - - const std::map> tensor_to_shape_ = {{"keypoints", {1, 5, 3}}, - {"keyarea_1", {1, 1, 4}}, - {"keyarea_2", {1, 1, 4}}, - {"keyarea_3", {1, 1, 4}}, - {"keyarea_4", {1, 1, 4}}, - {"keyarea_5", {1, 1, 4}}, - {"lines", {1, 5, 2}}, - {"logo", {320, 320, 4}}}; - const std::map tensor_to_index_ = { - {"keyarea_1", 1}, {"keyarea_2", 2}, {"keyarea_3", 3}, {"keyarea_4", 4}, {"keyarea_5", 5}}; - - const std::string path_to_logo_file_ = "../data/multiai_ultrasound/logo.txt"; - std::vector logo_image_; -}; - -} // namespace multiai -} // namespace holoscan -} // namespace nvidia - -#endif // NVIDIA_GXF_EXTENSIONS_VISUALIZER_ICARDIO_HPP_ diff --git a/gxf_extensions/visualizer_icardio/visualizer_icardio_extension.cpp b/gxf_extensions/visualizer_icardio/visualizer_icardio_extension.cpp deleted file mode 100644 index 6bab2463..00000000 --- a/gxf_extensions/visualizer_icardio/visualizer_icardio_extension.cpp +++ /dev/null @@ -1,34 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "visualizer_icardio.hpp" -#include "gxf/core/gxf.h" -#include "gxf/std/extension_factory_helper.hpp" - -extern "C" { - -GXF_EXT_FACTORY_BEGIN() - -GXF_EXT_FACTORY_SET_INFO(0xc09bcf6af8b2458f, 0xb9f3b0bd935a4124, "VisualizerICardio", - "Visualizer iCardio", "NVIDIA", "1.0.0", "LICENSE"); - -GXF_EXT_FACTORY_ADD(0x70cedec6a66e4812, 0x8114c6f9bce0048c, - nvidia::holoscan::multiai::VisualizerICardio, nvidia::gxf::Codelet, - "Visualizer iCardio Codelet."); - -GXF_EXT_FACTORY_END() -} diff --git a/gxf_extensions/visualizer_tool_tracking/CMakeLists.txt b/gxf_extensions/visualizer_tool_tracking/CMakeLists.txt deleted file mode 100644 index df2d536b..00000000 --- a/gxf_extensions/visualizer_tool_tracking/CMakeLists.txt +++ /dev/null @@ -1,62 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Create library -add_library(visualizer_tool_tracking_lib SHARED - frame_data.hpp - instrument_label.cpp - instrument_label.hpp - instrument_tip.cpp - instrument_tip.hpp - opengl_utils.cpp - opengl_utils.hpp - overlay_img_vis.cpp - overlay_img_vis.hpp - video_frame.cpp - video_frame.hpp - visualizer.cpp - visualizer.hpp -) -target_link_libraries(visualizer_tool_tracking_lib - PUBLIC - CUDA::cudart - glad::glad - glfw - GXF::std - GXF::multimedia - # OpenGL::GL # Using glad + patch instead for GL/gl.h - nanovg - yaml-cpp -) - -# Create extension -add_library(visualizer_tool_tracking SHARED - visualizer_ext.cpp -) -target_link_libraries(visualizer_tool_tracking - PUBLIC visualizer_tool_tracking_lib -) - -# Copy resources -file(COPY fonts DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) -file(COPY glsl DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) - -# Install GXF extension as a component 'holoscan-gxf_extensions' -install_gxf_extension(visualizer_tool_tracking) -file(RELATIVE_PATH _relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) -install(DIRECTORY fonts glsl - DESTINATION ${_relative_dest_path} - COMPONENT "holoscan-gxf_extensions" -) diff --git a/gxf_extensions/visualizer_tool_tracking/fonts/Roboto-Regular.ttf b/gxf_extensions/visualizer_tool_tracking/fonts/Roboto-Regular.ttf deleted file mode 100755 index 3e6e2e76..00000000 Binary files a/gxf_extensions/visualizer_tool_tracking/fonts/Roboto-Regular.ttf and /dev/null differ diff --git a/gxf_extensions/visualizer_tool_tracking/glsl/instrument_mask.frag b/gxf_extensions/visualizer_tool_tracking/glsl/instrument_mask.frag deleted file mode 100644 index 20b4c4be..00000000 --- a/gxf_extensions/visualizer_tool_tracking/glsl/instrument_mask.frag +++ /dev/null @@ -1,63 +0,0 @@ -#version 450 - -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Arrays in a UBO must use a constant expression for their size. -// Note: MAX_LAYERS should match the value used by the host code using this shader. -const int MAX_LAYERS = 64; - -in vec2 tex_coords; - -layout(binding = 0) uniform sampler2DArray mask; - -layout(location = 0) uniform uint num_layers; -layout(location = 1) uniform vec3 layer_colors[MAX_LAYERS]; - -layout(std430, binding = 0) buffer ConfidenceBlock { - float confidence[MAX_LAYERS]; -}; - -layout(location = 0) out vec4 out_color; - -float threshold(float v, float minV, float maxV) { - const float range = maxV - minV; - v = clamp(v, minV, maxV); - v -= minV; - v /= range; - return v; -} - -void main() { - vec4 dst = vec4(0.0); - - const float minV = 0.3; - const float maxV = 0.99; - - ivec3 tex_size = textureSize(mask, 0); - vec2 tex_coord_offset = vec2(1.0 / tex_size.xy); - - for (int i = 0; i != num_layers; ++i) { - if (confidence[i] > 0.5) { - float s = threshold(texture(mask, vec3(tex_coords - tex_coord_offset, i)).r, minV, maxV); - vec4 src = vec4(layer_colors[i], s * 0.7); - dst = (1.0 - src.a) * dst + src.a * src; - } - } - - out_color = dst; -}; diff --git a/gxf_extensions/visualizer_tool_tracking/glsl/instrument_tip.frag b/gxf_extensions/visualizer_tool_tracking/glsl/instrument_tip.frag deleted file mode 100644 index 1087d722..00000000 --- a/gxf_extensions/visualizer_tool_tracking/glsl/instrument_tip.frag +++ /dev/null @@ -1,39 +0,0 @@ -#version 450 - -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Arrays in a UBO must use a constant expression for their size. -// Note: MAX_TOOLS should match the value used by the host code using this shader. -const int MAX_TOOLS = 64; - -layout(location = 0) uniform vec3 tool_colors[MAX_TOOLS]; - -flat in float size; -flat in float linewidth; -flat in int instrument_idx; - -layout(location = 0) out vec4 out_color; - -void main() { - vec2 P = gl_PointCoord.xy - vec2(0.5, 0.5); - P *= size; - - if (abs(P.x) > 0.5 * linewidth && abs(P.y) > 0.5 * linewidth) { discard; } - - out_color = vec4(tool_colors[instrument_idx], 1.0); -} diff --git a/gxf_extensions/visualizer_tool_tracking/glsl/instrument_tip.vert b/gxf_extensions/visualizer_tool_tracking/glsl/instrument_tip.vert deleted file mode 100644 index a6afba74..00000000 --- a/gxf_extensions/visualizer_tool_tracking/glsl/instrument_tip.vert +++ /dev/null @@ -1,46 +0,0 @@ -#version 450 - -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -layout(location = 0) in vec2 position; -layout(location = 1) in float confidence; - -flat out float size; -flat out float linewidth; -flat out int instrument_idx; - -void main(void) { - // consts - const float M_SQRT_2 = 1.4142135623730951; - size = 30.0; - linewidth = 4.0; - - vec2 pos = position.xy; - // change direction of y coordinate - pos.y = 1.0f - pos.y; - // transform to NDC space with (0,0) in center and coordinate range [-1, 1 ] for x and y - pos.x = 2.0f * pos.x - 1.0f; - pos.y = 2.0f * pos.y - 1.0f; - - gl_Position = vec4(pos, 0.0, 1.0); - instrument_idx = gl_VertexID; - - float p_size = (confidence > 0.5) ? M_SQRT_2 * size : 0.0; - - gl_PointSize = p_size; -} diff --git a/gxf_extensions/visualizer_tool_tracking/glsl/viewport_filling_triangle.vert b/gxf_extensions/visualizer_tool_tracking/glsl/viewport_filling_triangle.vert deleted file mode 100644 index f78fd2c1..00000000 --- a/gxf_extensions/visualizer_tool_tracking/glsl/viewport_filling_triangle.vert +++ /dev/null @@ -1,50 +0,0 @@ -#version 450 - -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -out vec2 tex_coords; - -void main() -{ - /* - This vertex shader is used for rendering a single fullscreen - covering triangle. The vertex position in clip space and - 2D texture coordinates are computed from the vertex id. - - Vertex ID | Vertex Pos | Texture Coords - ---------------------------------------------- - 0 | [-1, -1] | [ 0, 0 ] - 1 | [ 3, -1] | [ 2, 0 ] - 2 | [-1, 3] | [ 0, -2 ] - - The triangle fully covers the clip space region and creates - fragments with vertex positions in the range [-1, 1] and - texture coordinates in the range X: [0, 1], Y: [0, -1]. - Please note that for display of surgical video the texture - coordinates y - component is flipped. - - The main benefit of using this shader is that no vertex and - vertex attribute buffers are required. Only a single call to - glDrawArrays to render one triangle is needed. - - */ - float x = -1.0 + float((gl_VertexID & 1) << 2); - float y = -1.0 + float((gl_VertexID & 2) << 1); - tex_coords = vec2(0.5, -0.5) * (vec2(x, y) + 1.0); - gl_Position = vec4(x, y, 0.0, 1.0); -} diff --git a/gxf_extensions/visualizer_tool_tracking/instrument_label.cpp b/gxf_extensions/visualizer_tool_tracking/instrument_label.cpp deleted file mode 100644 index 220a12d3..00000000 --- a/gxf_extensions/visualizer_tool_tracking/instrument_label.cpp +++ /dev/null @@ -1,91 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "instrument_label.hpp" - -#include -#define NANOVG_GL3_IMPLEMENTATION -#include -#include - -#include - -#include "opengl_utils.hpp" - -namespace nvidia { -namespace holoscan { -namespace visualizer_tool_tracking { - -gxf_result_t InstrumentLabel::start() { - nvg_ctx_ = nvgCreateGL3(NVG_ANTIALIAS | NVG_STENCIL_STROKES | NVG_DEBUG); - if (nvg_ctx_ == nullptr) { - GXF_LOG_ERROR("Could not init NANOVG.\n"); - return GXF_FAILURE; - } - - // ---------------------------------------------------------------------- - - int font; - font = nvgCreateFont(nvg_ctx_, "sans", label_sans_font_path_.c_str()); - if (font == -1) { - GXF_LOG_ERROR("Could not add font regular: %s\n", label_sans_font_path_.c_str()); - return GXF_FAILURE; - } - font = nvgCreateFont(nvg_ctx_, "sans-bold", label_sans_bold_font_path_.c_str()); - if (font == -1) { - GXF_LOG_ERROR("Could not add font bold: %s \n", label_sans_bold_font_path_.c_str()); - return GXF_FAILURE; - } - - // Add numbers as default values if labels are missing - tool_labels_or_numbers_ = tool_labels_; - for (uint32_t i = tool_labels_or_numbers_.size(); i < num_tool_classes_; ++i) { - tool_labels_or_numbers_.push_back(std::to_string(i)); - } - - return GXF_SUCCESS; -} - -gxf_result_t InstrumentLabel::tick(float width, float height) { - nvgBeginFrame(nvg_ctx_, width, height, width / height); - - nvgFontSize(nvg_ctx_, 25.0f); - nvgFontFace(nvg_ctx_, "sans-bold"); - nvgTextAlign(nvg_ctx_, NVG_ALIGN_LEFT | NVG_ALIGN_MIDDLE); - - for (uint32_t i = 0; i != num_tool_classes_; ++i) { - // skip non-present tools - if (frame_data_.confidence_host_[i] < 0.5) { continue; } - - const char* label_text = tool_labels_or_numbers_[i].c_str(); - float x = frame_data_.position_host_[i * num_tool_pos_components_] * width + 40; - float y = frame_data_.position_host_[i * num_tool_pos_components_ + 1] * height + 40; - nvgFillColor(nvg_ctx_, nvgRGBAf(1.0f, 1.0f, 1.0f, 0.9f)); - nvgText(nvg_ctx_, x, y, label_text, NULL); - } - - nvgEndFrame(nvg_ctx_); - - return GXF_SUCCESS; -} - -gxf_result_t InstrumentLabel::stop() { - return GXF_SUCCESS; -} - -} // namespace visualizer_tool_tracking -} // namespace holoscan -} // namespace nvidia diff --git a/gxf_extensions/visualizer_tool_tracking/instrument_label.hpp b/gxf_extensions/visualizer_tool_tracking/instrument_label.hpp deleted file mode 100644 index fbbab7e0..00000000 --- a/gxf_extensions/visualizer_tool_tracking/instrument_label.hpp +++ /dev/null @@ -1,62 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef NVIDIA_CLARA_HOLOSCAN_GXF_INSTRUMENT_LABEL_HPP_ -#define NVIDIA_CLARA_HOLOSCAN_GXF_INSTRUMENT_LABEL_HPP_ - -#include - -#include -#include - -#include "gxf/std/codelet.hpp" - -#include "frame_data.hpp" - -struct NVGcontext; - -namespace nvidia { -namespace holoscan { -namespace visualizer_tool_tracking { - -struct InstrumentLabel { - // owned externally - const FrameData& frame_data_; - // owned internally - NVGcontext* nvg_ctx_ = nullptr; - std::string label_sans_font_path_ = "UNDEFINED"; - std::string label_sans_bold_font_path_ = "UNDEFINED"; - uint32_t num_tool_classes_ = 0; - uint32_t num_tool_pos_components_ = 2; - std::vector tool_labels_ = {}; - - explicit InstrumentLabel(const FrameData& frame_data) : frame_data_(frame_data) {} - - // private - std::vector tool_labels_or_numbers_ = {}; - - InstrumentLabel(const InstrumentLabel&) = delete; - InstrumentLabel& operator=(const InstrumentLabel&) = delete; - - gxf_result_t start(); - gxf_result_t tick(float width, float height); - gxf_result_t stop(); -}; - -} // namespace visualizer_tool_tracking -} // namespace holoscan -} // namespace nvidia -#endif // NVIDIA_CLARA_HOLOSCAN_GXF_INSTRUMENT_LABEL_HPP_ diff --git a/gxf_extensions/visualizer_tool_tracking/instrument_tip.cpp b/gxf_extensions/visualizer_tool_tracking/instrument_tip.cpp deleted file mode 100644 index 620d4c7d..00000000 --- a/gxf_extensions/visualizer_tool_tracking/instrument_tip.cpp +++ /dev/null @@ -1,98 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "instrument_tip.hpp" - -#include - -#include "opengl_utils.hpp" - -namespace nvidia { -namespace holoscan { -namespace visualizer_tool_tracking { - -gxf_result_t InstrumentTip::start() { - if (num_tool_classes_ > MAX_TOOLS) { - GXF_LOG_ERROR("Number of layers (%d) exceeds maximum number of layers (%d)", num_tool_classes_, - MAX_TOOLS); - return GXF_FAILURE; - } - - if (num_tool_classes_ > tool_tip_colors_.size()) { - GXF_LOG_ERROR("Number of tools (%d) exceeds number of colors provided (%d)", num_tool_classes_, - tool_tip_colors_.size()); - return GXF_FAILURE; - } - - for (auto color : tool_tip_colors_) { - if (color.size() != 3) { - GXF_LOG_ERROR("Tool colors must be 3 elements (RGB)"); - return GXF_FAILURE; - } - } - - // generate and setup vertex array object - glGenVertexArrays(1, &vao_); - glBindVertexArray(vao_); - glBindBuffer(GL_ARRAY_BUFFER, frame_data_.position_); - glVertexAttribPointer(0, num_tool_pos_components_, GL_FLOAT, GL_FALSE, 0, 0); - - glBindBuffer(GL_ARRAY_BUFFER, frame_data_.confidence_); - glVertexAttribPointer(1, 1, GL_FLOAT, GL_FALSE, 0, 0); - - glEnableVertexAttribArray(0); - glEnableVertexAttribArray(1); - glBindVertexArray(0); - - if (!createGLSLShaderFromFile(GL_VERTEX_SHADER, vertex_shader_, vertex_shader_file_path_)) { - GXF_LOG_ERROR("Failed to create GLSL vertex shader"); - return GXF_FAILURE; - } - - if (!createGLSLShaderFromFile(GL_FRAGMENT_SHADER, fragment_shader_, fragment_shader_file_path_)) { - GXF_LOG_ERROR("Failed to create GLSL fragment shader"); - return GXF_FAILURE; - } - - if (!linkGLSLProgram(vertex_shader_, fragment_shader_, program_)) { - GXF_LOG_ERROR("Failed to link GLSL program."); - return GXF_FAILURE; - } - - // Initialize constant uniforms. - glUseProgram(program_); - for (size_t i = 0; i < num_tool_classes_; ++i) { glUniform3fv(i, 1, tool_tip_colors_[i].data()); } - glUseProgram(0); - - GXF_LOG_INFO("Build GLSL shaders and program successfully"); - return GXF_SUCCESS; -} - -gxf_result_t InstrumentTip::tick() { - glUseProgram(program_); - glBindVertexArray(vao_); - glDrawArrays(GL_POINTS, 0, num_tool_classes_); - - return GXF_SUCCESS; -} - -gxf_result_t InstrumentTip::stop() { - return GXF_SUCCESS; -} - -} // namespace visualizer_tool_tracking -} // namespace holoscan -} // namespace nvidia diff --git a/gxf_extensions/visualizer_tool_tracking/instrument_tip.hpp b/gxf_extensions/visualizer_tool_tracking/instrument_tip.hpp deleted file mode 100644 index 8c43b4e1..00000000 --- a/gxf_extensions/visualizer_tool_tracking/instrument_tip.hpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef NVIDIA_CLARA_HOLOSCAN_GXF_INSTRUMENT_TIP_HPP_ -#define NVIDIA_CLARA_HOLOSCAN_GXF_INSTRUMENT_TIP_HPP_ - -#include - -#include -#include - -#include "gxf/std/codelet.hpp" - -#include "frame_data.hpp" - -namespace nvidia { -namespace holoscan { -namespace visualizer_tool_tracking { - -// Note: MAX_TOOLS should match the value used in the fragment shader. -constexpr size_t MAX_TOOLS = 64; - -struct InstrumentTip { - // owned externally - const FrameData& frame_data_; - // owned internally - GLuint vao_ = 0; - GLuint vertex_shader_ = 0, fragment_shader_ = 0, program_ = 0; - std::string vertex_shader_file_path_; - std::string fragment_shader_file_path_; - uint32_t num_tool_classes_ = 0; - uint32_t num_tool_pos_components_ = 2; - std::vector> tool_tip_colors_; - - explicit InstrumentTip(const FrameData& frame_data) : frame_data_(frame_data) {} - - InstrumentTip(const InstrumentTip&) = delete; - InstrumentTip& operator=(const InstrumentTip&) = delete; - - gxf_result_t start(); - gxf_result_t tick(); - gxf_result_t stop(); -}; - -} // namespace visualizer_tool_tracking -} // namespace holoscan -} // namespace nvidia -#endif // NVIDIA_CLARA_HOLOSCAN_GXF_INSTRUMENT_TIP_HPP_ diff --git a/gxf_extensions/visualizer_tool_tracking/opengl_utils.cpp b/gxf_extensions/visualizer_tool_tracking/opengl_utils.cpp deleted file mode 100644 index d05bb6cc..00000000 --- a/gxf_extensions/visualizer_tool_tracking/opengl_utils.cpp +++ /dev/null @@ -1,168 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "opengl_utils.hpp" - -#include -#include -#include - -#include "common/assert.hpp" - -#ifndef UNUSED -#define UNUSED(NAME) (void)(NAME) -#endif - -// -------------------------------------------------------------------------------------- -// -// OpenGL Debug Output Helpers -// - -const char* glDebugSource2Str(GLenum source) { - switch (source) { - default: - break; - case GL_DEBUG_SOURCE_API: - return "API"; - case GL_DEBUG_SOURCE_WINDOW_SYSTEM: - return "Window Sys"; - case GL_DEBUG_SOURCE_SHADER_COMPILER: - return "Shader Compiler"; - case GL_DEBUG_SOURCE_THIRD_PARTY: - return "3rdparty"; - case GL_DEBUG_SOURCE_APPLICATION: - return "App"; - case GL_DEBUG_SOURCE_OTHER: - return "Other"; - } - return "Unknown"; -} - -const char* glDebugType2Str(GLenum type) { - switch (type) { - default: - break; - case GL_DEBUG_TYPE_ERROR: - return "Error"; - case GL_DEBUG_TYPE_DEPRECATED_BEHAVIOR: - return "Deprecated Behavior"; - case GL_DEBUG_TYPE_UNDEFINED_BEHAVIOR: - return "Undefined Behavior"; - case GL_DEBUG_TYPE_PORTABILITY: - return "Portability"; - case GL_DEBUG_TYPE_PERFORMANCE: - return "Performance"; - case GL_DEBUG_TYPE_OTHER: - return "Other"; - } - return "Unknown"; -} - -const char* glDebugSeverity2Str(GLenum severity) { - switch (severity) { - default: - break; - case GL_DEBUG_SEVERITY_HIGH: - return "High"; - case GL_DEBUG_SEVERITY_MEDIUM: - return "Medium"; - case GL_DEBUG_SEVERITY_LOW: - return "Low"; - case GL_DEBUG_SEVERITY_NOTIFICATION: - return "Notification"; - } - return "Unknown"; -} - -void OpenGLDebugMessageCallback(GLenum source, GLenum type, GLuint id, GLenum severity, - GLsizei length, const GLchar* message, const void* userParam) { - UNUSED(id); - UNUSED(length); - UNUSED(userParam); - - const char* source_str = glDebugSource2Str(source); - const char* type_str = glDebugType2Str(type); - const char* severity_str = glDebugSeverity2Str(severity); - - if (severity == GL_DEBUG_TYPE_ERROR) { - GXF_LOG_ERROR("GL CALLBACK: source = %s, type = %s, severity = %s, message = %s\n", source_str, - type_str, severity_str, message); - } else { - GXF_LOG_INFO("GL CALLBACK: source = %s, type = %s, severity = %s, message = %s\n", source_str, - type_str, severity_str, message); - } -} - -bool createGLSLShader(GLenum shader_type, GLuint& shader, const char* shader_src) { - shader = glCreateShader(shader_type); - glShaderSource(shader, 1, &shader_src, NULL); - glCompileShader(shader); - - GLint isCompiled = 0; - glGetShaderiv(shader, GL_COMPILE_STATUS, &isCompiled); - - if (isCompiled == GL_FALSE) { - GLint maxLength = 0; - glGetShaderiv(shader, GL_INFO_LOG_LENGTH, &maxLength); - - // The maxLength includes the NULL character - std::string compile_log; - compile_log.resize(maxLength); - glGetShaderInfoLog(shader, maxLength, NULL, &compile_log[0]); - - GXF_LOG_ERROR("Shader compilation failed: %s ", compile_log.c_str()); - return false; - } - return true; -} - -bool createGLSLShaderFromFile(GLenum shader_type, GLuint& shader, - const std::string& shader_filename) { - std::ifstream file(shader_filename); - if (!file.good()) { - GXF_LOG_ERROR("Failed to open GLSL shader file: %s ", shader_filename.c_str()); - return false; - } - std::string shader_src_str = - std::string(std::istreambuf_iterator(file), std::istreambuf_iterator()); - return createGLSLShader(shader_type, shader, shader_src_str.c_str()); -} - -bool linkGLSLProgram(const GLuint vertex_shader, const GLuint fragment_shader, GLuint& program) { - program = glCreateProgram(); - glAttachShader(program, vertex_shader); - glAttachShader(program, fragment_shader); - glLinkProgram(program); - - GLint isLinked = 0; - glGetProgramiv(program, GL_LINK_STATUS, &isLinked); - if (isLinked == GL_FALSE) { - GLint maxLength = 0; - glGetProgramiv(program, GL_INFO_LOG_LENGTH, &maxLength); - - // The maxLength includes the NULL character - std::string link_log; - link_log.resize(maxLength); - glGetProgramInfoLog(program, maxLength, &maxLength, &link_log[0]); - GXF_LOG_ERROR("Failed to link GLSL program. Log: %s", link_log.c_str()); - // The program is useless now. So delete it. - glDeleteProgram(program); - // Provide the infolog in whatever manner you deem best. - // Exit with failure. - return false; - } - return true; -} diff --git a/gxf_extensions/visualizer_tool_tracking/opengl_utils.hpp b/gxf_extensions/visualizer_tool_tracking/opengl_utils.hpp deleted file mode 100644 index b36c71ab..00000000 --- a/gxf_extensions/visualizer_tool_tracking/opengl_utils.hpp +++ /dev/null @@ -1,34 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef NVIDIA_CLARA_HOLOSCAN_GXF_OPENGL_UTILS_HPP_ -#define NVIDIA_CLARA_HOLOSCAN_GXF_OPENGL_UTILS_HPP_ - -#include - -#include - -void GLAPIENTRY OpenGLDebugMessageCallback(GLenum source, GLenum type, GLuint id, GLenum severity, - GLsizei length, const GLchar* message, - const void* userParam); - -bool createGLSLShader(GLenum shader_type, GLuint& shader, const char* shader_src); - -bool createGLSLShaderFromFile(GLenum shader_type, GLuint& shader, - const std::string& shader_filename); - -bool linkGLSLProgram(const GLuint vertex_shader, const GLuint fragment_shader, GLuint& program); -#endif // NVIDIA_CLARA_HOLOSCAN_GXF_OPENGL_UTILS_HPP_ diff --git a/gxf_extensions/visualizer_tool_tracking/overlay_img_vis.cpp b/gxf_extensions/visualizer_tool_tracking/overlay_img_vis.cpp deleted file mode 100644 index 33c55260..00000000 --- a/gxf_extensions/visualizer_tool_tracking/overlay_img_vis.cpp +++ /dev/null @@ -1,101 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "overlay_img_vis.hpp" - -#include - -#include "opengl_utils.hpp" - -namespace nvidia { -namespace holoscan { -namespace visualizer_tool_tracking { - -gxf_result_t OverlayImageVis::start() { - if (num_layers_ > MAX_LAYERS) { - GXF_LOG_ERROR("Number of layers (%d) exceeds maximum number of layers (%d)", num_layers_, - MAX_LAYERS); - return GXF_FAILURE; - } - - if (num_layers_ > layer_colors_.size()) { - GXF_LOG_ERROR("Number of layers (%d) exceeds number of colors provided (%d)", num_layers_, - layer_colors_.size()); - return GXF_FAILURE; - } - - for (auto color : layer_colors_) { - if (color.size() != 3) { - GXF_LOG_ERROR("Layer colors must be 3 elements (RGB)"); - return GXF_FAILURE; - } - } - - glGenVertexArrays(1, &vao_); - - glCreateSamplers(1, &sampler); - glSamplerParameteri(sampler, GL_TEXTURE_WRAP_S, GL_REPEAT); - glSamplerParameteri(sampler, GL_TEXTURE_WRAP_T, GL_REPEAT); - glSamplerParameteri(sampler, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - glSamplerParameteri(sampler, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - - if (!createGLSLShaderFromFile(GL_VERTEX_SHADER, vertex_shader_, vertex_shader_file_path_)) { - GXF_LOG_ERROR("Failed to create GLSL vertex shader"); - return GXF_FAILURE; - } - - if (!createGLSLShaderFromFile(GL_FRAGMENT_SHADER, fragment_shader_, fragment_shader_file_path_)) { - GXF_LOG_ERROR("Failed to create GLSL fragment shader"); - return GXF_FAILURE; - } - - if (!linkGLSLProgram(vertex_shader_, fragment_shader_, program_)) { - GXF_LOG_ERROR("Failed to link GLSL program."); - return GXF_FAILURE; - } - - // Initialize constant uniforms. - glUseProgram(program_); - glUniform1ui(0, num_layers_); - for (size_t i = 0; i < num_layers_; ++i) { glUniform3fv(1 + i, 1, layer_colors_[i].data()); } - glUseProgram(0); - - GXF_LOG_INFO("Build GLSL shaders and program successfully"); - - return GXF_SUCCESS; -} - -gxf_result_t OverlayImageVis::tick() { - glActiveTexture(GL_TEXTURE0); - glBindSampler(0, sampler); - glBindTexture(GL_TEXTURE_2D_ARRAY, overlay_img_tex_); - - glUseProgram(program_); - glBindBufferBase(GL_SHADER_STORAGE_BUFFER, 0, frame_data_.confidence_); - - glBindVertexArray(vao_); - glDrawArrays(GL_TRIANGLE_STRIP, 0, 3); - - return GXF_SUCCESS; -} - -gxf_result_t OverlayImageVis::stop() { - return GXF_SUCCESS; -} - -} // namespace visualizer_tool_tracking -} // namespace holoscan -} // namespace nvidia diff --git a/gxf_extensions/visualizer_tool_tracking/overlay_img_vis.hpp b/gxf_extensions/visualizer_tool_tracking/overlay_img_vis.hpp deleted file mode 100644 index f1a53fec..00000000 --- a/gxf_extensions/visualizer_tool_tracking/overlay_img_vis.hpp +++ /dev/null @@ -1,62 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef NVIDIA_CLARA_HOLOSCAN_GXF_OVERLAY_IMG_VIS_HPP_ -#define NVIDIA_CLARA_HOLOSCAN_GXF_OVERLAY_IMG_VIS_HPP_ - -#include - -#include -#include - -#include "gxf/std/codelet.hpp" - -#include "frame_data.hpp" - -namespace nvidia { -namespace holoscan { -namespace visualizer_tool_tracking { - -// Note: MAX_LAYERS should match the value used in the fragment shader. -constexpr size_t MAX_LAYERS = 64; - -struct OverlayImageVis { - // owned externally - const FrameData& frame_data_; - const GLuint& overlay_img_tex_; - // owned internally - GLuint vao_ = 0; - GLuint sampler = 0; - GLuint vertex_shader_ = 0, fragment_shader_ = 0, program_ = 0; - std::string vertex_shader_file_path_; - std::string fragment_shader_file_path_; - size_t num_layers_ = 0; - std::vector> layer_colors_; - - OverlayImageVis(const FrameData& frame_data, const GLuint& overlay_img_tex) - : frame_data_(frame_data), overlay_img_tex_(overlay_img_tex) {} - - OverlayImageVis(const OverlayImageVis&) = delete; - OverlayImageVis& operator=(const OverlayImageVis&) = delete; - - gxf_result_t start(); - gxf_result_t tick(); - gxf_result_t stop(); -}; -} // namespace visualizer_tool_tracking -} // namespace holoscan -} // namespace nvidia -#endif // NVIDIA_CLARA_HOLOSCAN_GXF_OVERLAY_IMG_VIS_HPP_ diff --git a/gxf_extensions/visualizer_tool_tracking/video_frame.cpp b/gxf_extensions/visualizer_tool_tracking/video_frame.cpp deleted file mode 100644 index fed8d464..00000000 --- a/gxf_extensions/visualizer_tool_tracking/video_frame.cpp +++ /dev/null @@ -1,73 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "video_frame.hpp" - -#include - -#include "opengl_utils.hpp" - -namespace nvidia { -namespace holoscan { -namespace visualizer_tool_tracking { - -gxf_result_t VideoFrame::start() { - glGenVertexArrays(1, &vao_); - - glCreateSamplers(1, &sampler_); - glSamplerParameteri(sampler_, GL_TEXTURE_WRAP_S, GL_REPEAT); - glSamplerParameteri(sampler_, GL_TEXTURE_WRAP_T, GL_REPEAT); - - if (!createGLSLShaderFromFile(GL_VERTEX_SHADER, vertex_shader_, vertex_shader_file_path_)) { - GXF_LOG_ERROR("Failed to create GLSLvertex shader"); - return GXF_FAILURE; - } - - if (!createGLSLShaderFromFile(GL_FRAGMENT_SHADER, fragment_shader_, fragment_shader_file_path_)) { - GXF_LOG_ERROR("Failed to create GLSL fragment shader"); - return GXF_FAILURE; - } - - if (!linkGLSLProgram(vertex_shader_, fragment_shader_, program_)) { - GXF_LOG_ERROR("Failed to link GLSL program."); - return GXF_FAILURE; - } - - GXF_LOG_INFO("Build GLSL shaders and program successfully"); - return GXF_SUCCESS; -} - -gxf_result_t VideoFrame::tick(GLuint tex, GLenum filter) { - glActiveTexture(GL_TEXTURE0); - glBindSampler(0, sampler_); - glSamplerParameteri(sampler_, GL_TEXTURE_MIN_FILTER, filter); - glSamplerParameteri(sampler_, GL_TEXTURE_MAG_FILTER, filter); - glBindTexture(GL_TEXTURE_2D, tex); - glUseProgram(program_); - glBindVertexArray(vao_); - glDrawArrays(GL_TRIANGLE_STRIP, 0, 3); - glBindTexture(GL_TEXTURE_2D, 0); - - return GXF_SUCCESS; -} - -gxf_result_t VideoFrame::stop() { - return GXF_SUCCESS; -} - -} // namespace visualizer_tool_tracking -} // namespace holoscan -} // namespace nvidia diff --git a/gxf_extensions/visualizer_tool_tracking/video_frame.hpp b/gxf_extensions/visualizer_tool_tracking/video_frame.hpp deleted file mode 100644 index 74140ac2..00000000 --- a/gxf_extensions/visualizer_tool_tracking/video_frame.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef NVIDIA_CLARA_HOLOSCAN_GXF_VIDEO_FRAME_HPP_ -#define NVIDIA_CLARA_HOLOSCAN_GXF_VIDEO_FRAME_HPP_ -#include - -#include - -#include "gxf/std/codelet.hpp" - -namespace nvidia { -namespace holoscan { -namespace visualizer_tool_tracking { - -struct VideoFrame { - // owned internally - GLuint vao_ = 0; - GLuint vertex_shader_ = 0, fragment_shader_ = 0, program_ = 0; - GLuint sampler_ = 0; - std::string vertex_shader_file_path_; - std::string fragment_shader_file_path_; - - VideoFrame() {} - - VideoFrame(const VideoFrame&) = delete; - VideoFrame& operator=(const VideoFrame&) = delete; - - gxf_result_t start(); - gxf_result_t tick(GLuint tex, GLenum filter); - gxf_result_t stop(); -}; - -} // namespace visualizer_tool_tracking -} // namespace holoscan -} // namespace nvidia -#endif // NVIDIA_CLARA_HOLOSCAN_GXF_VIDEO_FRAME_HPP_ diff --git a/gxf_extensions/visualizer_tool_tracking/visualizer.cpp b/gxf_extensions/visualizer_tool_tracking/visualizer.cpp deleted file mode 100644 index 4fd6f872..00000000 --- a/gxf_extensions/visualizer_tool_tracking/visualizer.cpp +++ /dev/null @@ -1,829 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "visualizer.hpp" - -#include -#include -#include - -#include -#include -#include -#include - -#include "gxf/multimedia/video.hpp" -#include "gxf/std/tensor.hpp" - -#include "opengl_utils.hpp" - -#define CUDA_TRY(stmt) \ - ({ \ - cudaError_t _holoscan_cuda_err = stmt; \ - if (cudaSuccess != _holoscan_cuda_err) { \ - GXF_LOG_ERROR("CUDA Runtime call %s in line %d of file %s failed with '%s' (%d).\n", #stmt, \ - __LINE__, __FILE__, cudaGetErrorString(_holoscan_cuda_err), \ - _holoscan_cuda_err); \ - } \ - _holoscan_cuda_err; \ - }) - -namespace nvidia { -namespace holoscan { -namespace visualizer_tool_tracking { - -constexpr int32_t DEFAULT_SRC_WIDTH = 640; -constexpr int32_t DEFAULT_SRC_HEIGHT = 480; -constexpr int16_t DEFAULT_SRC_CHANNELS = 3; -constexpr uint8_t DEFAULT_SRC_BYTES_PER_PIXEL = 1; -// 12 qualitative classes color scheme from colorbrewer2 -static const std::vector> DEFAULT_COLORS = { - {0.12f, 0.47f, 0.71f}, {0.20f, 0.63f, 0.17f}, {0.89f, 0.10f, 0.11f}, {1.00f, 0.50f, 0.00f}, - {0.42f, 0.24f, 0.60f}, {0.69f, 0.35f, 0.16f}, {0.65f, 0.81f, 0.89f}, {0.70f, 0.87f, 0.54f}, - {0.98f, 0.60f, 0.60f}, {0.99f, 0.75f, 0.44f}, {0.79f, 0.70f, 0.84f}, {1.00f, 1.00f, 0.60f}}; - -static const uint32_t NUM_POSITION_COMPONENTS = 2; -static const uint32_t NUM_TOOL_CLASSES = 7; -static const uint32_t POSITION_BUFFER_SIZE = - sizeof(float) * NUM_TOOL_CLASSES * NUM_POSITION_COMPONENTS; -static const uint32_t CONFIDENCE_BUFFER_SIZE = sizeof(float) * NUM_TOOL_CLASSES; - -static void glfwPrintErrorCallback(int error, const char* msg) { - std::cerr << " [" << error << "] " << msg << "\n"; -} - -// process all input: query GLFW whether relevant keys are pressed/released this frame and react -// accordingly -// --------------------------------------------------------------------------------------------- -static void glfwProcessInput(GLFWwindow* window_) { - if (glfwGetKey(window_, GLFW_KEY_ESCAPE) == GLFW_PRESS) glfwSetWindowShouldClose(window_, true); -} - -// whenever the window size changed (by OS or user resize) this callback function executes -// --------------------------------------------------------------------------------------------- -static void glfwFramebufferSizeCallback(GLFWwindow* window_, int width, int height) { - Sink* sink = static_cast(glfwGetWindowUserPointer(window_)); - if (sink) { sink->onFramebufferSizeCallback(width, height); } -} - -static void glfwKeyCallback(GLFWwindow* window_, int key, int scancode, int action, int mods) { - Sink* sink = static_cast(glfwGetWindowUserPointer(window_)); - if (sink) { sink->onKeyCallback(key, scancode, action, mods); } -} - -Sink::Sink() - : video_frame_vis_(), - tooltip_vis_(frame_data_), - label_vis_(frame_data_), - overlay_img_vis(frame_data_, overlay_img_tex_) {} - -gxf_result_t Sink::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - - result &= registrar->parameter(videoframe_vertex_shader_path_, "videoframe_vertex_shader_path", - "Videoframe GLSL Vertex Shader File Path", - "Path to vertex shader to be loaded"); - result &= registrar->parameter( - videoframe_fragment_shader_path_, "videoframe_fragment_shader_path", - "Videoframe GLSL Fragment Shader File Path", "Path to fragment shader to be loaded"); - - result &= registrar->parameter(tooltip_vertex_shader_path_, "tooltip_vertex_shader_path", - "Tool tip GLSL Vertex Shader File Path", - "Path to vertex shader to be loaded"); - result &= registrar->parameter(tooltip_fragment_shader_path_, "tooltip_fragment_shader_path", - "Tool tip GLSL Fragment Shader File Path", - "Path to fragment shader to be loaded"); - result &= registrar->parameter(num_tool_classes_, "num_tool_classes", "Tool Classes", - "Number of different tool classes"); - result &= registrar->parameter(num_tool_pos_components_, "num_tool_pos_components", - "Position Components", - "Number of components of the tool position vector", 2); - result &= registrar->parameter( - tool_tip_colors_, "tool_tip_colors", "Tool Tip Colors", - "Color of the tool tips, a list of RGB values with components between 0 and 1", - DEFAULT_COLORS); - - result &= registrar->parameter(overlay_img_vertex_shader_path_, "overlay_img_vertex_shader_path", - "Overlay Image GLSL Vertex Shader File Path", - "Path to vertex shader to be loaded"); - result &= registrar->parameter( - overlay_img_fragment_shader_path_, "overlay_img_fragment_shader_path", - "Overlay Image GLSL Fragment Shader File Path", "Path to fragment shader to be loaded"); - result &= registrar->parameter(overlay_img_width_, "overlay_img_width", "Overlay Image Width", - "Width of overlay image"); - result &= registrar->parameter(overlay_img_height_, "overlay_img_height", "Overlay Image Height", - "Height of overlay image"); - result &= - registrar->parameter(overlay_img_channels_, "overlay_img_channels", - "Number of Overlay Image Channels", "Number of Overlay Image Channels"); - result &= - registrar->parameter(overlay_img_layers_, "overlay_img_layers", - "Number of Overlay Image Layers", "Number of Overlay Image Layers"); - result &= registrar->parameter( - overlay_img_colors_, "overlay_img_colors", "Overlay Image Layer Colors", - "Color of the image overlays, a list of RGB values with components between 0 and 1", - DEFAULT_COLORS); - - result &= registrar->parameter( - tool_labels_, "tool_labels", "Tool Labels", "List of tool names.", - {}); // Default handled in instrument_label to dynamically adjust for the number of tools - result &= registrar->parameter(label_sans_font_path_, "label_sans_font_path", - "File path for sans font for displaying tool name", - "Path for sans font to be loaded"); - result &= registrar->parameter(label_sans_bold_font_path_, "label_sans_bold_font_path", - "File path for sans bold font for displaying tool name", - "Path for sans bold font to be loaded"); - - result &= registrar->parameter(in_, "in", "Input", "List of input channels"); - result &= registrar->parameter(in_tensor_names_, "in_tensor_names", "Input Tensor Names", - "Names of input tensors.", {std::string("")}); - result &= registrar->parameter(in_width_, "in_width", "InputWidth", "Width of the image.", - DEFAULT_SRC_WIDTH); - result &= registrar->parameter(in_height_, "in_height", "InputHeight", "Height of the image.", - DEFAULT_SRC_HEIGHT); - result &= registrar->parameter(in_channels_, "in_channels", "InputChannels", - "Number of channels.", DEFAULT_SRC_CHANNELS); - result &= - registrar->parameter(in_bytes_per_pixel_, "in_bytes_per_pixel", "InputBytesPerPixel", - "Number of bytes per pixel of the image.", DEFAULT_SRC_BYTES_PER_PIXEL); - result &= registrar->parameter(alpha_value_, "alpha_value", "Alpha value", - "Alpha value that can be used when converting RGB888 to RGBA8888.", - static_cast(255)); - result &= registrar->parameter(pool_, "pool", "Pool", "Pool to allocate the output message."); - result &= registrar->parameter( - window_close_scheduling_term_, "window_close_scheduling_term", "WindowCloseSchedulingTerm", - "BooleanSchedulingTerm to stop the codelet from ticking after all messages are published."); - - result &= registrar->parameter(overlay_buffer_input_, "overlay_buffer_input", - "OverlayBufferInput", "Input for an empty overlay buffer.", - gxf::Registrar::NoDefaultParameter(), - GXF_PARAMETER_FLAGS_OPTIONAL); - result &= registrar->parameter(overlay_buffer_output_, "overlay_buffer_output", - "OverlayBufferOutput", "Output for a filled overlay buffer.", - gxf::Registrar::NoDefaultParameter(), - GXF_PARAMETER_FLAGS_OPTIONAL); - - return gxf::ToResultCode(result); -} - -gxf_result_t Sink::start() { - glfwSetErrorCallback(glfwPrintErrorCallback); - - // Create window - // ------------- - // initialize and configure - if (!glfwInit()) { - GXF_LOG_ERROR("Failed to initialize GLFW"); - glfwTerminate(); - return GXF_FAILURE; - } - - glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 4); - glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 6); - glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_CORE_PROFILE); - window_ = glfwCreateWindow(in_width_, in_height_, "GXF Video Stream", NULL, NULL); - if (window_ == NULL) { - GXF_LOG_ERROR("Failed to create GLFW window"); - glfwTerminate(); - return GXF_FAILURE; - } - glfwSetWindowUserPointer(window_, this); - glfwSetFramebufferSizeCallback(window_, glfwFramebufferSizeCallback); - glfwSetKeyCallback(window_, glfwKeyCallback); - glfwMakeContextCurrent(window_); - - // propagate width, height manually as first framebuffer resize callback is not triggered - onFramebufferSizeCallback(in_width_, in_height_); - - // Load all OpenGL function pointers - GLADloadproc gl_loader = reinterpret_cast(glfwGetProcAddress); - if (!gladLoadGLLoader(gl_loader)) { - GXF_LOG_ERROR("Failed to initialize GLAD"); - return GXF_FAILURE; - } - - glEnable(GL_DEBUG_OUTPUT); - // disable frequent GL API notification messages, e.g. buffer usage info, to avoid spamming log - glDebugMessageControl(GL_DEBUG_SOURCE_API, GL_DEBUG_TYPE_OTHER, GL_DEBUG_SEVERITY_NOTIFICATION, 0, - 0, GL_FALSE); - glDebugMessageCallback(OpenGLDebugMessageCallback, 0); - glDisable(GL_BLEND); - glDisable(GL_DEPTH_TEST); - glEnable(GL_PROGRAM_POINT_SIZE); - glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); - - // CUDA GL Interop not support with 3 channel formats, e.g. GL_RGB - use_cuda_opengl_interop_ = in_channels_.get() != 3; - - GXF_LOG_INFO("#%d channels, CUDA / OpenGL interop %s", in_channels_.get(), - use_cuda_opengl_interop_ ? "enabled" : "disabled"); - - // Allocate host memory and OpenGL buffers, textures for video frame and inference results - // ---------------------------------------------------------------------------------- - - video_frame_buffer_host_.resize(in_width_ * in_height_ * in_channels_.get(), 0); - overlay_img_buffer_host_.resize(overlay_img_width_ * overlay_img_height_ * overlay_img_layers_, - 0.0f); - overlay_img_layered_host_.resize(overlay_img_width_ * overlay_img_height_ * overlay_img_layers_, - 0.0f); - frame_data_.confidence_host_.resize(NUM_TOOL_CLASSES, 0.0f); - frame_data_.position_host_.resize(NUM_TOOL_CLASSES * NUM_POSITION_COMPONENTS, 0.0f); - - glCreateBuffers(1, &frame_data_.position_); - glNamedBufferData(frame_data_.position_, POSITION_BUFFER_SIZE, NULL, GL_STREAM_DRAW); - glCreateBuffers(1, &frame_data_.confidence_); - glNamedBufferData(frame_data_.confidence_, CONFIDENCE_BUFFER_SIZE, NULL, GL_STREAM_DRAW); - - // register frame_data_.confidence_ and position_ with CUDA - { - cudaError_t cuda_status = CUDA_TRY(cudaGraphicsGLRegisterBuffer( - &cuda_confidence_resource_, frame_data_.confidence_, cudaGraphicsMapFlagsWriteDiscard)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to register confidence buffer for CUDA / OpenGL Interop"); - return GXF_FAILURE; - } - cuda_status = CUDA_TRY(cudaGraphicsGLRegisterBuffer( - &cuda_position_resource_, frame_data_.position_, cudaGraphicsMapFlagsWriteDiscard)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to register position buffer for CUDA / OpenGL Interop"); - return GXF_FAILURE; - } - } - - glActiveTexture(GL_TEXTURE0); - glGenTextures(1, &video_frame_tex_); - glBindTexture(GL_TEXTURE_2D, video_frame_tex_); - // allocate immutable texture storage ( if resize need to re-create texture object) - GLenum format = (in_channels_ == 4) ? GL_RGBA8 : GL_RGB8; - glTexStorage2D(GL_TEXTURE_2D, 1, format, in_width_, in_height_); - // set the texture wrapping parameters - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_REPEAT); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_REPEAT); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - glBindTexture(GL_TEXTURE_2D, 0); - - // register this texture with CUDA - if (use_cuda_opengl_interop_) { - cudaError_t cuda_status = - CUDA_TRY(cudaGraphicsGLRegisterImage(&cuda_video_frame_tex_resource_, video_frame_tex_, - GL_TEXTURE_2D, cudaGraphicsMapFlagsWriteDiscard)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to register video frame texture for CUDA / OpenGL Interop"); - return GXF_FAILURE; - } - } - - glGenTextures(1, &overlay_img_tex_); - glBindTexture(GL_TEXTURE_2D_ARRAY, overlay_img_tex_); - // allocate immutable texture storage ( if resize need to re-create texture object) - glTexStorage3D(GL_TEXTURE_2D_ARRAY, 1, GL_R32F, overlay_img_width_, overlay_img_height_, - overlay_img_layers_); - // set the texture wrapping parameters - glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_WRAP_S, GL_REPEAT); - glTexParameteri(GL_TEXTURE_2D_ARRAY, GL_TEXTURE_WRAP_T, GL_REPEAT); - glBindTexture(GL_TEXTURE_2D_ARRAY, 0); - - // CUDA / GL interop for overlay image tex - { - cudaError_t cuda_status = CUDA_TRY( - cudaGraphicsGLRegisterImage(&cuda_overlay_img_tex_resource_, overlay_img_tex_, - GL_TEXTURE_2D_ARRAY, cudaGraphicsMapFlagsWriteDiscard)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to register overlay image texture for CUDA / OpenGL Interop"); - return GXF_FAILURE; - } - } - - // Initialize helper class instancces - // ---------------------------------------------------------------------------------- - - video_frame_vis_.vertex_shader_file_path_ = videoframe_vertex_shader_path_.get(); - video_frame_vis_.fragment_shader_file_path_ = videoframe_fragment_shader_path_.get(); - - tooltip_vis_.vertex_shader_file_path_ = tooltip_vertex_shader_path_.get(); - tooltip_vis_.fragment_shader_file_path_ = tooltip_fragment_shader_path_.get(); - tooltip_vis_.num_tool_classes_ = num_tool_classes_.get(); - tooltip_vis_.num_tool_pos_components_ = num_tool_pos_components_.get(); - tooltip_vis_.tool_tip_colors_ = tool_tip_colors_.get(); - - overlay_img_vis.vertex_shader_file_path_ = overlay_img_vertex_shader_path_.get(); - overlay_img_vis.fragment_shader_file_path_ = overlay_img_fragment_shader_path_.get(); - overlay_img_vis.num_layers_ = overlay_img_layers_.get(); - overlay_img_vis.layer_colors_ = overlay_img_colors_.get(); - - label_vis_.label_sans_font_path_ = label_sans_font_path_.get(); - label_vis_.label_sans_bold_font_path_ = label_sans_bold_font_path_.get(); - label_vis_.num_tool_classes_ = num_tool_classes_.get(); - label_vis_.num_tool_pos_components_ = num_tool_pos_components_.get(); - label_vis_.tool_labels_ = tool_labels_.get(); - - gxf_result_t res = video_frame_vis_.start(); - if (res != GXF_SUCCESS) { return res; } - res = label_vis_.start(); - if (res != GXF_SUCCESS) { return res; } - res = overlay_img_vis.start(); - if (res != GXF_SUCCESS) { return res; } - res = tooltip_vis_.start(); - if (res != GXF_SUCCESS) { return res; } - - window_close_scheduling_term_->enable_tick(); - - return GXF_SUCCESS; -} - -gxf_result_t Sink::stop() { - // Free mem allocated in utility classes. - // ---------------------------------------------------------------------------------- - - video_frame_vis_.stop(); - label_vis_.stop(); - overlay_img_vis.stop(); - tooltip_vis_.stop(); - - // Free OpenGL buffer and texture memory - // ---------------------------------------------------------------------------------- - - if (cuda_overlay_renderbuffer_resource_) { - cudaError_t cuda_status = CUDA_TRY( - cudaGraphicsUnmapResources(1, &cuda_overlay_renderbuffer_resource_, 0)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to unmap CUDA overlay renderbuffer resource"); - } - } - - // terminate, clearing all previously allocated GLFW resources. - if (window_ != nullptr) { - glfwDestroyWindow(window_); - window_ = nullptr; - } - glfwTerminate(); - - return GXF_SUCCESS; -} - -gxf_result_t Sink::tick() { - // Grabs latest messages from all receivers - std::vector messages; - messages.reserve(in_.get().size()); - for (auto& rx : in_.get()) { - gxf::Expected maybe_message = rx->receive(); - if (maybe_message) { messages.push_back(std::move(maybe_message.value())); } - } - if (messages.empty()) { - GXF_LOG_ERROR("No message available."); - return GXF_CONTRACT_MESSAGE_NOT_AVAILABLE; - } - - glfwProcessInput(window_); - if (glfwWindowShouldClose(window_)) { - window_close_scheduling_term_->disable_tick(); - return GXF_SUCCESS; - } - - cudaError_t cuda_status = {}; - - // Read message from receiver, update buffers / memory - // ---------------------------------------------------------------------------------- - - gxf::Expected> maybe_tensor = gxf::Unexpected{GXF_UNINITIALIZED_VALUE}; - gxf::Expected> maybe_video = - gxf::Unexpected{GXF_UNINITIALIZED_VALUE}; - - // Pick one input tensor - for (const auto& tensor_name : in_tensor_names_.get()) { - for (auto& msg : messages) { - maybe_tensor = msg.get(tensor_name.c_str()); - if (maybe_tensor) { break; } - maybe_video = msg.get(); - if (maybe_video) { break; } - } - if (!maybe_tensor && !maybe_video) { - GXF_LOG_ERROR("Failed to retrieve Tensor(%s) or VideoBuffer", tensor_name.c_str()); - return GXF_FAILURE; - } - - // Pick only the first tensor from multiple input channels/tensors - break; - } - - uint8_t* in_tensor_ptr = nullptr; - uint64_t buffer_size = 0; - int32_t columns = 0; - int32_t rows = 0; - int16_t channels = 0; - if (maybe_video) { - { - auto frame = maybe_video.value(); - - // NOTE: VideoBuffer::moveToTensor() converts VideoBuffer instance to the Tensor instance - // with an unexpected shape: [width, height] or [width, height, num_planes]. - // And, if we use moveToTensor() to convert VideoBuffer to Tensor, we may lose the the - // original video buffer when the VideoBuffer instance is used in other places. For that - // reason, we directly access internal data of VideoBuffer instance to access Tensor data. - const auto& buffer_info = frame->video_frame_info(); - switch (buffer_info.color_format) { - case gxf::VideoFormat::GXF_VIDEO_FORMAT_RGBA: - break; - default: - GXF_LOG_ERROR("Unsupported input format: %d\n", buffer_info.color_format); - return GXF_FAILURE; - } - - columns = buffer_info.width; - rows = buffer_info.height; - channels = 4; // RGBA - - in_tensor_ptr = frame->pointer(); - } - } else { - // Get tensor attached to the message - auto in_tensor = maybe_tensor; - if (!in_tensor) { return in_tensor.error(); } - if (in_tensor.value()->storage_type() != gxf::MemoryStorageType::kDevice) { - return GXF_MEMORY_INVALID_STORAGE_MODE; - } - auto maybe_in_tensor_ptr = in_tensor.value()->data(); - if (!maybe_in_tensor_ptr) { return maybe_in_tensor_ptr.error(); } - - const gxf::Shape shape = in_tensor.value()->shape(); - - columns = shape.dimension(1); - rows = shape.dimension(0); - channels = shape.dimension(2); - - in_tensor_ptr = maybe_in_tensor_ptr.value(); - } - - if (in_channels_ != channels) { - GXF_LOG_ERROR( - "Received VideoBuffer/Tensor has a different number of channels (%d. Expected %d)", - channels, in_channels_.get()); - return GXF_FAILURE; - } - - buffer_size = columns * rows * channels; - - const int32_t in_height = in_height_; - const int32_t in_width = in_width_; - const int32_t in_channels = in_channels_; - - const uint64_t buffer_items_size = in_width * in_height * in_channels; - - if (in_height != rows || in_width != columns || in_channels != channels) { - GXF_LOG_ERROR("Received buffer size doesn't match."); - return GXF_FAILURE; - } - - // Set alignment requirement to 1 so that the tensor with any width can work. - glPixelStorei(GL_UNPACK_ALIGNMENT, 1); - - if (in_tensor_ptr && buffer_size > 0) { - // Video Frame - // -------------------------------------------------------------------------------------------- - if (use_cuda_opengl_interop_) { - cuda_status = CUDA_TRY(cudaGraphicsMapResources(1, &cuda_video_frame_tex_resource_, 0)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to map video frame texture via CUDA / OpenGL interop"); - return GXF_FAILURE; - } - cudaArray* texture_ptr = nullptr; - cuda_status = CUDA_TRY(cudaGraphicsSubResourceGetMappedArray( - &texture_ptr, cuda_video_frame_tex_resource_, 0, 0)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to get mapped array for video frame texture"); - return GXF_FAILURE; - } - size_t spitch = 4 * in_width_ * sizeof(GLubyte); - cuda_status = CUDA_TRY(cudaMemcpy2DToArray(texture_ptr, 0, 0, in_tensor_ptr, spitch, spitch, - in_height, cudaMemcpyDeviceToDevice)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to copy video frame to OpenGL texture via CUDA / OpenGL interop"); - return GXF_FAILURE; - } - cuda_status = CUDA_TRY(cudaGraphicsUnmapResources(1, &cuda_video_frame_tex_resource_, 0)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to unmap video frame texture via CUDA / OpenGL interop"); - return GXF_FAILURE; - } - } else { - const uint64_t buffer_size = buffer_items_size * in_bytes_per_pixel_; - cuda_status = CUDA_TRY(cudaMemcpy(video_frame_buffer_host_.data(), in_tensor_ptr, buffer_size, - cudaMemcpyDeviceToHost)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to copy video frame texture from Device to Host"); - return GXF_FAILURE; - } - // update data - glActiveTexture(GL_TEXTURE0); - glBindTexture(GL_TEXTURE_2D, video_frame_tex_); - GLenum format = (in_channels == 4) ? GL_RGBA : GL_RGB; - glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, in_width_, in_height_, format, GL_UNSIGNED_BYTE, - video_frame_buffer_host_.data()); - } - } // if (in_tensor_ptr && buffer_size > 0) - - if (messages.size() >= 2) { - const auto& inference_message = messages[1]; - // Confidence - // -------------------------------------------------------------------------------------------- - if (enable_tool_tip_vis_ || enable_tool_labels_ || enable_overlay_img_vis_) { - auto src = inference_message.get("probs").value()->data().value(); - - // download for use on host for e.g. controlling rendering of instrument names - cuda_status = CUDA_TRY(cudaMemcpy(frame_data_.confidence_host_.data(), src, - CONFIDENCE_BUFFER_SIZE, cudaMemcpyDeviceToHost)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to copy confidence buffer from Device to Host."); - return GXF_FAILURE; - } - // use CUDA / OpenGL interop to copy confidence tensor to confidence OpenGL buffer object - // with device to device memcpy - cuda_status = CUDA_TRY(cudaGraphicsMapResources(1, &cuda_confidence_resource_, 0)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to map confidence buffer via CUDA / OpenGL interop"); - return GXF_FAILURE; - } - float* dptr = nullptr; - size_t num_bytes = 0; - cuda_status = CUDA_TRY(cudaGraphicsResourceGetMappedPointer( - reinterpret_cast(&dptr), &num_bytes, cuda_confidence_resource_)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to get mapped pointer for confidence buffer"); - return GXF_FAILURE; - } - cuda_status = CUDA_TRY(cudaMemcpy(dptr, src, num_bytes, cudaMemcpyDeviceToDevice)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to copy confidence buffer to OpenGL via CUDA / OpenGL interop"); - return GXF_FAILURE; - } - cuda_status = CUDA_TRY(cudaGraphicsUnmapResources(1, &cuda_confidence_resource_, 0)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to unmap confidence buffer via CUDA / OpenGL interop"); - return GXF_FAILURE; - } - } - - // Position - // -------------------------------------------------------------------------------------------- - if (enable_tool_tip_vis_ || enable_tool_labels_) { - auto src = inference_message.get("scaled_coords").value()->data().value(); - // Data still needs to be copied to host for drawing text labels - cuda_status = CUDA_TRY(cudaMemcpy(frame_data_.position_host_.data(), src, - POSITION_BUFFER_SIZE, cudaMemcpyDeviceToHost)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to copy position buffer from Device to Host."); - return GXF_FAILURE; - } - - // use CUDA / OpenGL interop to copy position tensor to position OpenGL buffer object - // with device to device memcpy - cuda_status = CUDA_TRY(cudaGraphicsMapResources(1, &cuda_position_resource_, 0)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to map position buffer via CUDA / OpenGL interop"); - return GXF_FAILURE; - } - float* dptr = nullptr; - size_t num_bytes = 0; - cuda_status = CUDA_TRY(cudaGraphicsResourceGetMappedPointer( - reinterpret_cast(&dptr), &num_bytes, cuda_position_resource_)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to get mapped pointer for position buffer"); - return GXF_FAILURE; - } - cuda_status = CUDA_TRY(cudaMemcpy(dptr, src, num_bytes, cudaMemcpyDeviceToDevice)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to copy position buffer to OpenGL via CUDA / OpenGL interop"); - return GXF_FAILURE; - } - cuda_status = CUDA_TRY(cudaGraphicsUnmapResources(1, &cuda_position_resource_, 0)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to unmap position buffer via CUDA / OpenGL interop"); - return GXF_FAILURE; - } - } - - // Overlay Image Texture - // -------------------------------------------------------------------------------------------- - if (enable_overlay_img_vis_) { - auto src = inference_message.get("binary_masks").value()->data().value(); - cuda_status = CUDA_TRY(cudaGraphicsMapResources(1, &cuda_overlay_img_tex_resource_, 0)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to map overlay image texture via CUDA / OpenGL interop"); - return GXF_FAILURE; - } - - cudaArray* texture_ptr = nullptr; - cudaMemcpy3DParms copyParams = {0}; - // overlay image from inference - copyParams.srcPtr = make_cudaPitchedPtr(src, overlay_img_width_ * sizeof(float), - overlay_img_width_, overlay_img_height_); - copyParams.extent = make_cudaExtent(overlay_img_width_, overlay_img_height_, 1); - copyParams.kind = cudaMemcpyDeviceToDevice; - // copy overlay image layer by layer with data staying on GPU - for (int layer = 0; layer != overlay_img_layers_; ++layer) { - cuda_status = CUDA_TRY(cudaGraphicsSubResourceGetMappedArray( - &texture_ptr, cuda_overlay_img_tex_resource_, layer, 0)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to get mapped array for %d-th layer of overlay OpenGL texture", - layer); - return GXF_FAILURE; - } - // set dst array for cudaMemcpy3D, configure offset for src pointer to copy current layer to - // OpenGL texture - copyParams.dstArray = texture_ptr; - copyParams.srcPos = make_cudaPos(0, 0, layer); - - cuda_status = CUDA_TRY(cudaMemcpy3D(©Params)); - if (cuda_status) { - GXF_LOG_ERROR( - "Failed to copy %d-th layer of overlay image tensor to OpenGL texture via CUDA / " - "OpenGL interop", - layer); - return GXF_FAILURE; - } - } - // unmap the graphics resource again - cuda_status = CUDA_TRY(cudaGraphicsUnmapResources(1, &cuda_overlay_img_tex_resource_, 0)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to unmap overlay image texture via CUDA / OpenGL interop"); - return GXF_FAILURE; - } - } - } // if (messages.size() >= 2) - - // Draw Frame - // ---------------------------------------------------------------------------------- - - glViewport(0, 0, vp_width_, vp_height_); - video_frame_vis_.tick(video_frame_tex_, GL_LINEAR); - renderInferenceResults(vp_width_, vp_height_); - - // Overlay output - // ---------------------------------------------------------------------------------- - - if (overlay_buffer_input_.try_get() && overlay_buffer_output_.try_get()) { - const auto& overlay_buffer_input = overlay_buffer_input_.try_get().value()->receive(); - if (overlay_buffer_input) { - // Get the empty input buffer - const auto& overlay_buffer_in = overlay_buffer_input.value().get(); - if (overlay_buffer_in) { - auto info = overlay_buffer_in.value()->video_frame_info(); - - // Initialize the overlay rendering objects - if (overlay_render_fbo_ == 0) { - glGenFramebuffers(1, &overlay_render_fbo_); - glBindFramebuffer(GL_FRAMEBUFFER, overlay_render_fbo_); - glGenTextures(1, &overlay_render_texture_); - glBindTexture(GL_TEXTURE_2D, overlay_render_texture_); - glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGBA8, info.width, info.height); - glFramebufferTexture(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, overlay_render_texture_, 0); - - glGenFramebuffers(1, &overlay_output_fbo_); - glBindFramebuffer(GL_FRAMEBUFFER, overlay_output_fbo_); - glGenRenderbuffers(1, &overlay_output_renderbuffer_); - glBindRenderbuffer(GL_RENDERBUFFER, overlay_output_renderbuffer_); - glRenderbufferStorage(GL_RENDERBUFFER, GL_RGBA8, info.width, info.height); - glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_RENDERBUFFER, - overlay_output_renderbuffer_); - - cudaError_t cuda_status = - CUDA_TRY(cudaGraphicsGLRegisterImage(&cuda_overlay_renderbuffer_resource_, - overlay_output_renderbuffer_, - GL_RENDERBUFFER, - cudaGraphicsRegisterFlagsReadOnly)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to register overlay renderbuffer for CUDA / OpenGL Interop"); - return GXF_FAILURE; - } - - cuda_status = CUDA_TRY(cudaGraphicsMapResources(1, &cuda_overlay_renderbuffer_resource_)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to map overlay renderbuffer for CUDA / OpenGL interop"); - return GXF_FAILURE; - } - - cuda_status = CUDA_TRY(cudaGraphicsSubResourceGetMappedArray( - &cuda_overlay_array_, cuda_overlay_renderbuffer_resource_, 0, 0)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to get mapped CUDA array for overlay renderbuffer"); - return GXF_FAILURE; - } - } - - glViewport(0, 0, info.width, info.height); - - // Render the overlay to the texture - glBindFramebuffer(GL_FRAMEBUFFER, overlay_render_fbo_); - glClear(GL_COLOR_BUFFER_BIT); - renderInferenceResults(info.width, info.height); - - // Use the video renderer to vertically flip the overlay to the output renderbuffer - glBindFramebuffer(GL_FRAMEBUFFER, overlay_output_fbo_); - glClear(GL_COLOR_BUFFER_BIT); - video_frame_vis_.tick(overlay_render_texture_, GL_NEAREST); - - glBindFramebuffer(GL_FRAMEBUFFER, 0); - - // Copy the overlay to the output buffer - auto memcpyKind = (overlay_buffer_in.value()->storage_type() == - gxf::MemoryStorageType::kHost) - ? cudaMemcpyDeviceToHost : cudaMemcpyDeviceToDevice; - auto row_size = info.width * 4; - cudaError_t cuda_status = CUDA_TRY(cudaMemcpy2DFromArray( - overlay_buffer_in.value()->pointer(), - row_size, cuda_overlay_array_, 0, 0, row_size, info.height, memcpyKind)); - if (cuda_status) { - GXF_LOG_ERROR("Failed to copy overlay buffer via CUDA / OpenGL interop"); - return GXF_FAILURE; - } - - // Output the filled overlay buffer object - auto overlay_buffer_output = gxf::Entity::New(context()); - if (!overlay_buffer_output) { - GXF_LOG_ERROR("Failed to allocate overlay output; terminating."); - return GXF_FAILURE; - } - - auto overlay_buffer_out = overlay_buffer_output.value().add(); - overlay_buffer_out.value()->wrapMemory(info, - overlay_buffer_in.value()->size(), - overlay_buffer_in.value()->storage_type(), - overlay_buffer_in.value()->pointer(), nullptr); - - const auto result = overlay_buffer_output_.try_get().value()->publish( - std::move(overlay_buffer_output.value())); - if (GXF_SUCCESS != gxf::ToResultCode(result)) { - GXF_LOG_ERROR("Failed to publish overlay output buffer"); - return GXF_FAILURE; - } - } - } - } - - // swap buffers and poll IO events (keys pressed/released, mouse moved etc.) - // ------------------------------------------------------------------------------- - glfwSwapBuffers(window_); - glfwPollEvents(); - - return GXF_SUCCESS; -} - -void Sink::renderInferenceResults(int width, int height) { - // render inference results: Overlay Image, Tooltip, Tool Names - glEnable(GL_BLEND); - - if (enable_overlay_img_vis_) { overlay_img_vis.tick(); } - - if (enable_tool_tip_vis_) { tooltip_vis_.tick(); } - - glDisable(GL_BLEND); - - if (enable_tool_labels_) { - label_vis_.tick(width, height); - } -} - -void Sink::onKeyCallback(int key, int scancode, int action, int mods) { - if (action != GLFW_RELEASE) { return; } - - switch (key) { - default: - break; - case GLFW_KEY_1: - enable_tool_tip_vis_ = !enable_tool_tip_vis_; - break; - case GLFW_KEY_2: - enable_tool_labels_ = !enable_tool_labels_; - break; - case GLFW_KEY_3: - enable_overlay_img_vis_ = !enable_overlay_img_vis_; - break; - } -} - -void Sink::onFramebufferSizeCallback(int width, int height) { - if (width == vp_width_ && height == vp_height_) { return; } - - vp_width_ = width; - vp_height_ = height; -} - -} // namespace visualizer_tool_tracking -} // namespace holoscan -} // namespace nvidia diff --git a/gxf_extensions/visualizer_tool_tracking/visualizer.hpp b/gxf_extensions/visualizer_tool_tracking/visualizer.hpp deleted file mode 100644 index b3ca7b53..00000000 --- a/gxf_extensions/visualizer_tool_tracking/visualizer.hpp +++ /dev/null @@ -1,164 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef NVIDIA_CLARA_HOLOSCAN_GXF_VISUALIZER_HPP_ -#define NVIDIA_CLARA_HOLOSCAN_GXF_VISUALIZER_HPP_ -// clang-format off -#define GLFW_INCLUDE_NONE 1 -#include -#include // NOLINT(build/include_order) -// clang-format on - -#include - -#include -#include - -#include "gxf/std/codelet.hpp" -#include "gxf/std/memory_buffer.hpp" -#include "gxf/std/parameter_parser_std.hpp" -#include "gxf/std/receiver.hpp" -#include "gxf/std/scheduling_terms.hpp" - -#include "frame_data.hpp" -#include "instrument_label.hpp" -#include "instrument_tip.hpp" -#include "overlay_img_vis.hpp" -#include "video_frame.hpp" - -struct cudaGraphicsResource; - -namespace nvidia { -namespace holoscan { -namespace visualizer_tool_tracking { - -/// @brief Visualization Codelet for Instrument Tracking and Overlay -/// -/// This visualizer uses OpenGL/CUDA interopt for quickly passing data from the output of inference -/// to an OpenGL context for rendering. -/// The visualizer renders the location and text of an instrument and optionally displays the -/// model's confidence score. -class Sink : public gxf::Codelet { - public: - Sink(); - - gxf_result_t start() override; - gxf_result_t tick() override; - gxf_result_t stop() override; - - gxf_result_t registerInterface(gxf::Registrar* registrar) override; - - void onKeyCallback(int key, int scancode, int action, int mods); - void onFramebufferSizeCallback(int width, int height); - - void renderInferenceResults(int width, int height); - - private: - // GLFW members and callback funds - GLFWwindow* window_ = nullptr; - // GL viewport - int vp_width_ = 0; - int vp_height_ = 0; - FrameData frame_data_; - - cudaGraphicsResource* cuda_confidence_resource_ = nullptr; - cudaGraphicsResource* cuda_position_resource_ = nullptr; - - // Videoframe Vis related members - // -------------------------------------------------------------------- - - bool use_cuda_opengl_interop_ = true; - GLuint video_frame_tex_ = 0; - VideoFrame video_frame_vis_; - cudaGraphicsResource* cuda_video_frame_tex_resource_ = nullptr; - std::vector video_frame_buffer_host_; - - gxf::Parameter in_width_; - gxf::Parameter in_height_; - gxf::Parameter in_channels_; - gxf::Parameter in_bytes_per_pixel_; - gxf::Parameter alpha_value_; - - gxf::Parameter videoframe_vertex_shader_path_; - gxf::Parameter videoframe_fragment_shader_path_; - - // Tooltip Vis related members - // -------------------------------------------------------------------- - - - bool enable_tool_tip_vis_ = true; - InstrumentTip tooltip_vis_; - gxf::Parameter tooltip_vertex_shader_path_; - gxf::Parameter tooltip_fragment_shader_path_; - // Those two below apply to Tool Name (instrument labels) also - gxf::Parameter num_tool_classes_; - gxf::Parameter num_tool_pos_components_; - gxf::Parameter>> tool_tip_colors_; - - // Tool Name Vis related members - // -------------------------------------------------------------------- - - bool enable_tool_labels_ = true; - InstrumentLabel label_vis_; - gxf::Parameter> tool_labels_; - gxf::Parameter label_sans_font_path_; - gxf::Parameter label_sans_bold_font_path_; - - // Overlay Img Vis related members - // -------------------------------------------------------------------- - - bool enable_overlay_img_vis_ = true; - - gxf::Parameter overlay_img_width_; - gxf::Parameter overlay_img_height_; - gxf::Parameter overlay_img_layers_; - gxf::Parameter overlay_img_channels_; - gxf::Parameter>> overlay_img_colors_; - - GLuint overlay_img_tex_ = 0; - std::vector overlay_img_buffer_host_; - std::vector overlay_img_layered_host_; - OverlayImageVis overlay_img_vis; - - cudaGraphicsResource* cuda_overlay_img_tex_resource_ = nullptr; - gxf::Parameter overlay_img_vertex_shader_path_; - gxf::Parameter overlay_img_fragment_shader_path_; - - // Overlay output - // -------------------------------------------------------------------- - - gxf::Parameter> overlay_buffer_input_; - gxf::Parameter> overlay_buffer_output_; - GLuint overlay_render_fbo_ = 0; - GLuint overlay_render_texture_ = 0; - GLuint overlay_output_fbo_ = 0; - GLuint overlay_output_renderbuffer_ = 0; - cudaGraphicsResource* cuda_overlay_renderbuffer_resource_ = nullptr; - cudaArray* cuda_overlay_array_ = nullptr; - - // GFX Parameters - // -------------------------------------------------------------------- - - gxf::Parameter>> in_; - gxf::Parameter> in_tensor_names_; - gxf::Parameter> pool_; - gxf::Parameter> window_close_scheduling_term_; -}; - -} // namespace visualizer_tool_tracking -} // namespace holoscan -} // namespace nvidia -#endif // NVIDIA_CLARA_HOLOSCAN_GXF_VISUALIZER_HPP_ diff --git a/gxf_extensions/visualizer_tool_tracking/visualizer_ext.cpp b/gxf_extensions/visualizer_tool_tracking/visualizer_ext.cpp deleted file mode 100644 index 7c396a36..00000000 --- a/gxf_extensions/visualizer_tool_tracking/visualizer_ext.cpp +++ /dev/null @@ -1,28 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "gxf/std/extension_factory_helper.hpp" - -#include "visualizer.hpp" - -GXF_EXT_FACTORY_BEGIN() -GXF_EXT_FACTORY_SET_INFO(0xd3b5b6f7291e42bc, 0x85d4a2b01327913a, "VisualizerToolTrackingExtension", - "Holoscan Surgical Tool Tracking Visualizer extension", "NVIDIA", "0.2.0", - "LICENSE"); -GXF_EXT_FACTORY_ADD(0xab207890cc6b4391, 0x88e8d1dc47f3ae11, - nvidia::holoscan::visualizer_tool_tracking::Sink, nvidia::gxf::Codelet, - "Surgical Tool Tracking Viz codelet."); -GXF_EXT_FACTORY_END() diff --git a/include/holoscan/core/arg.hpp b/include/holoscan/core/arg.hpp index b6daaad3..b134e644 100644 --- a/include/holoscan/core/arg.hpp +++ b/include/holoscan/core/arg.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -43,7 +43,7 @@ namespace holoscan { enum class ArgElementType { kCustom, ///< Custom type kBoolean, ///< Boolean type (bool) - kInt8, ///< 8-bit integer type (uint8_t) + kInt8, ///< 8-bit integer type (int8_t) kUnsigned8, ///< 8-bit unsigned integer type (uint8_t) kInt16, ///< 16-bit integer type (int16_t) kUnsigned16, ///< Unsigned 16-bit integer (uint16_t) @@ -81,9 +81,10 @@ class ArgType { * * @param element_type The element type of the argument. * @param container_type The container type of the argument. + * @param dimension The dimension of the argument. */ - ArgType(ArgElementType element_type, ArgContainerType container_type) - : element_type_(element_type), container_type_(container_type) {} + ArgType(ArgElementType element_type, ArgContainerType container_type, int32_t dimension = 0) + : element_type_(element_type), container_type_(container_type), dimension_(dimension) {} /** * @brief Get the element type of the argument. @@ -114,10 +115,12 @@ class ArgType { return ArgType(get_element_type(index), ArgContainerType::kNative); } else if constexpr (holoscan::is_vector_v>) { auto elem_index = std::type_index(typeid(typename holoscan::type_info::element_type)); - return ArgType(get_element_type(elem_index), ArgContainerType::kVector); + return ArgType( + get_element_type(elem_index), ArgContainerType::kVector, holoscan::dimension_of_v); } else if constexpr (holoscan::is_array_v>) { auto elem_index = std::type_index(typeid(typename holoscan::type_info::element_type)); - return ArgType(get_element_type(elem_index), ArgContainerType::kArray); + return ArgType( + get_element_type(elem_index), ArgContainerType::kArray, holoscan::dimension_of_v); } else { HOLOSCAN_LOG_ERROR("No element type for '{}' exists", typeid(std::decay_t).name()); return ArgType(ArgElementType::kCustom, ArgContainerType::kNative); @@ -130,12 +133,26 @@ class ArgType { * @return The element type of the argument. */ ArgElementType element_type() const { return element_type_; } + /** * @brief Get the container type of the argument. * * @return The container type of the argument. */ ArgContainerType container_type() const { return container_type_; } + /** + * @brief Get the dimension of the argument. + * + * @return The dimension of the argument. + */ + int32_t dimension() const { return dimension_; } + + /** + * @brief Get a string representation of the argument type. + * + * @return String representation of the argument type. + */ + std::string to_string() const; private: template @@ -162,8 +179,29 @@ class ArgType { to_element_type_pair>(ArgElementType::kCondition), to_element_type_pair>(ArgElementType::kResource), }; + inline static const std::unordered_map element_type_name_map_{ + {ArgElementType::kCustom, "CustomType"}, + {ArgElementType::kBoolean, "bool"}, + {ArgElementType::kInt8, "int8_t"}, + {ArgElementType::kUnsigned8, "uint8_t"}, + {ArgElementType::kInt16, "int16_t"}, + {ArgElementType::kUnsigned16, "uint16_t"}, + {ArgElementType::kInt32, "int32_t"}, + {ArgElementType::kUnsigned32, "uint32_t"}, + {ArgElementType::kInt64, "int64_t"}, + {ArgElementType::kUnsigned64, "uint64_t"}, + {ArgElementType::kFloat32, "float"}, + {ArgElementType::kFloat64, "double"}, + {ArgElementType::kString, "std::string"}, + {ArgElementType::kHandle, "std::any"}, + {ArgElementType::kYAMLNode, "YAML::Node"}, + {ArgElementType::kIOSpec, "holoscan::IOSpec*"}, + {ArgElementType::kCondition, "std::shared_ptr"}, + {ArgElementType::kResource, "std::shared_ptr"}, + }; ArgElementType element_type_ = ArgElementType::kCustom; ArgContainerType container_type_ = ArgContainerType::kNative; + int32_t dimension_ = 0; }; /** @@ -256,6 +294,21 @@ class Arg { */ std::any& value() { return value_; } + /** + * @brief Get a YAML representation of the argument. + * + * @return YAML node including the name, type, and value of the argument. + */ + YAML::Node to_yaml_node() const; + + /** + * @brief Get a description of the argument. + * + * @see to_yaml_node() + * @return YAML string. + */ + std::string description() const; + private: std::string name_; ///< The name of the argument. ArgType arg_type_; ///< The type of the argument. @@ -470,6 +523,21 @@ class ArgList { */ const std::string& name() const { return name_; } + /** + * @brief Get a YAML representation of the argument list. + * + * @return YAML node including the name, and arguments of the argument list. + */ + YAML::Node to_yaml_node() const; + + /** + * @brief Get a description of the argument list. + * + * @see to_yaml_node() + * @return YAML string. + */ + std::string description() const; + private: std::string name_{"arglist"}; ///< The name of the argument list. std::vector args_; ///< The vector of arguments. diff --git a/include/holoscan/core/argument_setter.hpp b/include/holoscan/core/argument_setter.hpp index 184c4de5..9f42f51e 100644 --- a/include/holoscan/core/argument_setter.hpp +++ b/include/holoscan/core/argument_setter.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,9 +30,11 @@ #include #include -#include "./common.hpp" +#include + #include "../utils/yaml_parser.hpp" #include "./arg.hpp" +#include "./common.hpp" #include "./condition.hpp" #include "./parameter.hpp" #include "./resource.hpp" @@ -141,12 +143,19 @@ class ArgumentSetter { function_map_.try_emplace( std::type_index(typeid(typeT)), [](ParameterWrapper& param_wrap, Arg& arg) { std::any& any_param = param_wrap.value(); - std::any& any_arg = arg.value(); - // Note that the type of any_param is Parameter*, not Parameter. auto& param = *std::any_cast*>(any_param); + + // If arg has no name and value, that indicates that we want to set the default value for + // the native operator if it is not specified. + if (arg.name().empty() && !arg.has_value()) { + auto& param = *std::any_cast*>(any_param); + param.set_default_value(); + return; + } + + std::any& any_arg = arg.value(); const auto& arg_type = arg.arg_type(); - (void)param; auto element_type = arg_type.element_type(); auto container_type = arg_type.container_type(); @@ -155,17 +164,67 @@ class ArgumentSetter { switch (container_type) { case ArgContainerType::kNative: { switch (element_type) { + // Handle the argument with 'kInt64' type differently because the argument might + // come from Python, and Python only has 'int' type ('int64_t' in C++). + case ArgElementType::kInt64: { + if constexpr (holoscan::is_one_of_v) { + auto& arg_value = std::any_cast(any_arg); + param = static_cast(arg_value); + } else { + HOLOSCAN_LOG_ERROR( + "Unable to convert argument type '{}' to parameter type '{}' for '{}'", + any_arg.type().name(), + typeid(typeT).name(), + arg.name()); + } + break; + } + // Handle the argument with 'kFloat64' type differently because the argument might + // come from Python, and Python only has 'float' type ('double' in C++). + case ArgElementType::kFloat64: { + if constexpr (holoscan::is_one_of_v) { + auto& arg_value = std::any_cast(any_arg); + param = static_cast(arg_value); + } else { + HOLOSCAN_LOG_ERROR( + "Unable to convert argument type '{}' to parameter type '{}' for '{}'", + any_arg.type().name(), + typeid(typeT).name(), + arg.name()); + } + break; + } case ArgElementType::kBoolean: case ArgElementType::kInt8: case ArgElementType::kInt16: case ArgElementType::kInt32: - case ArgElementType::kInt64: case ArgElementType::kUnsigned8: case ArgElementType::kUnsigned16: case ArgElementType::kUnsigned32: case ArgElementType::kUnsigned64: case ArgElementType::kFloat32: - case ArgElementType::kFloat64: case ArgElementType::kString: case ArgElementType::kIOSpec: { if constexpr (holoscan::is_one_of_v void param(Parameter& parameter, const char* key, const char* headline, const char* description); - /** * @brief Define a parameter that has a default value. * @@ -111,6 +117,21 @@ class ComponentSpec { */ std::unordered_map& params() { return params_; } + /** + * @brief Get a YAML representation of the component spec. + * + * @return YAML node including the parameters of this component. + */ + virtual YAML::Node to_yaml_node() const; + + /** + * @brief Get a description of the component spec. + * + * @see to_yaml_node() + * @return YAML string. + */ + std::string description() const; + protected: Fragment* fragment_ = nullptr; ///< The pointer to the fragment that contains this component. std::unordered_map params_; ///< The parameters of this component. diff --git a/include/holoscan/core/condition.hpp b/include/holoscan/core/condition.hpp index 23e3b35c..a2d41504 100644 --- a/include/holoscan/core/condition.hpp +++ b/include/holoscan/core/condition.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -169,7 +169,7 @@ class Condition : public Component { * @param spec The component specification. * @return The reference to the condition. */ - Condition& spec(std::shared_ptr spec) { + Condition& spec(const std::shared_ptr& spec) { spec_ = spec; return *this; } @@ -180,6 +180,13 @@ class Condition : public Component { */ ComponentSpec* spec() { return spec_.get(); } + /** + * @brief Get the shared pointer to the component spec. + * + * @return The shared pointer to the component spec. + */ + std::shared_ptr spec_shared() { return spec_; } + using Component::add_arg; /** @@ -189,6 +196,13 @@ class Condition : public Component { */ virtual void setup(ComponentSpec& spec) { (void)spec; } + /** + * @brief Get a YAML representation of the condition. + * + * @return YAML node including spec of the condition in addition to the base component properties. + */ + YAML::Node to_yaml_node() const override; + protected: std::shared_ptr spec_; ///< The component specification. }; diff --git a/include/holoscan/core/conditions/gxf/boolean.hpp b/include/holoscan/core/conditions/gxf/boolean.hpp index 838449f9..27782e50 100644 --- a/include/holoscan/core/conditions/gxf/boolean.hpp +++ b/include/holoscan/core/conditions/gxf/boolean.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,6 +18,8 @@ #ifndef HOLOSCAN_CORE_CONDITIONS_GXF_BOOLEAN_HPP #define HOLOSCAN_CORE_CONDITIONS_GXF_BOOLEAN_HPP +#include + #include "../../gxf/gxf_condition.hpp" namespace holoscan { @@ -27,6 +29,7 @@ class BooleanCondition : public gxf::GXFCondition { HOLOSCAN_CONDITION_FORWARD_ARGS_SUPER(BooleanCondition, GXFCondition) explicit BooleanCondition(bool enable_tick = true) : enable_tick_(enable_tick) {} + BooleanCondition(const std::string& name, nvidia::gxf::BooleanSchedulingTerm* term); const char* gxf_typename() const override { return "nvidia::gxf::BooleanSchedulingTerm"; } diff --git a/include/holoscan/core/executor.hpp b/include/holoscan/core/executor.hpp index 35925832..900a41ac 100644 --- a/include/holoscan/core/executor.hpp +++ b/include/holoscan/core/executor.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,6 +26,7 @@ #include #include "./common.hpp" +#include "./extension_manager.hpp" #include "./operator.hpp" namespace holoscan { @@ -52,6 +53,13 @@ class Executor { */ virtual void run(Graph& graph) { (void)graph; } + /** + * @brief Set the pointer to the fragment of the executor. + * + * @param fragment The pointer to the fragment of the executor. + */ + void fragment(Fragment* fragment) { fragment_ = fragment; } + /** * @brief Get a pointer to Fragment object. * @@ -64,7 +72,7 @@ class Executor { * * @param context The context. */ - void context(void* context) { context_ = context; } + virtual void context(void* context) { context_ = context; } /** * @brief Get the context * @@ -76,6 +84,12 @@ class Executor { void context_uint64(uint64_t context) { context_ = reinterpret_cast(context); } uint64_t context_uint64() { return reinterpret_cast(context_); } + /** + * @brief Get the extension manager. + * @return The shared pointer of the extension manager. + */ + virtual std::shared_ptr extension_manager() { return extension_manager_; } + protected: friend class Fragment; // make Fragment a friend class to access protected members of Executor // (add_receivers()). @@ -127,6 +141,7 @@ class Executor { Fragment* fragment_ = nullptr; ///< The fragment of the executor. void* context_ = nullptr; ///< The context. + std::shared_ptr extension_manager_; ///< The extension manager. }; } // namespace holoscan diff --git a/include/holoscan/core/executors/gxf/gxf_executor.hpp b/include/holoscan/core/executors/gxf/gxf_executor.hpp index 6f8282b4..d06a32cc 100644 --- a/include/holoscan/core/executors/gxf/gxf_executor.hpp +++ b/include/holoscan/core/executors/gxf/gxf_executor.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,13 +20,15 @@ #include -#include +#include #include #include +#include #include -#include #include "../../executor.hpp" +#include "../../gxf/gxf_extension_manager.hpp" + namespace holoscan::gxf { /** @@ -35,7 +37,7 @@ namespace holoscan::gxf { class GXFExecutor : public holoscan::Executor { public: GXFExecutor() = delete; - explicit GXFExecutor(holoscan::Fragment* app); + explicit GXFExecutor(holoscan::Fragment* app, bool create_gxf_context = true); ~GXFExecutor() override; @@ -46,6 +48,26 @@ class GXFExecutor : public holoscan::Executor { */ void run(Graph& graph) override; + /** + * @brief Set the context. + * + * For GXF, GXFExtensionManager(gxf_extension_manager_) is initialized with the context. + * + * @param context The context. + */ + void context(void* context) override; + + // Inherit Executor::context(). + using Executor::context; + + /** + * @brief Get GXF extension manager. + * + * @return The GXF extension manager. + * @see GXFExtensionManager + */ + std::shared_ptr extension_manager() override; + /** * @brief Create and setup GXF components for input port. * @@ -64,9 +86,11 @@ class GXFExecutor : public holoscan::Executor { * @param gxf_context The GXF context. * @param eid The GXF entity ID. * @param io_spec The input port specification. + * @param bind_port If true, bind the port to the existing GXF Receiver component. Otherwise, + * create a new GXF Receiver component. */ static void create_input_port(Fragment* fragment, gxf_context_t gxf_context, gxf_uid_t eid, - IOSpec* io_spec); + IOSpec* io_spec, bool bind_port = false); /** * @brief Create and setup GXF components for output port. @@ -86,9 +110,35 @@ class GXFExecutor : public holoscan::Executor { * @param gxf_context The GXF context. * @param eid The GXF entity ID. * @param io_spec The output port specification. + * @param bind_port If true, bind the port to the existing GXF Transmitter component. Otherwise, + * create a new GXF Transmitter component. */ static void create_output_port(Fragment* fragment, gxf_context_t gxf_context, gxf_uid_t eid, - IOSpec* io_spec); + IOSpec* io_spec, bool bind_port = false); + + /** + * @brief Set the GXF entity ID of the operator initialized by this executor. + * + * If this is 0, a new entity is created for the operator. + * Otherwise, the operator as a codelet will be added to the existing entity specified by this ID. + * This is useful when initializing operators inside the existing entity. + * (e.g., when initializing an operator from `holoscan::gxf::OperatorWrapper` class) + * + * @param eid The GXF entity ID. + */ + void op_eid(gxf_uid_t eid) { op_eid_ = eid; } + + /** + * @brief Set the GXF component ID of the operator initialized by this executor. + * + * If this is 0, a new component is created for the operator. + * This is useful when initializing operators using the existing component inside the existing + * entity. + * (e.g., when initializing an operator from `holoscan::gxf::OperatorWrapper` class) + * + * @param cid The GXF component ID. + */ + void op_cid(gxf_uid_t cid) { op_cid_ = cid; } protected: bool initialize_operator(Operator* op) override; @@ -98,6 +148,12 @@ class GXFExecutor : public holoscan::Executor { private: void register_extensions(); + bool own_gxf_context_ = false; ///< Whether this executor owns the GXF context. + gxf_uid_t op_eid_ = 0; ///< The GXF entity ID of the operator. Create new entity for + ///< initializing a new operator if this is 0. + gxf_uid_t op_cid_ = 0; ///< The GXF component ID of the operator. Create new component for + ///< initializing a new operator if this is 0. + std::shared_ptr gxf_extension_manager_; ///< The GXF extension manager. }; } // namespace holoscan::gxf diff --git a/include/holoscan/core/executors/gxf/gxf_parameter_adaptor.hpp b/include/holoscan/core/executors/gxf/gxf_parameter_adaptor.hpp index 93fae2e2..fa2a68e6 100644 --- a/include/holoscan/core/executors/gxf/gxf_parameter_adaptor.hpp +++ b/include/holoscan/core/executors/gxf/gxf_parameter_adaptor.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -27,8 +27,8 @@ #include #include -#include "../../common.hpp" #include "../../arg.hpp" +#include "../../common.hpp" #include "../../gxf/gxf_condition.hpp" #include "../../gxf/gxf_resource.hpp" #include "../../gxf/gxf_utils.hpp" @@ -102,386 +102,370 @@ class GXFParameterAdaptor { template void add_param_handler() { - function_map_.try_emplace( - std::type_index(typeid(typeT)), - [](gxf_context_t context, - gxf_uid_t uid, - const char* key, - const ArgType& arg_type, - const std::any& any_value) { - (void)context; - (void)uid; - (void)key; - (void)arg_type; - (void)any_value; - try { - auto& param = *std::any_cast*>(any_value); + const AdaptFunc& func = [](gxf_context_t context, + gxf_uid_t uid, + const char* key, + const ArgType& arg_type, + const std::any& any_value) { + (void)context; // avoid `-Werror=unused-but-set-parameter` due to `constexpr` + (void)uid; // avoid `-Werror=unused-but-set-parameter` due to `constexpr` + try { + auto& param = *std::any_cast*>(any_value); - param.set_default_value(); // set default value if not set. + param.set_default_value(); // set default value if not set. - // If the parameter (any_value) is from the native operator - // (which means 'uid == -1'), return here without setting GXF parameter. - if (uid == -1) { return GXF_SUCCESS; } + if (!param.has_value()) { + HOLOSCAN_LOG_WARN( + "Unable to get argument for key '{}' with type '{}'", key, typeid(typeT).name()); + return GXF_FAILURE; + } - if (param.has_value()) { - auto& value = param.get(); - switch (arg_type.container_type()) { - case ArgContainerType::kNative: { - switch (arg_type.element_type()) { - case ArgElementType::kBoolean: { - if constexpr (std::is_same_v) { - return GxfParameterSetBool(context, uid, key, value); - } - break; - } - case ArgElementType::kInt8: { - HOLOSCAN_LOG_ERROR("GXF does not support int8_t parameter for key '{}'", key); - return GXF_FAILURE; - } - case ArgElementType::kUnsigned8: { - // GXF Doesn't support uint8_t parameter so use a workaround with - // GxfParameterSetFromYamlNode. - if constexpr (std::is_same_v) { - YAML::Node yaml_node; - // uint8_t is not supported natively by yaml-cpp so push it as a uint32_t - // so that GXF can handle it. - yaml_node.push_back(static_cast(value)); - YAML::Node value_node = yaml_node[0]; - return GxfParameterSetFromYamlNode(context, uid, key, &value_node, ""); - } - break; - } - case ArgElementType::kInt16: { - // GXF Doesn't support int16_t parameter so use a workaround with - // GxfParameterSetFromYamlNode. - if constexpr (std::is_same_v) { - YAML::Node yaml_node; - yaml_node.push_back(value); - YAML::Node value_node = yaml_node[0]; - return GxfParameterSetFromYamlNode(context, uid, key, &value_node, ""); - } - break; - } - case ArgElementType::kUnsigned16: { - if constexpr (std::is_same_v) { - return GxfParameterSetUInt16(context, uid, key, value); - } - break; - } - case ArgElementType::kInt32: { - if constexpr (std::is_same_v) { - return GxfParameterSetInt32(context, uid, key, value); - } - break; - } - case ArgElementType::kUnsigned32: { - if constexpr (std::is_same_v) { - return GxfParameterSetUInt32(context, uid, key, value); - } - break; - } - case ArgElementType::kInt64: { - if constexpr (std::is_same_v) { - return GxfParameterSetInt64(context, uid, key, value); - } - break; - } - case ArgElementType::kUnsigned64: { - if constexpr (std::is_same_v) { - return GxfParameterSetUInt64(context, uid, key, value); - } - break; - } - case ArgElementType::kFloat32: { - // GXF Doesn't support float parameter so use a workaround with - // GxfParameterSetFromYamlNode. - if constexpr (std::is_same_v) { - YAML::Node yaml_node; - yaml_node.push_back(value); - YAML::Node value_node = yaml_node[0]; - return GxfParameterSetFromYamlNode(context, uid, key, &value_node, ""); - } - break; - } - case ArgElementType::kFloat64: { - if constexpr (std::is_same_v) { - return GxfParameterSetFloat64(context, uid, key, value); - } - break; - } - case ArgElementType::kString: { - if constexpr (std::is_same_v) { - return GxfParameterSetStr(context, uid, key, value.c_str()); - } - break; - } - case ArgElementType::kHandle: { - HOLOSCAN_LOG_ERROR("Unable to set handle parameter for key '{}'", key); - return GXF_FAILURE; - } - case ArgElementType::kYAMLNode: { - HOLOSCAN_LOG_ERROR("Unable to handle ArgElementType::kYAMLNode for key '{}'", - key); - return GXF_FAILURE; - } - case ArgElementType::kIOSpec: { - if constexpr (std::is_same_v) { - if (value) { - auto gxf_resource = - std::dynamic_pointer_cast(value->resource()); - gxf_uid_t cid = gxf_resource->gxf_cid(); + auto& value = param.get(); + switch (arg_type.container_type()) { + case ArgContainerType::kNative: { + switch (arg_type.element_type()) { + case ArgElementType::kBoolean: { + if constexpr (std::is_same_v) { + return GxfParameterSetBool(context, uid, key, value); + } + break; + } + case ArgElementType::kInt8: { + HOLOSCAN_LOG_ERROR("GXF does not support int8_t parameter for key '{}'", key); + return GXF_FAILURE; + } + case ArgElementType::kUnsigned8: { + // GXF Doesn't support uint8_t parameter so use a workaround with + // GxfParameterSetFromYamlNode. + if constexpr (std::is_same_v) { + YAML::Node yaml_node; + // uint8_t is not supported natively by yaml-cpp so push it as a uint32_t + // so that GXF can handle it. + yaml_node.push_back(static_cast(value)); + YAML::Node value_node = yaml_node[0]; + return GxfParameterSetFromYamlNode(context, uid, key, &value_node, ""); + } + break; + } + case ArgElementType::kInt16: { + // GXF Doesn't support int16_t parameter so use a workaround with + // GxfParameterSetFromYamlNode. + if constexpr (std::is_same_v) { + YAML::Node yaml_node; + yaml_node.push_back(value); + YAML::Node value_node = yaml_node[0]; + return GxfParameterSetFromYamlNode(context, uid, key, &value_node, ""); + } + break; + } + case ArgElementType::kUnsigned16: { + if constexpr (std::is_same_v) { + return GxfParameterSetUInt16(context, uid, key, value); + } + break; + } + case ArgElementType::kInt32: { + if constexpr (std::is_same_v) { + return GxfParameterSetInt32(context, uid, key, value); + } + break; + } + case ArgElementType::kUnsigned32: { + if constexpr (std::is_same_v) { + return GxfParameterSetUInt32(context, uid, key, value); + } + break; + } + case ArgElementType::kInt64: { + if constexpr (std::is_same_v) { + return GxfParameterSetInt64(context, uid, key, value); + } + break; + } + case ArgElementType::kUnsigned64: { + if constexpr (std::is_same_v) { + return GxfParameterSetUInt64(context, uid, key, value); + } + break; + } + case ArgElementType::kFloat32: { + // GXF Doesn't support float parameter so use a workaround with + // GxfParameterSetFromYamlNode. + if constexpr (std::is_same_v) { + YAML::Node yaml_node; + yaml_node.push_back(value); + YAML::Node value_node = yaml_node[0]; + return GxfParameterSetFromYamlNode(context, uid, key, &value_node, ""); + } + break; + } + case ArgElementType::kFloat64: { + if constexpr (std::is_same_v) { + return GxfParameterSetFloat64(context, uid, key, value); + } + break; + } + case ArgElementType::kString: { + if constexpr (std::is_same_v) { + return GxfParameterSetStr(context, uid, key, value.c_str()); + } + break; + } + case ArgElementType::kHandle: { + HOLOSCAN_LOG_ERROR("Unable to set handle parameter for key '{}'", key); + return GXF_FAILURE; + } + case ArgElementType::kYAMLNode: { + if constexpr (std::is_same_v) { + return GxfParameterSetFromYamlNode(context, uid, key, &value, ""); + } else { + HOLOSCAN_LOG_ERROR("Unable to handle ArgElementType::kYAMLNode for key '{}'", + key); + return GXF_FAILURE; + } + } + case ArgElementType::kIOSpec: { + if constexpr (std::is_same_v) { + if (value) { + auto gxf_resource = std::dynamic_pointer_cast(value->resource()); + gxf_uid_t cid = gxf_resource->gxf_cid(); - return GxfParameterSetHandle(context, uid, key, cid); - } else { - // If the IOSpec is null, do not set the parameter. - return GXF_SUCCESS; - } - } - break; - } - case ArgElementType::kResource: { - if constexpr (std::is_same_v< - typename holoscan::type_info::element_type, - std::shared_ptr> && - holoscan::type_info::dimension == 0) { - // Set the handle parameter only if the resource is valid. - if (value) { - auto gxf_resource = std::dynamic_pointer_cast(value); - // Initialize GXF component if it is not already initialized. - if (gxf_resource->gxf_context() == nullptr) { - gxf_resource->gxf_eid(gxf::get_component_eid( - context, uid)); // set Entity ID of the component + return GxfParameterSetHandle(context, uid, key, cid); + } else { + // If the IOSpec is null, do not set the parameter. + return GXF_SUCCESS; + } + } + break; + } + case ArgElementType::kResource: { + if constexpr (std::is_same_v::element_type, + std::shared_ptr> && + holoscan::type_info::dimension == 0) { + // Set the handle parameter only if the resource is valid. + if (value) { + auto gxf_resource = std::dynamic_pointer_cast(value); + // Initialize GXF component if it is not already initialized. + if (gxf_resource->gxf_context() == nullptr) { + gxf_resource->gxf_eid( + gxf::get_component_eid(context, uid)); // set Entity ID of the component - gxf_resource->initialize(); - } - return GxfParameterSetHandle(context, uid, key, gxf_resource->gxf_cid()); - } else { - HOLOSCAN_LOG_TRACE( - "Resource is null for key '{}'. Not setting parameter.", key); - return GXF_SUCCESS; - } - } - HOLOSCAN_LOG_ERROR("Unable to handle ArgElementType::kResource for key '{}'", - key); - break; + gxf_resource->initialize(); } - case ArgElementType::kCondition: { - if constexpr (std::is_same_v< - typename holoscan::type_info::element_type, - std::shared_ptr> && - holoscan::type_info::dimension == 0) { - auto gxf_condition = std::dynamic_pointer_cast(value); - if (value) { - // Initialize GXF component if it is not already initialized. - if (gxf_condition->gxf_context() == nullptr) { - gxf_condition->gxf_eid(gxf::get_component_eid( - context, uid)); // set Entity ID of the component + return GxfParameterSetHandle(context, uid, key, gxf_resource->gxf_cid()); + } else { + HOLOSCAN_LOG_TRACE("Resource is null for key '{}'. Not setting parameter.", + key); + return GXF_SUCCESS; + } + } + HOLOSCAN_LOG_ERROR("Unable to handle ArgElementType::kResource for key '{}'", key); + break; + } + case ArgElementType::kCondition: { + if constexpr (std::is_same_v::element_type, + std::shared_ptr> && + holoscan::type_info::dimension == 0) { + auto gxf_condition = std::dynamic_pointer_cast(value); + if (value) { + // Initialize GXF component if it is not already initialized. + if (gxf_condition->gxf_context() == nullptr) { + gxf_condition->gxf_eid( + gxf::get_component_eid(context, uid)); // set Entity ID of the component - gxf_condition->initialize(); - } - return GxfParameterSetHandle(context, uid, key, gxf_condition->gxf_cid()); - } - HOLOSCAN_LOG_ERROR( - "Unable to handle ArgElementType::kCondition for key '{}'", key); - } - break; - } - case ArgElementType::kCustom: { - HOLOSCAN_LOG_ERROR("Unable to handle ArgElementType::kCustom for key '{}'", - key); - return GXF_FAILURE; + gxf_condition->initialize(); } + return GxfParameterSetHandle(context, uid, key, gxf_condition->gxf_cid()); } - break; + HOLOSCAN_LOG_ERROR("Unable to handle ArgElementType::kCondition for key '{}'", + key); } - case ArgContainerType::kVector: { - switch (arg_type.element_type()) { - case ArgElementType::kInt8: { - HOLOSCAN_LOG_ERROR( - "GXF does not support std::vector parameter " - "for key '{}'", - key); - return GXF_FAILURE; - } - case ArgElementType::kUnsigned8: { - // GXF Doesn't support std::vector parameter so use a workaround with - // GxfParameterSetFromYamlNode. - if constexpr (std::is_same_v>) { - // Create vector of Handles - YAML::Node yaml_node; - for (auto& item : value) { - // uint8_t is not supported natively by yaml-cpp so push it as a uint32_t - // so that GXF can handle it. - yaml_node.push_back(static_cast(item)); - } - return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); - } else if constexpr (std::is_same_v>>) { - YAML::Node yaml_node; - for (const std::vector& vec : value) { - for (uint32_t item : vec) { yaml_node.push_back(item); } - } - return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); - } - break; + break; + } + case ArgElementType::kCustom: { + HOLOSCAN_LOG_ERROR("Unable to handle ArgElementType::kCustom for key '{}'", key); + return GXF_FAILURE; + } + } + break; + } + case ArgContainerType::kVector: { + switch (arg_type.element_type()) { + case ArgElementType::kInt8: { + HOLOSCAN_LOG_ERROR( + "GXF does not support std::vector parameter " + "for key '{}'", + key); + return GXF_FAILURE; + } + case ArgElementType::kUnsigned8: { + // GXF Doesn't support std::vector parameter so use a workaround with + // GxfParameterSetFromYamlNode. + if constexpr (std::is_same_v>) { + // Create vector of Handles + YAML::Node yaml_node; + for (auto& item : value) { + // uint8_t is not supported natively by yaml-cpp so push it as a uint32_t + // so that GXF can handle it. + yaml_node.push_back(static_cast(item)); + } + return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); + } else if constexpr (std::is_same_v>>) { + YAML::Node yaml_node; + for (const std::vector& vec : value) { + for (uint32_t item : vec) { yaml_node.push_back(item); } + } + return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); + } + break; + } + case ArgElementType::kBoolean: + case ArgElementType::kInt16: + case ArgElementType::kUnsigned16: + case ArgElementType::kInt32: + case ArgElementType::kUnsigned32: + case ArgElementType::kInt64: + case ArgElementType::kUnsigned64: + case ArgElementType::kFloat32: + case ArgElementType::kFloat64: + case ArgElementType::kString: { + // GXF Doesn't support std::vector parameter so use a workaround with + // GxfParameterSetFromYamlNode. + if constexpr (holoscan::is_one_of_v< + typename holoscan::type_info::element_type, + bool, + int16_t, + uint16_t, + int32_t, + uint32_t, + int64_t, + uint64_t, + float, + double, + std::string>) { + if constexpr (holoscan::dimension_of_v == 1) { + // Create vector of Handles + YAML::Node yaml_node = YAML::Load("[]"); // Create an empty sequence + for (typename holoscan::type_info::element_type item : value) { + yaml_node.push_back(item); } - case ArgElementType::kBoolean: - case ArgElementType::kInt16: - case ArgElementType::kUnsigned16: - case ArgElementType::kInt32: - case ArgElementType::kUnsigned32: - case ArgElementType::kInt64: - case ArgElementType::kUnsigned64: - case ArgElementType::kFloat32: - case ArgElementType::kFloat64: - case ArgElementType::kString: { - // GXF Doesn't support std::vector parameter so use a workaround with - // GxfParameterSetFromYamlNode. - if constexpr (holoscan::is_one_of_v< - typename holoscan::type_info::element_type, - bool, - int16_t, - uint16_t, - int32_t, - uint32_t, - int64_t, - uint64_t, - float, - double, - std::string>) { - if constexpr (holoscan::dimension_of_v == 1) { - // Create vector of Handles - YAML::Node yaml_node = YAML::Load("[]"); // Create an empty sequence - for (typename holoscan::type_info::element_type item : value) { - yaml_node.push_back(item); - } - return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); - } else if constexpr (holoscan::dimension_of_v == 2) { - YAML::Node yaml_node = YAML::Load("[]"); // Create an empty sequence - for (std::vector::element_type>& vec : - value) { - YAML::Node inner_yaml_node = - YAML::Load("[]"); // Create an empty sequence - for (typename holoscan::type_info::element_type item : vec) { - inner_yaml_node.push_back(item); - } - if (inner_yaml_node.size() > 0) { - yaml_node.push_back(inner_yaml_node); - } - } - return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); - } + return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); + } else if constexpr (holoscan::dimension_of_v == 2) { + YAML::Node yaml_node = YAML::Load("[]"); // Create an empty sequence + for (std::vector::element_type>& vec : + value) { + YAML::Node inner_yaml_node = YAML::Load("[]"); // Create an empty sequence + for (typename holoscan::type_info::element_type item : vec) { + inner_yaml_node.push_back(item); } - break; + if (inner_yaml_node.size() > 0) { yaml_node.push_back(inner_yaml_node); } } - case ArgElementType::kHandle: { - HOLOSCAN_LOG_ERROR( - "Unable to handle vector of ArgElementType::kHandle for key '{}'", key); - return GXF_FAILURE; - } - case ArgElementType::kYAMLNode: { - HOLOSCAN_LOG_ERROR( - "Unable to handle vector of ArgElementType::kYAMLNode for key '{}'", key); - return GXF_FAILURE; - } - case ArgElementType::kIOSpec: { - if constexpr (std::is_same_v>) { - // Create vector of Handles - YAML::Node yaml_node = YAML::Load("[]"); // Create an empty sequence - for (auto& io_spec : value) { - if (io_spec) { // Only consider non-null IOSpecs - auto gxf_resource = - std::dynamic_pointer_cast(io_spec->resource()); - yaml_node.push_back(gxf_resource->gxf_cname()); - } - } - return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); - } - HOLOSCAN_LOG_ERROR( - "Unable to handle vector of std::vector> for key: " - "'{}'", - key); - break; + return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); + } + } + break; + } + case ArgElementType::kHandle: { + HOLOSCAN_LOG_ERROR( + "Unable to handle vector of ArgElementType::kHandle for key '{}'", key); + return GXF_FAILURE; + } + case ArgElementType::kYAMLNode: { + HOLOSCAN_LOG_ERROR( + "Unable to handle vector of ArgElementType::kYAMLNode for key '{}'", key); + return GXF_FAILURE; + } + case ArgElementType::kIOSpec: { + if constexpr (std::is_same_v>) { + // Create vector of Handles + YAML::Node yaml_node = YAML::Load("[]"); // Create an empty sequence + for (auto& io_spec : value) { + if (io_spec) { // Only consider non-null IOSpecs + auto gxf_resource = + std::dynamic_pointer_cast(io_spec->resource()); + yaml_node.push_back(gxf_resource->gxf_cname()); } - case ArgElementType::kResource: { - if constexpr (std::is_same_v< - typename holoscan::type_info::element_type, - std::shared_ptr> && - holoscan::type_info::dimension == 1) { - // Create vector of Handles - YAML::Node yaml_node; - for (auto& resource : value) { - auto gxf_resource = std::dynamic_pointer_cast(resource); - // Push back the resource's gxf_cname only if it is not null. - if (gxf_resource) { - gxf_uid_t resource_cid = gxf_resource->gxf_cid(); - std::string full_resource_name = - gxf::get_full_component_name(context, resource_cid); - yaml_node.push_back(full_resource_name.c_str()); - } else { - HOLOSCAN_LOG_TRACE( - "Resource item in the vector is null. Skipping it for key '{}'", - key); - } - } - return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); - } - HOLOSCAN_LOG_ERROR( - "Unable to handle vector of ArgElementType::kResource for key '{}'", key); - break; + } + return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); + } + HOLOSCAN_LOG_ERROR( + "Unable to handle vector of std::vector> for key: " + "'{}'", + key); + break; + } + case ArgElementType::kResource: { + if constexpr (std::is_same_v::element_type, + std::shared_ptr> && + holoscan::type_info::dimension == 1) { + // Create vector of Handles + YAML::Node yaml_node; + for (auto& resource : value) { + auto gxf_resource = std::dynamic_pointer_cast(resource); + // Push back the resource's gxf_cname only if it is not null. + if (gxf_resource) { + gxf_uid_t resource_cid = gxf_resource->gxf_cid(); + std::string full_resource_name = + gxf::get_full_component_name(context, resource_cid); + yaml_node.push_back(full_resource_name.c_str()); + } else { + HOLOSCAN_LOG_TRACE( + "Resource item in the vector is null. Skipping it for key '{}'", key); } - case ArgElementType::kCondition: { - if constexpr (std::is_same_v< - typename holoscan::type_info::element_type, - std::shared_ptr> && - holoscan::type_info::dimension == 1) { - // Create vector of Handles - YAML::Node yaml_node; - for (auto& condition : value) { - auto gxf_condition = std::dynamic_pointer_cast(condition); - // Initialize GXF component if it is not already initialized. - if (gxf_condition->gxf_context() == nullptr) { - gxf_condition->gxf_eid(gxf::get_component_eid( - context, uid)); // set Entity ID of the component + } + return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); + } + HOLOSCAN_LOG_ERROR( + "Unable to handle vector of ArgElementType::kResource for key '{}'", key); + break; + } + case ArgElementType::kCondition: { + if constexpr (std::is_same_v::element_type, + std::shared_ptr> && + holoscan::type_info::dimension == 1) { + // Create vector of Handles + YAML::Node yaml_node; + for (auto& condition : value) { + auto gxf_condition = std::dynamic_pointer_cast(condition); + // Initialize GXF component if it is not already initialized. + if (gxf_condition->gxf_context() == nullptr) { + gxf_condition->gxf_eid( + gxf::get_component_eid(context, uid)); // set Entity ID of the component - gxf_condition->initialize(); - } - gxf_uid_t condition_cid = gxf_condition->gxf_cid(); - std::string full_condition_name = - gxf::get_full_component_name(context, condition_cid); - yaml_node.push_back(full_condition_name.c_str()); - } - return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); - } - HOLOSCAN_LOG_ERROR( - "Unable to handle vector of ArgElementType::kCondition for key '{}'", - key); - break; - } - case ArgElementType::kCustom: { - HOLOSCAN_LOG_ERROR( - "Unable to handle vector of ArgElementType::kCustom type for key '{}'", - key); - return GXF_FAILURE; + gxf_condition->initialize(); } + gxf_uid_t condition_cid = gxf_condition->gxf_cid(); + std::string full_condition_name = + gxf::get_full_component_name(context, condition_cid); + yaml_node.push_back(full_condition_name.c_str()); } - break; - } - case ArgContainerType::kArray: { - HOLOSCAN_LOG_ERROR("Unable to handle ArgContainerType::kArray type for key '{}'", - key); - break; + return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); } + HOLOSCAN_LOG_ERROR( + "Unable to handle vector of ArgElementType::kCondition for key '{}'", key); + break; + } + case ArgElementType::kCustom: { + HOLOSCAN_LOG_ERROR( + "Unable to handle vector of ArgElementType::kCustom type for key '{}'", key); + return GXF_FAILURE; } } - HOLOSCAN_LOG_WARN( - "Unable to get argument for key '{}' with type '{}'", key, typeid(typeT).name()); - } catch (const std::bad_any_cast& e) { - HOLOSCAN_LOG_ERROR("Bad any cast exception: {}", e.what()); + break; + } + case ArgContainerType::kArray: { + HOLOSCAN_LOG_ERROR("Unable to handle ArgContainerType::kArray type for key '{}'", key); + break; } + } + } catch (const std::bad_any_cast& e) { + HOLOSCAN_LOG_ERROR("Bad any cast exception: {}", e.what()); + } - return GXF_FAILURE; - }); + return GXF_FAILURE; + }; + + function_map_.try_emplace(std::type_index(typeid(typeT)), func); } private: @@ -523,6 +507,7 @@ class GXFParameterAdaptor { add_param_handler>>(); add_param_handler>>(); + add_param_handler(); add_param_handler(); add_param_handler>(); diff --git a/include/holoscan/core/extension_manager.hpp b/include/holoscan/core/extension_manager.hpp new file mode 100644 index 00000000..17926529 --- /dev/null +++ b/include/holoscan/core/extension_manager.hpp @@ -0,0 +1,120 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HOLOSCAN_CORE_EXTENSION_MANAGER_HPP +#define HOLOSCAN_CORE_EXTENSION_MANAGER_HPP + +#include + +#include +#include + +#include "holoscan/core/common.hpp" + +namespace holoscan { + +/** + * @brief Class to manage extensions. + * + * This class is a helper class to manage extensions. + * + */ +class ExtensionManager { + public: + /** + * @brief Construct a new ExtensionManager object. + * + * @param context The context. + */ + explicit ExtensionManager(void* context) : context_(context) {} + + /** + * @brief Destroy the ExtensionManager object. + * + */ + virtual ~ExtensionManager() = default; + + /** + * @brief Refresh the extension list. + * + * Based on the current context, construct the internal extension list. + */ + virtual void refresh() {} + + /** + * @brief Load an extension. + * + * This method loads an extension and stores the extension handler so that it can be + * unloaded when the class is destroyed. + * + * @param file_name The file name of the extension (e.g. libmyextension.so). + * @param no_error_message If true, no error message will be printed if the extension is not + * found. + * @param search_path_envs The environment variable names that contains the search paths for the + * extension. The environment variable names are separated by a comma (,). (default: + * "HOLOSCAN_LIB_PATH"). + * @return true if the extension is loaded successfully, false otherwise. + */ + virtual bool load_extension(const std::string& file_name, bool no_error_message = false, + const std::string& search_path_envs = "HOLOSCAN_LIB_PATH") { + (void)file_name; + (void)no_error_message; + (void)search_path_envs; + return false; + } + + /** + * @brief Load extensions from a yaml file. + * + * The yaml file should contain a list of extension file names under the key "extensions". + * + * For example: + * + * ```yaml + * extensions: + * - /path/to/extension1.so + * - /path/to/extension2.so + * - /path/to/extension3.so + * ``` + * + * @param node The yaml node. + * @param no_error_message If true, no error message will be printed if the extension is not + * found. + * @param search_path_envs The environment variable names that contains the search paths for the + * extension. The environment variable names are separated by a comma (,). (default: + * "HOLOSCAN_LIB_PATH"). + * @param key The key in the yaml node that contains the extension file names (default: + * "extensions"). + * @return true if the extension is loaded successfully, false otherwise. + */ + virtual bool load_extensions_from_yaml(const YAML::Node& node, bool no_error_message = false, + const std::string& search_path_envs = "HOLOSCAN_LIB_PATH", + const std::string& key = "extensions") { + (void)node; + (void)no_error_message; + (void)search_path_envs; + (void)key; + return false; + } + + protected: + void* context_ = nullptr; ///< The context +}; + +} // namespace holoscan + +#endif /* HOLOSCAN_CORE_EXTENSION_MANAGER_HPP */ diff --git a/include/holoscan/core/forward_def.hpp b/include/holoscan/core/forward_def.hpp index 0f2e1a59..640fa904 100644 --- a/include/holoscan/core/forward_def.hpp +++ b/include/holoscan/core/forward_def.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -34,6 +34,7 @@ class Config; class Component; class ComponentSpec; class ExecutionContext; +class ExtensionManager; class Executor; class Fragment; enum class FlowType; @@ -65,6 +66,7 @@ class GXFCondition; class GXFInputContext; class GXFOutputContext; class GXFResource; +class GXFExtensionManager; } // namespace gxf // holoscan::ops diff --git a/include/holoscan/core/fragment.hpp b/include/holoscan/core/fragment.hpp index 288e602e..3a8b4d5d 100644 --- a/include/holoscan/core/fragment.hpp +++ b/include/holoscan/core/fragment.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,7 +25,10 @@ #include // for std::enable_if_t, std::is_constructible #include // for std::pair -#include "./common.hpp" +#include "common.hpp" +#include "config.hpp" +#include "executor.hpp" +#include "graph.hpp" namespace holoscan { @@ -85,13 +88,18 @@ class Fragment { * * The `extensions` field in the YAML configuration file is a list of GXF extension paths. * The paths can be absolute or relative to the current working directory, considering paths in - * `LD_LIBRARY_PATH` environment variable. The paths consists of the following parts: + * `LD_LIBRARY_PATH` environment variable. + * + * The paths can consist of the following parts: * * - GXF core extensions * - built-in extensions such as `libgxf_std.so` and `libgxf_cuda.so`. + * - `libgxf_std.so`, `libgxf_cuda.so`, `libgxf_multimedia.so`, `libgxf_serialization.so` are + * always loaded by default. * - GXF core extensions are copied to the `lib` directory of the build/installation directory. * - Other GXF extensions * - GXF extensions that are required for operators that this fragment uses. + * - some core GXF extensions such as `libgxf_stream_playback.so` are always loaded by default. * - these paths are usually relative to the build/installation directory. * * The extension paths are used to load dependent GXF extensions at runtime when @@ -104,12 +112,7 @@ class Fragment { * * ```yaml * extensions: - * - libgxf_std.so - * - libgxf_cuda.so - * - libgxf_multimedia.so - * - libgxf_serialization.so * - libmy_recorder.so - * - libstream_playback.so * * replayer: * directory: "../data/endoscopy/video" @@ -231,7 +234,7 @@ class Fragment { /** * @brief Create a new (operator) resource. * - * @tparam OperatorT The type of the resource. + * @tparam ResourceT The type of the resource. * @param name The name of the resource. * @param args The arguments for the resource. * @return The shared pointer to the resource. @@ -254,7 +257,7 @@ class Fragment { /** * @brief Create a new (operator) resource. * - * @tparam OperatorT The type of the resource. + * @tparam ResourceT The type of the resource. * @param args The arguments for the resource. * @return The shared pointer to the resource. */ @@ -268,7 +271,7 @@ class Fragment { /** * @brief Create a new condition. * - * @tparam OperatorT The type of the condition. + * @tparam ConditionT The type of the condition. * @param name The name of the condition. * @param args The arguments for the condition. * @return The shared pointer to the condition. @@ -292,7 +295,7 @@ class Fragment { /** * @brief Create a new condition. * - * @tparam OperatorT The type of the condition. + * @tparam ConditionT The type of the condition. * @param args The arguments for the condition. * @return The shared pointer to the condition. */ @@ -393,8 +396,8 @@ class Fragment { * add_flow(visualizer_format_converter, visualizer, {{"", "receivers"}}); * * add_flow(source, format_converter); - * add_flow(format_converter, lstm_inferer); - * add_flow(lstm_inferer, visualizer, {{"", "receivers"}}); + * add_flow(format_converter, multiai_inference); + * add_flow(multiai_inference, visualizer, {{"", "receivers"}}); * * Instead of: * @@ -402,8 +405,8 @@ class Fragment { * add_flow(visualizer_format_converter, visualizer, {{"", "source_video"}}); * * add_flow(source, format_converter); - * add_flow(format_converter, lstm_inferer); - * add_flow(lstm_inferer, visualizer, {{"", "tensor"}}); + * add_flow(format_converter, multiai_inference); + * add_flow(multiai_inference, visualizer, {{"", "tensor"}}); * * By using the parameter (`receivers`) with `std::vector` type, the framework * creates input ports (`receivers:0` and `receivers:1`) implicitly and connects them (and adds @@ -448,6 +451,11 @@ class Fragment { return std::make_unique(this); } + template + std::unique_ptr make_executor(ArgsT&&... args) { + return std::make_unique(std::forward(args)...); + } + std::string name_; ///< The name of the fragment. Application* app_ = nullptr; ///< The application that this fragment belongs to. std::unique_ptr config_; ///< The configuration of the fragment. diff --git a/include/holoscan/core/gxf/entity.hpp b/include/holoscan/core/gxf/entity.hpp index 7f1a651e..80229ac4 100644 --- a/include/holoscan/core/gxf/entity.hpp +++ b/include/holoscan/core/gxf/entity.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -27,6 +27,7 @@ #include "gxf/core/entity.hpp" #pragma GCC diagnostic pop +#include "gxf/multimedia/video.hpp" #include "holoscan/core/gxf/gxf_tensor.hpp" #include "holoscan/core/type_traits.hpp" @@ -50,12 +51,22 @@ class Entity : public nvidia::gxf::Entity { // Creates a new entity static Entity New(ExecutionContext* context); + /** + * @brief Return true if the entity is not null. + * + * Calling this method on an entity object from {cpp:func}`holoscan::IOContext::receive` will + * return false if there is no entity to receive. + * + * @return true if the entity is not null. Otherwise, false. + */ + operator bool() const { return !is_null(); } + // Gets a component by type. Asserts if no such component. // Adds a component with given type template && holoscan::is_one_of_v>> - std::shared_ptr get(const char* name = nullptr) const { + std::shared_ptr get(const char* name = nullptr, bool log_errors = true) const { bool is_holoscan_gxf_tensor = true; // We should use nullptr as a default name because In GXF, 'nullptr' should be used with // GxfComponentFind() if we want to get the first component of the given type. @@ -65,7 +76,9 @@ class Entity : public nvidia::gxf::Entity { auto tid_result = GxfComponentTypeId(context(), nvidia::TypenameAsString(), &tid); if (tid_result != GXF_SUCCESS) { - HOLOSCAN_LOG_ERROR("Unable to get component type id: {}", tid_result); + if (log_errors) { + HOLOSCAN_LOG_ERROR("Unable to get component type id: {}", tid_result); + } return nullptr; } @@ -76,9 +89,11 @@ class Entity : public nvidia::gxf::Entity { tid_result = GxfComponentTypeId(context(), nvidia::TypenameAsString(), &tid); if (tid_result != GXF_SUCCESS) { - HOLOSCAN_LOG_ERROR( - "Unable to get component type id from 'nvidia::gxf::Tensor' (error code: {})", - tid_result); + if (log_errors) { + HOLOSCAN_LOG_ERROR( + "Unable to get component type id from 'nvidia::gxf::Tensor' (error code: {})", + tid_result); + } return nullptr; } @@ -87,9 +102,11 @@ class Entity : public nvidia::gxf::Entity { } if (cid_result != GXF_SUCCESS) { - HOLOSCAN_LOG_ERROR("Unable to find component from the name '{}' (error code: {})", - name == nullptr ? "" : name, - cid_result); + if (log_errors) { + HOLOSCAN_LOG_ERROR("Unable to find component from the name '{}' (error code: {})", + name == nullptr ? "" : name, + cid_result); + } return nullptr; } @@ -135,10 +152,16 @@ class Entity : public nvidia::gxf::Entity { holoscan::gxf::GXFTensor* tensor_ptr = handle->get(); // Copy the member data (std::shared_ptr) from the Tensor to GXFTensor - *tensor_ptr = data->dl_ctx(); + *tensor_ptr = GXFTensor(data->dl_ctx()); } }; +// Modified version of the Tensor version of gxf::Entity::get +// Retrieves a VideoBuffer instead +// TODO: Support gxf::VideoBuffer natively in Holoscan +nvidia::gxf::Handle get_videobuffer(Entity entity, + const char* name = nullptr); + } // namespace holoscan::gxf #endif /* HOLOSCAN_CORE_GXF_ENTITY_HPP */ diff --git a/include/holoscan/core/gxf/gxf_condition.hpp b/include/holoscan/core/gxf/gxf_condition.hpp index a2a46ddd..a3f1ec74 100644 --- a/include/holoscan/core/gxf/gxf_condition.hpp +++ b/include/holoscan/core/gxf/gxf_condition.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,15 +18,20 @@ #ifndef HOLOSCAN_CORE_GXF_GXF_CONDITION_HPP #define HOLOSCAN_CORE_GXF_GXF_CONDITION_HPP +#include + #include "../condition.hpp" #include "./gxf_component.hpp" +#include "gxf/std/scheduling_terms.hpp" + namespace holoscan::gxf { class GXFCondition : public holoscan::Condition, public gxf::GXFComponent { public: HOLOSCAN_CONDITION_FORWARD_ARGS_SUPER(GXFCondition, holoscan::Condition) GXFCondition() = default; + GXFCondition(const std::string& name, nvidia::gxf::SchedulingTerm* term); void initialize() override; }; diff --git a/include/holoscan/core/gxf/gxf_extension_manager.hpp b/include/holoscan/core/gxf/gxf_extension_manager.hpp new file mode 100644 index 00000000..c8c4334c --- /dev/null +++ b/include/holoscan/core/gxf/gxf_extension_manager.hpp @@ -0,0 +1,183 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef INCLUDE_HOLOSCAN_CORE_GXF_GXF_EXTENSION_MANAGER_HPP +#define INCLUDE_HOLOSCAN_CORE_GXF_GXF_EXTENSION_MANAGER_HPP + +#include + +#include +#include +#include + +#include "gxf/core/gxf.h" +#include "gxf/std/extension.hpp" + +#include "holoscan/core/common.hpp" +#include "holoscan/core/extension_manager.hpp" + +namespace holoscan::gxf { + +namespace { +// Method name to get the GXF extension factory +constexpr const char* kGxfExtensionFactoryName = "GxfExtensionFactory"; +// Max size of extensions +constexpr int kGXFExtensionsMaxSize = 1024; +// Method signature for the GXF extension factory +using GxfExtensionFactory = gxf_result_t(void**); +} // namespace + +/** + * @brief Class to manage GXF extensions. + * + * This class is a helper class to manage GXF extensions. + * + * Since GXF API doesn't provide a way to ignore duplicate extensions, this class is used to + * manage the extensions, prevent duplicate extensions from being loaded, and unload the + * extension handlers when the class is destroyed. + * + * Example: + * + * ```cpp + * #include + * #include "holoscan/core/gxf/gxf_extension_manager.hpp" + * ... + * + * holoscan::gxf::GXFExtensionManager extension_manager(reinterpret_cast(context)); + * // Load extensions from a file + * for (const auto& extension_filename : extension_filenames) { + * extension_manager.load_extension(extension_filename); + * } + * // Load extensions from a yaml node (YAML::Node object) + * for (const auto& manifest_filename : manifest_filenames) { + * auto node = YAML::LoadFile(manifest_filename); + * extension_manager.load_extensions_from_yaml(node); + * } + * ``` + */ +class GXFExtensionManager : public ExtensionManager { + public: + /** + * @brief Construct a new GXFExtensionManager object. + * + * @param context The GXF context + */ + explicit GXFExtensionManager(gxf_context_t context); + + /** + * @brief Destroy the GXFExtensionManager object. + * + * This method closes all the extension handles that are loaded by this class. + * Note that the shared library is opened with `RTLD_NODELETE` flag, so the library is not + * unloaded when the handle is closed. + */ + ~GXFExtensionManager() override; + + /** + * @brief Refresh the extension list. + * + * Based on the current GXF context, it gets the list of extensions and stores the type IDs + * of the extensions so that duplicate extensions can be ignored. + */ + void refresh() override; + + /** + * @brief Load an extension. + * + * This method loads an extension and stores the extension handler so that it can be + * unloaded when the class is destroyed. + * + * @param file_name The file name of the extension (e.g. libgxf_std.so). + * @param no_error_message If true, no error message will be printed if the extension is not + * found. + * @param search_path_envs The environment variable names that contains the search paths for the + * extension. The environment variable names are separated by a comma (,). (default: + * "HOLOSCAN_LIB_PATH"). + * @return true if the extension is loaded successfully, false otherwise. + */ + bool load_extension(const std::string& file_name, bool no_error_message = false, + const std::string& search_path_envs = "HOLOSCAN_LIB_PATH") override; + + /** + * @brief Load extensions from a yaml file. + * + * The yaml file should contain a list of extension file names under the key "extensions". + * + * For example: + * + * ```yaml + * extensions: + * - /path/to/extension1.so + * - /path/to/extension2.so + * - /path/to/extension3.so + * ``` + * + * @param node The yaml node. + * @param no_error_message If true, no error message will be printed if the extension is not + * found. + * @param search_path_envs The environment variable names that contains the search paths for the + * extension. The environment variable names are separated by a comma (,). (default: + * "HOLOSCAN_LIB_PATH"). + * @param key The key in the yaml node that contains the extension file names (default: + * "extensions"). + * @return true if the extension is loaded successfully, false otherwise. + */ + bool load_extensions_from_yaml(const YAML::Node& node, bool no_error_message = false, + const std::string& search_path_envs = "HOLOSCAN_LIB_PATH", + const std::string& key = "extensions") override; + + /** + * @brief Load an extension from a pointer. + * + * `GxfLoadExtensionFromPointer()` API is used to register the extension programmatically. + * + * @param extension The extension pointer to load. + * @param handle The handle of the extension library. + * @return true if the extension is loaded successfully. + */ + bool load_extension(nvidia::gxf::Extension* extension, void* handle = nullptr); + + /** + * @brief Check if the extension is loaded. + * + * @param tid The type ID of the extension. + * @return true if the extension is loaded. false otherwise. + */ + bool is_extension_loaded(gxf_tid_t tid); + + /** + * @brief Tokenize a string. + * + * @param str The string to tokenize. + * @param delimiters The delimiters. + * @return The vector of tokens. + */ + static std::vector tokenize(const std::string& str, const std::string& delimiters); + + protected: + /// Storage for the extension TIDs + gxf_tid_t extension_tid_list_[kGXFExtensionsMaxSize] = {}; + /// request/response structure for the runtime info + gxf_runtime_info runtime_info_{nullptr, kGXFExtensionsMaxSize, extension_tid_list_}; + + std::set extension_tids_; ///< Set of extension TIDs + std::set extension_handles_; ///< Set of extension handles +}; + +} // namespace holoscan::gxf + +#endif /* INCLUDE_HOLOSCAN_CORE_GXF_GXF_EXTENSION_MANAGER_HPP */ diff --git a/include/holoscan/core/gxf/gxf_extension_registrar.hpp b/include/holoscan/core/gxf/gxf_extension_registrar.hpp index bb3103e8..a149d793 100644 --- a/include/holoscan/core/gxf/gxf_extension_registrar.hpp +++ b/include/holoscan/core/gxf/gxf_extension_registrar.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -173,9 +173,10 @@ class GXFExtensionRegistrar { /** * @brief Register the extension. * + * @param out_extension_ptr If provided, the pointer to the extension is set to this pointer. * @return true If the extension is registered successfully. Otherwise, false. */ - bool register_extension() { + bool register_extension(nvidia::gxf::Extension** out_extension_ptr = nullptr) { if (!factory_) { HOLOSCAN_LOG_ERROR("GXF Extension factory is not initialized"); return false; @@ -188,9 +189,19 @@ class GXFExtensionRegistrar { } nvidia::gxf::Extension* extension = factory_.release(); + + // Set the extension pointer if provided. + if (out_extension_ptr != nullptr) { + if (extension != nullptr) { + *out_extension_ptr = extension; + } else { + *out_extension_ptr = nullptr; + } + } + gxf_result_t result = GxfLoadExtensionFromPointer(context_, extension); if (result != GXF_SUCCESS) { - HOLOSCAN_LOG_ERROR("Unable to register the GXF extension: {}", result); + HOLOSCAN_LOG_ERROR("Unable to register the GXF extension: {}", GxfResultStr(result)); return false; } return true; diff --git a/include/holoscan/core/gxf/gxf_resource.hpp b/include/holoscan/core/gxf/gxf_resource.hpp index 60612997..93a66aa1 100644 --- a/include/holoscan/core/gxf/gxf_resource.hpp +++ b/include/holoscan/core/gxf/gxf_resource.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,6 +18,10 @@ #ifndef HOLOSCAN_CORE_GXF_GXF_RESOURCE_HPP #define HOLOSCAN_CORE_GXF_GXF_RESOURCE_HPP +#include + +#include + #include "../resource.hpp" #include "./gxf_component.hpp" @@ -29,6 +33,7 @@ class GXFResource : public holoscan::Resource, public gxf::GXFComponent { public: HOLOSCAN_RESOURCE_FORWARD_ARGS_SUPER(GXFResource, holoscan::Resource) GXFResource() = default; + GXFResource(const std::string& name, nvidia::gxf::Component* component); void initialize() override; }; diff --git a/include/holoscan/core/gxf/gxf_tensor.hpp b/include/holoscan/core/gxf/gxf_tensor.hpp index 67079907..b727f2fe 100644 --- a/include/holoscan/core/gxf/gxf_tensor.hpp +++ b/include/holoscan/core/gxf/gxf_tensor.hpp @@ -19,6 +19,8 @@ #define HOLOSCAN_CORE_GXF_GXF_TENSOR_HPP #include +#include +#include #include "gxf/std/tensor.hpp" @@ -46,14 +48,14 @@ class GXFTensor : public nvidia::gxf::Tensor { * * @param tensor Tensor to wrap. */ - GXFTensor(nvidia::gxf::Tensor& tensor); + explicit GXFTensor(nvidia::gxf::Tensor& tensor); /** * @brief Construct a new GXFTensor object. * * @param dl_ctx DLManagedTensorCtx object to wrap. */ - GXFTensor(std::shared_ptr& dl_ctx); + explicit GXFTensor(std::shared_ptr& dl_ctx); /** * @brief Get DLDevice object from the GXF Tensor. @@ -116,7 +118,7 @@ class GXFMemoryBuffer : public nvidia::gxf::MemoryBuffer { public: using nvidia::gxf::MemoryBuffer::MemoryBuffer; - GXFMemoryBuffer(nvidia::gxf::MemoryBuffer&& other) + explicit GXFMemoryBuffer(nvidia::gxf::MemoryBuffer&& other) : nvidia::gxf::MemoryBuffer(std::forward(other)) {} nvidia::gxf::Tensor::stride_array_t gxf_strides; ///< Strides of the GXF Tensor. diff --git a/include/holoscan/core/gxf/gxf_utils.hpp b/include/holoscan/core/gxf/gxf_utils.hpp index 43f377f2..c72c61ab 100644 --- a/include/holoscan/core/gxf/gxf_utils.hpp +++ b/include/holoscan/core/gxf/gxf_utils.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,6 +24,10 @@ #include #include +#include + +#include "holoscan/logger/logger.hpp" + namespace holoscan::gxf { /** @@ -115,6 +119,108 @@ inline std::string create_name(const char* prefix, const std::string& name) { return sstream.str(); } +/** + * @brief Return a component ID from the handle name. + * + * This method parses the handle name and interprets it as either a component name in + * the current entity, or as a composed string of the form 'entity_name/component_name'. + * This method reuses a logic from the GXF SDK. + * + * @param context The GXF context. + * @param component_uid The component ID of the component that contains the parameter. + * @param key The key of the parameter. + * @param tag The handle name. + * @param prefix The prefix of the component name. + * @return The component ID referenced by the key. + */ +template +inline gxf_uid_t find_component_handle(gxf_context_t context, gxf_uid_t component_uid, + const char* key, const std::string& tag, + const std::string& prefix) { + gxf_uid_t eid; + std::string component_name; + + const size_t pos = tag.find('/'); + if (pos == std::string::npos) { + // Get the entity of this component + const gxf_result_t result_1 = GxfComponentEntity(context, component_uid, &eid); + if (result_1 != GXF_SUCCESS) { return 0; } + component_name = tag; + } else { + component_name = tag.substr(pos + 1); + + // Get the entity + gxf_result_t result_1_with_prefix = GXF_FAILURE; + // Try using entity name with prefix + if (!prefix.empty()) { + const std::string entity_name = prefix + tag.substr(0, pos); + result_1_with_prefix = GxfEntityFind(context, entity_name.c_str(), &eid); + if (result_1_with_prefix != GXF_SUCCESS) { + HOLOSCAN_LOG_WARN( + "Could not find entity (with prefix) '{}' while parsing parameter '{}' " + "of component {}", + entity_name.c_str(), + key, + component_uid); + } + } + // Try using entity name without prefix, if lookup with prefix failed + if (result_1_with_prefix != GXF_SUCCESS) { + const std::string entity_name = tag.substr(0, pos); + const gxf_result_t result_1_no_prefix = GxfEntityFind(context, entity_name.c_str(), &eid); + if (result_1_no_prefix != GXF_SUCCESS) { + HOLOSCAN_LOG_ERROR( + "Could not find entity '{}' while parsing parameter '{}' of component {}", + entity_name.c_str(), + key, + component_uid); + return 0; + } else if (!prefix.empty()) { + HOLOSCAN_LOG_WARN( + "Found entity (without prefix) '{}' while parsing parameter '{}' " + "of component {} in a subgraph, however the approach is deprecated," + " please use prerequisites instead", + entity_name.c_str(), + key, + component_uid); + } + } + } + + // Get the type id of the component we are are looking for. + gxf_tid_t tid; + const gxf_result_t result_2 = GxfComponentTypeId(context, ::nvidia::TypenameAsString(), &tid); + if (result_2 != GXF_SUCCESS) { return 0; } + + // Find the component in the indicated entity + gxf_uid_t cid; + const gxf_result_t result_3 = + GxfComponentFind(context, eid, tid, component_name.c_str(), nullptr, &cid); + if (result_3 != GXF_SUCCESS) { + if (component_name == "") { + HOLOSCAN_LOG_DEBUG( + "Using an handle in entity {} while parsing parameter '{}'" + " of component {}. This handle must be set to a valid component before graph activation", + eid, + key, + component_uid); + return 0; + } else { + HOLOSCAN_LOG_WARN( + "Could not find component '{}' in entity {} while parsing parameter '{}' " + "of component {}", + component_name.c_str(), + eid, + key, + component_uid); + } + + return 0; + } + + return cid; +} + } // namespace holoscan::gxf #endif /* HOLOSCAN_CORE_GXF_GXF_UTILS_HPP */ diff --git a/include/holoscan/core/io_context.hpp b/include/holoscan/core/io_context.hpp index efec4db2..82121c5a 100644 --- a/include/holoscan/core/io_context.hpp +++ b/include/holoscan/core/io_context.hpp @@ -98,8 +98,9 @@ class InputContext { * void compute(InputContext& op_input, OutputContext&, ExecutionContext&) override { * // The type of `value` is `std::shared_ptr` * auto value = op_input.receive("in"); - * - * HOLOSCAN_LOG_INFO("Message received (value: {})", value->data()); + * if (value) { + * HOLOSCAN_LOG_INFO("Message received (value: {})", value->data()); + * } * } * }; * ``` @@ -113,6 +114,10 @@ class InputContext { !holoscan::is_one_of_v>> std::shared_ptr receive(const char* name = nullptr) { auto value = receive_impl(name); + + // If the received data is nullptr, return a null shared pointer. + if (value.type() == typeid(nullptr_t)) { return nullptr; } + try { return std::any_cast>(value); } catch (const std::bad_any_cast& e) { @@ -133,7 +138,8 @@ class InputContext { * * If the input port with the given name and the type (`nvidia::gxf::Entity`) is available, * it will return the message data (`holoscan::gxf::Entity`) from the input - * port. Otherwise, it will throw an exception. + * port. Otherwise, it will return an empty Entity or throw an exception if it fails to parse + * the message data. * * Example: * @@ -151,9 +157,10 @@ class InputContext { * void compute(InputContext& op_input, OutputContext&, ExecutionContext&) override { * // The type of `in_entity` is 'holoscan::gxf::Entity'. * auto in_entity = op_input.receive("in"); - * - * // Process with `in_entity`. - * // ... + * if (in_entity) { + * // Process with `in_entity`. + * // ... + * } * } * }; * ``` @@ -161,13 +168,17 @@ class InputContext { * @tparam DataT The type of the data to receive. It should be `holoscan::gxf::Entity`. * @param name The name of the input port to receive the data from. * @return The entity object (`holoscan::gxf::Entity`). - * @throws std::runtime_error if the input port with the given name and the type (`DataT`) is - * not available. + * @throws std::runtime_error if it fails to parse the message data from the input port with the + * given type (`DataT`). */ template >> DataT receive(const char* name) { auto value = receive_impl(name); + + // If the received data is nullptr, return an empty entity + if (value.type() == typeid(nullptr_t)) { return {}; } + try { return std::any_cast(value); } catch (const std::bad_any_cast& e) { @@ -205,16 +216,20 @@ class InputContext { * void compute(InputContext& op_input, OutputContext&, ExecutionContext&) override { * // The type of `in_any` is 'std::any'. * auto in_any = op_input.receive("in"); - * if (in_any.type() == typeid(holoscan::gxf::Entity)) { + * auto& in_any_type = in_any.type(); + * + * if (in_any_type == typeid(holoscan::gxf::Entity)) { * auto in_entity = std::any_cast(in_any); * // Process with `in_entity`. * // ... - * } else if (in_any.type() == typeid(std::shared_ptr)) { + * } else if (in_any_type == typeid(std::shared_ptr)) { * auto in_message = std::any_cast>(in_any); * // Process with `in_message`. * // ... + * } else if (in_any_type == typeid(nullptr_t)) { + * // No message is available. * } else { - * HOLOSCAN_LOG_ERROR("Message is not available"); + * HOLOSCAN_LOG_ERROR("Invalid message type: {}", in_any_type.name()); * return; * } * } @@ -238,7 +253,8 @@ class InputContext { * If the parameter (of type `std::vector`) with the given name is available, * it will return a vector of the shared pointers to the data * (`std::vector>`) from the input port. Otherwise, it will return an - * empty vector. + * empty vector. The vector can have empty shared pointers if the data of the corresponding input + * port is not available. * * Example: * @@ -259,9 +275,10 @@ class InputContext { * HOLOSCAN_LOG_INFO("Received {} messages", in_messages.size()); * // The type of `in_message` is 'std::vector>'. * auto& in_message = in_messages[0]; - * - * // Process with `in_message`. - * // ... + * if (in_message) { + * // Process with `in_message`. + * // ... + * } * } * * private: @@ -289,14 +306,14 @@ class InputContext { if (it == params.end()) { HOLOSCAN_LOG_ERROR( - "Unable to find input parameter with type 'std::vector' and name '{}'", name); + "Unable to find input parameter with name '{}'", name); return input_vector; } auto& param_wrapper = it->second; auto& arg_type = param_wrapper.arg_type(); if ((arg_type.element_type() != ArgElementType::kIOSpec) || (arg_type.container_type() != ArgContainerType::kVector)) { - HOLOSCAN_LOG_ERROR("Input parameter with name {} is not of type 'std::vector'", + HOLOSCAN_LOG_ERROR("Input parameter with name '{}' is not of type 'std::vector'", name); return input_vector; } @@ -314,6 +331,13 @@ class InputContext { // (':'. e.g, 'receivers:0') to return an object with // 'std::vector' type. auto value = receive_impl(fmt::format("{}:{}", name, index).c_str(), true); + + // If the received data is nullptr, add a null shared pointer. + if (value.type() == typeid(nullptr_t)) { + input_vector.emplace_back(nullptr); + continue; + } + try { auto casted_value = std::any_cast>(value); input_vector.push_back(std::move(casted_value)); @@ -338,7 +362,8 @@ class InputContext { * If the parameter (of type `std::vector`) with * the given name is available, it will return a vector of entities * (`std::vector`). Otherwise, it will return an - * empty vector. + * empty vector. The vector can have empty entities if the data of the corresponding + * input port is not available. * * Example: * @@ -359,8 +384,10 @@ class InputContext { * HOLOSCAN_LOG_INFO("Received {} messages", in_messages.size()); * // in_message's type is 'holoscan::gxf::Entity'. * auto& in_message = in_messages[0]; - * - * // Process with 'in_message' here. + * if (in_message) + * { + * // Process with 'in_message' here. + * } * } * * private: @@ -385,14 +412,14 @@ class InputContext { if (it == params.end()) { HOLOSCAN_LOG_ERROR( - "Unable to find input parameter with type 'std::vector' and name '{}'", name); + "Unable to find input parameter with name '{}'", name); return input_vector; } auto& param_wrapper = it->second; auto& arg_type = param_wrapper.arg_type(); if ((arg_type.element_type() != ArgElementType::kIOSpec) || (arg_type.container_type() != ArgContainerType::kVector)) { - HOLOSCAN_LOG_ERROR("Input parameter with name {} is not of type 'std::vector'", + HOLOSCAN_LOG_ERROR("Input parameter with name '{}' is not of type 'std::vector'", name); return input_vector; } @@ -410,6 +437,13 @@ class InputContext { // (':'. e.g, 'receivers:0') to return an object with // 'std::vector' type. auto value = receive_impl(fmt::format("{}:{}", name, index).c_str(), true); + + // If the received data is nullptr, add an empty entity. + if (value.type() == typeid(nullptr_t)) { + input_vector.emplace_back(); + continue; + } + try { auto casted_value = std::any_cast(value); input_vector.push_back(std::move(casted_value)); @@ -431,7 +465,7 @@ class InputContext { * * If the parameter (of type `std::vector`) with * the given name is available, it will return a vector of entities - * (`std::vector`). Otherwise, it will return an + * (`std::vector`) from the receivers. Otherwise, it will return an * empty vector. * * Example: @@ -459,17 +493,20 @@ class InputContext { * * // in_any's type is 'std::any'. * auto& in_any = in_any_vector[0]; + * auto& in_any_type = in_any.type(); * - * if (in_any.type() == typeid(holoscan::gxf::Entity)) { + * if (in_any_type == typeid(holoscan::gxf::Entity)) { * auto in_entity = std::any_cast(in_any); * // Process with `in_entity`. * // ... - * } else if (in_any.type() == typeid(std::shared_ptr)) { + * } else if (in_any_type == typeid(std::shared_ptr)) { * auto in_message = std::any_cast>(in_any); * // Process with `in_message`. * // ... + * } else if (in_any_type == typeid(nullptr_t)) { + * // No message is available. * } else { - * HOLOSCAN_LOG_ERROR("Unable to parse message"); + * HOLOSCAN_LOG_ERROR("Invalid message type: {}", in_any_type.name()); * return; * } * } @@ -495,14 +532,14 @@ class InputContext { if (it == params.end()) { HOLOSCAN_LOG_ERROR( - "Unable to find input parameter with type 'std::vector' and name '{}'", name); + "Unable to find input parameter with name '{}'", name); return input_vector; } auto& param_wrapper = it->second; auto& arg_type = param_wrapper.arg_type(); if ((arg_type.element_type() != ArgElementType::kIOSpec) || (arg_type.container_type() != ArgContainerType::kVector)) { - HOLOSCAN_LOG_ERROR("Input parameter with name {} is not of type 'std::vector'", + HOLOSCAN_LOG_ERROR("Input parameter with name '{}' is not of type 'std::vector'", name); return input_vector; } diff --git a/include/holoscan/core/operator.hpp b/include/holoscan/core/operator.hpp index e8738d63..5924858b 100644 --- a/include/holoscan/core/operator.hpp +++ b/include/holoscan/core/operator.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -39,9 +39,9 @@ template > && \ - (std::is_same_v> || \ - std::is_same_v> || \ + !std::is_base_of_v> && \ + (std::is_same_v> || \ + std::is_same_v> || \ std::is_base_of_v::derived_type> || \ std::is_base_of_v>& conditions() { return conditions_; } + /** * @brief Get the resources of the operator. * @@ -395,6 +396,14 @@ class Operator : public Component { register_argument_setter(); } + /** + * @brief Get a YAML representation of the operator. + * + * @return YAML node including type, specs, conditions and resources of the operator in addition + * to the base component properties. + */ + YAML::Node to_yaml_node() const override; + protected: /** * @brief Register the argument setter for the given type. @@ -408,6 +417,15 @@ class Operator : public Component { ArgumentSetter::get_instance().add_argument_setter( [](ParameterWrapper& param_wrap, Arg& arg) { std::any& any_param = param_wrap.value(); + + // If arg has no name and value, that indicates that we want to set the default value for + // the native operator if it is not specified. + if (arg.name().empty() && !arg.has_value()) { + auto& param = *std::any_cast*>(any_param); + param.set_default_value(); + return; + } + std::any& any_arg = arg.value(); // Note that the type of any_param is Parameter*, not Parameter. @@ -421,8 +439,8 @@ class Operator : public Component { HOLOSCAN_LOG_DEBUG( "Registering converter for parameter {} (element_type: {}, container_type: {})", arg.name(), - (int)element_type, - (int)container_type); + static_cast(element_type), + static_cast(container_type)); if (element_type == ArgElementType::kYAMLNode) { auto& arg_value = std::any_cast(any_arg); diff --git a/include/holoscan/core/operator_spec.hpp b/include/holoscan/core/operator_spec.hpp index 640c611f..d83a8b0c 100644 --- a/include/holoscan/core/operator_spec.hpp +++ b/include/holoscan/core/operator_spec.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -185,57 +185,15 @@ class OperatorSpec : public ComponentSpec { } /** - * @brief Define a resource that is required by this operator. + * @brief Get a YAML representation of the operator spec. * - * @tparam ResourceT The type of the resource. - * @param name The name of the resource. - * @param args The arguments to construct the resource. + * @return YAML node including the inputs, outputs, and parameters of this operator. */ - template >> - void resource(const StringT& name, ArgsT&&... args) { - resources_.emplace(name, std::make_shared(std::forward(args)...)); - } - - /** - * @brief Define a resource that is required by this operator. - * - * @param args The arguments to construct the resource. - */ - template - void resource(ArgsT&&... args) { - resource("", std::forward(args)...); - } - - /** - * @brief Define a condition that is required by this operator. - * - * @tparam ConditionT The type of the condition. - * @param name The name of the condition. - * @param args The arguments to construct the condition. - */ - template >> - void condition(const StringT& name, ArgsT&&... args) { - conditions_.emplace(name, std::make_shared(std::forward(args)...)); - } - - /** - * @brief Define a condition that is required by this operator. - * - * @param args The arguments to construct the condition. - */ - template - void condition(ArgsT&&... args) { - condition("", std::forward(args)...); - } + YAML::Node to_yaml_node() const override; protected: std::unordered_map> inputs_; ///< Input specs std::unordered_map> outputs_; ///< Outputs specs - // TODO(gbae): use these for operator spec. - std::unordered_map> conditions_; ///< Conditions - std::unordered_map> resources_; ///< Resources }; } // namespace holoscan diff --git a/include/holoscan/core/parameter.hpp b/include/holoscan/core/parameter.hpp index e9d14ebb..a1964aa8 100644 --- a/include/holoscan/core/parameter.hpp +++ b/include/holoscan/core/parameter.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,8 +26,8 @@ #include #include -#include "./common.hpp" #include "./arg.hpp" +#include "./common.hpp" namespace holoscan { @@ -55,8 +55,10 @@ class ParameterWrapper { */ template explicit ParameterWrapper(Parameter& param) - : type_(&typeid(typeT)), arg_type_(ArgType::create()), value_(¶m) {} - + : type_(&typeid(typeT)), + arg_type_(ArgType::create()), + value_(¶m), + storage_ptr_(static_cast(¶m)) {} /** * @brief Construct a new ParameterWrapper object. @@ -91,10 +93,18 @@ class ParameterWrapper { */ std::any& value() { return value_; } + /** + * @brief Get the pointer to the parameter storage. + * + * @return The pointer to the parameter storage. + */ + void* storage_ptr() const { return storage_ptr_; } + private: - const std::type_info* type_ = nullptr; /// element type of Parameter - ArgType arg_type_; /// type of argument - std::any value_; + const std::type_info* type_ = nullptr; ///< The element type of Parameter + ArgType arg_type_; ///< The type of the argument + std::any value_; ///< The value of the parameter + void* storage_ptr_ = nullptr; ///< The pointer to the parameter storage }; /** @@ -146,6 +156,20 @@ class MetaParameter { */ const std::string& key() const { return key_; } + /** + * @brief Get the headline of the parameter. + * + * @return The headline of the parameter. + */ + const std::string& headline() const { return headline_; } + + /** + * @brief Get the description of the parameter. + * + * @return The description of the parameter. + */ + const std::string& description() const { return description_; } + /** * @brief Check whether the parameter contains a value. * @@ -182,7 +206,7 @@ class MetaParameter { return default_value_.value(); } else { throw std::runtime_error( - fmt::format("MetaParameter: default value for '{}' is not set", key_)); + fmt::format("MetaParameter: default value for '{}' is not set", key_)); } } @@ -203,12 +227,12 @@ class MetaParameter { private: friend class ComponentSpec; friend class OperatorSpec; - std::optional value_; - std::optional default_value_; std::string key_; std::string headline_; std::string description_; ParameterFlag flag_ = ParameterFlag::kNone; + std::optional value_; + std::optional default_value_; }; } // namespace holoscan diff --git a/include/holoscan/core/resource.hpp b/include/holoscan/core/resource.hpp index 4c47718c..157d69d8 100644 --- a/include/holoscan/core/resource.hpp +++ b/include/holoscan/core/resource.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -145,7 +145,7 @@ class Resource : public Component { * @param spec The component specification. * @return The reference to the resource. */ - Resource& spec(std::shared_ptr spec) { + Resource& spec(const std::shared_ptr& spec) { spec_ = spec; return *this; } @@ -156,6 +156,13 @@ class Resource : public Component { */ ComponentSpec* spec() { return spec_.get(); } + /** + * @brief Get the shared pointer to the component spec. + * + * @return The shared pointer to the component spec. + */ + std::shared_ptr spec_shared() { return spec_; } + using Component::add_arg; /** @@ -165,6 +172,13 @@ class Resource : public Component { */ virtual void setup(ComponentSpec& spec) { (void)spec; } + /** + * @brief Get a YAML representation of the resource. + * + * @return YAML node including spec of the resource in addition to the base component properties. + */ + YAML::Node to_yaml_node() const override; + protected: std::shared_ptr spec_; ///< The component specification. }; diff --git a/include/holoscan/core/resources/gxf/allocator.hpp b/include/holoscan/core/resources/gxf/allocator.hpp index 0f57c56f..43926398 100644 --- a/include/holoscan/core/resources/gxf/allocator.hpp +++ b/include/holoscan/core/resources/gxf/allocator.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,6 +18,10 @@ #ifndef HOLOSCAN_CORE_RESOURCES_GXF_ALLOCATOR_HPP #define HOLOSCAN_CORE_RESOURCES_GXF_ALLOCATOR_HPP +#include + +#include + #include "../../gxf/gxf_resource.hpp" namespace holoscan { @@ -28,22 +32,16 @@ class Allocator : public gxf::GXFResource { public: HOLOSCAN_RESOURCE_FORWARD_ARGS_SUPER(Allocator, GXFResource) Allocator() = default; + Allocator(const std::string& name, nvidia::gxf::Allocator* component); const char* gxf_typename() const override { return "nvidia::gxf::Allocator"; } - virtual bool is_available(uint64_t size) { - (void)size; - return true; - } + virtual bool is_available(uint64_t size); // TODO(gbae): Introduce expected<> type - std::byte* allocate(uint64_t size, MemoryStorageType type) { - (void)size; - (void)type; - return nullptr; - } + virtual nvidia::byte* allocate(uint64_t size, MemoryStorageType type); - void free(std::byte* pointer) { (void)pointer; } + virtual void free(nvidia::byte* pointer); }; } // namespace holoscan diff --git a/include/holoscan/core/resources/gxf/block_memory_pool.hpp b/include/holoscan/core/resources/gxf/block_memory_pool.hpp index 5925ae47..e11aff27 100644 --- a/include/holoscan/core/resources/gxf/block_memory_pool.hpp +++ b/include/holoscan/core/resources/gxf/block_memory_pool.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,6 +18,49 @@ #ifndef HOLOSCAN_CORE_RESOURCES_GXF_BLOCK_MEMORY_POOL_HPP #define HOLOSCAN_CORE_RESOURCES_GXF_BLOCK_MEMORY_POOL_HPP +#include +// gxf/std/block_memory_pool.hpp is missing in the GXF SDK +// The following is a copy of the file from the GXF SDK until it is fixed +// TODO: Remove this file once the issue is fixed +//////////////////////////////////////////////////////////////////////////////////////////////////// +#include +#include +#include +#include "gxf/std/allocator.hpp" + +namespace nvidia { +namespace gxf { + +class FixedPoolUint64; + +// A memory pools which provides a maximum number of equally sized blocks of +// memory. +class BlockMemoryPool : public Allocator { + public: + BlockMemoryPool(); + ~BlockMemoryPool(); + + gxf_result_t registerInterface(Registrar* registrar) override; + gxf_result_t initialize() override; + gxf_result_t is_available_abi(uint64_t size) override; + gxf_result_t allocate_abi(uint64_t size, int32_t type, void** pointer) override; + gxf_result_t free_abi(void* pointer) override; + gxf_result_t deinitialize() override; + + private: + Parameter storage_type_; + Parameter block_size_; + Parameter num_blocks_; + + void* pointer_; + std::unique_ptr stack_; + std::mutex stack_mutex_; +}; + +} // namespace gxf +} // namespace nvidia +//////////////////////////////////////////////////////////////////////////////////////////////////// + #include "./allocator.hpp" namespace holoscan { @@ -29,6 +72,7 @@ class BlockMemoryPool : public Allocator { BlockMemoryPool() = default; BlockMemoryPool(int32_t storage_type, uint64_t block_size, uint64_t num_blocks) : storage_type_(storage_type), block_size_(block_size), num_blocks_(num_blocks) {} + BlockMemoryPool(const std::string& name, nvidia::gxf::BlockMemoryPool* component); const char* gxf_typename() const override { return "nvidia::gxf::BlockMemoryPool"; } diff --git a/include/holoscan/core/resources/gxf/cuda_stream_pool.hpp b/include/holoscan/core/resources/gxf/cuda_stream_pool.hpp index 61c6970e..8147a944 100644 --- a/include/holoscan/core/resources/gxf/cuda_stream_pool.hpp +++ b/include/holoscan/core/resources/gxf/cuda_stream_pool.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,6 +18,10 @@ #ifndef HOLOSCAN_CORE_RESOURCES_CUDA_STREAM_POOL_HPP #define HOLOSCAN_CORE_RESOURCES_CUDA_STREAM_POOL_HPP +#include + +#include + #include "./allocator.hpp" namespace holoscan { @@ -32,6 +36,7 @@ class CudaStreamPool : public Allocator { stream_priority_(stream_priority), reserved_size_(reserved_size), max_size_(max_size) {} + CudaStreamPool(const std::string& name, nvidia::gxf::CudaStreamPool* component); const char* gxf_typename() const override { return "nvidia::gxf::CudaStreamPool"; } diff --git a/include/holoscan/core/resources/gxf/double_buffer_receiver.hpp b/include/holoscan/core/resources/gxf/double_buffer_receiver.hpp index a327e8e4..e77dc9f6 100644 --- a/include/holoscan/core/resources/gxf/double_buffer_receiver.hpp +++ b/include/holoscan/core/resources/gxf/double_buffer_receiver.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,6 +18,10 @@ #ifndef HOLOSCAN_CORE_RESOURCES_GXF_DOUBLE_BUFFER_RECEIVER_HPP #define HOLOSCAN_CORE_RESOURCES_GXF_DOUBLE_BUFFER_RECEIVER_HPP +#include + +#include + #include "./receiver.hpp" namespace holoscan { @@ -26,6 +30,7 @@ class DoubleBufferReceiver : public Receiver { public: HOLOSCAN_RESOURCE_FORWARD_ARGS_SUPER(DoubleBufferReceiver, Receiver) DoubleBufferReceiver() = default; + DoubleBufferReceiver(const std::string& name, nvidia::gxf::DoubleBufferReceiver* component); const char* gxf_typename() const override { return "nvidia::gxf::DoubleBufferReceiver"; } diff --git a/include/holoscan/core/resources/gxf/double_buffer_transmitter.hpp b/include/holoscan/core/resources/gxf/double_buffer_transmitter.hpp index ba727d8e..cb56a05b 100644 --- a/include/holoscan/core/resources/gxf/double_buffer_transmitter.hpp +++ b/include/holoscan/core/resources/gxf/double_buffer_transmitter.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,6 +18,62 @@ #ifndef HOLOSCAN_CORE_RESOURCES_GXF_DOUBLE_BUFFER_TRANSMITTER_HPP #define HOLOSCAN_CORE_RESOURCES_GXF_DOUBLE_BUFFER_TRANSMITTER_HPP +#include +// gxf/std/double_buffer_transmitter.hpp is missing in the GXF SDK +// The following is a copy of the file from the GXF SDK until it is fixed +// TODO: Remove this file once the issue is fixed +//////////////////////////////////////////////////////////////////////////////////////////////////// +#include + +#include "gxf/core/component.hpp" +#include "gxf/core/entity.hpp" +#include "gxf/core/handle.hpp" +#include "gxf/std/gems/staging_queue/staging_queue.hpp" +#include "gxf/std/transmitter.hpp" + +namespace nvidia { +namespace gxf { + +// A transmitter which uses a double-buffered queue where messages are pushed to a backstage after +// they are published. Outgoing messages are not immediately available and need to be +// moved to the backstage first. +class DoubleBufferTransmitter : public Transmitter { + public: + using queue_t = ::gxf::staging_queue::StagingQueue; + + gxf_result_t registerInterface(Registrar* registrar) override; + + gxf_result_t initialize() override; + + gxf_result_t deinitialize() override; + + gxf_result_t pop_abi(gxf_uid_t* uid) override; + + gxf_result_t push_abi(gxf_uid_t other) override; + + gxf_result_t peek_abi(gxf_uid_t* uid, int32_t index) override; + + size_t capacity_abi() override; + + size_t size_abi() override; + + gxf_result_t publish_abi(gxf_uid_t uid) override; + + size_t back_size_abi() override; + + gxf_result_t sync_abi() override; + + Parameter capacity_; + Parameter policy_; + + private: + std::unique_ptr queue_; +}; + +} // namespace gxf +} // namespace nvidia +//////////////////////////////////////////////////////////////////////////////////////////////////// + #include "./transmitter.hpp" namespace holoscan { @@ -26,6 +82,7 @@ class DoubleBufferTransmitter : public Transmitter { public: HOLOSCAN_RESOURCE_FORWARD_ARGS_SUPER(DoubleBufferTransmitter, Transmitter) DoubleBufferTransmitter() = default; + DoubleBufferTransmitter(const std::string& name, nvidia::gxf::DoubleBufferTransmitter* component); const char* gxf_typename() const override { return "nvidia::gxf::DoubleBufferTransmitter"; } diff --git a/include/holoscan/core/resources/gxf/receiver.hpp b/include/holoscan/core/resources/gxf/receiver.hpp index 088cb86c..64ae56f7 100644 --- a/include/holoscan/core/resources/gxf/receiver.hpp +++ b/include/holoscan/core/resources/gxf/receiver.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,6 +18,10 @@ #ifndef HOLOSCAN_CORE_RESOURCES_GXF_RECEIVER_HPP #define HOLOSCAN_CORE_RESOURCES_GXF_RECEIVER_HPP +#include + +#include + #include "../../gxf/gxf_resource.hpp" namespace holoscan { @@ -26,6 +30,7 @@ class Receiver : public gxf::GXFResource { public: HOLOSCAN_RESOURCE_FORWARD_ARGS_SUPER(Receiver, GXFResource) Receiver() = default; + Receiver(const std::string& name, nvidia::gxf::Receiver* component); const char* gxf_typename() const override { return "nvidia::gxf::Receiver"; } }; diff --git a/include/holoscan/core/resources/gxf/transmitter.hpp b/include/holoscan/core/resources/gxf/transmitter.hpp index 422ff00e..02563fcf 100644 --- a/include/holoscan/core/resources/gxf/transmitter.hpp +++ b/include/holoscan/core/resources/gxf/transmitter.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,6 +18,10 @@ #ifndef HOLOSCAN_CORE_RESOURCES_GXF_TRANSMITTER_HPP #define HOLOSCAN_CORE_RESOURCES_GXF_TRANSMITTER_HPP +#include + +#include + #include "../../gxf/gxf_resource.hpp" namespace holoscan { @@ -26,6 +30,7 @@ class Transmitter : public gxf::GXFResource { public: HOLOSCAN_RESOURCE_FORWARD_ARGS_SUPER(Transmitter, GXFResource) Transmitter() = default; + Transmitter(const std::string& name, nvidia::gxf::Transmitter* component); const char* gxf_typename() const override { return "nvidia::gxf::Transmitter"; } }; diff --git a/include/holoscan/core/resources/gxf/unbounded_allocator.hpp b/include/holoscan/core/resources/gxf/unbounded_allocator.hpp index e344df94..40a74f92 100644 --- a/include/holoscan/core/resources/gxf/unbounded_allocator.hpp +++ b/include/holoscan/core/resources/gxf/unbounded_allocator.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,8 +15,12 @@ * limitations under the License. */ -#ifndef HOLOSCAN_CORE_RESOURCES_UNBOUNDED_ALLOCATOR_HPP -#define HOLOSCAN_CORE_RESOURCES_UNBOUNDED_ALLOCATOR_HPP +#ifndef HOLOSCAN_CORE_RESOURCES_GXF_UNBOUNDED_ALLOCATOR_HPP +#define HOLOSCAN_CORE_RESOURCES_GXF_UNBOUNDED_ALLOCATOR_HPP + +#include + +#include #include "./allocator.hpp" @@ -26,10 +30,12 @@ class UnboundedAllocator : public Allocator { public: HOLOSCAN_RESOURCE_FORWARD_ARGS_SUPER(UnboundedAllocator, Allocator) UnboundedAllocator() = default; + UnboundedAllocator(const std::string& name, nvidia::gxf::UnboundedAllocator* component) + : Allocator(name, component) {} const char* gxf_typename() const override { return "nvidia::gxf::UnboundedAllocator"; } }; } // namespace holoscan -#endif /* HOLOSCAN_CORE_RESOURCES_UNBOUNDED_ALLOCATOR_HPP */ +#endif /* HOLOSCAN_CORE_RESOURCES_GXF_UNBOUNDED_ALLOCATOR_HPP */ diff --git a/include/holoscan/core/type_traits.hpp b/include/holoscan/core/type_traits.hpp index f1ec7afb..d5233502 100644 --- a/include/holoscan/core/type_traits.hpp +++ b/include/holoscan/core/type_traits.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,6 +19,7 @@ #define HOLOSCAN_CORE_TYPES_HPP #include +#include #include #include #include @@ -34,23 +35,20 @@ struct array_type : std::integral_constant {}; // base_type/base_type_t template -struct _base_type { +struct base_type { using type = std::decay_t; }; template -struct _base_type>>> { +struct base_type>>> { using type = Resource; }; template -struct _base_type>>> { +struct base_type>>> { using type = Condition; }; -template -using base_type = _base_type>; - template using base_type_t = typename base_type::type; @@ -61,7 +59,7 @@ struct _type_info { using container_type = scalar_type; using element_type = base_type_t; using derived_type = std::decay_t; - static constexpr int dimension = 0; + static constexpr int32_t dimension = 0; }; template @@ -69,7 +67,7 @@ struct _type_info> { using container_type = scalar_type; using element_type = std::shared_ptr>; using derived_type = std::decay_t; - static constexpr int dimension = 0; + static constexpr int32_t dimension = 0; }; template @@ -77,7 +75,7 @@ struct _type_info> { using container_type = vector_type; using element_type = base_type_t; using derived_type = std::decay_t; - static constexpr int dimension = 1; + static constexpr int32_t dimension = 1; }; template @@ -85,7 +83,7 @@ struct _type_info>> { using container_type = vector_type; using element_type = std::shared_ptr>; using derived_type = std::decay_t; - static constexpr int dimension = 1; + static constexpr int32_t dimension = 1; }; template @@ -93,7 +91,7 @@ struct _type_info>> { using container_type = vector_type; using element_type = base_type_t; using derived_type = std::decay_t; - static constexpr int dimension = 2; + static constexpr int32_t dimension = 2; }; template @@ -101,7 +99,7 @@ struct _type_info>>> { using container_type = vector_type; using element_type = std::shared_ptr>; using derived_type = std::decay_t; - static constexpr int dimension = 2; + static constexpr int32_t dimension = 2; }; template @@ -109,7 +107,7 @@ struct _type_info> { using container_type = array_type; using element_type = base_type_t; using derived_type = std::decay_t; - static constexpr int dimension = 1; + static constexpr int32_t dimension = 1; }; template @@ -117,7 +115,7 @@ struct _type_info>> { using container_type = scalar_type; using element_type = std::shared_ptr>; using derived_type = std::decay_t; - static constexpr int dimension = 0; + static constexpr int32_t dimension = 0; }; template @@ -125,7 +123,7 @@ struct _type_info>> { using container_type = vector_type; using element_type = base_type_t; using derived_type = std::decay_t; - static constexpr int dimension = 1; + static constexpr int32_t dimension = 1; }; template @@ -133,7 +131,7 @@ struct _type_info>>> { using container_type = vector_type; using element_type = base_type_t; using derived_type = std::decay_t; - static constexpr int dimension = 2; + static constexpr int32_t dimension = 2; }; template @@ -141,7 +139,7 @@ struct _type_info>> { using container_type = array_type; using element_type = base_type_t; using derived_type = std::decay_t; - static constexpr int dimension = 1; + static constexpr int32_t dimension = 1; }; template @@ -214,7 +212,7 @@ inline constexpr bool is_one_of_derived_v = ((std::is_base_of_v || ... // dimension_of_v template -inline constexpr int dimension_of_v = type_info::dimension; +inline constexpr int32_t dimension_of_v = type_info::dimension; } // namespace holoscan diff --git a/include/holoscan/core/logger.hpp b/include/holoscan/logger/logger.hpp similarity index 98% rename from include/holoscan/core/logger.hpp rename to include/holoscan/logger/logger.hpp index f27c0c64..5fdc2cb9 100644 --- a/include/holoscan/core/logger.hpp +++ b/include/holoscan/logger/logger.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/include/holoscan/operators/README.md b/include/holoscan/operators/README.md index e53632d5..a53e7a7d 100644 --- a/include/holoscan/operators/README.md +++ b/include/holoscan/operators/README.md @@ -1,21 +1,16 @@ # Operators -## Folder Structure +These are the operators included as part of the Holoscan SDK: -It currently includes Operators (components) that are needed to create the endoscopy and ultrasound sample applications. - -- aja_source -- custom_lstm_inference -- format_converter -- holoviz -- segmentation_postprocessor - - postprocessing used in the ultrasound sample application -- stream_playback - - This folder has both VideoStreamReplayerOp and VideoStreamRecorderOp - - Good candidates to build a GXF extension that has multiple Operators - - VideoStreamReplayerOp is using Holoscan API and VideoStreamRecorderOp is generated by a GXF extension -- tensor_rt - - This folder has GXF's built-in TensorRT Codelet(Operator). -- tool_tracking_postprocessor - - postprocessing used in the endoscopy sample application -- visualizer_tool_tracking +- **aja_source**: support AJA capture card as source +- **bayer_demosaic**: perform color filter array (CFA) interpolation for 1-channel inputs of 8 or 16-bit unsigned integer and outputs an RGB or RGBA image +- **format_converter**: provides common video or tensor operations in inference pipelines to change datatypes, resize images, reorder channels, and normalize and scale values. +- **holoviz**: handles compositing, blending, and visualization of RGB or RGBA images, masks, geometric primitives, text and depth maps +- **multiai_inference**: perform AI inference +- **multiai_postprocessor**: perform AI postprocessing +- **ping_rx**: "receive" and log an int value +- **ping_tx**: "transmit" an int value +- **segmentation_postprocessor**: generic AI postprocessing operator +- **tensor_rt** *(deprecated)*: perform AI inference with TensorRT +- **video_stream_recorder**: write a video stream output as `.gxf_entities` + `.gxf_index` files on disk +- **video_stream_replayer**: read `.gxf_entities` + `.gxf_index` files on disk as a video stream input diff --git a/include/holoscan/operators/aja_source/aja_source.hpp b/include/holoscan/operators/aja_source/aja_source.hpp index 6ebfe5cf..ae330768 100644 --- a/include/holoscan/operators/aja_source/aja_source.hpp +++ b/include/holoscan/operators/aja_source/aja_source.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,33 +18,47 @@ #ifndef HOLOSCAN_OPERATORS_AJA_SOURCE_AJA_SOURCE_HPP #define HOLOSCAN_OPERATORS_AJA_SOURCE_AJA_SOURCE_HPP +#include +#include +#include + #include #include #include -#include "../../core/gxf/gxf_operator.hpp" +#include "holoscan/core/gxf/gxf_operator.hpp" #include "./ntv2channel.hpp" namespace holoscan::ops { /** * @brief Operator class to get the video stream from AJA capture card. - * - * This wraps a GXF Codelet(`nvidia::holoscan::AJASource`). */ -class AJASourceOp : public holoscan::ops::GXFOperator { +class AJASourceOp : public holoscan::Operator { public: - HOLOSCAN_OPERATOR_FORWARD_ARGS_SUPER(AJASourceOp, holoscan::ops::GXFOperator) - - AJASourceOp() = default; + HOLOSCAN_OPERATOR_FORWARD_ARGS(AJASourceOp) - const char* gxf_typename() const override { return "nvidia::holoscan::AJASource"; } + AJASourceOp(); void setup(OperatorSpec& spec) override; void initialize() override; + void start() override; + void compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) override; + void stop() override; private: + AJAStatus DetermineVideoFormat(); + AJAStatus OpenDevice(); + AJAStatus SetupVideo(); + AJAStatus SetupBuffers(); + AJAStatus StartAutoCirculate(); + bool AllocateBuffers(std::vector& buffers, size_t num_buffers, size_t buffer_size, + bool rdma); + void FreeBuffers(std::vector& buffers, bool rdma); + bool GetNTV2VideoFormatTSI(NTV2VideoFormat* format); + Parameter video_buffer_output_; Parameter device_specifier_; Parameter channel_; @@ -57,6 +71,20 @@ class AJASourceOp : public holoscan::ops::GXFOperator { Parameter overlay_rdma_; Parameter overlay_buffer_input_; Parameter overlay_buffer_output_; + + // internal state + CNTV2Card device_; + NTV2DeviceID device_id_; + NTV2VideoFormat video_format_; + NTV2PixelFormat pixel_format_ = NTV2_FBF_ABGR; + bool use_tsi_ = false; + bool is_kona_hdmi_ = false; + + std::vector buffers_; + std::vector overlay_buffers_; + uint8_t current_buffer_ = 0; + uint8_t current_hw_frame_ = 0; + uint8_t current_overlay_hw_frame_ = 0; }; } // namespace holoscan::ops diff --git a/include/holoscan/operators/aja_source/ntv2channel.hpp b/include/holoscan/operators/aja_source/ntv2channel.hpp index dadf601c..d8fc61e3 100644 --- a/include/holoscan/operators/aja_source/ntv2channel.hpp +++ b/include/holoscan/operators/aja_source/ntv2channel.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,23 +18,12 @@ #ifndef HOLOSCAN_OPERATORS_AJA_SOURCE_NTV2CHANNEL_HPP #define HOLOSCAN_OPERATORS_AJA_SOURCE_NTV2CHANNEL_HPP +#include #include + #include #include -typedef enum { - NTV2_CHANNEL1, ///< @brief Specifies channel or Frame Store 1 (or the first item). - NTV2_CHANNEL2, ///< @brief Specifies channel or Frame Store 2 (or the 2nd item). - NTV2_CHANNEL3, ///< @brief Specifies channel or Frame Store 3 (or the 3rd item). - NTV2_CHANNEL4, ///< @brief Specifies channel or Frame Store 4 (or the 4th item). - NTV2_CHANNEL5, ///< @brief Specifies channel or Frame Store 5 (or the 5th item). - NTV2_CHANNEL6, ///< @brief Specifies channel or Frame Store 6 (or the 6th item). - NTV2_CHANNEL7, ///< @brief Specifies channel or Frame Store 7 (or the 7th item). - NTV2_CHANNEL8, ///< @brief Specifies channel or Frame Store 8 (or the 8th item). - NTV2_MAX_NUM_CHANNELS, // Always last! - NTV2_CHANNEL_INVALID = NTV2_MAX_NUM_CHANNELS -} NTV2Channel; - template <> struct YAML::convert { static Node encode(const NTV2Channel& rhs) { diff --git a/include/holoscan/operators/bayer_demosaic/bayer_demosaic.hpp b/include/holoscan/operators/bayer_demosaic/bayer_demosaic.hpp index c31387a7..57003fc3 100644 --- a/include/holoscan/operators/bayer_demosaic/bayer_demosaic.hpp +++ b/include/holoscan/operators/bayer_demosaic/bayer_demosaic.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +21,7 @@ #include #include -#include "../../core/gxf/gxf_operator.hpp" +#include "holoscan/core/gxf/gxf_operator.hpp" namespace holoscan::ops { @@ -58,4 +58,3 @@ class BayerDemosaicOp : public holoscan::ops::GXFOperator { } // namespace holoscan::ops #endif /* HOLOSCAN_OPERATORS_BAYER_DEMOSAIC_HPP */ - diff --git a/include/holoscan/operators/custom_lstm_inference/lstm_tensor_rt_inference.hpp b/include/holoscan/operators/custom_lstm_inference/lstm_tensor_rt_inference.hpp deleted file mode 100644 index 8486ea3a..00000000 --- a/include/holoscan/operators/custom_lstm_inference/lstm_tensor_rt_inference.hpp +++ /dev/null @@ -1,74 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef HOLOSCAN_OPERATORS_CUSTOM_LSTM_INFERENCE_LSTM_TENSOR_RT_INFERENCE_HPP -#define HOLOSCAN_OPERATORS_CUSTOM_LSTM_INFERENCE_LSTM_TENSOR_RT_INFERENCE_HPP - -#include -#include -#include - -#include "../../core/gxf/gxf_operator.hpp" - -namespace holoscan::ops { - -/** - * @brief Operator class to perform the inference of the LSTM model. - * - * This wraps a GXF Codelet(`nvidia::holoscan::custom_lstm_inference::TensorRtInference`). - */ -class LSTMTensorRTInferenceOp : public holoscan::ops::GXFOperator { - public: - HOLOSCAN_OPERATOR_FORWARD_ARGS_SUPER(LSTMTensorRTInferenceOp, holoscan::ops::GXFOperator) - - LSTMTensorRTInferenceOp() = default; - - const char* gxf_typename() const override { - return "nvidia::holoscan::custom_lstm_inference::TensorRtInference"; - } - - // TODO(gbae): use std::expected - void setup(OperatorSpec& spec) override; - - private: - Parameter model_file_path_; - Parameter engine_cache_dir_; - Parameter plugins_lib_namespace_; - Parameter force_engine_update_; - Parameter> input_tensor_names_; - Parameter> input_state_tensor_names_; - Parameter> input_binding_names_; - Parameter> output_tensor_names_; - Parameter> output_state_tensor_names_; - Parameter> output_binding_names_; - Parameter> pool_; - Parameter> cuda_stream_pool_; - Parameter max_workspace_size_; - Parameter dla_core_; - Parameter max_batch_size_; - Parameter enable_fp16_; - Parameter relaxed_dimension_check_; - Parameter verbose_; - Parameter> clock_; - - Parameter> rx_; - Parameter tx_; -}; - -} // namespace holoscan::ops - -#endif /* HOLOSCAN_OPERATORS_CUSTOM_LSTM_INFERENCE_LSTM_TENSOR_RT_INFERENCE_HPP */ diff --git a/include/holoscan/operators/emergent_source/emergent_source.hpp b/include/holoscan/operators/emergent_source/emergent_source.hpp deleted file mode 100644 index f08c42eb..00000000 --- a/include/holoscan/operators/emergent_source/emergent_source.hpp +++ /dev/null @@ -1,55 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef HOLOSCAN_OPERATORS_EMERGENT_SOURCE_HPP -#define HOLOSCAN_OPERATORS_EMERGENT_SOURCE_HPP - -#include "../../core/gxf/gxf_operator.hpp" - - -namespace holoscan::ops { - -/** - * @brief Operator class to get the video stream from Emergent Vision Technologies - * camera using MLNX ConnectX SmartNIC. - * - * This wraps a GXF Codelet(`nvidia::holoscan::EmergentSource`). - */ -class EmergentSourceOp : public holoscan::ops::GXFOperator { - public: - HOLOSCAN_OPERATOR_FORWARD_ARGS_SUPER(EmergentSourceOp, holoscan::ops::GXFOperator) - - EmergentSourceOp() = default; - - const char* gxf_typename() const override { return "nvidia::holoscan::EmergentSource"; } - - void setup(OperatorSpec& spec) override; - - void initialize() override; - - private: - Parameter signal_; - Parameter width_; - Parameter height_; - Parameter framerate_; - Parameter use_rdma_; -}; - -} // namespace holoscan::ops - -#endif /* HOLOSCAN_OPERATORS_EMERGENT_SOURCE_HPP */ - diff --git a/include/holoscan/operators/format_converter/format_converter.hpp b/include/holoscan/operators/format_converter/format_converter.hpp index 8de7b117..9f84e538 100644 --- a/include/holoscan/operators/format_converter/format_converter.hpp +++ b/include/holoscan/operators/format_converter/format_converter.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,31 +18,57 @@ #ifndef HOLOSCAN_OPERATORS_FORMAT_CONVERTER_FORMAT_CONVERTER_HPP #define HOLOSCAN_OPERATORS_FORMAT_CONVERTER_FORMAT_CONVERTER_HPP +#include + #include #include #include -#include "../../core/gxf/gxf_operator.hpp" +#include "holoscan/core/gxf/gxf_operator.hpp" +#include "holoscan/utils/cuda_stream_handler.hpp" namespace holoscan::ops { +enum class FormatDType { kUnknown, kRGB888, kRGBA8888, kUnsigned8, kFloat32, kYUV420, kNV12 }; + +enum class FormatConversionType { + kUnknown, + kNone, + kUnsigned8ToFloat32, + kFloat32ToUnsigned8, + kRGB888ToRGBA8888, + kRGBA8888ToRGB888, + kRGBA8888ToFloat32, + kRGB888ToYUV420, + kYUV420ToRGBA8888, + kYUV420ToRGB888, + kNV12ToRGB888 +}; + /** * @brief Operator class to convert the data format of the input data. - * - * This wraps a GXF Codelet(`nvidia::holoscan::formatconverter::FormatConverter`). */ -class FormatConverterOp : public holoscan::ops::GXFOperator { +class FormatConverterOp : public holoscan::Operator { public: - HOLOSCAN_OPERATOR_FORWARD_ARGS_SUPER(FormatConverterOp, holoscan::ops::GXFOperator) + HOLOSCAN_OPERATOR_FORWARD_ARGS(FormatConverterOp) FormatConverterOp() = default; - const char* gxf_typename() const override { - return "nvidia::holoscan::formatconverter::FormatConverter"; - } - // TODO(gbae): use std::expected void setup(OperatorSpec& spec) override; + void initialize() override; + void start() override; + void compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) override; + void stop() override; + + nvidia::gxf::Expected resizeImage(const void* in_tensor_data, const int32_t rows, + const int32_t columns, const int16_t channels, + const nvidia::gxf::PrimitiveType primitive_type, + const int32_t resize_width, const int32_t resize_height); + void convertTensorFormat(const void* in_tensor_data, void* out_tensor_data, const int32_t rows, + const int32_t columns, const int16_t in_channels, + const int16_t out_channels); private: Parameter in_; @@ -62,6 +88,20 @@ class FormatConverterOp : public holoscan::ops::GXFOperator { Parameter in_dtype_str_; Parameter out_dtype_str_; + + // internal state + FormatDType in_dtype_ = FormatDType::kUnknown; + FormatDType out_dtype_ = FormatDType::kUnknown; + nvidia::gxf::PrimitiveType in_primitive_type_ = nvidia::gxf::PrimitiveType::kCustom; + nvidia::gxf::PrimitiveType out_primitive_type_ = nvidia::gxf::PrimitiveType::kCustom; + FormatConversionType format_conversion_type_ = FormatConversionType::kUnknown; + + nvidia::gxf::MemoryBuffer resize_buffer_; + nvidia::gxf::MemoryBuffer channel_buffer_; + nvidia::gxf::MemoryBuffer device_scratch_buffer_; + NppStreamContext npp_stream_ctx_{}; + + CudaStreamHandler cuda_stream_handler_; }; } // namespace holoscan::ops diff --git a/include/holoscan/operators/holoviz/holoviz.hpp b/include/holoscan/operators/holoviz/holoviz.hpp index 91575dcb..71ef785d 100644 --- a/include/holoscan/operators/holoviz/holoviz.hpp +++ b/include/holoscan/operators/holoviz/holoviz.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,21 +22,113 @@ #include #include -#include "../../core/gxf/gxf_operator.hpp" +#include "holoscan/core/gxf/gxf_operator.hpp" +#include "holoscan/utils/cuda_stream_handler.hpp" namespace holoscan::ops { -class HolovizOp : public holoscan::ops::GXFOperator { +/** + * @brief Operator class for data visualization. + * + * This high-speed viewer handles compositing, blending, and visualization of RGB or RGBA images, + * masks, geometric primitives, text and depth maps. The operator can auto detect the format of the + * input tensors when only the `receivers` parameter list is specified. + * + * 1. Displaying Color Images + * + * Image data can either be on host or device (GPU). Multiple image formats are supported + * - R 8 bit unsigned + * - R 16 bit unsigned + * - R 16 bit float + * - R 32 bit unsigned + * - R 32 bit float + * - RGB 8 bit unsigned + * - BGR 8 bit unsigned + * - RGBA 8 bit unsigned + * - BGRA 8 bit unsigned + * - RGBA 16 bit unsigned + * - RGBA 16 bit float + * - RGBA 32 bit float + * + * When the `type` parameter is set to `color_lut` the final color is looked up using the values + * from the `color_lut` parameter. For color lookups these image formats are supported + * - R 8 bit unsigned + * - R 16 bit unsigned + * - R 32 bit unsigned + * + * 2. Drawing Geometry + * + * In all cases, `x` and `y` are normalized coordinates in the range `[0, 1]`. The `x` and `y` + * correspond to the horizontal and vertical axes of the display, respectively. The origin `(0, 0)` + * is at the top left of the display. All coordinates should be defined using a single precision + * float data type. Geometric primitives outside of the visible area are clipped. Coordinate arrays + * are expected to have the shape `(1, N, C)` where `N` is the coordinate count and `C` is the + * component count for each coordinate. + * + * - Points are defined by a `(x, y)` coordinate pair. + * - Lines are defined by a set of two `(x, y)` coordinate pairs. + * - Lines strips are defined by a sequence of `(x, y)` coordinate pairs. The first two coordinates + * define the first line, each additional coordinate adds a line connecting to the previous + * coordinate. + * - Triangles are defined by a set of three `(x, y)` coordinate pairs. + * - Crosses are defined by `(x, y, size)` tuples. `size` specifies the size of the cross in the `x` + * direction and is optional, if omitted it's set to `0.05`. The size in the `y` direction is + * calculated using the aspect ratio of the window to make the crosses square. + * - Rectangles (bounding boxes) are defined by a pair of 2-tuples defining the upper-left and + * lower-right coordinates of a box: `(x1, y1), (x2, y2)`. + * - Ovals are defined by `(x, y, size_x, size_y)` tuples. `size_x` and `size_y` are optional, if + * omitted they are set to `0.05`. + * - Texts are defined by `(x, y, size)` tuples. `size` specifies the size of the text in `y` + * direction and is optional, if omitted it's set to `0.05`. The size in the `x` direction is + * calculated using the aspect ratio of the window. The index of each coordinate references a text + * string from the `text` parameter and the index is clamped to the size of the text array. For + * example, if there is one item set for the `text` parameter, e.g. `text=['my_text']` and three + * coordinates, then `my_text` is rendered three times. If `text=['first text', 'second text']` and + * three coordinates are specified, then `first text` is rendered at the first coordinate, `second + * text` at the second coordinate and then `second text` again at the third coordinate. The `text` + * string array is fixed and can't be changed after initialization. To hide text which should not be + * displayed, specify coordinates greater than `(1.0, 1.0)` for the text item, the text is then + * clipped away. + * + * 3. Displaying Depth Maps + * + * When `type` is `depth_map` the provided data is interpreted as a rectangular array of depth + * values. Additionally a 2d array with a color value for each point in the grid can be specified by + * setting `type` to `depth_map_color`. + * + * The type of geometry drawn can be selected by setting `depth_map_render_mode`. + * + * Depth maps are rendered in 3D and support camera movement. The camera is controlled using the + * mouse: + * - Orbit (LMB) + * - Pan (LMB + CTRL | MMB) + * - Dolly (LMB + SHIFT | RMB | Mouse wheel) + * - Look Around (LMB + ALT | LMB + CTRL + SHIFT) + * - Zoom (Mouse wheel + SHIFT) + * + * 4. Output + * + * By default a window is opened to display the rendering, but the extension can also be run in + * headless mode with the `headless` parameter. + * + * Using a display in exclusive mode is also supported with the `use_exclusive_display` parameter. + * This reduces the latency by avoiding the desktop compositor. + * + * The rendered framebuffer can be output to `render_buffer_output`. + * + */ +class HolovizOp : public Operator { public: - HOLOSCAN_OPERATOR_FORWARD_ARGS_SUPER(HolovizOp, holoscan::ops::GXFOperator) + HOLOSCAN_OPERATOR_FORWARD_ARGS(HolovizOp) HolovizOp() = default; - const char* gxf_typename() const override { return "nvidia::holoscan::Holoviz"; } - void setup(OperatorSpec& spec) override; - void initialize() override; + void start() override; + void compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) override; + void stop() override; /** * Input type. @@ -60,8 +152,28 @@ class HolovizOp : public holoscan::ops::GXFOperator { ///< (xi, yi) and (xi+1, yi+1) OVALS, ///< oval primitive, an oval primitive is defined by the center coordinate and the axis ///< sizes (xi, yi, sxi, syi) - TEXT ///< text is defined by the top left coordinate and the size (x, y, s) per string, text + TEXT, ///< text is defined by the top left coordinate and the size (x, y, s) per string, text ///< strings are define by InputSpec::text_ + DEPTH_MAP, ///< single channel 2d array where each element represents a depth value. The data + ///< is rendered as a 3d object using points, lines or triangles. The color for the + ///< elements can be specified through `DEPTH_MAP_COLOR`. + ///< Supported format: + ///< 8-bit unsigned normalized format that has a single 8-bit depth component + DEPTH_MAP_COLOR ///< RGBA 2d image, same size as the depth map. One color value for each + ///< element of the depth map grid. + ///< Supported format: + ///< 32-bit unsigned normalized format that has an 8-bit R component in byte + ///> 0, an 8-bit G component in byte 1, an 8-bit B component in byte 2, + ///< and an 8-bit A component in byte 3unsigned 8-bit RGBA + }; + + /** + * Depth map render mode. + */ + enum class DepthMapRenderMode { + POINTS, ///< render points + LINES, ///< render lines + TRIANGLES ///< render triangles }; /** @@ -86,11 +198,15 @@ class HolovizOp : public holoscan::ops::GXFOperator { std::vector color_{1.f, 1.f, 1.f, 1.f}; ///< color of rendered geometry float line_width_ = 1.f; ///< line width for geometry made of lines float point_size_ = 1.f; ///< point size for geometry made of points - std::vector text_; ///< array of text strings, used when type_ is text. + std::vector text_; ///< array of text strings, used when type_ is TEXT. + DepthMapRenderMode depth_map_render_mode_ = + DepthMapRenderMode::POINTS; ///< depth map render mode, used if type_ is + ///< DEPTH_MAP or DEPTH_MAP_COLOR. }; private: - void enable_conditional_port(const std::string &name); + void enable_conditional_port(const std::string& name); + bool check_port_enabled(const std::string& name); Parameter> receivers_; @@ -105,13 +221,21 @@ class HolovizOp : public holoscan::ops::GXFOperator { Parameter display_name_; Parameter width_; Parameter height_; - Parameter framerate_; + Parameter framerate_; Parameter use_exclusive_display_; Parameter fullscreen_; Parameter headless_; - Parameter> window_close_scheduling_term_; - + Parameter> window_close_scheduling_term_; Parameter> allocator_; + Parameter font_path_; + + // internal state + std::vector lut_; + std::vector input_spec_; + CudaStreamHandler cuda_stream_handler_; + bool render_buffer_input_enabled_; + bool render_buffer_output_enabled_; + bool is_first_tick_ = true; }; } // namespace holoscan::ops diff --git a/include/holoscan/operators/multiai_inference/multiai_inference.hpp b/include/holoscan/operators/multiai_inference/multiai_inference.hpp index fc3369aa..47fbb6c4 100644 --- a/include/holoscan/operators/multiai_inference/multiai_inference.hpp +++ b/include/holoscan/operators/multiai_inference/multiai_inference.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,7 +23,13 @@ #include #include -#include "../../core/gxf/gxf_operator.hpp" +#include "holoscan/core/gxf/gxf_operator.hpp" + +#include "holoinfer.hpp" +#include "holoinfer_utils.hpp" +#include "holoinfer_buffer.hpp" + +namespace HoloInfer = holoscan::inference; namespace holoscan::ops { /** @@ -31,18 +37,18 @@ namespace holoscan::ops { * * Class wraps a GXF Codelet(`nvidia::holoscan::multiai::MultiAIInference`). */ -class MultiAIInferenceOp : public holoscan::ops::GXFOperator { +class MultiAIInferenceOp : public holoscan::Operator { public: - HOLOSCAN_OPERATOR_FORWARD_ARGS_SUPER(MultiAIInferenceOp, holoscan::ops::GXFOperator) + HOLOSCAN_OPERATOR_FORWARD_ARGS(MultiAIInferenceOp) MultiAIInferenceOp() = default; - const char* gxf_typename() const override { - return "nvidia::holoscan::multiai::MultiAIInference"; - } - void setup(OperatorSpec& spec) override; void initialize() override; + void start() override; + void compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) override; + void stop() override; /** * DataMap specification @@ -123,6 +129,24 @@ class MultiAIInferenceOp : public holoscan::ops::GXFOperator { /// @brief Output transmitter. Single transmitter supported. Parameter> transmitter_; + + // Internal state + + /// Pointer to inference context. + std::unique_ptr holoscan_infer_context_; + + /// Pointer to multi ai inference specifications + std::shared_ptr multiai_specs_; + + /// Data type of message to be transmitted. Supported value: float32 + nvidia::gxf::PrimitiveType data_type_ = nvidia::gxf::PrimitiveType::kFloat32; + + /// Map holding dimensions per model. Key is model name and value is a vector with + /// dimensions. + std::map> dims_per_tensor_; + + /// Codelet Identifier, used in reporting. + const std::string module_{"Multi AI Inference Codelet"}; }; } // namespace holoscan::ops diff --git a/include/holoscan/operators/multiai_postprocessor/multiai_postprocessor.hpp b/include/holoscan/operators/multiai_postprocessor/multiai_postprocessor.hpp index a98f3cb7..23b858ee 100644 --- a/include/holoscan/operators/multiai_postprocessor/multiai_postprocessor.hpp +++ b/include/holoscan/operators/multiai_postprocessor/multiai_postprocessor.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,7 +23,13 @@ #include #include -#include "../../core/gxf/gxf_operator.hpp" +#include "holoscan/core/gxf/gxf_operator.hpp" + +#include "holoinfer.hpp" +#include "holoinfer_utils.hpp" +#include "holoinfer_buffer.hpp" + +namespace HoloInfer = holoscan::inference; namespace holoscan::ops { /** @@ -31,18 +37,17 @@ namespace holoscan::ops { * * Class wraps a GXF Codelet(`nvidia::holoscan::multiai::MultiAIPostprocessor`). */ -class MultiAIPostprocessorOp : public holoscan::ops::GXFOperator { +class MultiAIPostprocessorOp : public holoscan::Operator { public: - HOLOSCAN_OPERATOR_FORWARD_ARGS_SUPER(MultiAIPostprocessorOp, holoscan::ops::GXFOperator) + HOLOSCAN_OPERATOR_FORWARD_ARGS(MultiAIPostprocessorOp) MultiAIPostprocessorOp() = default; - const char* gxf_typename() const override { - return "nvidia::holoscan::multiai::MultiAIPostprocessor"; - } - void setup(OperatorSpec& spec) override; void initialize() override; + void start() override; + void compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) override; /** * DataMap specification @@ -106,6 +111,23 @@ class MultiAIPostprocessorOp : public holoscan::ops::GXFOperator { /// @brief Output transmitter. Single transmitter supported. Parameter> transmitter_; + + void conditional_disable_output_port(const std::string& name); + + // Internal state + + /// Pointer to Data Processor context. + std::unique_ptr holoscan_postprocess_context_; + + /// Map holding data per input tensor. + HoloInfer::DataMap data_per_tensor_; + + /// Map holding dimensions per model. Key is model name and value is a vector with + /// dimensions. + std::map> dims_per_tensor_; + + /// Codelet Identifier, used in reporting. + const std::string module_{"Multi AI Postprocessor Codelet"}; }; } // namespace holoscan::ops diff --git a/gxf_extensions/holoviz/holoviz_ext.cpp b/include/holoscan/operators/ping_rx/ping_rx.hpp similarity index 54% rename from gxf_extensions/holoviz/holoviz_ext.cpp rename to include/holoscan/operators/ping_rx/ping_rx.hpp index 428dc475..ab369309 100644 --- a/gxf_extensions/holoviz/holoviz_ext.cpp +++ b/include/holoscan/operators/ping_rx/ping_rx.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,13 +15,25 @@ * limitations under the License. */ -#include "gxf/std/extension_factory_helper.hpp" +#ifndef HOLOSCAN_OPERATORS_PING_RX_HPP +#define HOLOSCAN_OPERATORS_PING_RX_HPP -#include "holoviz.hpp" +#include -GXF_EXT_FACTORY_BEGIN() -GXF_EXT_FACTORY_SET_INFO(0x1c3764640543460f, 0x95c882615dbf23d9, "Holoviz", - "Holoviz extension", "NVIDIA", "0.1.0", "LICENSE"); -GXF_EXT_FACTORY_ADD(0x9c6153f82d0a4fdf, 0xa7df68545c44ccb3, nvidia::holoscan::Holoviz, - nvidia::gxf::Codelet, "Holoviz."); -GXF_EXT_FACTORY_END() +namespace holoscan::ops { + +class PingRxOp : public Operator { + public: + HOLOSCAN_OPERATOR_FORWARD_ARGS(PingRxOp) + + PingRxOp() = default; + + void setup(OperatorSpec& spec) override; + + void compute(InputContext& op_input, OutputContext&, ExecutionContext&) override; +}; + + +} // namespace holoscan::ops + +#endif /* HOLOSCAN_OPERATORS_PING_RX_HPP */ diff --git a/gxf_extensions/aja/aja_ext.cpp b/include/holoscan/operators/ping_tx/ping_tx.hpp similarity index 53% rename from gxf_extensions/aja/aja_ext.cpp rename to include/holoscan/operators/ping_tx/ping_tx.hpp index 679d5455..e1dfad8a 100644 --- a/gxf_extensions/aja/aja_ext.cpp +++ b/include/holoscan/operators/ping_tx/ping_tx.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,13 +14,28 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "gxf/std/extension_factory_helper.hpp" -#include "aja_source.hpp" +#ifndef HOLOSCAN_OPERATORS_PING_TX_HPP +#define HOLOSCAN_OPERATORS_PING_TX_HPP -GXF_EXT_FACTORY_BEGIN() -GXF_EXT_FACTORY_SET_INFO(0x1e53e49e24024aab, 0xb9b08b08768f3817, "AJA", "AJA Extension", "NVIDIA", - "1.0.0", "LICENSE"); -GXF_EXT_FACTORY_ADD(0x3ef12fe13d704e2d, 0x848f4d8dedba9d24, nvidia::holoscan::AJASource, - nvidia::gxf::Codelet, "AJA Source Codelet"); -GXF_EXT_FACTORY_END() +#include + +namespace holoscan::ops { + +class PingTxOp : public Operator { + public: + HOLOSCAN_OPERATOR_FORWARD_ARGS(PingTxOp) + + PingTxOp() = default; + + void setup(OperatorSpec& spec) override; + + void compute(InputContext&, OutputContext& op_output, ExecutionContext&) override; + + private: + int index_ = 1; +}; + +} // namespace holoscan::ops + +#endif /* HOLOSCAN_OPERATORS_PING_TX_HPP */ diff --git a/gxf_extensions/segmentation_postprocessor/segmentation_postprocessor.cu.hpp b/include/holoscan/operators/segmentation_postprocessor/segmentation_postprocessor.cuh similarity index 84% rename from gxf_extensions/segmentation_postprocessor/segmentation_postprocessor.cu.hpp rename to include/holoscan/operators/segmentation_postprocessor/segmentation_postprocessor.cuh index 9b183e2a..11b6eb64 100644 --- a/gxf_extensions/segmentation_postprocessor/segmentation_postprocessor.cu.hpp +++ b/include/holoscan/operators/segmentation_postprocessor/segmentation_postprocessor.cuh @@ -14,11 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +#include + #include -#include -namespace nvidia { -namespace holoscan { +namespace holoscan::ops { namespace segmentation_postprocessor { struct Shape { @@ -40,11 +41,9 @@ enum DataFormat { typedef uint8_t output_type_t; -static constexpr size_t kMaxChannelCount = std::numeric_limits::max(); - void cuda_postprocess(enum NetworkOutputType network_output_type, enum DataFormat data_format, - Shape shape, const float* input, output_type_t* output); + Shape shape, const float* input, output_type_t* output, + cudaStream_t cuda_stream = cudaStreamDefault); } // namespace segmentation_postprocessor -} // namespace holoscan -} // namespace nvidia +} // namespace holoscan::ops diff --git a/include/holoscan/operators/segmentation_postprocessor/segmentation_postprocessor.hpp b/include/holoscan/operators/segmentation_postprocessor/segmentation_postprocessor.hpp index 76f94ae1..d074f3a1 100644 --- a/include/holoscan/operators/segmentation_postprocessor/segmentation_postprocessor.hpp +++ b/include/holoscan/operators/segmentation_postprocessor/segmentation_postprocessor.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,24 +22,31 @@ #include #include -#include "../../core/gxf/gxf_operator.hpp" +#include "holoscan/core/operator.hpp" +#include "holoscan/utils/cuda_stream_handler.hpp" +#include "segmentation_postprocessor.cuh" + +using holoscan::ops::segmentation_postprocessor::DataFormat; +using holoscan::ops::segmentation_postprocessor::NetworkOutputType; namespace holoscan::ops { -class SegmentationPostprocessorOp : public holoscan::ops::GXFOperator { +class SegmentationPostprocessorOp : public Operator { public: - HOLOSCAN_OPERATOR_FORWARD_ARGS_SUPER(SegmentationPostprocessorOp, holoscan::ops::GXFOperator) + HOLOSCAN_OPERATOR_FORWARD_ARGS(SegmentationPostprocessorOp) SegmentationPostprocessorOp() = default; - const char* gxf_typename() const override { - return "nvidia::holoscan::segmentation_postprocessor::Postprocessor"; - } - // TODO(gbae): use std::expected void setup(OperatorSpec& spec) override; + void start() override; + void compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) override; private: + NetworkOutputType network_output_type_value_; + DataFormat data_format_value_; + Parameter in_; Parameter out_; @@ -48,6 +55,8 @@ class SegmentationPostprocessorOp : public holoscan::ops::GXFOperator { Parameter in_tensor_name_; Parameter network_output_type_; Parameter data_format_; + + CudaStreamHandler cuda_stream_handler_; }; } // namespace holoscan::ops diff --git a/include/holoscan/operators/tensor_rt/tensor_rt_inference.hpp b/include/holoscan/operators/tensor_rt/tensor_rt_inference.hpp index f917dd74..83b2f1d1 100644 --- a/include/holoscan/operators/tensor_rt/tensor_rt_inference.hpp +++ b/include/holoscan/operators/tensor_rt/tensor_rt_inference.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/include/holoscan/operators/tool_tracking_postprocessor/tool_tracking_postprocessor.hpp b/include/holoscan/operators/tool_tracking_postprocessor/tool_tracking_postprocessor.hpp deleted file mode 100644 index ff807895..00000000 --- a/include/holoscan/operators/tool_tracking_postprocessor/tool_tracking_postprocessor.hpp +++ /dev/null @@ -1,53 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef HOLOSCAN_OPERATORS_TOOL_TRACKING_POSTPROCESSOR_TOOL_TRACKING_POSTPROCESSOR_HPP -#define HOLOSCAN_OPERATORS_TOOL_TRACKING_POSTPROCESSOR_TOOL_TRACKING_POSTPROCESSOR_HPP - -#include -#include -#include - -#include "../../core/gxf/gxf_operator.hpp" -namespace holoscan::ops { - -class ToolTrackingPostprocessorOp : public holoscan::ops::GXFOperator { - public: - HOLOSCAN_OPERATOR_FORWARD_ARGS_SUPER(ToolTrackingPostprocessorOp, holoscan::ops::GXFOperator) - - ToolTrackingPostprocessorOp() = default; - - const char* gxf_typename() const override { - return "nvidia::holoscan::tool_tracking_postprocessor::Postprocessor"; - } - - void setup(OperatorSpec& spec) override; - - private: - Parameter in_; - Parameter out_; - - Parameter min_prob_; - Parameter>> overlay_img_colors_; - - Parameter> host_allocator_; - Parameter> device_allocator_; -}; - -} // namespace holoscan::ops - -#endif /* HOLOSCAN_OPERATORS_TOOL_TRACKING_POSTPROCESSOR_TOOL_TRACKING_POSTPROCESSOR_HPP */ diff --git a/include/holoscan/operators/stream_playback/video_stream_recorder.hpp b/include/holoscan/operators/video_stream_recorder/video_stream_recorder.hpp similarity index 64% rename from include/holoscan/operators/stream_playback/video_stream_recorder.hpp rename to include/holoscan/operators/video_stream_recorder/video_stream_recorder.hpp index 6f27b3cb..7b84abc4 100644 --- a/include/holoscan/operators/stream_playback/video_stream_recorder.hpp +++ b/include/holoscan/operators/video_stream_recorder/video_stream_recorder.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,26 +23,29 @@ #include #include -#include "../../core/gxf/gxf_operator.hpp" -#include "../../core/fragment.hpp" +#include "holoscan/core/gxf/gxf_operator.hpp" +#include "holoscan/core/fragment.hpp" +#include "gxf/serialization/file_stream.hpp" + namespace holoscan::ops { /** * @brief Operator class to record the video stream to a file. - * - * This wraps a GXF Codelet(`nvidia::gxf::EntityRecorder`). */ -class VideoStreamRecorderOp : public holoscan::ops::GXFOperator { +class VideoStreamRecorderOp : public holoscan::Operator { public: - HOLOSCAN_OPERATOR_FORWARD_ARGS_SUPER(VideoStreamRecorderOp, holoscan::ops::GXFOperator) + HOLOSCAN_OPERATOR_FORWARD_ARGS(VideoStreamRecorderOp) VideoStreamRecorderOp() = default; - const char* gxf_typename() const override { return "nvidia::gxf::EntityRecorder"; } + ~VideoStreamRecorderOp() override; void setup(OperatorSpec& spec) override; void initialize() override; + // void deinitialize() override; + void compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) override; private: Parameter receiver_; @@ -50,6 +53,13 @@ class VideoStreamRecorderOp : public holoscan::ops::GXFOperator { Parameter directory_; Parameter basename_; Parameter flush_on_tick_; + + // File stream for data index + nvidia::gxf::FileStream index_file_stream_; + // File stream for binary data + nvidia::gxf::FileStream binary_file_stream_; + // Offset into binary file + size_t binary_file_offset_; }; } // namespace holoscan::ops diff --git a/include/holoscan/operators/stream_playback/video_stream_replayer.hpp b/include/holoscan/operators/video_stream_replayer/video_stream_replayer.hpp similarity index 61% rename from include/holoscan/operators/stream_playback/video_stream_replayer.hpp rename to include/holoscan/operators/video_stream_replayer/video_stream_replayer.hpp index c31e9d3c..e7a41e4c 100644 --- a/include/holoscan/operators/stream_playback/video_stream_replayer.hpp +++ b/include/holoscan/operators/video_stream_replayer/video_stream_replayer.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,33 +23,32 @@ #include #include -#include "../../core/gxf/gxf_operator.hpp" +#include "holoscan/core/gxf/gxf_operator.hpp" +#include "gxf/serialization/file_stream.hpp" namespace holoscan::ops { /** * @brief Operator class to replay a video stream from a file. - * - * This wraps a GXF Codelet(`nvidia::holoscan::stream_playback::VideoStreamReplayer`). */ -class VideoStreamReplayerOp : public holoscan::ops::GXFOperator { +class VideoStreamReplayerOp : public holoscan::Operator { public: - HOLOSCAN_OPERATOR_FORWARD_ARGS_SUPER(VideoStreamReplayerOp, holoscan::ops::GXFOperator) + HOLOSCAN_OPERATOR_FORWARD_ARGS(VideoStreamReplayerOp) VideoStreamReplayerOp() = default; - const char* gxf_typename() const override { - return "nvidia::holoscan::stream_playback::VideoStreamReplayer"; - } + ~VideoStreamReplayerOp() override; void setup(OperatorSpec& spec) override; void initialize() override; + void compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) override; private: Parameter transmitter_; Parameter> entity_serializer_; - Parameter> boolean_scheduling_term_; + Parameter> boolean_scheduling_term_; Parameter directory_; Parameter basename_; Parameter batch_size_; @@ -58,6 +57,20 @@ class VideoStreamReplayerOp : public holoscan::ops::GXFOperator { Parameter realtime_; Parameter repeat_; Parameter count_; + + // Internal state + // File stream for entities + nvidia::gxf::FileStream entity_file_stream_; + // File stream for index + nvidia::gxf::FileStream index_file_stream_; + + uint64_t playback_index_ = 0; + uint64_t playback_count_ = 0; + uint64_t index_start_timestamp_ = 0; + uint64_t index_last_timestamp_ = 0; + uint64_t index_timestamp_duration_ = 0; + uint64_t index_frame_count_ = 1; + uint64_t playback_start_timestamp_ = 0; }; } // namespace holoscan::ops diff --git a/include/holoscan/operators/visualizer_icardio/visualizer_icardio.hpp b/include/holoscan/operators/visualizer_icardio/visualizer_icardio.hpp deleted file mode 100644 index ce099e25..00000000 --- a/include/holoscan/operators/visualizer_icardio/visualizer_icardio.hpp +++ /dev/null @@ -1,56 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef HOLOSCAN_OPERATORS_VISUALIZER_ICARDIO_HPP -#define HOLOSCAN_OPERATORS_VISUALIZER_ICARDIO_HPP - -#include -#include -#include - -#include "../../core/gxf/gxf_operator.hpp" - -namespace holoscan::ops { -/** - * @brief Visualizer iCardio Operator class to generate data for visualization - * - * Class wraps a GXF Codelet(`nvidia::holoscan::multiai::VisualizerICardio`). - */ -class VisualizerICardioOp : public holoscan::ops::GXFOperator { - public: - HOLOSCAN_OPERATOR_FORWARD_ARGS_SUPER(VisualizerICardioOp, holoscan::ops::GXFOperator) - - VisualizerICardioOp() = default; - - const char* gxf_typename() const override { - return "nvidia::holoscan::multiai::VisualizerICardio"; - } - - void setup(OperatorSpec& spec) override; - - private: - Parameter> in_tensor_names_; - Parameter> out_tensor_names_; - Parameter> allocator_; - Parameter> receivers_; - Parameter> transmitters_; - Parameter input_on_cuda_; -}; - -} // namespace holoscan::ops - -#endif /* HOLOSCAN_OPERATORS_VISUALIZER_ICARDIO_HPP */ diff --git a/include/holoscan/operators/visualizer_tool_tracking/visualizer_tool_tracking.hpp b/include/holoscan/operators/visualizer_tool_tracking/visualizer_tool_tracking.hpp deleted file mode 100644 index 5d95d91c..00000000 --- a/include/holoscan/operators/visualizer_tool_tracking/visualizer_tool_tracking.hpp +++ /dev/null @@ -1,88 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef HOLOSCAN_OPERATORS_VISUALIZER_TOOL_TRACKING_VISUALIZER_TOOL_TRACKING_HPP -#define HOLOSCAN_OPERATORS_VISUALIZER_TOOL_TRACKING_VISUALIZER_TOOL_TRACKING_HPP - -#include -#include -#include - -#include "../../core/gxf/gxf_operator.hpp" - -namespace holoscan::ops { - -/** - * @brief Operator class to visualize the tool tracking results. - * - * This wraps a GXF Codelet(`nvidia::holoscan::visualizer_tool_tracking::Sink`). - */ -class ToolTrackingVizOp : public holoscan::ops::GXFOperator { - public: - HOLOSCAN_OPERATOR_FORWARD_ARGS_SUPER(ToolTrackingVizOp, holoscan::ops::GXFOperator) - - ToolTrackingVizOp() = default; - - const char* gxf_typename() const override { - return "nvidia::holoscan::visualizer_tool_tracking::Sink"; - } - - void setup(OperatorSpec& spec) override; - - void initialize() override; - - private: - Parameter in_width_; - Parameter in_height_; - Parameter in_channels_; - Parameter in_bytes_per_pixel_; - Parameter alpha_value_; - - Parameter videoframe_vertex_shader_path_; - Parameter videoframe_fragment_shader_path_; - - Parameter tooltip_vertex_shader_path_; - Parameter tooltip_fragment_shader_path_; - Parameter num_tool_classes_; - Parameter num_tool_pos_components_; - Parameter>> tool_tip_colors_; - - Parameter> tool_labels_; - Parameter label_sans_font_path_; - Parameter label_sans_bold_font_path_; - - Parameter overlay_img_width_; - Parameter overlay_img_height_; - Parameter overlay_img_layers_; - Parameter overlay_img_channels_; - Parameter>> overlay_img_colors_; - - Parameter overlay_img_vertex_shader_path_; - Parameter overlay_img_fragment_shader_path_; - - Parameter> in_; - Parameter> in_tensor_names_; - Parameter> pool_; - Parameter> window_close_scheduling_term_; - - Parameter overlay_buffer_input_; - Parameter overlay_buffer_output_; -}; - -} // namespace holoscan::ops - -#endif /* HOLOSCAN_OPERATORS_VISUALIZER_TOOL_TRACKING_VISUALIZER_TOOL_TRACKING_HPP */ diff --git a/include/holoscan/std_ops.hpp b/include/holoscan/std_ops.hpp deleted file mode 100644 index f11bc834..00000000 --- a/include/holoscan/std_ops.hpp +++ /dev/null @@ -1,39 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef HOLOSCAN_STD_OPS_HPP -#define HOLOSCAN_STD_OPS_HPP - -#include "holoscan/operators/aja_source/aja_source.hpp" -#include "holoscan/operators/bayer_demosaic/bayer_demosaic.hpp" -#include "holoscan/operators/custom_lstm_inference/lstm_tensor_rt_inference.hpp" -#if HOLOSCAN_BUILD_EMERGENT == 1 - #include "holoscan/operators/emergent_source/emergent_source.hpp" -#endif -#include "holoscan/operators/format_converter/format_converter.hpp" -#include "holoscan/operators/holoviz/holoviz.hpp" -#include "holoscan/operators/multiai_inference/multiai_inference.hpp" -#include "holoscan/operators/multiai_postprocessor/multiai_postprocessor.hpp" -#include "holoscan/operators/segmentation_postprocessor/segmentation_postprocessor.hpp" -#include "holoscan/operators/stream_playback/video_stream_recorder.hpp" -#include "holoscan/operators/stream_playback/video_stream_replayer.hpp" -#include "holoscan/operators/tensor_rt/tensor_rt_inference.hpp" -#include "holoscan/operators/tool_tracking_postprocessor/tool_tracking_postprocessor.hpp" -#include "holoscan/operators/visualizer_icardio/visualizer_icardio.hpp" -#include "holoscan/operators/visualizer_tool_tracking/visualizer_tool_tracking.hpp" - -#endif /* HOLOSCAN_STD_OPS_HPP */ diff --git a/include/holoscan/utils/cuda_stream_handler.hpp b/include/holoscan/utils/cuda_stream_handler.hpp new file mode 100644 index 00000000..e8da959c --- /dev/null +++ b/include/holoscan/utils/cuda_stream_handler.hpp @@ -0,0 +1,289 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef INCLUDE_HOLOSCAN_UTILS_CUDA_STREAM_HANDLER_HPP +#define INCLUDE_HOLOSCAN_UTILS_CUDA_STREAM_HANDLER_HPP + +#include +#include +#include + +#include "../core/operator_spec.hpp" +#include "../core/parameter.hpp" +#include "../core/resources/gxf/cuda_stream_pool.hpp" +#include "gxf/cuda/cuda_stream.hpp" +#include "gxf/cuda/cuda_stream_id.hpp" +#include "gxf/cuda/cuda_stream_pool.hpp" + +namespace holoscan { + +/** + * This class handles usage of CUDA streams for operators. + * + * When using CUDA operations the default stream '0' synchronizes with all other streams in the same + * context, see + * https://docs.nvidia.com/cuda/cuda-runtime-api/stream-sync-behavior.html#stream-sync-behavior. + * This can reduce performance. The CudaStreamHandler class manages streams across operators and + * makes sure that CUDA operations are properly chained. + * + * Usage: + * - add an instance of CudaStreamHandler to your operator + * - call CudaStreamHandler::registerInterface(spec) from the operator setup() function + * - in the compute() function call CudaStreamHandler::fromMessage(), this will get the CUDA stream + * from the message of the previous operator. When the operator receives multiple messages, then + * call CudaStreamHandler::fromMessages(). This will synchronize with multiple streams. + * - when executing CUDA functions CudaStreamHandler::get() to get the CUDA stream which should + * be used by your CUDA function + * - before publishing the output message(s) of your operator call CudaStreamHandler::toMessage() on + * each message. This will add the CUDA stream used by the CUDA functions in your operator to + * the output message. + */ +class CudaStreamHandler { + public: + /** + * @brief Destroy the CudaStreamHandler object + */ + ~CudaStreamHandler() { + for (auto&& event : cuda_events_) { + const cudaError_t result = cudaEventDestroy(event); + if (cudaSuccess != result) { + GXF_LOG_ERROR("Failed to destroy CUDA event: %s", cudaGetErrorString(result)); + } + } + cuda_events_.clear(); + } + + /** + * Define the parameters used by this class. + * + * @param spec OperatorSpec to define the cuda_stream_pool parameter + * @param required if set then it's required that the CUDA stream pool is specified + */ + void defineParams(OperatorSpec& spec, bool required = false) { + spec.param(cuda_stream_pool_, + "cuda_stream_pool", + "CUDA Stream Pool", + "Instance of gxf::CudaStreamPool."); + cuda_stream_pool_required_ = required; + } + + /** + * Get the CUDA stream for the operation from the incoming message + * + * @param context + * @param message + * @return gxf_result_t + */ + gxf_result_t fromMessage(gxf_context_t context, + const nvidia::gxf::Expected& message) { + // if the message contains a stream use this + const auto maybe_cuda_stream_id = message.value().get(); + if (maybe_cuda_stream_id) { + const auto maybe_cuda_stream_handle = nvidia::gxf::Handle::Create( + context, maybe_cuda_stream_id.value()->stream_cid); + if (maybe_cuda_stream_handle) { + message_cuda_stream_handle_ = maybe_cuda_stream_handle.value(); + } + } else { + // if no stream had been found, allocate a stream and use that + gxf_result_t result = allocateInternalStream(context); + if (result != GXF_SUCCESS) { return result; } + message_cuda_stream_handle_ = cuda_stream_handle_; + } + return GXF_SUCCESS; + } + + /** + * Get the CUDA stream for the operation from the incoming messages + * + * @param context + * @param messages + * @return gxf_result_t + */ + gxf_result_t fromMessages(gxf_context_t context, + const std::vector& messages) { + const gxf_result_t result = allocateInternalStream(context); + if (result != GXF_SUCCESS) { return result; } + + if (!cuda_stream_handle_) { + // if no CUDA stream can be allocated because no stream pool is set, then don't sync + // with incoming streams. CUDA operations of this operator will use the default stream + // which sync with all other streams by default. + return GXF_SUCCESS; + } + + // iterate through all messages and use events to chain incoming streams with the internal + // stream + auto event_it = cuda_events_.begin(); + for (auto& msg : messages) { + const auto maybe_cuda_stream_id = msg.get(); + if (maybe_cuda_stream_id) { + const auto maybe_cuda_stream_handle = nvidia::gxf::Handle::Create( + context, maybe_cuda_stream_id.value()->stream_cid); + if (maybe_cuda_stream_handle) { + const cudaStream_t cuda_stream = maybe_cuda_stream_handle.value()->stream().value(); + cudaError_t result; + + // allocate a new event if needed + if (event_it == cuda_events_.end()) { + cudaEvent_t cuda_event; + result = cudaEventCreateWithFlags(&cuda_event, cudaEventDisableTiming); + if (cudaSuccess != result) { + GXF_LOG_ERROR("Failed to create input CUDA event: %s", cudaGetErrorString(result)); + return GXF_FAILURE; + } + cuda_events_.push_back(cuda_event); + event_it = cuda_events_.end(); + --event_it; + } + + result = cudaEventRecord(*event_it, cuda_stream); + if (cudaSuccess != result) { + GXF_LOG_ERROR("Failed to record event for message stream: %s", + cudaGetErrorString(result)); + return GXF_FAILURE; + } + result = cudaStreamWaitEvent(cuda_stream_handle_->stream().value(), *event_it); + if (cudaSuccess != result) { + GXF_LOG_ERROR("Failed to record wait on message event: %s", cudaGetErrorString(result)); + return GXF_FAILURE; + } + ++event_it; + } + } + } + message_cuda_stream_handle_ = cuda_stream_handle_; + return GXF_SUCCESS; + } + + /** + * Add the used CUDA stream to the outgoing message + * + * @param message + * @return gxf_result_t + */ + gxf_result_t toMessage(nvidia::gxf::Expected& message) { + if (message_cuda_stream_handle_) { + const auto maybe_stream_id = message.value().add(); + if (!maybe_stream_id) { + GXF_LOG_ERROR("Failed to add CUDA stream id to output message."); + return nvidia::gxf::ToResultCode(maybe_stream_id); + } + maybe_stream_id.value()->stream_cid = message_cuda_stream_handle_.cid(); + } + return GXF_SUCCESS; + } + + /** + * Get the CUDA stream handle which should be used for CUDA commands + * + * @param context + * @return nvidia::gxf::Handle + */ + nvidia::gxf::Handle getStreamHandle(gxf_context_t context) { + // If there is a message stream handle, return this + if (message_cuda_stream_handle_) { return message_cuda_stream_handle_; } + + // else allocate an internal CUDA stream and return it + allocateInternalStream(context); + return cuda_stream_handle_; + } + + /** + * Get the CUDA stream which should be used for CUDA commands. + * + * If no message stream is set and no stream can be allocated, return the default stream. + * + * @param context + * @return cudaStream_t + */ + cudaStream_t getCudaStream(gxf_context_t context) { + const nvidia::gxf::Handle cuda_stream_handle = + getStreamHandle(context); + if (cuda_stream_handle) { return cuda_stream_handle->stream().value(); } + if (!default_stream_warning_) { + default_stream_warning_ = true; + GXF_LOG_WARNING( + "Parameter `cuda_stream_pool` is not set, using the default CUDA stream for CUDA " + "operations."); + } + return cudaStreamDefault; + } + + private: + /** + * Allocate the internal CUDA stream + * + * @param context + * @return gxf_result_t + */ + gxf_result_t allocateInternalStream(gxf_context_t context) { + // Create the CUDA stream if it does not yet exist. + if (!cuda_stream_handle_) { + // Check if a cuda stream pool is given. + const bool has_cuda_stream_pool_ = cuda_stream_pool_.has_value() && cuda_stream_pool_.get(); + if (!has_cuda_stream_pool_) { + // If the cuda stream pool is required return an error + if (cuda_stream_pool_required_) { + GXF_LOG_ERROR("'cuda_stream_pool' is required but not set."); + return GXF_FAILURE; + } + return GXF_SUCCESS; + } + + // get Handle to underlying nvidia::gxf::CudaStreamPool from + // std::shared_ptr + const auto cuda_stream_pool = nvidia::gxf::Handle::Create( + context, cuda_stream_pool_.get()->gxf_cid()); + if (cuda_stream_pool) { + // allocate a stream + auto maybe_stream = cuda_stream_pool.value()->allocateStream(); + if (!maybe_stream) { + GXF_LOG_ERROR("Failed to allocate CUDA stream"); + return nvidia::gxf::ToResultCode(maybe_stream); + } + cuda_stream_handle_ = std::move(maybe_stream.value()); + } + } + return GXF_SUCCESS; + } + + /// if set then it's required that the CUDA stream pool is specified, if this is not the case + /// an error is generated + bool cuda_stream_pool_required_ = false; + + /// CUDA stream pool used to allocate the internal CUDA stream + Parameter> cuda_stream_pool_; + + /// If the CUDA stream pool is not set and we can't use the incoming CUDA stream, issue + /// a warning once. + bool default_stream_warning_ = false; + + /// Array of CUDA events used to synchronize the internal CUDA stream with multiple incoming + /// streams + std::vector cuda_events_; + + /// The CUDA stream which is attached to the incoming message + nvidia::gxf::Handle message_cuda_stream_handle_; + + /// Allocated internal CUDA stream handle + nvidia::gxf::Handle cuda_stream_handle_; +}; + +} // namespace holoscan + +#endif /* INCLUDE_HOLOSCAN_UTILS_CUDA_STREAM_HANDLER_HPP */ diff --git a/include/holoscan/utils/holoinfer_utils.hpp b/include/holoscan/utils/holoinfer_utils.hpp new file mode 100644 index 00000000..7f448a0c --- /dev/null +++ b/include/holoscan/utils/holoinfer_utils.hpp @@ -0,0 +1,48 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HOLOSCAN_UTILS_HOLOINFER_HPP +#define HOLOSCAN_UTILS_HOLOINFER_HPP + +#include +#include +#include + +#include "holoscan/core/io_context.hpp" + +#include + +namespace HoloInfer = holoscan::inference; + +namespace holoscan::utils { + +gxf_result_t multiai_get_data_per_model(InputContext& op_input, + const std::vector& in_tensors, + HoloInfer::DataMap& data_per_input_tensor, + std::map>& dims_per_tensor, + bool cuda_buffer_out, const std::string& module); + +gxf_result_t multiai_transmit_data_per_model( + gxf_context_t& cont, const HoloInfer::Mappings& model_to_tensor_map, + HoloInfer::DataMap& input_data_map, OutputContext& op_output, + const std::vector& out_tensors, HoloInfer::DimType& tensor_out_dims_map, + bool cuda_buffer_in, bool cuda_buffer_out, const nvidia::gxf::PrimitiveType& element_type, + const nvidia::gxf::Handle& allocator_, const std::string& module); + +} // namespace holoscan::utils + +#endif /* HOLOSCAN_UTILS_HOLOINFER_HPP */ diff --git a/include/holoscan/utils/yaml_parser.hpp b/include/holoscan/utils/yaml_parser.hpp index 23177016..c4c66b99 100644 --- a/include/holoscan/utils/yaml_parser.hpp +++ b/include/holoscan/utils/yaml_parser.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -46,6 +46,20 @@ struct YAMLNodeParser { } }; +template <> +struct YAMLNodeParser { + static uint8_t parse(const YAML::Node& node) { + try { + return static_cast(node.as()); + } catch (...) { + std::stringstream ss; + ss << node; + HOLOSCAN_LOG_ERROR("Unable to parse YAML node: '{}'", ss.str()); + return 0; + } + } +}; + template <> struct YAMLNodeParser { static uint8_t parse(const YAML::Node& node) { diff --git a/modules/CMakeLists.txt b/modules/CMakeLists.txt index 693cb513..39b26d1b 100644 --- a/modules/CMakeLists.txt +++ b/modules/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,9 +22,6 @@ cmake_minimum_required(VERSION 3.22) # before this folder is processed.) set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME "holoscan-modules") -# We set the default RPATH to $ORIGIN to make sure we load the installed libraries -set(CMAKE_INSTALL_RPATH \$ORIGIN) - include(FetchContent) FetchContent_Declare( diff --git a/modules/holoinfer/CMakeLists.txt b/modules/holoinfer/CMakeLists.txt index e3c30001..66be748c 100644 --- a/modules/holoinfer/CMakeLists.txt +++ b/modules/holoinfer/CMakeLists.txt @@ -19,14 +19,4 @@ project(holoscan_inference_toolkit LANGUAGES CXX CUDA ) -set(CMAKE_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/package") -# avoid 'Up-to-date' install messages -set(CMAKE_INSTALL_MESSAGE LAZY) -# strip release binaries -if (CMAKE_BUILD_TYPE STREQUAL "Release") - set(_INSTALL_TARGET "install/strip") -else() - set(_INSTALL_TARGET "install") -endif() - add_subdirectory(src) diff --git a/modules/holoinfer/src/CMakeLists.txt b/modules/holoinfer/src/CMakeLists.txt index 30478171..b20f2a31 100644 --- a/modules/holoinfer/src/CMakeLists.txt +++ b/modules/holoinfer/src/CMakeLists.txt @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,15 +15,12 @@ # limitations under the License. # - -file(READ ${CMAKE_CURRENT_SOURCE_DIR}/VERSION HOLOINFER_VERSION) - -project(holoinfer VERSION ${HOLOINFER_VERSION} LANGUAGES CXX CUDA) +project(holoinfer VERSION ${HOLOSCAN_BUILD_VERSION} LANGUAGES CXX CUDA) find_package(CUDAToolkit REQUIRED) find_package(ONNXRuntime REQUIRED) -add_library(holoinfer SHARED +add_library(${PROJECT_NAME} SHARED infer/onnx/core.cpp infer/trt/core.cpp infer/trt/utils.cpp @@ -32,44 +29,60 @@ add_library(holoinfer SHARED manager/infer_manager.cpp manager/process_manager.cpp utils/infer_utils.cpp - utils/infer_buffer.cpp) + utils/infer_buffer.cpp + ) +add_library(holoscan::infer ALIAS ${PROJECT_NAME}) -target_include_directories(holoinfer - PRIVATE +target_include_directories(${PROJECT_NAME} + PRIVATE $ $ PUBLIC - $/include - $ + $ + $ ) - -target_link_libraries(holoinfer PUBLIC - ONNXRuntime::ONNXRuntime - CUDA::cudart - GXF::cuda - GXF::std - GXF::multimedia - yaml-cpp - CUDA::nppidei - CUDA::nppig - TensorRT::nvonnxparser - ) + +target_link_libraries(${PROJECT_NAME} + PUBLIC + GXF::core + GXF::cuda + GXF::multimedia + GXF::std + yaml-cpp + PRIVATE + CUDA::cudart + CUDA::nppidei + CUDA::nppig + holoscan::logger + TensorRT::nvonnxparser + ONNXRuntime::ONNXRuntime +) + +set_target_properties(${PROJECT_NAME} PROPERTIES + OUTPUT_NAME holoscan_infer + EXPORT_NAME infer + SOVERSION ${PROJECT_VERSION_MAJOR} + VERSION ${PROJECT_VERSION} +) + +install(TARGETS ${PROJECT_NAME} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ) + +install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/${PROJECT_NAME} + ) # install the ONNX runtime library get_target_property(ONNXRuntimeLibLocation ONNXRuntime::ONNXRuntime LOCATION) get_filename_component(ONNXRuntimeLocation ${ONNXRuntimeLibLocation} REALPATH) + +# export the name of the lib +get_filename_component(ONNXRuntimeLibName ${ONNXRuntimeLocation} NAME) +set(ONNXRUNTIME_LIBRARY_NAME ${ONNXRuntimeLibName} CACHE INTERNAL "ONNXRuntime library name") + install(FILES "${ONNXRuntimeLocation}" DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT "holoscan-dependencies" ) - -install(TARGETS holoinfer - EXPORT ${PROJECT_NAME}Config - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} -) - -# TODO: Commenting the install since it's included in the SDK -#install(EXPORT ${PROJECT_NAME}Config -# DESTINATION cmake -#) \ No newline at end of file diff --git a/modules/holoinfer/src/VERSION b/modules/holoinfer/src/VERSION deleted file mode 100644 index 60a2d3e9..00000000 --- a/modules/holoinfer/src/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.4.0 \ No newline at end of file diff --git a/modules/holoinfer/src/include/holoinfer_buffer.hpp b/modules/holoinfer/src/include/holoinfer_buffer.hpp index 5149bcf5..3d300dba 100644 --- a/modules/holoinfer/src/include/holoinfer_buffer.hpp +++ b/modules/holoinfer/src/include/holoinfer_buffer.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -182,12 +182,20 @@ struct MultiAISpecs { */ Mappings get_path_map() const { return model_path_map_; } - /// @brief Flag showing if inference on CUDA. Default is True. - bool oncuda_ = true; + std::string backend_type_{"trt"}; + + /// @brief Map with key as model name and value as model file path + Mappings model_path_map_; + + /// @brief Map with key as model name and value as inferred tensor name + Mappings inference_map_; /// @brief Flag showing if input model path is path to engine files bool is_engine_path_ = false; + /// @brief Flag showing if inference on CUDA. Default is True. + bool oncuda_ = true; + /// @brief Flag to enable parallel inference. Default is True. bool parallel_processing_ = false; @@ -200,12 +208,6 @@ struct MultiAISpecs { /// @brief Flag showing if output buffers are on CUDA. Default is True. bool cuda_buffer_out_ = true; - /// @brief Map with key as model name and value as model file path - Mappings model_path_map_; - - /// @brief Map with key as model name and value as inferred tensor name - Mappings inference_map_; - /// @brief Input Data Map with key as model name and value as DataBuffer DataMap data_per_model_; @@ -214,8 +216,6 @@ struct MultiAISpecs { /// @brief Output Data Map with key as tensor name and value as DataBuffer DataMap output_per_model_; - - std::string backend_type_{"trt"}; }; /** diff --git a/modules/holoinfer/src/include/holoinfer_constants.hpp b/modules/holoinfer/src/include/holoinfer_constants.hpp index 817fac06..42c7949e 100644 --- a/modules/holoinfer/src/include/holoinfer_constants.hpp +++ b/modules/holoinfer/src/include/holoinfer_constants.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -32,6 +32,8 @@ #include #include +#include + #define _HOLOSCAN_EXTERNAL_API_ __attribute__((visibility("default"))) namespace holoscan { @@ -58,8 +60,19 @@ class _HOLOSCAN_EXTERNAL_API_ InferStatus { std::string get_message() const { return _message; } void set_code(const holoinfer_code& _c) { _code = _c; } void set_message(const std::string& _m) { _message = _m; } - void display_message() const { std::cout << _message << " \n"; } - + void display_message() const { + switch (_code) { + case holoinfer_code::H_SUCCESS: + default: { + HOLOSCAN_LOG_INFO(_message); + break; + } + case holoinfer_code::H_ERROR: { + HOLOSCAN_LOG_ERROR(_message); + break; + } + } + } InferStatus(const holoinfer_code& code = holoinfer_code::H_SUCCESS, const std::string& message = "") : _code(code), _message(message) {} diff --git a/modules/holoinfer/src/include/holoinfer_utils.hpp b/modules/holoinfer/src/include/holoinfer_utils.hpp index 6c9899e2..ad04821b 100644 --- a/modules/holoinfer/src/include/holoinfer_utils.hpp +++ b/modules/holoinfer/src/include/holoinfer_utils.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,6 +18,7 @@ #ifndef _HOLOSCAN_INFER_UTILS_API_H #define _HOLOSCAN_INFER_UTILS_API_H +#include #include #include #include @@ -45,6 +46,8 @@ namespace holoscan { namespace inference { +void timer_init(TimePoint& _t); + using GXFTransmitters = std::vector>; using GXFReceivers = std::vector>; @@ -113,6 +116,14 @@ InferStatus map_data_to_model_from_tensor(const MultiMappings& model_data_mappin gxf_result_t _HOLOSCAN_EXTERNAL_API_ report_error(const std::string& module, const std::string& submodule); +/** + * Raise error with module, submodule and message + * + * @param module Module of error occurrence + * @param submodule Submodule/Function of error occurrence with the error message (as string) + */ +void _HOLOSCAN_EXTERNAL_API_ raise_error(const std::string& module, const std::string& submodule); + /** * @brief Checks for correctness of inference parameters from configuration. * @param model_path_map Map with model name as key, path to model as value @@ -138,6 +149,11 @@ InferStatus multiai_processor_validity_check(const Mappings& processed_map, const std::vector& in_tensor_names, const std::vector& out_tensor_names); +/** + * @brief Checks if the processor is arm based + */ +bool is_platform_aarch64(); + } // namespace inference } // namespace holoscan #endif diff --git a/modules/holoinfer/src/infer/onnx/core.cpp b/modules/holoinfer/src/infer/onnx/core.cpp index c642fc2d..766d75ca 100644 --- a/modules/holoinfer/src/infer/onnx/core.cpp +++ b/modules/holoinfer/src/infer/onnx/core.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,27 +20,14 @@ namespace holoscan { namespace inference { void OnnxInfer::print_model_details() { - std::cout << "Input node count: " << input_nodes_ << std::endl; - std::cout << "Output node count: " << output_nodes_ << std::endl; - - std::cout << "Input names: ["; - for (const auto& in : input_names_) std::cout << in << " "; - std::cout << "]" << std::endl; - - std::cout << "Input Dimension : [ "; - for (const auto& sh : input_dims_) std::cout << sh << " "; - std::cout << "]" << std::endl; - - std::cout << "Input Type: " << input_type_ << std::endl; - - std::cout << "Output names: ["; - for (const auto& out : output_names_) std::cout << out << " "; - std::cout << "]" << std::endl; - - std::cout << "Output Dimension : [ "; - for (const auto& sh : output_dims_) std::cout << sh << " "; - std::cout << "]" << std::endl; - std::cout << "Output Type: " << output_type_ << std::endl; + HOLOSCAN_LOG_INFO("Input node count: {}", input_nodes_); + HOLOSCAN_LOG_INFO("Output node count: {}", output_nodes_); + HOLOSCAN_LOG_INFO("Input names: [{}]", fmt::join(input_names_, ", ")); + HOLOSCAN_LOG_INFO("Input dims: [{}]", fmt::join(input_dims_, ", ")); + HOLOSCAN_LOG_INFO("Input Type: {}", input_type_); + HOLOSCAN_LOG_INFO("Output names: [{}]", fmt::join(output_names_, ", ")); + HOLOSCAN_LOG_INFO("Output dims: [{}]", fmt::join(output_dims_, ", ")); + HOLOSCAN_LOG_INFO("Output Type: {}", output_type_); } void OnnxInfer::populate_model_details() { diff --git a/modules/holoinfer/src/infer/trt/core.cpp b/modules/holoinfer/src/infer/trt/core.cpp index 242aa641..8030facc 100644 --- a/modules/holoinfer/src/infer/trt/core.cpp +++ b/modules/holoinfer/src/infer/trt/core.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,6 +17,8 @@ #include "core.hpp" +#include + namespace holoscan { namespace inference { @@ -32,7 +34,7 @@ TrtInfer::TrtInfer(const std::string& model_path, const std::string& model_name, initLibNvInferPlugins(nullptr, ""); if (!is_engine_path_) { - std::cout << "TRT Inference: converting ONNX model at " << model_path_ << "\n"; + HOLOSCAN_LOG_INFO("TRT Inference: converting ONNX model at {}", model_path_); bool status = generate_engine_path(network_options_, model_path_, engine_path_); if (!status) { throw std::runtime_error("TRT Inference: could not generate TRT engine path."); } @@ -55,50 +57,50 @@ TrtInfer::~TrtInfer() { } bool TrtInfer::load_engine() { - std::cout << "Loading Engine: " << engine_path_ << "\n"; + HOLOSCAN_LOG_INFO("Loading Engine: {}", engine_path_); std::ifstream file(engine_path_, std::ios::binary | std::ios::ate); std::streamsize size = file.tellg(); file.seekg(0, std::ios::beg); std::vector buffer(size); if (!file.read(buffer.data(), size)) { - std::cout << "Load Engine: File read error: " << engine_path_ << "\n"; + HOLOSCAN_LOG_ERROR("Load Engine: File read error: {}", engine_path_); return false; } std::unique_ptr runtime{nvinfer1::createInferRuntime(logger_)}; if (!runtime) { - std::cout << "Load Engine: Error in creating inference runtime. \n"; + HOLOSCAN_LOG_ERROR("Load Engine: Error in creating inference runtime."); return false; } // Set the device index auto status = cudaSetDevice(network_options_.device_index); if (status != 0) { - std::cout << "Load Engine: Setting cuda device failed. \n"; + HOLOSCAN_LOG_ERROR("Load Engine: Setting cuda device failed."); throw std::runtime_error("Error setting cuda device in load engine."); } engine_ = std::unique_ptr( runtime->deserializeCudaEngine(buffer.data(), buffer.size())); if (!engine_) { - std::cout << "Load Engine: Error in deserializing cuda engine. \n"; + HOLOSCAN_LOG_ERROR("Load Engine: Error in deserializing cuda engine."); return false; } context_ = std::unique_ptr(engine_->createExecutionContext()); if (!context_) { - std::cout << "Load Engine: Error in creating execution context. \n"; + HOLOSCAN_LOG_ERROR("Load Engine: Error in creating execution context."); return false; } status = cudaStreamCreate(&cuda_stream_); if (status != 0) { - std::cout << "Load Engine: Cuda stream creation failed. \n"; + HOLOSCAN_LOG_ERROR("Load Engine: Cuda stream creation failed."); throw std::runtime_error("Unable to create cuda stream"); } - std::cout << "Engine loaded: " << engine_path_ << "\n"; + HOLOSCAN_LOG_INFO("Engine loaded: {}", engine_path_); return true; } diff --git a/modules/holoinfer/src/infer/trt/core.hpp b/modules/holoinfer/src/infer/trt/core.hpp index b9a0e3f7..ed492ffd 100644 --- a/modules/holoinfer/src/infer/trt/core.hpp +++ b/modules/holoinfer/src/infer/trt/core.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +21,6 @@ #include #include -#include #include #include diff --git a/modules/holoinfer/src/infer/trt/utils.cpp b/modules/holoinfer/src/infer/trt/utils.cpp index a3b7003a..2210075a 100644 --- a/modules/holoinfer/src/infer/trt/utils.cpp +++ b/modules/holoinfer/src/infer/trt/utils.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,16 +15,19 @@ * limitations under the License. */ +#include "utils.hpp" + #include -#include "utils.hpp" +#include +#include namespace holoscan { namespace inference { cudaError_t check_cuda(cudaError_t result) { if (result != cudaSuccess) { - std::cerr << "Cuda runtime error: " << cudaGetErrorString(result); + HOLOSCAN_LOG_ERROR("Cuda runtime error: {}", cudaGetErrorString(result)); assert(result == cudaSuccess); } return result; @@ -39,7 +42,7 @@ bool generate_engine_path(const NetworkOptions& options, const std::string& onnx cudaDeviceProp device_prop; auto status = cudaGetDeviceProperties(&device_prop, options.device_index); if (status != cudaSuccess) { - std::cerr << "Error in getting device properties\n"; + HOLOSCAN_LOG_ERROR("Error in getting device properties."); return false; } @@ -73,12 +76,14 @@ void set_dimensions_for_profile(const char* input_name, nvinfer1::IOptimizationP bool build_engine(const std::string& onnx_model_path, const std::string& engine_path, const NetworkOptions& network_options, Logger& logger) { if (valid_file_path(engine_path)) { - std::cout << "Cached engine found: " << engine_path << std::endl; + HOLOSCAN_LOG_INFO("Cached engine found: {}", engine_path); return true; } - std::cout << - "Engine file missing at " << engine_path << ". Starting generation process." << std::endl; + HOLOSCAN_LOG_INFO( + "Engine file missing at {}. Starting generation process.\nNOTE: This could take a couple of " + "minutes depending on your model size and GPU!", + engine_path); auto builder = std::unique_ptr(nvinfer1::createInferBuilder(logger)); if (!builder) { return false; } @@ -146,7 +151,7 @@ bool build_engine(const std::string& onnx_model_path, const std::string& engine_ std::ofstream outfile(engine_path, std::ofstream::binary); outfile.write(reinterpret_cast(plan->data()), plan->size()); - std::cout << "Engine file generated, saved as: " << engine_path << std::endl; + HOLOSCAN_LOG_INFO("Engine file generated, saved as: {}", engine_path); return true; } diff --git a/modules/holoinfer/src/infer/trt/utils.hpp b/modules/holoinfer/src/infer/trt/utils.hpp index c2250b6b..0bb579bd 100644 --- a/modules/holoinfer/src/infer/trt/utils.hpp +++ b/modules/holoinfer/src/infer/trt/utils.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -33,9 +33,7 @@ #include #include -#include -#include -#include +#include namespace holoscan { namespace inference { @@ -44,7 +42,7 @@ namespace inference { */ class Logger : public nvinfer1::ILogger { void log(Severity severity, const char* msg) noexcept override { - if (severity <= Severity::kWARNING) { std::cout << msg << std::endl; } + if (severity <= Severity::kWARNING) { HOLOSCAN_LOG_INFO(msg); } }; }; diff --git a/modules/holoinfer/src/manager/infer_manager.cpp b/modules/holoinfer/src/manager/infer_manager.cpp index 5475b607..ea315e75 100644 --- a/modules/holoinfer/src/manager/infer_manager.cpp +++ b/modules/holoinfer/src/manager/infer_manager.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -111,6 +111,13 @@ InferStatus ManagerInfer::set_inference_params(std::shared_ptr& mu "Inference manager, Engine path cannot be true with onnx runtime backend"); return status; } + + bool is_aarch64 = is_platform_aarch64(); + if (is_aarch64 && multiai_specs->oncuda_) { + status.set_message("Onnxruntime with CUDA not supported on aarch64."); + return status; + } + holo_infer_context_.insert( {model_map.first, std::make_unique(model_map.second, multiai_specs->oncuda_)}); diff --git a/modules/holoinfer/src/manager/infer_manager.hpp b/modules/holoinfer/src/manager/infer_manager.hpp index 7f69ff32..d759d9eb 100644 --- a/modules/holoinfer/src/manager/infer_manager.hpp +++ b/modules/holoinfer/src/manager/infer_manager.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,6 +26,7 @@ #include #include +#include #include #include #include diff --git a/modules/holoinfer/src/manager/process_manager.cpp b/modules/holoinfer/src/manager/process_manager.cpp index 2382a9ff..164de059 100644 --- a/modules/holoinfer/src/manager/process_manager.cpp +++ b/modules/holoinfer/src/manager/process_manager.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -34,7 +34,6 @@ InferStatus ManagerProcessor::process( DataMap& inferred_result_map, const std::map>& dimension_map) { for (const auto& tensor_to_ops : tensor_oper_map) { auto tensor_name = tensor_to_ops.first; - if (inferred_result_map.find(tensor_name) == inferred_result_map.end()) { return InferStatus( holoinfer_code::H_ERROR, @@ -62,13 +61,32 @@ InferStatus ManagerProcessor::process( processed_dims_map_.clear(); - for (const auto& operation : operations) { + for (auto& operation : operations) { std::vector processed_dims; std::vector process_vector; + std::vector custom_strings; // if operation is print, then no need to allocate output memory - if (operation.compare("print") == 0) { - std::cout << "Printing results from " << tensor_name << " -> "; + if (operation.find("print") != std::string::npos) { + if (operation.compare("print") == 0) { + std::cout << "Printing results from " << tensor_name << " -> "; + } else { + if (operation.find("custom") != std::string::npos) { + std::istringstream cstrings(operation); + + std::string custom_string; + while (std::getline(cstrings, custom_string, ',')) { + custom_strings.push_back(custom_string); + } + if (custom_strings.size() != 3) { + return InferStatus( + holoinfer_code::H_ERROR, + "Process manager, Custom binary print operation must generate 3 strings"); + } + operation = custom_strings.at(0); + custom_strings.erase(custom_strings.begin()); + } + } } else { if (in_out_tensor_map.find(tensor_name) == in_out_tensor_map.end()) { return InferStatus( @@ -84,7 +102,7 @@ InferStatus ManagerProcessor::process( } InferStatus status = infer_data_->process_operation( - operation, dimensions, out_result, processed_dims, process_vector); + operation, dimensions, out_result, processed_dims, process_vector, custom_strings); if (status.get_code() != holoinfer_code::H_SUCCESS) { return InferStatus(holoinfer_code::H_ERROR, @@ -117,7 +135,7 @@ ProcessorContext::ProcessorContext() { try { process_manager = std::make_unique(); } catch (const std::bad_alloc&) { - std::cerr << "Holoscan Outdata context: Memory allocation error\n"; + HOLOSCAN_LOG_ERROR("Holoscan Outdata context: Memory allocation error."); throw; } } diff --git a/modules/holoinfer/src/process/data_processor.cpp b/modules/holoinfer/src/process/data_processor.cpp index a5e0f682..a5db35a1 100644 --- a/modules/holoinfer/src/process/data_processor.cpp +++ b/modules/holoinfer/src/process/data_processor.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,9 +28,17 @@ InferStatus DataProcessor::initialize(const MultiMappings& process_operations) { } for (const auto& _op : _operations) { - if (supported_operations_.find(_op) == supported_operations_.end()) { - return InferStatus(holoinfer_code::H_ERROR, - "Data processor, Operation " + _op + " not supported."); + if (_op.find("print") == std::string::npos) { + if (supported_operations_.find(_op) == supported_operations_.end()) { + return InferStatus(holoinfer_code::H_ERROR, + "Data processor, Operation " + _op + " not supported."); + } + } else { + if ((_op.compare("print") != 0) && + (_op.find("print_custom_binary_classification") == std::string::npos)) { + return InferStatus(holoinfer_code::H_ERROR, + "Data processor, Print operation: " + _op + " not supported."); + } } } } @@ -45,6 +53,26 @@ void DataProcessor::print_results(const std::vector& dimensions, std::cout << indata[dsize - 1] << "\n"; } +void DataProcessor::print_custom_binary_classification( + const std::vector& dimensions, const std::vector& indata, + const std::vector& custom_strings) { + size_t dsize = indata.size(); + + if (dsize == 2) { + auto first_value = 1.0 / (1 + exp(-indata.at(0))); + auto second_value = 1.0 / (1 + exp(-indata.at(1))); + + if (first_value > second_value) { + std::cout << custom_strings[0] << ". Confidence: " << first_value << "\n"; + } else { + std::cout << custom_strings[1] << ". Confidence: " << second_value << "\n"; + } + } else { + HOLOSCAN_LOG_INFO("Input data size: {}", dsize); + HOLOSCAN_LOG_INFO("This is binary classification custom print, size must be 2."); + } +} + void DataProcessor::compute_max_per_channel_cpu(const std::vector& dimensions, const std::vector& outvector, std::vector& processed_dims, @@ -90,18 +118,20 @@ InferStatus DataProcessor::process_operation(const std::string& operation, const std::vector& indims, const std::vector& indata, std::vector& processed_dims, - std::vector& processed_data) { + std::vector& processed_data, + const std::vector& custom_strings) { if (oper_to_fp_.find(operation) == oper_to_fp_.end()) return InferStatus(holoinfer_code::H_ERROR, "Data processor, Operation " + operation + " not found in map"); try { - oper_to_fp_.at(operation)(indims, indata, processed_dims, processed_data); + oper_to_fp_.at(operation)(indims, indata, processed_dims, processed_data, custom_strings); } catch (...) { return InferStatus(holoinfer_code::H_ERROR, "Data processor, Exception in running " + operation); } - if (operation.compare("print") != 0 && processed_data.size() == 0) + if (operation.find("print") == std::string::npos && processed_data.size() == 0) { return InferStatus(holoinfer_code::H_ERROR, "Data processor, Processed data map empty"); + } return InferStatus(); } diff --git a/modules/holoinfer/src/process/data_processor.hpp b/modules/holoinfer/src/process/data_processor.hpp index ddde83c5..488e9da3 100644 --- a/modules/holoinfer/src/process/data_processor.hpp +++ b/modules/holoinfer/src/process/data_processor.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,10 +17,12 @@ #ifndef _HOLOSCAN_DATA_PROCESSOR_H #define _HOLOSCAN_DATA_PROCESSOR_H +#include #include #include #include #include +#include #include #include @@ -29,8 +31,9 @@ namespace holoscan { namespace inference { /// Declaration of function callback used by DataProcessor -using processor_FP = std::function&, const std::vector&, - std::vector&, std::vector&)>; +using processor_FP = + std::function&, const std::vector&, std::vector&, + std::vector&, const std::vector&)>; /** * @brief Data Processor class that processes one operations per tensor. Currently supports CPU @@ -62,11 +65,13 @@ class DataProcessor { * @param in_data Input data buffer * @param out_dims Dimension of the output tensor * @param out_data Output data buffer + * @param custom_strings Strings to display for custom print operations * @returns InferStatus with appropriate code and message */ InferStatus process_operation(const std::string& operation, const std::vector& in_dims, const std::vector& in_data, std::vector& out_dims, - std::vector& out_data); + std::vector& out_data, + const std::vector& custom_strings); /** * @brief Computes max per channel in input data and scales it to [0, 1]. (CPU based) @@ -89,27 +94,49 @@ class DataProcessor { */ void print_results(const std::vector& in_dims, const std::vector& in_data); + /** + * @brief Print custom text for binary classification results in the input buffer. + * + * @param in_dims Dimension of the input tensor + * @param in_data Input data buffer + * @param custom_strings Strings to display for custom print operations + */ + void print_custom_binary_classification(const std::vector& in_dims, + const std::vector& in_data, + const std::vector& custom_strings); + private: /// Map defining supported operations by DataProcessor Class. /// Keyword in this map must be used exactly by the user in configuration. /// Operation is the key and its related implementation platform as the value. const std::map supported_operations_{ {"max_per_channel_scaled", holoinfer_data_processor::HOST}, - {"print", holoinfer_data_processor::HOST}}; + {"print", holoinfer_data_processor::HOST}, + {"print_custom_binary_classification", holoinfer_data_processor::HOST}}; /// Mapped function call for the function pointer of max_per_channel_scaled processor_FP max_per_channel_scaled_fp_ = [this](auto& in_dims, auto& in_data, auto& out_dims, - auto& out_data) { + auto& out_data, auto& custom_strings) { compute_max_per_channel_cpu(in_dims, in_data, out_dims, out_data); }; /// Mapped function call for the function pointer of print processor_FP print_results_fp_ = [this](auto& in_dims, auto& in_data, auto& out_dims, - auto& out_data) { print_results(in_dims, in_data); }; + auto& out_data, auto& custom_strings) { + print_results(in_dims, in_data); + }; + + /// Mapped function call for the function pointer of printing custom binary classification results + processor_FP print_custom_binary_classification_fp_ = + [this](auto& in_dims, auto& in_data, auto& out_dims, auto& out_data, auto& custom_strings) { + print_custom_binary_classification(in_dims, in_data, custom_strings); + }; /// Map with supported operation as the key and related function pointer as value const std::map oper_to_fp_{ - {"max_per_channel_scaled", max_per_channel_scaled_fp_}, {"print", print_results_fp_}}; + {"max_per_channel_scaled", max_per_channel_scaled_fp_}, + {"print", print_results_fp_}, + {"print_custom_binary_classification", print_custom_binary_classification_fp_}}; }; } // namespace inference diff --git a/modules/holoinfer/src/utils/infer_utils.cpp b/modules/holoinfer/src/utils/infer_utils.cpp index 717db412..1ba1e519 100644 --- a/modules/holoinfer/src/utils/infer_utils.cpp +++ b/modules/holoinfer/src/utils/infer_utils.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,6 +26,11 @@ gxf_result_t report_error(const std::string& module, const std::string& submodul return GXF_FAILURE; } +void raise_error(const std::string& module, const std::string& message) { + std::string error_string{"Error in " + module + ", Sub-module->" + message}; + throw std::runtime_error(error_string); +} + void timer_init(TimePoint& _t) { _t = std::chrono::steady_clock::now(); } @@ -37,6 +42,20 @@ gxf_result_t timer_check(TimePoint& start, TimePoint& end, const std::string& mo return GXF_SUCCESS; } +bool is_platform_aarch64() { + struct utsname buffer; + + if (uname(&buffer) == 0) { + std::string machine(buffer.machine); + + if (machine.find("arm") != std::string::npos || machine.find("aarch64") != std::string::npos) { + return true; + } + } + // Return false in all other conditions. + return false; +} + InferStatus map_data_to_model_from_tensor(const MultiMappings& model_data_mapping, DataMap& data_per_model, DataMap& data_per_input_tensor) { InferStatus status = InferStatus(holoinfer_code::H_ERROR); @@ -499,17 +518,16 @@ InferStatus multiai_processor_validity_check(const Mappings& processed_map, const std::vector& out_tensor_names) { InferStatus status = InferStatus(holoinfer_code::H_ERROR); - auto l_status = check_mappings_size_value(processed_map, "processed_map"); - if (l_status.get_code() == holoinfer_code::H_ERROR) { return l_status; } - if (in_tensor_names.empty()) { status.set_message("Input tensor names cannot be empty"); return status; } if (out_tensor_names.empty()) { - status.set_message("Output tensor names cannot be empty"); - return status; + status.set_message("WARNING: Output tensor names empty"); + } else { + auto l_status = check_mappings_size_value(processed_map, "processed_map"); + if (l_status.get_code() == holoinfer_code::H_ERROR) { return l_status; } } if (!check_equality(processed_map.size(), out_tensor_names.size())) { diff --git a/modules/holoviz/.clang-format b/modules/holoviz/.clang-format deleted file mode 100644 index 26034dc2..00000000 --- a/modules/holoviz/.clang-format +++ /dev/null @@ -1,105 +0,0 @@ ---- -AccessModifierOffset: -4 -AlignAfterOpenBracket: Align -AlignConsecutiveAssignments: true -AlignConsecutiveDeclarations: false -AlignEscapedNewlines: Left -AlignOperands: true -AlignTrailingComments: true -AllowAllParametersOfDeclarationOnNextLine: false -AllowShortBlocksOnASingleLine: false -AllowShortCaseLabelsOnASingleLine: false -AllowShortFunctionsOnASingleLine: Empty -AllowShortIfStatementsOnASingleLine: false -AllowShortLoopsOnASingleLine: false -AlwaysBreakAfterDefinitionReturnType: None -AlwaysBreakAfterReturnType: None -AlwaysBreakBeforeMultilineStrings: true -AlwaysBreakTemplateDeclarations: true -BinPackArguments: true -BinPackParameters: true -BraceWrapping: - AfterClass: true - AfterControlStatement: true - AfterEnum: true - AfterFunction: true - AfterNamespace: true - AfterObjCDeclaration: false - AfterStruct: true - AfterUnion: true - BeforeCatch: true - BeforeElse: true - IndentBraces: false - SplitEmptyFunction: true - SplitEmptyRecord: true - SplitEmptyNamespace: true -BreakBeforeBinaryOperators: None -BreakBeforeBraces: Custom -BreakBeforeInheritanceComma: true -BreakBeforeTernaryOperators: true -BreakConstructorInitializersBeforeComma: true -BreakConstructorInitializers: BeforeComma -BreakAfterJavaFieldAnnotations: false -BreakStringLiterals: true -ColumnLimit: 120 -CommentPragmas: '^ IWYU pragma:' -CompactNamespaces: false -ConstructorInitializerAllOnOneLineOrOnePerLine: false -ConstructorInitializerIndentWidth: 4 -ContinuationIndentWidth: 4 -Cpp11BracedListStyle: true -DerivePointerAlignment: false -DisableFormat: false -ExperimentalAutoDetectBinPacking: false -FixNamespaceComments: true -ForEachMacros: - - foreach - - Q_FOREACH - - BOOST_FOREACH -IncludeCategories: - - Regex: '^<.*\.hpp>' - Priority: 1 - - Regex: '^<.*' - Priority: 2 - - Regex: '.*' - Priority: 3 -IncludeIsMainRegex: '([-_](test|unittest))?$' -IndentCaseLabels: false -IndentWidth: 4 -IndentWrappedFunctionNames: true -JavaScriptQuotes: Leave -JavaScriptWrapImports: true -KeepEmptyLinesAtTheStartOfBlocks: true -MacroBlockBegin: '' -MacroBlockEnd: '' -MaxEmptyLinesToKeep: 1 -NamespaceIndentation: None -ObjCBlockIndentWidth: 4 -ObjCSpaceAfterProperty: false -ObjCSpaceBeforeProtocolList: false -PenaltyBreakAssignment: 2 -PenaltyBreakBeforeFirstCallParameter: 1 -PenaltyBreakComment: 300 -PenaltyBreakFirstLessLess: 120 -PenaltyBreakString: 1000 -PenaltyExcessCharacter: 1000000 -PenaltyReturnTypeOnItsOwnLine: 200 -PointerAlignment: Right -ReflowComments: false -SortIncludes: false -SortUsingDeclarations: true -SpaceAfterCStyleCast: false -SpaceAfterTemplateKeyword: false -SpaceBeforeAssignmentOperators: true -SpaceBeforeParens: ControlStatements -SpaceInEmptyParentheses: false -SpacesBeforeTrailingComments: 1 -SpacesInAngles: false -SpacesInContainerLiterals: false -SpacesInCStyleCastParentheses: false -SpacesInParentheses: false -SpacesInSquareBrackets: false -Standard: Cpp11 -TabWidth: 8 -UseTab: Never -... diff --git a/modules/holoviz/.dockerignore b/modules/holoviz/.dockerignore deleted file mode 100644 index f9db036d..00000000 --- a/modules/holoviz/.dockerignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!docker \ No newline at end of file diff --git a/modules/holoviz/.gitignore b/modules/holoviz/.gitignore deleted file mode 100644 index a8904476..00000000 --- a/modules/holoviz/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -build*/ -.vscode diff --git a/modules/holoviz/CMakeLists.txt b/modules/holoviz/CMakeLists.txt index f2bd960a..3d9867d5 100644 --- a/modules/holoviz/CMakeLists.txt +++ b/modules/holoviz/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,39 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -cmake_minimum_required(VERSION 3.17) - -file(READ ${CMAKE_CURRENT_SOURCE_DIR}/VERSION CLARA_HOLOVIZ_VERSION) - -project(ClaraHoloViz - DESCRIPTION "Clara Holoviz" - VERSION ${CLARA_HOLOVIZ_VERSION} +project(holoviz + DESCRIPTION "Holoviz" + VERSION ${HOLOSCAN_BUILD_VERSION} LANGUAGES CXX CUDA ) -set(CMAKE_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/package") -# avoid 'Up-to-date' install messages -set(CMAKE_INSTALL_MESSAGE LAZY) -# strip release binaries -if(CMAKE_BUILD_TYPE STREQUAL "Release") - set(_INSTALL_TARGET "install/strip") -else() - set(_INSTALL_TARGET "install") -endif() -# show progress when populating external content -set(FETCHCONTENT_QUIET OFF) - -# this generates a 'compile_commands.json' file which can be read by VSCode to -#configure the include paths for IntelliSense -set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - -# support Maxwell and up -set(CMAKE_CUDA_ARCHITECTURES 50) - add_subdirectory(thirdparty) if(HOLOSCAN_BUILD_EXAMPLES) - add_subdirectory(examples) + add_subdirectory(examples) endif() add_subdirectory(src) diff --git a/modules/holoviz/README.md b/modules/holoviz/README.md index 15bad8a3..7eea4ef4 100644 --- a/modules/holoviz/README.md +++ b/modules/holoviz/README.md @@ -1,66 +1,3 @@ -# Holoviz SDK +# Holoviz -> Note: Holoviz will be tightly coupled to the Holoscan SDK in a future release, and will not be an independant SDK - -## Prerequisites - -- Install [NVIDIA Docker](https://github.com/NVIDIA/nvidia-docker) - -## Building - -Holoviz is buillt with a docker container. - -All build operations are executed using the 'run.sh' batch file, use - -```shell -$ ./run.sh -h -``` - -to see the options. The script supports - -- creating the docker image needed for building Holoviz -- building Holoviz - -The docker image _clara-holoviz-dev_ARCH:?.?.?_ (where `ARCH` is `x86_64` -or `aarch64` and `?.?.?` is the version) is used to build the Holoviz -executable. - -First the _dev_ docker image needs to be build by, this need to be done only once or when the dev docker image changes. - -```shell -$ ./run.sh image_build_dev -``` - -When this is done the Holoviz executable can be build. - -```shell -$ ./run.sh build -``` - -The build results are written to the `build_x86_64` or `build_aarch64` directory, dependent on the target architecture. - -The final binary package is written to the `package` directory in the build directory. It includes the Holoviz shared library, header files, CMake setup files to include Holoviz in CMake based projects, a static library of ImGUI and example source files. - -To build the examples, go to the `package/src/examples/xyz` directory (where `xyx` is the name of the example) and execute these steps: - -```shell -$ cmake -B build . -$ cmake --build build/ -``` - -### Cross-compilation: x86_64 to aarch64 - -While the development dockerfile does not currently support true cross-compilation, -you can compile for Holoviz for aarch64 from a x86_64 host using an emulation environment. - -Run the following steps on your host. - -Follow the [installation steps for `qemu`](https://docs.nvidia.com/datacenter/cloud-native/playground/x-arch.html#emulation-environment) - -Set the `TARGET_ARCH` environment variable: - -```shell -$ export TARGET_ARCH=aarch64 -``` - -Follow the build steps above by first building the docker _dev_ image and then building Clara Holoviz. +Holoviz is a module of the Holoscan SDK built for visualizing data. Holoviz composites real time streams of frames with multiple different other layers like segmentation mask layers, geometry layers and GUI layers. diff --git a/modules/holoviz/VERSION b/modules/holoviz/VERSION deleted file mode 100644 index 60a2d3e9..00000000 --- a/modules/holoviz/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.4.0 \ No newline at end of file diff --git a/modules/holoviz/docker/DEV_VERSION b/modules/holoviz/docker/DEV_VERSION deleted file mode 100644 index 1cc5f657..00000000 --- a/modules/holoviz/docker/DEV_VERSION +++ /dev/null @@ -1 +0,0 @@ -1.1.0 \ No newline at end of file diff --git a/modules/holoviz/docker/Dockerfile.dev b/modules/holoviz/docker/Dockerfile.dev deleted file mode 100644 index 13ae39d9..00000000 --- a/modules/holoviz/docker/Dockerfile.dev +++ /dev/null @@ -1,81 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# **IMPORTANT*** If you change this file, you *must* increment the DEB_VERSION -# file in this directory for the changes to take!! - -ARG CUDA_IMAGE - -FROM ${CUDA_IMAGE} - -ARG TARGET_ARCH - -# update the the CUDA signing key -# add the kitware repo for latest cmake -# install dependencies -# * git, openssh-client: used by cmake to clone git projects -# * build-essential, cmake: for building -# * libxrandr-dev, libxxf86vm1, libxinerama-dev, libxcursor-dev, libxi-dev, libxext-dev, libgl-dev: for nvpro_core -RUN . /etc/lsb-release \ - && DISTRIB_ID=$(echo "${DISTRIB_ID}" | sed -e 's/\(.*\)/\L\1/') \ - && DISTRIB_RELEASE=$(echo "${DISTRIB_RELEASE}" | sed -e 's/\.//g') \ - && if [ "${TARGET_ARCH}" = "aarch64" ]; then CUDA_ARCH="sbsa"; else CUDA_ARCH="${TARGET_ARCH}"; fi \ - && apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/${DISTRIB_ID}${DISTRIB_RELEASE}/${CUDA_ARCH}/3bf863cc.pub \ - && apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - ca-certificates wget gnupg2 \ - && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null \ - && echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ focal main' | tee /etc/apt/sources.list.d/kitware.list >/dev/null \ - && apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - git openssh-client \ - build-essential \ - cmake \ - libxrandr-dev libxxf86vm-dev libxinerama-dev libxcursor-dev libxi-dev libxext-dev libgl-dev \ - && rm -rf /var/lib/apt/lists/* - -# Build the Vulkan SDK from source -# Note there is no arm64 version and the build script always saves the artifacts to the `x86_64` directory, therefore we don't -# use the TARGET_ARCH env variable here -ENV VULKAN_SDK_VERSION=1.3.216.0 -RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - libglm-dev cmake libxcb-dri3-0 libxcb-present0 libpciaccess0 \ - libpng-dev libxcb-keysyms1-dev libxcb-dri3-dev libx11-dev g++ gcc \ - libmirclient-dev libwayland-dev libxrandr-dev libxcb-randr0-dev libxcb-ewmh-dev \ - git python python3 bison libx11-xcb-dev liblz4-dev libzstd-dev python3-distutils \ - qt5-default ocaml-core ninja-build pkg-config libxml2-dev wayland-protocols \ - && rm -rf /var/lib/apt/lists/* - -RUN cd /tmp \ - && wget --inet4-only -nv --show-progress --progress=dot:giga https://sdk.lunarg.com/sdk/download/${VULKAN_SDK_VERSION}/linux/vulkansdk-linux-x86_64-${VULKAN_SDK_VERSION}.tar.gz \ - && tar -xzf vulkansdk-linux-x86_64-${VULKAN_SDK_VERSION}.tar.gz \ - && rm vulkansdk-linux-x86_64-${VULKAN_SDK_VERSION}.tar.gz \ - && cd ${VULKAN_SDK_VERSION} \ - && rm -rf x86_64 \ - && ./vulkansdk shaderc glslang headers loader \ - && rm -rf source - -ENV VULKAN_SDK=/tmp/${VULKAN_SDK_VERSION}/x86_64 -ENV PATH="$VULKAN_SDK/bin:${PATH}" - -# all devices should be visible -ENV NVIDIA_VISIBLE_DEVICES all -# set 'compute' driver cap to use Cuda -# set 'video' driver cap to use the video encoder -# set 'graphics' driver cap to use OpenGL/EGL/Vulkan -# set 'display' for X11 -ENV NVIDIA_DRIVER_CAPABILITIES compute,video,graphics,display diff --git a/modules/holoviz/examples/CMakeLists.txt b/modules/holoviz/examples/CMakeLists.txt index 71ba7823..cec5ed04 100644 --- a/modules/holoviz/examples/CMakeLists.txt +++ b/modules/holoviz/examples/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,4 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -add_subdirectory(demo) \ No newline at end of file +add_subdirectory(demo) +add_subdirectory(depth_map) \ No newline at end of file diff --git a/modules/holoviz/examples/demo/CMakeLists.txt b/modules/holoviz/examples/demo/CMakeLists.txt index 4abe854e..8b96a52a 100644 --- a/modules/holoviz/examples/demo/CMakeLists.txt +++ b/modules/holoviz/examples/demo/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -set(PROJECT_NAME ExampleDemo) +set(PROJECT_NAME holoviz_demo) # fetch the dependencies include(FetchContent) @@ -24,7 +24,7 @@ FetchContent_Declare( GIT_REPOSITORY https://github.com/nothings/stb.git GIT_TAG af1a5bc352164740c1cc1354942b1c6b72eacb8a GIT_PROGRESS TRUE -) + ) FetchContent_MakeAvailable(stb) add_executable(${PROJECT_NAME}) @@ -44,25 +44,3 @@ target_link_libraries(${PROJECT_NAME} holoscan::viz holoscan::viz::imgui ) - -# install source files -install( - FILES Main.cpp - DESTINATION src/examples/demo - COMPONENT holoviz-examples - ) - -# install data files -install( - FILES nv_logo.png - DESTINATION src/examples/demo - COMPONENT holoviz-examples - ) - -# install the public CMakeLists.txt file -install( - FILES CMakeLists.txt.public - DESTINATION src/examples/demo - RENAME CMakeLists.txt - COMPONENT holoviz-examples - ) \ No newline at end of file diff --git a/modules/holoviz/examples/demo/Main.cpp b/modules/holoviz/examples/demo/Main.cpp index bea27f9d..7decef02 100644 --- a/modules/holoviz/examples/demo/Main.cpp +++ b/modules/holoviz/examples/demo/Main.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,10 +15,10 @@ * limitations under the License. */ -#include +#include #include #include -#include +#include #define STB_IMAGE_IMPLEMENTATION #include #define STB_IMAGE_WRITE_IMPLEMENTATION @@ -35,16 +35,12 @@ namespace viz = holoscan::viz; -static const char *source_items[]{"Host", "Device", "Array"}; -static const char *format_items[]{"R8_UINT", "R8G8B8_UNORM", "R8G8B8A8_UNORM"}; -static viz::ImageFormat formats[]{viz::ImageFormat::R8_UINT, viz::ImageFormat::R8G8B8_UNORM, - viz::ImageFormat::R8G8B8A8_UNORM}; +static const char* source_items[]{"Host", "Device", "Array"}; +static const char* format_items[]{"R8_UINT", "R8G8B8_UNORM", "R8G8B8A8_UNORM"}; +static viz::ImageFormat formats[]{ + viz::ImageFormat::R8_UINT, viz::ImageFormat::R8G8B8_UNORM, viz::ImageFormat::R8G8B8A8_UNORM}; -enum class Source { - HOST, - DEVICE, - ARRAY -}; +enum class Source { HOST, DEVICE, ARRAY }; // options bool benchmark_mode = false; @@ -52,24 +48,24 @@ bool headless_mode = false; bool show_ui = true; -bool show_image_layer = true; -Source current_source = Source::DEVICE; +bool show_image_layer = true; +Source current_source = Source::DEVICE; uint32_t current_format_index = 2; -float image_layer_opacity = 1.f; -int image_layer_priority = 0; +float image_layer_opacity = 1.f; +int image_layer_priority = 0; -bool show_geometry_layer = true; +bool show_geometry_layer = true; float geometry_layer_opacity = 1.f; -int geometry_layer_priority = 1; +int geometry_layer_priority = 1; -uint32_t width = 1920; +uint32_t width = 1920; uint32_t height = 1080; // timing std::chrono::steady_clock::time_point start; std::chrono::milliseconds elapsed; uint32_t iterations = 0; -float fps = 0.f; +float fps = 0.f; // memory std::unique_ptr host_mem_r8; @@ -79,432 +75,422 @@ std::unique_ptr host_mem_r8g8b8a8; std::vector palette; // cuda -CUcontext cuda_context = nullptr; -CUdeviceptr cu_device_mem_r8 = 0; -CUdeviceptr cu_device_mem_r8g8b8 = 0; +CUcontext cuda_context = nullptr; +CUdeviceptr cu_device_mem_r8 = 0; +CUdeviceptr cu_device_mem_r8g8b8 = 0; CUdeviceptr cu_device_mem_r8g8b8a8 = 0; void tick() { - if (start.time_since_epoch().count() == 0) { - start = std::chrono::steady_clock::now(); - } + if (start.time_since_epoch().count() == 0) { start = std::chrono::steady_clock::now(); } - viz::Begin(); - if (show_ui) { - // UI - viz::BeginImGuiLayer(); - - viz::LayerPriority(11); - - ImGui::Begin("Options"); - - ImGui::Checkbox("Image layer", &show_image_layer); - if (show_image_layer) { - ImGui::Combo("Source", reinterpret_cast(¤t_source), source_items, - IM_ARRAYSIZE(source_items)); - ImGui::Combo("Format", reinterpret_cast(¤t_format_index), format_items, - IM_ARRAYSIZE(format_items)); - ImGui::SliderFloat("Opacity##image", &image_layer_opacity, 0.f, 1.f); - ImGui::SliderInt("Priority##image", &image_layer_priority, -10, 10); - - // color picker for first item of LUT - if (formats[current_format_index] == viz::ImageFormat::R8_UINT) { - static int color_index = 0; - ImGui::SliderInt("LUT index", &color_index, 0, palette.size() - 1); - - uint32_t &item = palette[color_index]; - float color[]{(item & 0xFF) / 255.f, ((item >> 8) & 0xFF) / 255.f, - ((item >> 16) & 0xFF) / 255.f, - ((item >> 24) & 0xFF) / 255.f}; - ImGui::ColorEdit4("##color", color, ImGuiColorEditFlags_DefaultOptions_); - item = static_cast((color[0] * 255.f) + 0.5f) + - (static_cast((color[1] * 255.f) + 0.5f) << 8) + - (static_cast((color[2] * 255.f) + 0.5f) << 16) + - (static_cast((color[3] * 255.f) + 0.5f) << 24); - } - } - ImGui::Separator(); - ImGui::Checkbox("Geometry layer", &show_geometry_layer); - if (show_geometry_layer) { - ImGui::SliderFloat("Opacity##geom", &geometry_layer_opacity, 0.f, 1.f); - ImGui::SliderInt("Priority##geom", &geometry_layer_priority, -10, 10); - } + viz::Begin(); + if (show_ui) { + // UI + viz::BeginImGuiLayer(); - ImGui::Text("%.1f frames/s", fps); + viz::LayerPriority(11); - ImGui::End(); - - viz::EndLayer(); - } + ImGui::Begin("Options", nullptr, ImGuiWindowFlags_AlwaysAutoResize); + ImGui::Checkbox("Image layer", &show_image_layer); if (show_image_layer) { - viz::BeginImageLayer(); - viz::LayerOpacity(image_layer_opacity); - viz::LayerPriority(image_layer_priority); - - if ((formats[current_format_index] == viz::ImageFormat::R8G8B8_UNORM) || - (formats[current_format_index] == viz::ImageFormat::R8G8B8A8_UNORM)) { - // Color image - - // host memory - switch (current_source) { - case Source::HOST: { - const void *data; - switch (formats[current_format_index]) { - case viz::ImageFormat::R8G8B8_UNORM: - data = host_mem_r8g8b8.get(); - break; - case viz::ImageFormat::R8G8B8A8_UNORM: - data = host_mem_r8g8b8a8.get(); - break; - } - viz::ImageHost(width, height, formats[current_format_index], data); - break; - } - case Source::DEVICE: { - CUdeviceptr device_ptr; - switch (formats[current_format_index]) { - case viz::ImageFormat::R8G8B8_UNORM: - device_ptr = cu_device_mem_r8g8b8; - break; - case viz::ImageFormat::R8G8B8A8_UNORM: - device_ptr = cu_device_mem_r8g8b8a8; - break; - } - viz::ImageCudaDevice(width, height, formats[current_format_index], device_ptr); - break; - } - } - } else { - // Image with LUT - viz::LUT(palette.size(), viz::ImageFormat::R8G8B8A8_UNORM, - palette.size() * sizeof(uint32_t), palette.data()); - - if (current_source == Source::DEVICE) { - viz::ImageCudaDevice(width, height, formats[current_format_index], cu_device_mem_r8); - } else { - viz::ImageHost(width, height, formats[current_format_index], host_mem_r8.get()); - } - } - - viz::EndLayer(); + ImGui::Combo("Source", + reinterpret_cast(¤t_source), + source_items, + IM_ARRAYSIZE(source_items)); + ImGui::Combo("Format", + reinterpret_cast(¤t_format_index), + format_items, + IM_ARRAYSIZE(format_items)); + ImGui::SliderFloat("Opacity##image", &image_layer_opacity, 0.f, 1.f); + ImGui::SliderInt("Priority##image", &image_layer_priority, -10, 10); + + // color picker for first item of LUT + if (formats[current_format_index] == viz::ImageFormat::R8_UINT) { + static int color_index = 0; + ImGui::SliderInt("LUT index", &color_index, 0, palette.size() - 1); + + uint32_t& item = palette[color_index]; + float color[]{(item & 0xFF) / 255.f, + ((item >> 8) & 0xFF) / 255.f, + ((item >> 16) & 0xFF) / 255.f, + ((item >> 24) & 0xFF) / 255.f}; + ImGui::ColorEdit4("##color", color, ImGuiColorEditFlags_DefaultOptions_); + item = static_cast((color[0] * 255.f) + 0.5f) + + (static_cast((color[1] * 255.f) + 0.5f) << 8) + + (static_cast((color[2] * 255.f) + 0.5f) << 16) + + (static_cast((color[3] * 255.f) + 0.5f) << 24); + } } - + ImGui::Separator(); + ImGui::Checkbox("Geometry layer", &show_geometry_layer); if (show_geometry_layer) { - viz::BeginGeometryLayer(); - viz::LayerOpacity(geometry_layer_opacity); - viz::LayerPriority(geometry_layer_priority); - - viz::Color(1.f, 0.f, 0.f, 1.f); - viz::Text(0.75f, 0.05f, 0.02f, "POINT_LIST"); - { - const float data[]{0.9f, 0.1f, 0.95f, 0.05f}; - viz::PointSize(5.f); - viz::Primitive(viz::PrimitiveTopology::POINT_LIST, 2, - sizeof(data) / sizeof(data[0]), data); - } - - viz::Color(0.f, 1.f, 0.f, 1.f); - viz::Text(0.75f, 0.2f, 0.02f, "LINE_LIST"); - { - const float data[]{0.9f, 0.25f, 0.95f, 0.2f, 0.92f, 0.27f, 0.93f, 0.23f}; - viz::LineWidth(2.f); - viz::Primitive(viz::PrimitiveTopology::LINE_LIST, 2, - sizeof(data) / sizeof(data[0]), data); - } + ImGui::SliderFloat("Opacity##geom", &geometry_layer_opacity, 0.f, 1.f); + ImGui::SliderInt("Priority##geom", &geometry_layer_priority, -10, 10); + } - viz::Color(1.f, 1.f, 0.f, 1.f); - viz::Text(0.75f, 0.35f, 0.02f, "LINE_STRIP"); - { - const float data[]{0.9f, 0.35f, 0.95f, 0.3f, 0.97f, 0.37f, 0.93f, 0.35f}; - viz::LineWidth(1.f); - viz::Primitive(viz::PrimitiveTopology::LINE_STRIP, 3, - sizeof(data) / sizeof(data[0]), data); + ImGui::Text("%.1f frames/s", fps); + + ImGui::End(); + + viz::EndLayer(); + } + + if (show_image_layer) { + viz::BeginImageLayer(); + viz::LayerOpacity(image_layer_opacity); + viz::LayerPriority(image_layer_priority); + + if ((formats[current_format_index] == viz::ImageFormat::R8G8B8_UNORM) || + (formats[current_format_index] == viz::ImageFormat::R8G8B8A8_UNORM)) { + // Color image + + // host memory + switch (current_source) { + case Source::HOST: { + const void* data; + switch (formats[current_format_index]) { + case viz::ImageFormat::R8G8B8_UNORM: + data = host_mem_r8g8b8.get(); + break; + case viz::ImageFormat::R8G8B8A8_UNORM: + data = host_mem_r8g8b8a8.get(); + break; + } + viz::ImageHost(width, height, formats[current_format_index], data); + break; } - - viz::Color(0.f, 0.f, 1.f, 1.f); - viz::Text(0.75f, 0.5f, 0.02f, "TRIANGLE_LIST"); - { - const float data[]{0.9f, 0.45f, 0.92f, 0.45f, 0.91f, 0.5f, 0.95f, 0.45f, - 0.95f, 0.55f, 0.975f, 0.50f}; - viz::Primitive(viz::PrimitiveTopology::TRIANGLE_LIST, 2, - sizeof(data) / sizeof(data[0]), data); + case Source::DEVICE: { + CUdeviceptr device_ptr; + switch (formats[current_format_index]) { + case viz::ImageFormat::R8G8B8_UNORM: + device_ptr = cu_device_mem_r8g8b8; + break; + case viz::ImageFormat::R8G8B8A8_UNORM: + device_ptr = cu_device_mem_r8g8b8a8; + break; + } + viz::ImageCudaDevice(width, height, formats[current_format_index], device_ptr); + break; } + } + } else { + // Image with LUT + viz::LUT(palette.size(), + viz::ImageFormat::R8G8B8A8_UNORM, + palette.size() * sizeof(uint32_t), + palette.data()); + + if (current_source == Source::DEVICE) { + viz::ImageCudaDevice(width, height, formats[current_format_index], cu_device_mem_r8); + } else { + viz::ImageHost(width, height, formats[current_format_index], host_mem_r8.get()); + } + } - viz::Color(1.f, 0.f, 1.f, 1.f); - viz::Text(0.75f, 0.65f, 0.02f, "CROSS_LIST"); - { - const float data[]{0.9f, 0.7f, 0.08f, 0.95f, 0.65f, 0.05f}; - viz::Primitive(viz::PrimitiveTopology::CROSS_LIST, 2, - sizeof(data) / sizeof(data[0]), data); - } + viz::EndLayer(); + } - viz::Color(0.f, 1.f, 1.f, 1.f); - viz::Text(0.75f, 0.8f, 0.02f, "RECTANGLE_LIST"); - { - const float data[]{0.9f, 0.75f, 0.98f, 0.85f, 0.95f, 0.8f, 0.97f, 0.83f}; - viz::Primitive(viz::PrimitiveTopology::RECTANGLE_LIST, 2, - sizeof(data) / sizeof(data[0]), data); - } + if (show_geometry_layer) { + viz::BeginGeometryLayer(); + viz::LayerOpacity(geometry_layer_opacity); + viz::LayerPriority(geometry_layer_priority); - viz::Color(1.f, 1.f, 1.f, 1.f); - viz::Text(0.75f, 0.95f, 0.02f, "OVAL_LIST"); - { - const float data[]{0.9f, 0.95f, 0.1f, 0.1f, 0.95f, 0.975f, 0.05f, 0.1f}; - viz::LineWidth(3.f); - viz::Primitive(viz::PrimitiveTopology::OVAL_LIST, 2, - sizeof(data) / sizeof(data[0]), data); - } + const float text_size = 0.05f; - viz::EndLayer(); + viz::Color(1.f, 0.f, 0.f, 1.f); + viz::Text(0.65f, 0.05f, text_size, "POINT_LIST"); + { + const float data[]{0.9f, 0.1f, 0.95f, 0.05f}; + viz::PointSize(5.f); + viz::Primitive(viz::PrimitiveTopology::POINT_LIST, 2, sizeof(data) / sizeof(data[0]), data); } - viz::End(); - - // timing - ++iterations; - elapsed = std::chrono::duration_cast - (std::chrono::steady_clock::now() - start); - if (!benchmark_mode && (elapsed.count() > 1000)) { - fps = static_cast(iterations) / - (static_cast(elapsed.count()) / 1000.f); - start = std::chrono::steady_clock::now(); - iterations = 0; + viz::Color(0.f, 1.f, 0.f, 1.f); + viz::Text(0.65f, 0.2f, text_size, "LINE_LIST"); + { + const float data[]{0.9f, 0.25f, 0.95f, 0.2f, 0.92f, 0.27f, 0.93f, 0.23f}; + viz::LineWidth(2.f); + viz::Primitive(viz::PrimitiveTopology::LINE_LIST, 2, sizeof(data) / sizeof(data[0]), data); } -} -void initCuda() { - if (cuInit(0) != CUDA_SUCCESS) { - throw std::runtime_error("cuInit failed."); + viz::Color(1.f, 1.f, 0.f, 1.f); + viz::Text(0.65f, 0.35f, text_size, "LINE_STRIP"); + { + const float data[]{0.9f, 0.35f, 0.95f, 0.3f, 0.97f, 0.37f, 0.93f, 0.35f}; + viz::LineWidth(1.f); + viz::Primitive(viz::PrimitiveTopology::LINE_STRIP, 3, sizeof(data) / sizeof(data[0]), data); } - if (cuDevicePrimaryCtxRetain(&cuda_context, 0) != CUDA_SUCCESS) { - throw std::runtime_error("cuDevicePrimaryCtxRetain failed."); + viz::Color(0.f, 0.f, 1.f, 1.f); + viz::Text(0.65f, 0.5f, text_size, "TRIANGLE_LIST"); + { + const float data[]{ + 0.9f, 0.45f, 0.92f, 0.45f, 0.91f, 0.5f, 0.95f, 0.45f, 0.95f, 0.55f, 0.975f, 0.50f}; + viz::Primitive( + viz::PrimitiveTopology::TRIANGLE_LIST, 2, sizeof(data) / sizeof(data[0]), data); } - if (cuCtxPushCurrent(cuda_context) != CUDA_SUCCESS) { - throw std::runtime_error("cuDevicePrimaryCtxRetain failed."); + viz::Color(1.f, 0.f, 1.f, 1.f); + viz::Text(0.65f, 0.65f, text_size, "CROSS_LIST"); + { + const float data[]{0.9f, 0.7f, 0.08f, 0.95f, 0.65f, 0.05f}; + viz::Primitive(viz::PrimitiveTopology::CROSS_LIST, 2, sizeof(data) / sizeof(data[0]), data); } -} -void cleanupCuda() { - if (cu_device_mem_r8) { - if (cuMemFree(cu_device_mem_r8) != CUDA_SUCCESS) { - throw std::runtime_error("cuMemFree failed."); - } - } - if (cu_device_mem_r8g8b8) { - if (cuMemFree(cu_device_mem_r8g8b8) != CUDA_SUCCESS) { - throw std::runtime_error("cuMemFree failed."); - } - } - if (cu_device_mem_r8g8b8a8) { - if (cuMemFree(cu_device_mem_r8g8b8a8) != CUDA_SUCCESS) { - throw std::runtime_error("cuMemFree failed."); - } + viz::Color(0.f, 1.f, 1.f, 1.f); + viz::Text(0.65f, 0.8f, text_size, "RECTANGLE_LIST"); + { + const float data[]{0.9f, 0.75f, 0.98f, 0.85f, 0.95f, 0.8f, 0.97f, 0.83f}; + viz::Primitive( + viz::PrimitiveTopology::RECTANGLE_LIST, 2, sizeof(data) / sizeof(data[0]), data); } - if (cuda_context) { - if (cuCtxPopCurrent(&cuda_context) != CUDA_SUCCESS) { - throw std::runtime_error("cuCtxPopCurrent failed."); - } - cuda_context = nullptr; - if (cuDevicePrimaryCtxRelease(0) != CUDA_SUCCESS) { - throw std::runtime_error("cuDevicePrimaryCtxRelease failed."); - } + viz::Color(1.f, 1.f, 1.f, 1.f); + viz::Text(0.65f, 0.95f, text_size, "OVAL_LIST"); + { + const float data[]{0.9f, 0.95f, 0.1f, 0.1f, 0.95f, 0.975f, 0.05f, 0.1f}; + viz::LineWidth(3.f); + viz::Primitive(viz::PrimitiveTopology::OVAL_LIST, 2, sizeof(data) / sizeof(data[0]), data); } -} -void loadImage() { - int components; + viz::EndLayer(); + } - unsigned char *image_data = - stbi_load("nv_logo.png", reinterpret_cast(&width), reinterpret_cast(&height), - &components, 0); - if (!image_data) { - throw std::runtime_error("Loading image failed."); - } + viz::End(); - // allocate and set host memory - host_mem_r8.reset(new uint8_t[width * height]); - host_mem_r8g8b8.reset(new uint8_t[width * height * 3]); - host_mem_r8g8b8a8.reset(new uint8_t[width * height * 4]); - - uint8_t const *src = image_data; - - uint8_t *dst_r8 = host_mem_r8.get(); - uint8_t *dst_r8g8b8a8 = host_mem_r8g8b8a8.get(); - uint8_t *dst_r8g8b8 = host_mem_r8g8b8.get(); - for (uint32_t i = 0; i < width * height; ++i) { - dst_r8g8b8[0] = src[0]; - dst_r8g8b8[1] = src[1]; - dst_r8g8b8[2] = src[2]; - dst_r8g8b8 += 3; - - dst_r8g8b8a8[0] = src[0]; - dst_r8g8b8a8[1] = src[1]; - dst_r8g8b8a8[2] = src[2]; - dst_r8g8b8a8[3] = (components == 4) ? src[3] : 0xFF; - const uint32_t pixel = *reinterpret_cast(dst_r8g8b8a8); - dst_r8g8b8a8 += 4; - - std::vector::iterator it = std::find(palette.begin(), palette.end(), pixel); - if (it == palette.end()) { - palette.push_back(pixel); - it = --palette.end(); - } - dst_r8[0] = std::distance(palette.begin(), it); - dst_r8 += 1; + // timing + ++iterations; + elapsed = std::chrono::duration_cast(std::chrono::steady_clock::now() - + start); + if (!benchmark_mode && (elapsed.count() > 1000)) { + fps = static_cast(iterations) / (static_cast(elapsed.count()) / 1000.f); + start = std::chrono::steady_clock::now(); + iterations = 0; + } +} - src += components; - } +void initCuda() { + if (cuInit(0) != CUDA_SUCCESS) { throw std::runtime_error("cuInit failed."); } - stbi_image_free(image_data); + if (cuDevicePrimaryCtxRetain(&cuda_context, 0) != CUDA_SUCCESS) { + throw std::runtime_error("cuDevicePrimaryCtxRetain failed."); + } - // allocate and set device memory - if (cuMemAlloc(&cu_device_mem_r8, width * height) != CUDA_SUCCESS) { - throw std::runtime_error("cuMemAlloc failed."); - } - if (cuMemcpyHtoD(cu_device_mem_r8, host_mem_r8.get(), width * height) != CUDA_SUCCESS) { - throw std::runtime_error("cuMemcpyHtoD failed."); + if (cuCtxPushCurrent(cuda_context) != CUDA_SUCCESS) { + throw std::runtime_error("cuDevicePrimaryCtxRetain failed."); + } +} + +void cleanupCuda() { + if (cu_device_mem_r8) { + if (cuMemFree(cu_device_mem_r8) != CUDA_SUCCESS) { + throw std::runtime_error("cuMemFree failed."); } - if (cuMemAlloc(&cu_device_mem_r8g8b8, width * height * 3) != CUDA_SUCCESS) { - throw std::runtime_error("cuMemAlloc failed."); + } + if (cu_device_mem_r8g8b8) { + if (cuMemFree(cu_device_mem_r8g8b8) != CUDA_SUCCESS) { + throw std::runtime_error("cuMemFree failed."); } - if (cuMemcpyHtoD(cu_device_mem_r8g8b8, host_mem_r8g8b8.get(), width * height * 3) - != CUDA_SUCCESS) { - throw std::runtime_error("cuMemcpyHtoD failed."); + } + if (cu_device_mem_r8g8b8a8) { + if (cuMemFree(cu_device_mem_r8g8b8a8) != CUDA_SUCCESS) { + throw std::runtime_error("cuMemFree failed."); } - if (cuMemAlloc(&cu_device_mem_r8g8b8a8, width * height * 4) != CUDA_SUCCESS) { - throw std::runtime_error("cuMemAlloc failed."); + } + if (cuda_context) { + if (cuCtxPopCurrent(&cuda_context) != CUDA_SUCCESS) { + throw std::runtime_error("cuCtxPopCurrent failed."); } - if (cuMemcpyHtoD(cu_device_mem_r8g8b8a8, host_mem_r8g8b8a8.get(), width * height * 4) - != CUDA_SUCCESS) { - throw std::runtime_error("cuMemcpyHtoD failed."); + cuda_context = nullptr; + + if (cuDevicePrimaryCtxRelease(0) != CUDA_SUCCESS) { + throw std::runtime_error("cuDevicePrimaryCtxRelease failed."); } + } } -int main(int argc, char **argv) { - struct option long_options[] = {{"help", no_argument, 0, 'h'}, - {"bench", no_argument, 0, 'b'}, - {"headless", no_argument, 0, 'l'}, - {"display", required_argument, 0, 'd'}, - {0, 0, 0, 0}}; - std::string display_name; - // parse options - while (true) { - int option_index = 0; - const int c = getopt_long(argc, argv, "hbld:", long_options, &option_index); - - if (c == -1) { - break; - } - - const std::string argument(optarg ? optarg : ""); - switch (c) { - case 'h': - std::cout << "Usage: " << argv[0] << " [options]" << std::endl - << "Options:" << std::endl - << " -h, --help display this information" << std::endl - << " -b, --bench benchmark mode" << std::endl - << " -l, --headless headless mode" << std::endl - << " -d, --display name of the display to use in exclusive mode" - " (either EDID or xrandr name)" - << std::endl; - return EXIT_SUCCESS; - - case 'b': - benchmark_mode = true; - show_ui = false; - show_geometry_layer = false; - break; - case 'l': - headless_mode = true; - show_ui = false; - break; - case 'd': - display_name = argument; - break; - default: - throw std::runtime_error("Unhandled option "); - } +void loadImage() { + int components; + + unsigned char* image_data = stbi_load("nv_logo.png", + reinterpret_cast(&width), + reinterpret_cast(&height), + &components, + 0); + if (!image_data) { throw std::runtime_error("Loading image failed."); } + + // allocate and set host memory + host_mem_r8.reset(new uint8_t[width * height]); + host_mem_r8g8b8.reset(new uint8_t[width * height * 3]); + host_mem_r8g8b8a8.reset(new uint8_t[width * height * 4]); + + uint8_t const* src = image_data; + + uint8_t* dst_r8 = host_mem_r8.get(); + uint8_t* dst_r8g8b8a8 = host_mem_r8g8b8a8.get(); + uint8_t* dst_r8g8b8 = host_mem_r8g8b8.get(); + for (uint32_t i = 0; i < width * height; ++i) { + dst_r8g8b8[0] = src[0]; + dst_r8g8b8[1] = src[1]; + dst_r8g8b8[2] = src[2]; + dst_r8g8b8 += 3; + + dst_r8g8b8a8[0] = src[0]; + dst_r8g8b8a8[1] = src[1]; + dst_r8g8b8a8[2] = src[2]; + dst_r8g8b8a8[3] = (components == 4) ? src[3] : 0xFF; + const uint32_t pixel = *reinterpret_cast(dst_r8g8b8a8); + dst_r8g8b8a8 += 4; + + std::vector::iterator it = std::find(palette.begin(), palette.end(), pixel); + if (it == palette.end()) { + palette.push_back(pixel); + it = --palette.end(); } + dst_r8[0] = std::distance(palette.begin(), it); + dst_r8 += 1; + + src += components; + } + + stbi_image_free(image_data); + + // allocate and set device memory + if (cuMemAlloc(&cu_device_mem_r8, width * height) != CUDA_SUCCESS) { + throw std::runtime_error("cuMemAlloc failed."); + } + if (cuMemcpyHtoD(cu_device_mem_r8, host_mem_r8.get(), width * height) != CUDA_SUCCESS) { + throw std::runtime_error("cuMemcpyHtoD failed."); + } + if (cuMemAlloc(&cu_device_mem_r8g8b8, width * height * 3) != CUDA_SUCCESS) { + throw std::runtime_error("cuMemAlloc failed."); + } + if (cuMemcpyHtoD(cu_device_mem_r8g8b8, host_mem_r8g8b8.get(), width * height * 3) != + CUDA_SUCCESS) { + throw std::runtime_error("cuMemcpyHtoD failed."); + } + if (cuMemAlloc(&cu_device_mem_r8g8b8a8, width * height * 4) != CUDA_SUCCESS) { + throw std::runtime_error("cuMemAlloc failed."); + } + if (cuMemcpyHtoD(cu_device_mem_r8g8b8a8, host_mem_r8g8b8a8.get(), width * height * 4) != + CUDA_SUCCESS) { + throw std::runtime_error("cuMemcpyHtoD failed."); + } +} - initCuda(); - loadImage(); - - // If using ImGui, create a context and pass it to Holoviz, do this before calling viz::Init(). - ImGui::CreateContext(); - viz::ImGuiSetCurrentContext(ImGui::GetCurrentContext()); - - // set the Cuda stream to be used by Holoviz - viz::SetCudaStream(CU_STREAM_PER_THREAD); - - uint32_t display_width, display_height; - - // setup the window - if (!display_name.empty()) { - display_width = width; - display_height = height; - viz::Init(display_name.c_str()); - } else { - viz::InitFlags flags = viz::InitFlags::NONE; - - if (headless_mode) { - flags = viz::InitFlags::HEADLESS; - } - - display_width = 1024; - display_height = uint32_t(static_cast(height) / static_cast(width) * 1024.f); - viz::Init(display_width, display_height, "Holoviz Example", flags); +int main(int argc, char** argv) { + struct option long_options[] = {{"help", no_argument, 0, 'h'}, + {"bench", no_argument, 0, 'b'}, + {"headless", no_argument, 0, 'l'}, + {"display", required_argument, 0, 'd'}, + {0, 0, 0, 0}}; + std::string display_name; + // parse options + while (true) { + int option_index = 0; + const int c = getopt_long(argc, argv, "hbld:", long_options, &option_index); + + if (c == -1) { break; } + + const std::string argument(optarg ? optarg : ""); + switch (c) { + case 'h': + std::cout << "Usage: " << argv[0] << " [options]" << std::endl + << "Options:" << std::endl + << " -h, --help display this information" << std::endl + << " -b, --bench benchmark mode" << std::endl + << " -l, --headless headless mode" << std::endl + << " -d, --display name of the display to use in exclusive mode" + " (either EDID or xrandr name)" + << std::endl; + return EXIT_SUCCESS; + + case 'b': + benchmark_mode = true; + show_ui = false; + show_geometry_layer = false; + break; + case 'l': + headless_mode = true; + show_ui = false; + break; + case 'd': + display_name = argument; + break; + default: + throw std::runtime_error("Unhandled option "); + } + } + + initCuda(); + loadImage(); + + // If using ImGui, create a context and pass it to Holoviz, do this before calling viz::Init(). + ImGui::CreateContext(); + viz::ImGuiSetCurrentContext(ImGui::GetCurrentContext()); + + // set the Cuda stream to be used by Holoviz + viz::SetCudaStream(CU_STREAM_PER_THREAD); + + uint32_t display_width, display_height; + + // setup the window + if (!display_name.empty()) { + display_width = width; + display_height = height; + viz::Init(display_name.c_str()); + } else { + viz::InitFlags flags = viz::InitFlags::NONE; + + if (headless_mode) { flags = viz::InitFlags::HEADLESS; } + + display_width = 1024; + display_height = uint32_t(static_cast(height) / static_cast(width) * 1024.f); + viz::Init(display_width, display_height, "Holoviz Example", flags); + } + + if (benchmark_mode) { + do { tick(); } while (elapsed.count() < 2000); + std::cout << float(iterations) / (float(elapsed.count()) / 1000.f) << " fps" << std::endl; + } else if (headless_mode) { + tick(); + + // allocate a cuda buffer to hold the framebuffer data + const size_t data_size = display_width * display_height * 4 * sizeof(uint8_t); + CUdeviceptr read_buffer; + if (cuMemAlloc(&read_buffer, data_size) != CUDA_SUCCESS) { + throw std::runtime_error("cuMemAlloc failed."); } - if (benchmark_mode) { - do { - tick(); - } while (elapsed.count() < 2000); - std::cout << float(iterations) / (float(elapsed.count()) / 1000.f) << " fps" << std::endl; - } else if (headless_mode) { - tick(); - - // allocate a cuda buffer to hold the framebuffer data - const size_t data_size = display_width * display_height * 4 * sizeof(uint8_t); - CUdeviceptr read_buffer; - if (cuMemAlloc(&read_buffer, data_size) != CUDA_SUCCESS) { - throw std::runtime_error("cuMemAlloc failed."); - } - - // read back the framebuffer - viz::ReadFramebuffer(viz::ImageFormat::R8G8B8A8_UNORM, data_size, read_buffer); + // read back the framebuffer + viz::ReadFramebuffer( + viz::ImageFormat::R8G8B8A8_UNORM, display_width, display_height, data_size, read_buffer); - std::vector data(data_size); - if (cuMemcpyDtoHAsync(data.data(), read_buffer, data_size, CU_STREAM_PER_THREAD) - != CUDA_SUCCESS) { - throw std::runtime_error("cuMemcpyDtoHAsync failed."); - } - if (cuStreamSynchronize(CU_STREAM_PER_THREAD) != CUDA_SUCCESS) { - throw std::runtime_error("cuStreamSynchronize failed."); - } + std::vector data(data_size); + if (cuMemcpyDtoHAsync(data.data(), read_buffer, data_size, CU_STREAM_PER_THREAD) != + CUDA_SUCCESS) { + throw std::runtime_error("cuMemcpyDtoHAsync failed."); + } + if (cuStreamSynchronize(CU_STREAM_PER_THREAD) != CUDA_SUCCESS) { + throw std::runtime_error("cuStreamSynchronize failed."); + } - // write to a file - const char* filename = "framebuffer.png"; - std::cout << "Writing image to " << filename << "." << std::endl; - stbi_write_png("framebuffer.png", display_width, display_height, 4, data.data(), 0); + // write to a file + const char* filename = "framebuffer.png"; + std::cout << "Writing image to " << filename << "." << std::endl; + stbi_write_png("framebuffer.png", display_width, display_height, 4, data.data(), 0); - if (cuMemFree(read_buffer) != CUDA_SUCCESS) { - throw std::runtime_error("cuMemFree failed."); - } - } else { - while (!viz::WindowShouldClose()) { - if (!viz::WindowIsMinimized()) { - tick(); - } - } + if (cuMemFree(read_buffer) != CUDA_SUCCESS) { throw std::runtime_error("cuMemFree failed."); } + } else { + while (!viz::WindowShouldClose()) { + if (!viz::WindowIsMinimized()) { tick(); } } + } - viz::Shutdown(); + viz::Shutdown(); - cleanupCuda(); + cleanupCuda(); - return EXIT_SUCCESS; + return EXIT_SUCCESS; } diff --git a/modules/holoviz/examples/demo/CMakeLists.txt.public b/modules/holoviz/examples/depth_map/CMakeLists.txt similarity index 68% rename from modules/holoviz/examples/demo/CMakeLists.txt.public rename to modules/holoviz/examples/depth_map/CMakeLists.txt index 5830e5fe..af2b7a30 100644 --- a/modules/holoviz/examples/demo/CMakeLists.txt.public +++ b/modules/holoviz/examples/depth_map/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,17 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -cmake_minimum_required(VERSION 3.16) - -include(GNUInstallDirs) - -project(ExampleDemo) - -set(CMAKE_PREFIX_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../../cmake") - -find_package(clara_holoviz REQUIRED) -find_package(clara_holoviz_imgui REQUIRED) -find_package(CUDAToolkit REQUIRED) +set(PROJECT_NAME holoviz_depth_map) # fetch the dependencies include(FetchContent) @@ -34,7 +24,7 @@ FetchContent_Declare( GIT_REPOSITORY https://github.com/nothings/stb.git GIT_TAG af1a5bc352164740c1cc1354942b1c6b72eacb8a GIT_PROGRESS TRUE -) + ) FetchContent_MakeAvailable(stb) add_executable(${PROJECT_NAME}) @@ -51,10 +41,6 @@ target_include_directories(${PROJECT_NAME} target_link_libraries(${PROJECT_NAME} PRIVATE - clara_holoviz - clara_holoviz_imgui + holoscan::viz + holoscan::viz::imgui ) - -install(TARGETS ${PROJECT_NAME} - DESTINATION ${CMAKE_INSTALL_BINDIR} - ) \ No newline at end of file diff --git a/modules/holoviz/examples/depth_map/Main.cpp b/modules/holoviz/examples/depth_map/Main.cpp new file mode 100644 index 00000000..196e9a81 --- /dev/null +++ b/modules/holoviz/examples/depth_map/Main.cpp @@ -0,0 +1,426 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#define STB_IMAGE_IMPLEMENTATION +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace viz = holoscan::viz; + +uint32_t width; +uint32_t height; +uint32_t frame_index = 0; + +std::vector palette{0xFF000000, 0xFF7F7F7F, 0xFFFFFFFF}; + +// UI state +enum class SourceMode { GENERATED, FILES }; +static const char* source_mode_items[]{"Generated", "Files"}; +SourceMode current_source_mode = SourceMode::GENERATED; + +char depth_dir[256]; +char color_dir[256]; + +enum class RenderMode { POINTS, LINES, TRIANGLES, DEPTH }; +static const char* render_mode_items[]{"Points", "Lines", "Triangles", "Depth"}; +RenderMode current_render_mode = RenderMode::TRIANGLES; + +bool unlimited_fps = false; +float fps = 15.f; +int color_index = 0; +float line_width = 1.f; +float point_size = 1.f; + +// cuda +CUcontext cuda_context = nullptr; +std::vector depth_mems; +std::vector color_mems; + +void loadImage(const std::string& filename, CUdeviceptr* cu_device_mem, int* width, int* height, + int* components, bool convert_to_intensity) { + const unsigned char* image_data = + stbi_load(filename.c_str(), width, height, components, convert_to_intensity ? 0 : 4); + if (!image_data) { throw std::runtime_error("Loading image failed."); } + + std::vector tmp_image_data; + if ((*components != 1) && (convert_to_intensity)) { + tmp_image_data.reserve(*width * *height); + for (int index = 0; index < *width * *height; ++index) { + const uint8_t* src = &image_data[index * *components]; + tmp_image_data.push_back(static_cast(0.2126f * static_cast(src[0]) + + 0.7152f * static_cast(src[1]) + + 0.0722f * static_cast(src[2]) + 0.5f)); + } + image_data = tmp_image_data.data(); + *components = 1; + } else { + *components = 4; + } + + if (cuMemAlloc(cu_device_mem, *width * *height * *components) != CUDA_SUCCESS) { + throw std::runtime_error("cuMemAlloc failed."); + } + if (cuMemcpyHtoD(*cu_device_mem, image_data, *width * *height * *components) != CUDA_SUCCESS) { + throw std::runtime_error("cuMemcpyHtoD failed."); + } +} + +void freeSourceData() { + for (auto&& depth_mem : depth_mems) { + if (cuMemFree(depth_mem) != CUDA_SUCCESS) { throw std::runtime_error("cuMemFree failed."); } + } + depth_mems.clear(); + for (auto&& color_mem : color_mems) { + if (cuMemFree(color_mem) != CUDA_SUCCESS) { throw std::runtime_error("cuMemFree failed."); } + } + color_mems.clear(); +} + +void loadSourceData(const char* depth_dir, const char* color_dir) { + std::vector depth_files; + for (auto const& dir_entry : std::filesystem::directory_iterator{depth_dir}) { + depth_files.push_back(dir_entry.path()); + } + std::sort(depth_files.begin(), depth_files.end()); + + bool first = true; + CUdeviceptr cu_device_mem; + int cur_width, cur_height, cur_components; + + for (auto&& file_name : depth_files) { + std::cout << "\rReading depth image " << file_name << std::flush; + loadImage(file_name, &cu_device_mem, &cur_width, &cur_height, &cur_components, true); + if (first) { + width = cur_width; + height = cur_height; + first = false; + } else if ((cur_width != width) || (cur_height != height) || (cur_components != 1)) { + throw std::runtime_error("Inconsistent depth image sequence"); + } + depth_mems.push_back(cu_device_mem); + } + std::cout << std::endl; + + std::vector color_files; + for (auto const& dir_entry : std::filesystem::directory_iterator{color_dir}) { + color_files.push_back(dir_entry.path()); + } + std::sort(color_files.begin(), color_files.end()); + + for (auto&& file_name : color_files) { + std::cout << "\rReading color image " << file_name << std::flush; + loadImage(file_name, &cu_device_mem, &cur_width, &cur_height, &cur_components, false); + if (first) { + width = cur_width; + height = cur_height; + first = false; + } else if ((cur_width != width) || (cur_height != height) || (cur_components != 4)) { + throw std::runtime_error("Inconsistent color image sequence"); + } + color_mems.push_back(cu_device_mem); + } + std::cout << std::endl; +} + +void generateSourceData(uint32_t frame_index) { + const uint32_t images = 100; + CUdeviceptr cu_device_mem; + + width = 64; + height = 64; + + if (depth_mems.empty()) { + for (uint32_t index = 0; index < images; ++index) { + if (cuMemAlloc(&cu_device_mem, width * height * sizeof(uint8_t)) != CUDA_SUCCESS) { + throw std::runtime_error("cuMemAlloc failed."); + } + depth_mems.push_back(cu_device_mem); + } + } + if (color_mems.empty()) { + for (uint32_t index = 0; index < images; ++index) { + if (cuMemAlloc(&cu_device_mem, width * height * sizeof(uint32_t)) != CUDA_SUCCESS) { + throw std::runtime_error("cuMemAlloc failed."); + } + color_mems.push_back(cu_device_mem); + } + } + + std::vector depth_data(width * height); + std::vector color_data(width * height); + for (uint32_t index = 0; index < images; ++index) { + const float offset = float(frame_index) / float(images); + + for (uint32_t y = 0; y < height; ++y) { + for (uint32_t x = 0; x < width; ++x) { + const uint8_t depth = (std::sin((float(x) / float(width)) * 3.14f * 4.f) * + std::cos((float(y) / float(height)) * 3.14f * 3.f) + + 1.f) * offset * + 63.f; + + depth_data[y * width + x] = depth; + color_data[y * width + x] = depth | ((depth << (8 + (x & 1))) & 0xFF00) | + ((depth << (16 + (y & 1) * 2)) & 0xFF0000) | 0xFF204060; + } + } + + if (cuMemcpyHtoD(depth_mems[index], depth_data.data(), depth_data.size() * sizeof(uint8_t)) != + CUDA_SUCCESS) { + throw std::runtime_error("cuMemcpyHtoD failed."); + } + + if (cuMemcpyHtoD(color_mems[index], color_data.data(), color_data.size() * sizeof(uint32_t)) != + CUDA_SUCCESS) { + throw std::runtime_error("cuMemcpyHtoD failed."); + } + } +} + +void tick() { + viz::Begin(); + + // UI + viz::BeginImGuiLayer(); + + viz::LayerPriority(11); + + ImGui::Begin("Options", nullptr, ImGuiWindowFlags_AlwaysAutoResize); + + bool regenerate = depth_mems.empty(); + regenerate |= ImGui::Combo("Source Mode", + reinterpret_cast(¤t_source_mode), + source_mode_items, + IM_ARRAYSIZE(source_mode_items)); + + if (current_source_mode == SourceMode::FILES) { + ImGui::InputText("Depth Dir", depth_dir, sizeof(depth_dir)); + ImGui::InputText("Color Dir", color_dir, sizeof(color_dir)); + regenerate |= ImGui::Button("Load"); + } + + ImGui::Separator(); + ImGui::Combo("Render Mode", + reinterpret_cast(¤t_render_mode), + render_mode_items, + IM_ARRAYSIZE(render_mode_items)); + + switch (current_render_mode) { + case RenderMode::DEPTH: + if (ImGui::Button("+")) { + palette.push_back(static_cast(std::rand()) | 0xFF000000); + } + ImGui::SameLine(); + if (ImGui::Button("-") && (palette.size() > 1)) { palette.pop_back(); } + ImGui::SliderInt("LUT index", &color_index, 0, palette.size() - 1); + + { + uint32_t& item = palette[color_index]; + float color[]{(item & 0xFF) / 255.f, + ((item >> 8) & 0xFF) / 255.f, + ((item >> 16) & 0xFF) / 255.f, + ((item >> 24) & 0xFF) / 255.f}; + ImGui::ColorEdit4("##color", color, ImGuiColorEditFlags_DefaultOptions_); + item = static_cast((color[0] * 255.f) + 0.5f) + + (static_cast((color[1] * 255.f) + 0.5f) << 8) + + (static_cast((color[2] * 255.f) + 0.5f) << 16) + + (static_cast((color[3] * 255.f) + 0.5f) << 24); + } + break; + case RenderMode::LINES: + ImGui::SliderFloat("Line width", &line_width, 1.f, 20.f); + break; + case RenderMode::POINTS: + ImGui::SliderFloat("Point size", &point_size, 1.f, 20.f); + break; + } + + ImGui::End(); + + viz::EndLayer(); + + // regenerate source data if needed + if (regenerate) { + frame_index = 0; + freeSourceData(); + + switch (current_source_mode) { + case SourceMode::FILES: + loadSourceData(depth_dir, color_dir); + break; + case SourceMode::GENERATED: + generateSourceData(frame_index); + break; + } + } else if (current_source_mode == SourceMode::GENERATED) { + generateSourceData(frame_index); + } + + if (!depth_mems.empty()) { + switch (current_render_mode) { + case RenderMode::DEPTH: + viz::BeginImageLayer(); + + // Image with LUT + viz::LUT(palette.size(), + viz::ImageFormat::R8G8B8A8_UNORM, + palette.size() * sizeof(uint32_t), + palette.data(), + true); + + viz::ImageCudaDevice(width, height, viz::ImageFormat::R8_UNORM, depth_mems[frame_index]); + + viz::EndLayer(); + break; + case RenderMode::POINTS: + case RenderMode::LINES: + case RenderMode::TRIANGLES: + viz::BeginGeometryLayer(); + + viz::DepthMapRenderMode render_mode; + switch (current_render_mode) { + case RenderMode::POINTS: + render_mode = viz::DepthMapRenderMode::POINTS; + viz::PointSize(point_size); + break; + case RenderMode::LINES: + render_mode = viz::DepthMapRenderMode::LINES; + viz::LineWidth(line_width); + break; + case RenderMode::TRIANGLES: + render_mode = viz::DepthMapRenderMode::TRIANGLES; + break; + default: + throw std::runtime_error("Unhandled mode."); + } + viz::DepthMap(render_mode, + width, + height, + viz::ImageFormat::R8_UNORM, + depth_mems[frame_index], + viz::ImageFormat::R8G8B8A8_UNORM, + color_mems.size() > frame_index ? color_mems[frame_index] : 0); + + viz::EndLayer(); + + break; + } + ++frame_index; + if (frame_index >= depth_mems.size()) { frame_index = 0; } + } + + viz::End(); +} + +void initCuda() { + if (cuInit(0) != CUDA_SUCCESS) { throw std::runtime_error("cuInit failed."); } + + if (cuDevicePrimaryCtxRetain(&cuda_context, 0) != CUDA_SUCCESS) { + throw std::runtime_error("cuDevicePrimaryCtxRetain failed."); + } + + if (cuCtxPushCurrent(cuda_context) != CUDA_SUCCESS) { + throw std::runtime_error("cuDevicePrimaryCtxRetain failed."); + } +} + +void cleanupCuda() { + freeSourceData(); + if (cuda_context) { + if (cuCtxPopCurrent(&cuda_context) != CUDA_SUCCESS) { + throw std::runtime_error("cuDevicePrimaryCtxRetain failed."); + } + cuda_context = nullptr; + + if (cuDevicePrimaryCtxRelease(0) != CUDA_SUCCESS) { + throw std::runtime_error("cuDevicePrimaryCtxRelease failed."); + } + } +} + +int main(int argc, char** argv) { + struct option long_options[] = {{"help", no_argument, 0, 'h'}, + {"depth_dir", required_argument, 0, 'd'}, + {"color_dir", required_argument, 0, 'c'}, + {0, 0, 0, 0}}; + + // parse options + while (true) { + int option_index = 0; + const int c = getopt_long(argc, argv, "hd:c:", long_options, &option_index); + + if (c == -1) { break; } + + const std::string argument(optarg ? optarg : ""); + switch (c) { + case 'h': + std::cout << "Usage: " << argv[0] << " [options]" << std::endl + << "Options:" << std::endl + << " -d DIR, --depth_dir DIR directory to load depth images from" << std::endl + << " -c DIR, --color_dir DIR directory to load color images from" << std::endl + << " -h, --help display this information" << std::endl + << std::endl; + return EXIT_SUCCESS; + case 'd': + std::strncpy(depth_dir, argument.c_str(), sizeof(depth_dir)); + break; + case 'c': + std::strncpy(color_dir, argument.c_str(), sizeof(color_dir)); + break; + default: + throw std::runtime_error("Unhandled option "); + } + } + + initCuda(); + // If using ImGui, create a context and pass it to Holoviz, do this before calling viz::Init(). + ImGui::CreateContext(); + viz::ImGuiSetCurrentContext(ImGui::GetCurrentContext()); + + viz::Init(1024, 768, "Holoviz Depth Render Example"); + + while (!viz::WindowShouldClose()) { + if (!viz::WindowIsMinimized()) { + tick(); + if (!unlimited_fps) { + std::this_thread::sleep_for(std::chrono::duration(1000.f / fps)); + } + } + } + + viz::Shutdown(); + + cleanupCuda(); + + return EXIT_SUCCESS; +} diff --git a/modules/holoviz/examples/depth_map/README.md b/modules/holoviz/examples/depth_map/README.md new file mode 100644 index 00000000..8c190757 --- /dev/null +++ b/modules/holoviz/examples/depth_map/README.md @@ -0,0 +1,15 @@ +# Holoviz Depth Map Demo + +This example shows how to render depth maps in Holoviz. + +Depth maps are rectangular 2D arrays where each element represents a depth value. The data is +rendered as a 3D object using points, lines or triangles. +Additionally a 2D array with a color value for each point in the grid can be specified. + +Depth maps are rendered in 3D and support camera movement. +The camera is operated using the mouse. + - Orbit (LMB) + - Pan (LMB + CTRL | MMB) + - Dolly (LMB + SHIFT | RMB | Mouse wheel) + - Look Around (LMB + ALT | LMB + CTRL + SHIFT) + - Zoom (Mouse wheel + SHIFT) diff --git a/modules/holoviz/run.sh b/modules/holoviz/run.sh deleted file mode 100755 index 710d435b..00000000 --- a/modules/holoviz/run.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/bash -e -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -COMMAND="" - -TARGET_ARCH=${TARGET_ARCH:-$(uname -m)} -CUDA_VERSION="11.4.0" -DISTRIB_ID="ubuntu" -DISTRIB_RELEASE="20.04" - -# colors -RED="\033[0;31m" -GREEN="\033[0;32m" -NC="\033[0m" # No Color - -# print a failure message -function failMsg -{ - echo -e "$RED$1$NC$2" -} - -function image_build_dev -{ - # CUDA base images - if [ ${TARGET_ARCH} == "aarch64" ]; then - CUDA_BASE_IMAGE=nvidia/cuda-arm64 - else - CUDA_BASE_IMAGE=nvidia/cuda - fi - CUDA_IMAGE=${CUDA_BASE_IMAGE}:${CUDA_VERSION}-devel-${DISTRIB_ID}${DISTRIB_RELEASE} - - ${DOCKER_CMD} build --network host ${DOCKER_BUILD_ARGS} --build-arg CUDA_IMAGE=${CUDA_IMAGE} --build-arg TARGET_ARCH=${TARGET_ARCH} \ - . -f ./docker/Dockerfile.dev \ - -t ${DOCKER_DEV_IMAGE}:${DEV_IMAGE_TAG} $@ -} - -function build -{ - ${DOCKER_CMD} run --network host --rm ${DOCKER_INTERACTIVE} ${DOCKER_TTY} -u $(id -u):$(id -g) \ - -v $PWD:$PWD \ - -w $PWD \ - -e CLARA_HOLOVIZ_VERSION=${CLARA_HOLOVIZ_VERSION} \ - ${DOCKER_DEV_IMAGE}:${DEV_IMAGE_TAG} ./scripts/build.sh -o ${BUILD_DIR} $@ -} - -while (($#)); do -case $1 in - -h|--help) - echo "Usage: $0 COMMAND [-h|--help]" - echo "" - echo " -h, --help" - echo " display this help message" - echo " --image_build_dev" - echo " build the development docker container" - echo " --build" - echo " build Holoviz" - exit 1 - ;; - image_build_dev|build) - COMMAND=$1 - shift - break - ;; - *) - failMsg "Unknown option '"$1"'" - exit 1 - ;; -esac -done - -SDK_TOP=$PWD - -# version -CLARA_HOLOVIZ_VERSION="$(cat ${SDK_TOP}/VERSION)" - -# build output directory -BUILD_DIR=${BUILD_DIR:-build_${TARGET_ARCH}} - -# docker dev image tag -DEV_IMAGE_TAG="$(cat ${SDK_TOP}/docker/DEV_VERSION)" -# docker dev image name -DOCKER_DEV_IMAGE=clara-holoviz-dev_${TARGET_ARCH} - -# check if the current user is part of the docker group -# and we can run docker without sudo -if id | grep &>/dev/null '\bdocker\b' || ! command -v sudo >/dev/null; then - DOCKER_CMD=docker -else - echo "The current user is not in the 'docker' group, running 'docker' with 'sudo'" - DOCKER_CMD="sudo docker" -fi - -DOCKER_BUILD_ARGS="--pull" - -# enable terminal usage to be able to Ctrl-C a build or test -if [[ ! -z ${JENKINS_URL} ]]; then - # there is no terminal when running in Jenkins - DOCKER_INTERACTIVE= - DOCKER_TTY=-t - # always build official image without cache - DOCKER_BUILD_ARGS="${DOCKER_BUILD_ARGS} --no-cache" -else - DOCKER_INTERACTIVE=-i - DOCKER_TTY=-t -fi - -${COMMAND} $@ diff --git a/modules/holoviz/scripts/build.sh b/modules/holoviz/scripts/build.sh deleted file mode 100755 index 3f9b82ed..00000000 --- a/modules/holoviz/scripts/build.sh +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/bash -e -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -SRC=$PWD -# by default output is the current directory -OUT=$(readlink -f $SRC) - -# colors -RED="\033[0;31m" -GREEN="\033[0;32m" -NC="\033[0m" # No Color - -# print a failure message -function failMsg -{ - echo -e "$RED$1$NC$2" -} - -if [[ -z ${CLARA_HOLOVIZ_VERSION} ]]; then - failMsg "The version string is not defined." -fi - -while (($#)); do -case $1 in - -o|--output) - shift - if [ -z $1 ]; then - failMsg "Missing argument" - exit 1 - fi - OUT=$(readlink -f $1) - shift - ;; - - -h|--help) - echo "Usage: $0 [-c|-coverage] [-o|--output]" - echo "" - echo " -o, --output" - echo " output directory, default is current directory" - echo " -h, --help" - echo " display this help message" - exit 1 - ;; - - *) - failMsg "Unknown option '"$1"'" - exit 1 - ;; -esac -done - -mkdir -p $OUT -cd $OUT - -# initial call to cmake to generate the build files -if [ ! -f CMakeCache.txt ]; then - cmake -DCLARA_HOLOVIZ_VERSION=${CLARA_HOLOVIZ_VERSION} ${CMAKE_ARGS} $SRC -fi - -# build -make -j - -# install -cmake --install . diff --git a/modules/holoviz/src/CMakeLists.txt b/modules/holoviz/src/CMakeLists.txt index fd4e6052..923e3b86 100644 --- a/modules/holoviz/src/CMakeLists.txt +++ b/modules/holoviz/src/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,12 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -set(PROJECT_NAME clara_holoviz) +set(PROJECT_NAME holoviz) include(GNUInstallDirs) +include(GenHeaderFromBinaryFile) find_package(CUDAToolkit REQUIRED) find_package(X11 REQUIRED) +find_package(Vulkan REQUIRED) add_library(${PROJECT_NAME} SHARED) add_library(holoscan::viz ALIAS ${PROJECT_NAME}) @@ -27,6 +29,7 @@ add_library(holoscan::viz ALIAS ${PROJECT_NAME}) include("${nvpro_core_CMAKE_DIR}/utilities.cmake") set(GLSL_SOURCE_FILES) +list(APPEND GLSL_SOURCE_FILES "${CMAKE_CURRENT_SOURCE_DIR}/vulkan/shaders/geometry_color_shader.glsl.vert") list(APPEND GLSL_SOURCE_FILES "${CMAKE_CURRENT_SOURCE_DIR}/vulkan/shaders/geometry_shader.glsl.frag") list(APPEND GLSL_SOURCE_FILES "${CMAKE_CURRENT_SOURCE_DIR}/vulkan/shaders/geometry_shader.glsl.vert") list(APPEND GLSL_SOURCE_FILES "${CMAKE_CURRENT_SOURCE_DIR}/vulkan/shaders/geometry_text_shader.glsl.frag") @@ -36,6 +39,7 @@ list(APPEND GLSL_SOURCE_FILES "${CMAKE_CURRENT_SOURCE_DIR}/vulkan/shaders/image_ list(APPEND GLSL_SOURCE_FILES "${CMAKE_CURRENT_SOURCE_DIR}/vulkan/shaders/image_shader.glsl.frag") list(APPEND GLSL_SOURCE_FILES "${CMAKE_CURRENT_SOURCE_DIR}/vulkan/shaders/image_shader.glsl.vert") +set(GLSLANGVALIDATOR ${Vulkan_GLSLANG_VALIDATOR_EXECUTABLE}) compile_glsl( SOURCE_FILES ${GLSL_SOURCE_FILES} DST "${CMAKE_CURRENT_BINARY_DIR}/vulkan/spv" @@ -45,6 +49,9 @@ compile_glsl( set_source_files_properties(${GLSL_SOURCE_FILES} PROPERTIES GENERATED TRUE) +# generate the header file to embed the font +gen_header_from_binary_file(TARGET ${PROJECT_NAME} FILE_PATH "fonts/Roboto-Bold.ttf") + # sources target_sources(${PROJECT_NAME} PRIVATE @@ -63,11 +70,16 @@ target_sources(${PROJECT_NAME} layers/layer.cpp vulkan/framebuffer_sequence.cpp - vulkan/vulkan.cpp + vulkan/vulkan_app.cpp ${GLSL_SOURCE_FILES} ) +target_compile_definitions(${PROJECT_NAME} + PUBLIC + VULKAN_HPP_DISPATCH_LOADER_DYNAMIC=1 + ) + target_include_directories(${PROJECT_NAME} PRIVATE $ @@ -81,9 +93,12 @@ target_include_directories(${PROJECT_NAME} target_link_libraries(${PROJECT_NAME} PRIVATE + Vulkan::Vulkan + glfw X11::X11 nvpro_core holoscan::viz::imgui + holoscan::logger PUBLIC CUDA::cuda_driver @@ -94,26 +109,24 @@ set(EXPORT_MAP_FILE ${CMAKE_CURRENT_SOURCE_DIR}/export.map) set_target_properties(${PROJECT_NAME} PROPERTIES LINK_DEPENDS ${EXPORT_MAP_FILE} -) + ) set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--version-script=${EXPORT_MAP_FILE}") -# set library version -string(REGEX REPLACE "^([0-9]+)\\.([0-9]+)\\.([0-9]+)$" "\\1" SOVERSION ${CLARA_HOLOVIZ_VERSION}) +# set library name and version set_target_properties(${PROJECT_NAME} PROPERTIES - VERSION ${CLARA_HOLOVIZ_VERSION} - SOVERSION ${SOVERSION}) + OUTPUT_NAME holoscan_viz + EXPORT_NAME viz + SOVERSION ${PROJECT_VERSION_MAJOR} + VERSION ${PROJECT_VERSION} +) -# install +# install binaries install(TARGETS ${PROJECT_NAME} EXPORT ${PROJECT_NAME}Config ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} -) - -# TODO: Commenting the install since it's included in the SDK -#install(EXPORT ${PROJECT_NAME}Config -# DESTINATION cmake -#) + ) +# install headers install( DIRECTORY holoviz diff --git a/modules/holoviz/src/context.cpp b/modules/holoviz/src/context.cpp index 39e8be13..7bb62d1f 100644 --- a/modules/holoviz/src/context.cpp +++ b/modules/holoviz/src/context.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,243 +22,278 @@ #include #include +#include #include + #include "exclusive_window.hpp" #include "glfw_window.hpp" #include "headless_window.hpp" #include "layers/geometry_layer.hpp" #include "layers/im_gui_layer.hpp" #include "layers/image_layer.hpp" -#include "vulkan/vulkan.hpp" +#include "vulkan/vulkan_app.hpp" + +namespace { + +void nvprint_callback(int level, const char* fmt) { + // nvpro_core requires a new-line, our logging is automatically adding it. + // Remove the trailing newline if there is one. + std::string str(fmt); + if (str.back() == '\n') { str.pop_back(); } + + switch (level) { + case LOGLEVEL_INFO: + case LOGLEVEL_OK: + HOLOSCAN_LOG_INFO(str.c_str()); + break; + case LOGLEVEL_WARNING: + HOLOSCAN_LOG_WARN(str.c_str()); + break; + case LOGLEVEL_ERROR: + HOLOSCAN_LOG_ERROR(str.c_str()); + break; + case LOGLEVEL_DEBUG: + HOLOSCAN_LOG_DEBUG(str.c_str()); + break; + default: + case LOGLEVEL_STATS: + HOLOSCAN_LOG_TRACE(str.c_str()); + break; + } + + // nvpro_core prints the log message even if a callback is set. Set the string to '\0' to avoid + // the extra print. + char* buffer = const_cast(fmt); + *buffer = '\0'; +} + +} // namespace namespace holoscan::viz { struct Context::Impl { - InitFlags flags_ = InitFlags::NONE; - CUstream cuda_stream_ = 0; + InitFlags flags_ = InitFlags::NONE; + CUstream cuda_stream_ = 0; + std::string font_path_; + float font_size_in_pixels_ = 0.f; - std::unique_ptr window_; + std::unique_ptr window_; - std::unique_ptr vulkan_; + std::unique_ptr vulkan_; - /** - * We need to call ImGui::NewFrame() once for the first ImGUILayer, this is set to 'false' at - * begin in to 'true' when the first ImGUI layer had been created. - */ - bool imgui_new_frame_ = false; + /** + * We need to call ImGui::NewFrame() once for the first ImGUILayer, this is set to 'false' at + * begin in to 'true' when the first ImGUI layer had been created. + */ + bool imgui_new_frame_ = false; - std::unique_ptr active_layer_; ///< currently active layer + std::unique_ptr active_layer_; ///< currently active layer - std::list> layers_; ///< the list of the layers of the current frame + std::list> layers_; ///< the list of the layers of the current frame - std::list> layer_cache_; ///< layers of the previous frame, - /// most likely they can be reused + std::list> layer_cache_; ///< layers of the previous frame, + /// most likely they can be reused }; -Context &Context::get() { - // since C++11 static variables a thread-safe - static Context instance; +Context& Context::get() { + // since C++11 static variables a thread-safe + static Context instance; - return instance; + return instance; } -Context::Context() - : impl_(new Impl) { - // disable nvpro_core file logging - nvprintSetFileLogging(false); +Context::Context() : impl_(new Impl) { + // disable nvpro_core file logging + nvprintSetFileLogging(false); + // and use the callback to use our logging functions + nvprintSetCallback(nvprint_callback); } -void Context::init(GLFWwindow *window, InitFlags flags) { - impl_->window_.reset(new GLFWWindow(window)); - impl_->vulkan_.reset(new Vulkan); - impl_->vulkan_->setup(impl_->window_.get()); - impl_->flags_ = flags; +void Context::init(GLFWwindow* window, InitFlags flags) { + impl_->window_.reset(new GLFWWindow(window)); + impl_->vulkan_.reset(new Vulkan); + impl_->vulkan_->setup(impl_->window_.get(), impl_->font_path_, impl_->font_size_in_pixels_); + impl_->flags_ = flags; } -void Context::init(uint32_t width, uint32_t height, const char *title, InitFlags flags) { - if (flags & InitFlags::HEADLESS) { - impl_->window_.reset(new HeadlessWindow(width, height, flags)); - } else { - impl_->window_.reset(new GLFWWindow(width, height, title, flags)); - } - impl_->vulkan_.reset(new Vulkan); - impl_->vulkan_->setup(impl_->window_.get()); - impl_->flags_ = flags; +void Context::init(uint32_t width, uint32_t height, const char* title, InitFlags flags) { + if (flags & InitFlags::HEADLESS) { + impl_->window_.reset(new HeadlessWindow(width, height, flags)); + } else { + impl_->window_.reset(new GLFWWindow(width, height, title, flags)); + } + impl_->vulkan_.reset(new Vulkan); + impl_->vulkan_->setup(impl_->window_.get(), impl_->font_path_, impl_->font_size_in_pixels_); + impl_->flags_ = flags; } -void Context::init(const char *display_name, uint32_t width, uint32_t height, - uint32_t refresh_rate, InitFlags flags) { - impl_->window_.reset(new ExclusiveWindow(display_name, width, height, refresh_rate, flags)); - impl_->vulkan_.reset(new Vulkan); - impl_->vulkan_->setup(impl_->window_.get()); - impl_->flags_ = flags; +void Context::init(const char* display_name, uint32_t width, uint32_t height, uint32_t refresh_rate, + InitFlags flags) { + impl_->window_.reset(new ExclusiveWindow(display_name, width, height, refresh_rate, flags)); + impl_->vulkan_.reset(new Vulkan); + impl_->vulkan_->setup(impl_->window_.get(), impl_->font_path_, impl_->font_size_in_pixels_); + impl_->flags_ = flags; } void Context::shutdown() { - impl_->active_layer_.reset(); - impl_->layers_.clear(); - impl_->layer_cache_.clear(); - impl_->vulkan_.reset(); - impl_->window_.reset(); + // this should rather destroy the Context instance instead of resetting all members + impl_->active_layer_.reset(); + impl_->layers_.clear(); + impl_->layer_cache_.clear(); + impl_->vulkan_.reset(); + impl_->window_.reset(); + impl_->font_size_in_pixels_ = 0.f; + impl_->font_path_.clear(); + impl_->cuda_stream_ = 0; } -Window *Context::get_window() const { - if (!impl_->window_) { - throw std::runtime_error("There is no window set."); - } +Window* Context::get_window() const { + if (!impl_->window_) { throw std::runtime_error("There is no window set."); } - return impl_->window_.get(); + return impl_->window_.get(); } -void Context::im_gui_set_current_context(ImGuiContext *context) { - ImGui::SetCurrentContext(context); +void Context::im_gui_set_current_context(ImGuiContext* context) { + ImGui::SetCurrentContext(context); } void Context::set_cuda_stream(CUstream stream) { - impl_->cuda_stream_ = stream; + impl_->cuda_stream_ = stream; } CUstream Context::get_cuda_stream() const { - return impl_->cuda_stream_; + return impl_->cuda_stream_; +} + +void Context::set_font(const char* path, float size_in_pixels) { + if (impl_->vulkan_) { + throw std::runtime_error("The font has to be set before Init() is called"); + } + impl_->font_path_ = path; + impl_->font_size_in_pixels_ = size_in_pixels; } void Context::begin() { - impl_->imgui_new_frame_ = false; - impl_->window_->begin(); + impl_->imgui_new_frame_ = false; + impl_->window_->begin(); - // start the transfer pass, layers transfer their data on EndLayer(), layers are drawn on End() - impl_->vulkan_->begin_transfer_pass(); + // start the transfer pass, layers transfer their data on EndLayer(), layers are drawn on End() + impl_->vulkan_->begin_transfer_pass(); } void Context::end() { - impl_->window_->end(); + impl_->window_->end(); - // end the transfer pass - impl_->vulkan_->end_transfer_pass(); + // end the transfer pass + impl_->vulkan_->end_transfer_pass(); - // draw the layers - impl_->vulkan_->begin_render_pass(); + // draw the layers + impl_->vulkan_->begin_render_pass(); - // sort layers (inverse because highest priority is drawn last) - std::list sorted_layers; - for (auto &&item : impl_->layers_) { - sorted_layers.emplace_back(item.get()); - } - sorted_layers.sort([](Layer *a, Layer *b) { return a->get_priority() < b->get_priority(); }); + // sort layers (inverse because highest priority is drawn last) + std::list sorted_layers; + for (auto&& item : impl_->layers_) { sorted_layers.emplace_back(item.get()); } + sorted_layers.sort([](Layer* a, Layer* b) { return a->get_priority() < b->get_priority(); }); - // render - for (auto &&layer : sorted_layers) { - layer->render(impl_->vulkan_.get()); - } + // render + for (auto&& layer : sorted_layers) { layer->render(impl_->vulkan_.get()); } - // rendering is done - impl_->vulkan_->end_render_pass(); + // rendering is done + impl_->vulkan_->end_render_pass(); - // if the call sequence changed, then unused items remained in the cache, delete them - impl_->layer_cache_.clear(); + // if the call sequence changed, then unused items remained in the cache, delete them + impl_->layer_cache_.clear(); - // move the items to the layer cache for reuse in the next frame - impl_->layer_cache_.splice(impl_->layer_cache_.begin(), impl_->layers_); + // move the items to the layer cache for reuse in the next frame + impl_->layer_cache_.splice(impl_->layer_cache_.begin(), impl_->layers_); } void Context::begin_image_layer() { - if (impl_->active_layer_) { - throw std::runtime_error("There already is an active layer."); - } + if (impl_->active_layer_) { throw std::runtime_error("There already is an active layer."); } - impl_->active_layer_.reset(new ImageLayer()); + impl_->active_layer_.reset(new ImageLayer()); } void Context::begin_geometry_layer() { - if (impl_->active_layer_) { - throw std::runtime_error("There already is an active layer."); - } + if (impl_->active_layer_) { throw std::runtime_error("There already is an active layer."); } - impl_->active_layer_.reset(new GeometryLayer()); + impl_->active_layer_.reset(new GeometryLayer()); } void Context::begin_im_gui_layer() { - if (!ImGui::GetCurrentContext()) { - throw std::runtime_error( - "ImGui had not been setup, please call ImGuiSetCurrentContext() before calling Init()."); - } - - if (impl_->active_layer_) { - throw std::runtime_error("There already is an active layer."); - } - - if (!impl_->imgui_new_frame_) { - // Start the Dear ImGui frame - impl_->window_->im_gui_new_frame(); - impl_->imgui_new_frame_ = true; - } else { - throw std::runtime_error("Multiple ImGui layers are not supported"); - } - - impl_->active_layer_.reset(new ImGuiLayer()); + if (!ImGui::GetCurrentContext()) { + throw std::runtime_error( + "ImGui had not been setup, please call ImGuiSetCurrentContext() before calling Init()."); + } + + if (impl_->active_layer_) { throw std::runtime_error("There already is an active layer."); } + + if (!impl_->imgui_new_frame_) { + // Start the Dear ImGui frame + impl_->window_->im_gui_new_frame(); + impl_->imgui_new_frame_ = true; + } else { + throw std::runtime_error("Multiple ImGui layers are not supported"); + } + + impl_->active_layer_.reset(new ImGuiLayer()); } void Context::end_layer() { - if (!impl_->active_layer_) { - throw std::runtime_error("There is no active layer."); - } - - // scan the layer cache to check if the active layer already had been seen before - for (auto it = impl_->layer_cache_.begin(); it != impl_->layer_cache_.end(); ++it) { - if (impl_->active_layer_->can_be_reused(*it->get())) { - // set the 'soft' parameters - /// @todo this is error prone and needs to be resolved in a better way, - /// maybe have different categories (sections) of parameters and then copy - /// the 'soft' parameters to the re-used layer - (*it)->set_opacity(impl_->active_layer_->get_opacity()); - (*it)->set_priority(impl_->active_layer_->get_priority()); - - // replace the current active layer with the cached item - impl_->active_layer_ = std::move(*it); - - // remove from the cache - impl_->layer_cache_.erase(it); - break; - } + if (!impl_->active_layer_) { throw std::runtime_error("There is no active layer."); } + + // scan the layer cache to check if the active layer already had been seen before + for (auto it = impl_->layer_cache_.begin(); it != impl_->layer_cache_.end(); ++it) { + if (impl_->active_layer_->can_be_reused(*it->get())) { + // set the 'soft' parameters + /// @todo this is error prone and needs to be resolved in a better way, + /// maybe have different categories (sections) of parameters and then copy + /// the 'soft' parameters to the re-used layer + (*it)->set_opacity(impl_->active_layer_->get_opacity()); + (*it)->set_priority(impl_->active_layer_->get_priority()); + + // replace the current active layer with the cached item + impl_->active_layer_ = std::move(*it); + + // remove from the cache + impl_->layer_cache_.erase(it); + break; } + } - impl_->active_layer_->end(impl_->vulkan_.get()); - impl_->layers_.push_back(std::move(impl_->active_layer_)); + impl_->active_layer_->end(impl_->vulkan_.get()); + impl_->layers_.push_back(std::move(impl_->active_layer_)); } -void Context::read_framebuffer(ImageFormat fmt, size_t buffer_size, CUdeviceptr device_ptr) { - impl_->vulkan_->read_framebuffer(fmt, buffer_size, device_ptr, impl_->cuda_stream_); +void Context::read_framebuffer(ImageFormat fmt, uint32_t width, uint32_t height, size_t buffer_size, + CUdeviceptr device_ptr) { + impl_->vulkan_->read_framebuffer( + fmt, width, height, buffer_size, device_ptr, impl_->cuda_stream_); } -Layer *Context::get_active_layer() const { - if (!impl_->active_layer_) { - throw std::runtime_error("There is no active layer."); - } - return impl_->active_layer_.get(); +Layer* Context::get_active_layer() const { + if (!impl_->active_layer_) { throw std::runtime_error("There is no active layer."); } + return impl_->active_layer_.get(); } -ImageLayer *Context::get_active_image_layer() const { - if (!impl_->active_layer_) { - throw std::runtime_error("There is no active layer."); - } +ImageLayer* Context::get_active_image_layer() const { + if (!impl_->active_layer_) { throw std::runtime_error("There is no active layer."); } - if (impl_->active_layer_->get_type() != Layer::Type::Image) { - throw std::runtime_error("The active layer is not an image layer."); - } + if (impl_->active_layer_->get_type() != Layer::Type::Image) { + throw std::runtime_error("The active layer is not an image layer."); + } - return static_cast(impl_->active_layer_.get()); + return static_cast(impl_->active_layer_.get()); } -GeometryLayer *Context::get_active_geometry_layer() const { - if (!impl_->active_layer_) { - throw std::runtime_error("There is no active layer."); - } +GeometryLayer* Context::get_active_geometry_layer() const { + if (!impl_->active_layer_) { throw std::runtime_error("There is no active layer."); } - if (impl_->active_layer_->get_type() != Layer::Type::Geometry) { - throw std::runtime_error("The active layer is not a geometry layer."); - } + if (impl_->active_layer_->get_type() != Layer::Type::Geometry) { + throw std::runtime_error("The active layer is not a geometry layer."); + } - return static_cast(impl_->active_layer_.get()); + return static_cast(impl_->active_layer_.get()); } } // namespace holoscan::viz diff --git a/modules/holoviz/src/context.hpp b/modules/holoviz/src/context.hpp index f3b24ede..597d1d01 100644 --- a/modules/holoviz/src/context.hpp +++ b/modules/holoviz/src/context.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -42,147 +42,160 @@ class Layer; */ class Context : public NonCopyable { public: - /** - * @returns the context instance - */ - static Context &get(); - - /** - * Initialize the context with an existing GLFWwindow object - * - * @param window existing GLFWwindow window - * @param flags init flags - */ - void init(GLFWwindow *window, InitFlags flags); - - /** - * Initialize the context and create a window with the given properties. - * - * @param width, height window size - * @param title window title - * @param flags init flags - */ - void init(uint32_t width, uint32_t height, const char *title, InitFlags flags); - - /** - * Initialize the context and create a exclusive window with the given properties. - * - * @param display_name name of the display, this can either be the EDID name as displayed - * in the NVIDIA Settings, or the output name used by xrandr, - * if nullptr then the first display is selected. - * @param width desired width, ignored if 0 - * @param height desired height, ignored if 0 - * @param refresh_rate desired refresh rate (number of times the display is refreshed - * each second multiplied by 1000), ignored if 0 - * @param flags init flags - */ - void init(const char *display_name, uint32_t width, uint32_t height, - uint32_t refresh_rate, InitFlags flags); - - /** - * Shutdown and cleanup the global context. - */ - void shutdown(); - - /** - * Get the window object. - * - * @return window object - */ - Window *get_window() const; - - /** - * If using ImGui, create a context and pass it to Holoviz, do this before calling viz::Init(). - * - * Background: The ImGui context is a global variable and global variables are not shared - * across so/DLL boundaries. Therefore the app needs to create the ImGui context first and - * then provides the pointer to Holoviz like this: - * @code{.cpp} - * ImGui::CreateContext(); - * holoscan::viz::ImGuiSetCurrentContext(ImGui::GetCurrentContext()); - * @endcode - * - * @param context ImGui context - */ - void im_gui_set_current_context(ImGuiContext *context); - - /** - * Set the Cuda stream used by Holoviz for Cuda operations. - * - * The default stream is 0, i.e. non-concurrent mode. All Cuda commands issued by Holoviz - * then provides the pointer use that stream. - * The stream can be changed any time. - * - * @param stream Cuda stream to use - */ - void set_cuda_stream(CUstream stream); - - /** - * @returns the currently active cuda stream - */ - CUstream get_cuda_stream() const; - - /** - * Start recording layer definitions. - */ - void begin(); - - /** - * End recording and output the composited layers. - */ - void end(); - - /** - * Begin an image layer definition. - */ - void begin_image_layer(); - - /** - * Begin a geometry layer definition. - */ - void begin_geometry_layer(); - - /** - * Begin an ImGui layer definition. - */ - void begin_im_gui_layer(); - - /** - * End the current layer. - */ - void end_layer(); - - /** - * Read the framebuffer and store it to cuda device memory. - * - * Can only be called outside of Begin()/End(). - * - * @param fmt image format, currently only R8G8B8A8_UNORM is supported. - * @param buffer_size size of the storage buffer in bytes - * @param device_ptr pointer to Cuda device memory to store the framebuffer into - */ - void read_framebuffer(ImageFormat fmt, size_t buffer_size, CUdeviceptr device_ptr); - - /** - * @returns the active layer - */ - Layer *get_active_layer() const; - - /** - * @returns the active image layer - */ - ImageLayer *get_active_image_layer() const; - - /** - * @returns the active geometry layer - */ - GeometryLayer *get_active_geometry_layer() const; + /** + * @returns the context instance + */ + static Context& get(); + + /** + * Initialize the context with an existing GLFWwindow object + * + * @param window existing GLFWwindow window + * @param flags init flags + */ + void init(GLFWwindow* window, InitFlags flags); + + /** + * Initialize the context and create a window with the given properties. + * + * @param width, height window size + * @param title window title + * @param flags init flags + */ + void init(uint32_t width, uint32_t height, const char* title, InitFlags flags); + + /** + * Initialize the context and create a exclusive window with the given properties. + * + * @param display_name name of the display, this can either be the EDID name as displayed + * in the NVIDIA Settings, or the output name used by xrandr, + * if nullptr then the first display is selected. + * @param width desired width, ignored if 0 + * @param height desired height, ignored if 0 + * @param refresh_rate desired refresh rate (number of times the display is refreshed + * each second multiplied by 1000), ignored if 0 + * @param flags init flags + */ + void init(const char* display_name, uint32_t width, uint32_t height, uint32_t refresh_rate, + InitFlags flags); + + /** + * Shutdown and cleanup the global context. + */ + void shutdown(); + + /** + * Get the window object. + * + * @return window object + */ + Window* get_window() const; + + /** + * If using ImGui, create a context and pass it to Holoviz, do this before calling viz::Init(). + * + * Background: The ImGui context is a global variable and global variables are not shared + * across so/DLL boundaries. Therefore the app needs to create the ImGui context first and + * then provides the pointer to Holoviz like this: + * @code{.cpp} + * ImGui::CreateContext(); + * holoscan::viz::ImGuiSetCurrentContext(ImGui::GetCurrentContext()); + * @endcode + * + * @param context ImGui context + */ + void im_gui_set_current_context(ImGuiContext* context); + + /** + * Set the Cuda stream used by Holoviz for Cuda operations. + * + * The default stream is 0, i.e. non-concurrent mode. All Cuda commands issued by Holoviz + * then provides the pointer use that stream. + * The stream can be changed any time. + * + * @param stream Cuda stream to use + */ + void set_cuda_stream(CUstream stream); + + /** + * @returns the currently active cuda stream + */ + CUstream get_cuda_stream() const; + + /** + * Set the font used to render text, do this before calling viz::Init(). + * + * The font is converted to bitmaps, these bitmaps are scaled to the final size when rendering. + * + * @param path path to TTF font file + * @param size_in_pixels size of the font bitmaps + */ + void set_font(const char* path, float size_in_pixels); + + /** + * Start recording layer definitions. + */ + void begin(); + + /** + * End recording and output the composited layers. + */ + void end(); + + /** + * Begin an image layer definition. + */ + void begin_image_layer(); + + /** + * Begin a geometry layer definition. + */ + void begin_geometry_layer(); + + /** + * Begin an ImGui layer definition. + */ + void begin_im_gui_layer(); + + /** + * End the current layer. + */ + void end_layer(); + + /** + * Read the framebuffer and store it to cuda device memory. + * + * Can only be called outside of Begin()/End(). + * + * @param fmt image format, currently only R8G8B8A8_UNORM is supported. + * @param width, height width and height of the region to read back, will be limited to the + * framebuffer size if the framebuffer is smaller than that + * @param buffer_size size of the storage buffer in bytes + * @param device_ptr pointer to Cuda device memory to store the framebuffer into + */ + void read_framebuffer(ImageFormat fmt, uint32_t width, uint32_t height, size_t buffer_size, + CUdeviceptr device_ptr); + + /** + * @returns the active layer + */ + Layer* get_active_layer() const; + + /** + * @returns the active image layer + */ + ImageLayer* get_active_image_layer() const; + + /** + * @returns the active geometry layer + */ + GeometryLayer* get_active_geometry_layer() const; private: - Context(); + Context(); - struct Impl; - std::shared_ptr impl_; + struct Impl; + std::shared_ptr impl_; }; } // namespace holoscan::viz diff --git a/modules/holoviz/src/cuda/convert.cu b/modules/holoviz/src/cuda/convert.cu index dce3522a..46e6f80a 100644 --- a/modules/holoviz/src/cuda/convert.cu +++ b/modules/holoviz/src/cuda/convert.cu @@ -26,40 +26,38 @@ namespace { /** * Convert from R8G8B8 to R8G8B8A8 (set alpha to 0xFF) */ -__global__ void ConvertR8G8B8ToR8G8B8A8Kernel(uint32_t width, uint32_t height, const uint8_t *src, +__global__ void ConvertR8G8B8ToR8G8B8A8Kernel(uint32_t width, uint32_t height, const uint8_t* src, size_t src_pitch, CUsurfObject dst_surface) { - const uint2 launch_index = make_uint2(blockIdx.x * blockDim.x + threadIdx.x, blockIdx.y - * blockDim.y + threadIdx.y); - if ((launch_index.x >= width) || (launch_index.y >= height)) { - return; - } + const uint2 launch_index = + make_uint2(blockIdx.x * blockDim.x + threadIdx.x, blockIdx.y * blockDim.y + threadIdx.y); + if ((launch_index.x >= width) || (launch_index.y >= height)) { return; } - const size_t src_offset = launch_index.x * 3 + launch_index.y * src_pitch; + const size_t src_offset = launch_index.x * 3 + launch_index.y * src_pitch; - const uchar4 data{src[src_offset + 0], src[src_offset + 1], src[src_offset + 2], 0xFF}; - surf2Dwrite(data, dst_surface, launch_index.x * sizeof(uchar4), launch_index.y); + const uchar4 data{src[src_offset + 0], src[src_offset + 1], src[src_offset + 2], 0xFF}; + surf2Dwrite(data, dst_surface, launch_index.x * sizeof(uchar4), launch_index.y); } } // namespace void ConvertR8G8B8ToR8G8B8A8(uint32_t width, uint32_t height, CUdeviceptr src, size_t src_pitch, CUarray dst, CUstream stream) { - UniqueCUsurfObject dst_surface; - - dst_surface.reset([dst] { - CUDA_RESOURCE_DESC res_desc{}; - res_desc.resType = CU_RESOURCE_TYPE_ARRAY; - res_desc.res.array.hArray = dst; - CUsurfObject surf_object; - CudaCheck(cuSurfObjectCreate(&surf_object, &res_desc)); - return surf_object; - }()); - - const dim3 block_dim(32, 32); - const dim3 launch_grid((width + (block_dim.x - 1)) / block_dim.x, (height + (block_dim.y - 1)) - / block_dim.y); - ConvertR8G8B8ToR8G8B8A8Kernel<<>>( - width, height, reinterpret_cast(src), src_pitch, dst_surface.get()); + UniqueCUsurfObject dst_surface; + + dst_surface.reset([dst] { + CUDA_RESOURCE_DESC res_desc{}; + res_desc.resType = CU_RESOURCE_TYPE_ARRAY; + res_desc.res.array.hArray = dst; + CUsurfObject surf_object; + CudaCheck(cuSurfObjectCreate(&surf_object, &res_desc)); + return surf_object; + }()); + + const dim3 block_dim(32, 32); + const dim3 launch_grid((width + (block_dim.x - 1)) / block_dim.x, + (height + (block_dim.y - 1)) / block_dim.y); + ConvertR8G8B8ToR8G8B8A8Kernel<<>>( + width, height, reinterpret_cast(src), src_pitch, dst_surface.get()); } namespace { @@ -67,33 +65,35 @@ namespace { /** * Convert from B8G8R8A8 to R8G8B8A8 */ -__global__ void ConvertB8G8R8A8ToR8G8B8A8Kernel(uint32_t width, uint32_t height, const uint8_t *src, - size_t src_pitch, uint8_t *dst, size_t dst_pitch) { - const uint2 launch_index = make_uint2(blockIdx.x * blockDim.x + threadIdx.x, blockIdx.y - * blockDim.y + threadIdx.y); - if ((launch_index.x >= width) || (launch_index.y >= height)) { - return; - } - - const size_t src_offset = launch_index.x * 4 + launch_index.y * src_pitch; - const size_t dst_offset = launch_index.x * 4 + launch_index.y * dst_pitch; - - dst[dst_offset + 2] = src[src_offset + 0]; - dst[dst_offset + 1] = src[src_offset + 1]; - dst[dst_offset + 0] = src[src_offset + 2]; - dst[dst_offset + 3] = src[src_offset + 3]; +__global__ void ConvertB8G8R8A8ToR8G8B8A8Kernel(uint32_t width, uint32_t height, const uint8_t* src, + size_t src_pitch, uint8_t* dst, size_t dst_pitch) { + const uint2 launch_index = + make_uint2(blockIdx.x * blockDim.x + threadIdx.x, blockIdx.y * blockDim.y + threadIdx.y); + if ((launch_index.x >= width) || (launch_index.y >= height)) { return; } + + const size_t src_offset = launch_index.x * 4 + launch_index.y * src_pitch; + const size_t dst_offset = launch_index.x * 4 + launch_index.y * dst_pitch; + + dst[dst_offset + 2] = src[src_offset + 0]; + dst[dst_offset + 1] = src[src_offset + 1]; + dst[dst_offset + 0] = src[src_offset + 2]; + dst[dst_offset + 3] = src[src_offset + 3]; } } // namespace void ConvertB8G8R8A8ToR8G8B8A8(uint32_t width, uint32_t height, CUdeviceptr src, size_t src_pitch, CUdeviceptr dst, size_t dst_pitch, CUstream stream) { - const dim3 block_dim(32, 32); - const dim3 launch_grid((width + (block_dim.x - 1)) / block_dim.x, (height + (block_dim.y - 1)) - / block_dim.y); - ConvertB8G8R8A8ToR8G8B8A8Kernel<<>>( - width, height, reinterpret_cast(src), src_pitch, - reinterpret_cast(dst), dst_pitch); + const dim3 block_dim(32, 32); + const dim3 launch_grid((width + (block_dim.x - 1)) / block_dim.x, + (height + (block_dim.y - 1)) / block_dim.y); + ConvertB8G8R8A8ToR8G8B8A8Kernel<<>>( + width, + height, + reinterpret_cast(src), + src_pitch, + reinterpret_cast(dst), + dst_pitch); } } // namespace holoscan::viz diff --git a/modules/holoviz/src/cuda/convert.hpp b/modules/holoviz/src/cuda/convert.hpp index c99b08d7..f00be933 100644 --- a/modules/holoviz/src/cuda/convert.hpp +++ b/modules/holoviz/src/cuda/convert.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef HOLOSCAN_VIZ_CUDA_CONVERT_HPP -#define HOLOSCAN_VIZ_CUDA_CONVERT_HPP +#ifndef HOLOVIZ_SRC_CUDA_CONVERT_HPP +#define HOLOVIZ_SRC_CUDA_CONVERT_HPP #include @@ -48,8 +48,8 @@ void ConvertR8G8B8ToR8G8B8A8(uint32_t width, uint32_t height, CUdeviceptr src, s * @param stream stream to use for this operation */ void ConvertB8G8R8A8ToR8G8B8A8(uint32_t width, uint32_t height, CUdeviceptr src, size_t src_pitch, - CUdeviceptr dst, size_t dst_pitch, CUstream stream); + CUdeviceptr dst, size_t dst_pitch, CUstream stream); } // namespace holoscan::viz -#endif /* HOLOSCAN_VIZ_CUDA_CONVERT_HPP */ +#endif /* HOLOVIZ_SRC_CUDA_CONVERT_HPP */ diff --git a/modules/holoviz/src/cuda/cuda_service.cpp b/modules/holoviz/src/cuda/cuda_service.cpp index 79f1f5e5..998524fd 100644 --- a/modules/holoviz/src/cuda/cuda_service.cpp +++ b/modules/holoviz/src/cuda/cuda_service.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,66 +17,63 @@ #include "cuda_service.hpp" -#include +#include namespace holoscan::viz { static std::unique_ptr g_cuda_service; struct CudaService::Impl { - CUdevice device_ = 0; - CUcontext cuda_context_ = nullptr; + CUdevice device_ = 0; + CUcontext cuda_context_ = nullptr; }; -CudaService::CudaService() - : impl_(new Impl) { - /// @todo make configurable - const uint32_t device_ordinal = 0; +CudaService::CudaService() : impl_(new Impl) { + /// @todo make configurable + const uint32_t device_ordinal = 0; - CudaCheck(cuInit(0)); - CudaCheck(cuDeviceGet(&impl_->device_, device_ordinal)); - CudaCheck(cuDevicePrimaryCtxRetain(&impl_->cuda_context_, impl_->device_)); + CudaCheck(cuInit(0)); + CudaCheck(cuDeviceGet(&impl_->device_, device_ordinal)); + CudaCheck(cuDevicePrimaryCtxRetain(&impl_->cuda_context_, impl_->device_)); } CudaService::~CudaService() { - if (impl_->cuda_context_) { - // avoid CudaCheck() here, the driver might already be uninitialized - // when global variables are destroyed - cuDevicePrimaryCtxRelease(impl_->device_); - } + if (impl_->cuda_context_) { + // avoid CudaCheck() here, the driver might already be uninitialized + // when global variables are destroyed + cuDevicePrimaryCtxRelease(impl_->device_); + } } -CudaService &CudaService::get() { - if (!g_cuda_service) { - g_cuda_service.reset(new CudaService()); - } - return *g_cuda_service; +CudaService& CudaService::get() { + if (!g_cuda_service) { g_cuda_service.reset(new CudaService()); } + return *g_cuda_service; } class CudaService::ScopedPushImpl { public: - ScopedPushImpl() { - // might be called from a different thread than the thread - // which constructed CudaPrimaryContext, therefore call cuInit() - CudaCheck(cuInit(0)); - CudaCheck(cuCtxPushCurrent(CudaService::get().impl_->cuda_context_)); - } + ScopedPushImpl() { + // might be called from a different thread than the thread + // which constructed CudaPrimaryContext, therefore call cuInit() + CudaCheck(cuInit(0)); + CudaCheck(cuCtxPushCurrent(CudaService::get().impl_->cuda_context_)); + } - ~ScopedPushImpl() { - try { - CUcontext popped_context; - CudaCheck(cuCtxPopCurrent(&popped_context)); - if (popped_context != CudaService::get().impl_->cuda_context_) { - LOGE("Cuda: Unexpected context popped\n"); - } - } catch (const std::exception &e) { - LOGE("ScopedPush destructor failed with %s\n", e.what()); - } + ~ScopedPushImpl() { + try { + CUcontext popped_context; + CudaCheck(cuCtxPopCurrent(&popped_context)); + if (popped_context != CudaService::get().impl_->cuda_context_) { + HOLOSCAN_LOG_ERROR("Cuda: Unexpected context popped"); + } + } catch (const std::exception& e) { + HOLOSCAN_LOG_ERROR("ScopedPush destructor failed with {}", e.what()); } + } }; CudaService::ScopedPush CudaService::PushContext() { - return std::make_shared(); + return std::make_shared(); } } // namespace holoscan::viz diff --git a/modules/holoviz/src/cuda/cuda_service.hpp b/modules/holoviz/src/cuda/cuda_service.hpp index b49d8213..a10c76d7 100644 --- a/modules/holoviz/src/cuda/cuda_service.hpp +++ b/modules/holoviz/src/cuda/cuda_service.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef HOLOSCAN_VIZ_CUDA_CUDA_SERVICE_HPP -#define HOLOSCAN_VIZ_CUDA_CUDA_SERVICE_HPP +#ifndef HOLOVIZ_SRC_CUDA_CUDA_SERVICE_HPP +#define HOLOVIZ_SRC_CUDA_CUDA_SERVICE_HPP #include @@ -30,78 +30,78 @@ namespace holoscan::viz { /** * Cuda driver API error check helper */ -#define CudaCheck(FUNC) { \ - \ - const CUresult result = FUNC; \ - if (result != CUDA_SUCCESS) { \ - const char *error_name = ""; \ - cuGetErrorName(result, &error_name); \ - const char *error_string = ""; \ - cuGetErrorString(result, &error_string); \ - std::stringstream buf; \ - buf << "Cuda driver error " << result << " (" << error_name << "): " << error_string; \ - throw std::runtime_error(buf.str().c_str()); \ - } \ - } +#define CudaCheck(FUNC) \ + { \ + const CUresult result = FUNC; \ + if (result != CUDA_SUCCESS) { \ + const char* error_name = ""; \ + cuGetErrorName(result, &error_name); \ + const char* error_string = ""; \ + cuGetErrorString(result, &error_string); \ + std::stringstream buf; \ + buf << "Cuda driver error " << result << " (" << error_name << "): " << error_string; \ + throw std::runtime_error(buf.str().c_str()); \ + } \ + } /** * UniqueValue's for a Cuda objects */ /**@{*/ -using UniqueCUdeviceptr = UniqueValue; -using UniqueCUsurfObject = UniqueValue; -using UniqueCUtexObject = UniqueValue; +using UniqueCUdeviceptr = UniqueValue; +using UniqueCUsurfObject = + UniqueValue; +using UniqueCUtexObject = + UniqueValue; using UniqueCUexternalMemory = UniqueValue; using UniqueCUmipmappedArray = UniqueValue; using UniqueCUexternalSemaphore = UniqueValue; + &cuDestroyExternalSemaphore>; /**@}*/ /// Global Cuda service class class CudaService { private: - class ScopedPushImpl; ///< Internal class handling the lifetime of the pushed context. + class ScopedPushImpl; ///< Internal class handling the lifetime of the pushed context. public: - /** - * Destroy the Cuda service object. - */ - ~CudaService(); - - /** - * Get the global Cuda service, if there is no global Cuda service yet it will be created. - * - * @return global Cuda service - */ - static CudaService &get(); - - /** - * Shutdown and cleanup the global Cuda service. - */ - static void shutdown(); - - /// RAII type object to pop the Cuda primiary context on destruction - typedef std::shared_ptr ScopedPush; - - /** - * Push the primary Cuda context. - * - * @return ScopedPush RAII type object to pop the Cuda primiary context on destruction. - */ - ScopedPush PushContext(); + /** + * Destroy the Cuda service object. + */ + ~CudaService(); + + /** + * Get the global Cuda service, if there is no global Cuda service yet it will be created. + * + * @return global Cuda service + */ + static CudaService& get(); + + /** + * Shutdown and cleanup the global Cuda service. + */ + static void shutdown(); + + /// RAII type object to pop the Cuda primiary context on destruction + typedef std::shared_ptr ScopedPush; + + /** + * Push the primary Cuda context. + * + * @return ScopedPush RAII type object to pop the Cuda primiary context on destruction. + */ + ScopedPush PushContext(); private: - CudaService(); + CudaService(); - struct Impl; - std::shared_ptr impl_; + struct Impl; + std::shared_ptr impl_; }; } // namespace holoscan::viz -#endif /* HOLOSCAN_VIZ_CUDA_CUDA_SERVICE_HPP */ +#endif /* HOLOVIZ_SRC_CUDA_CUDA_SERVICE_HPP */ diff --git a/modules/holoviz/src/exclusive_window.cpp b/modules/holoviz/src/exclusive_window.cpp index b00df146..1aef5f74 100644 --- a/modules/holoviz/src/exclusive_window.cpp +++ b/modules/holoviz/src/exclusive_window.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,14 +18,15 @@ #include "exclusive_window.hpp" #include -#include #include +#include #include #include #include #include +#include #include #include @@ -35,240 +36,237 @@ namespace holoscan::viz { * ExclusiveWindow implementation details */ struct ExclusiveWindow::Impl { - std::string display_name_; - uint32_t desired_width_ = 0; - uint32_t desired_height_ = 0; - uint32_t desired_refresh_rate_ = 0; + std::string display_name_; + uint32_t desired_width_ = 0; + uint32_t desired_height_ = 0; + uint32_t desired_refresh_rate_ = 0; - uint32_t width_ = 0; - uint32_t height_ = 0; - uint32_t refresh_rate_ = 0; + uint32_t width_ = 0; + uint32_t height_ = 0; + uint32_t refresh_rate_ = 0; - Display *dpy_ = nullptr; + Display* dpy_ = nullptr; }; ExclusiveWindow::~ExclusiveWindow() { - if (impl_->dpy_) { - XCloseDisplay(impl_->dpy_); - } + if (impl_->dpy_) { XCloseDisplay(impl_->dpy_); } } -ExclusiveWindow::ExclusiveWindow(const char *display_name, uint32_t width, uint32_t height, +ExclusiveWindow::ExclusiveWindow(const char* display_name, uint32_t width, uint32_t height, uint32_t refresh_rate, InitFlags flags) : impl_(new Impl) { - impl_->display_name_ = display_name; - impl_->desired_width_ = width; - impl_->desired_height_ = height; - impl_->desired_refresh_rate_ = refresh_rate; + impl_->display_name_ = display_name; + impl_->desired_width_ = width; + impl_->desired_height_ = height; + impl_->desired_refresh_rate_ = refresh_rate; } void ExclusiveWindow::init_im_gui() {} -void ExclusiveWindow::setup_callbacks(std::function - frame_buffer_size_cb) {} +void ExclusiveWindow::setup_callbacks( + std::function frame_buffer_size_cb) {} -const char **ExclusiveWindow::get_required_instance_extensions(uint32_t *count) { - static char const *extensions[]{VK_KHR_SURFACE_EXTENSION_NAME, VK_KHR_DISPLAY_EXTENSION_NAME, - VK_EXT_ACQUIRE_XLIB_DISPLAY_EXTENSION_NAME, - VK_EXT_DIRECT_MODE_DISPLAY_EXTENSION_NAME}; +const char** ExclusiveWindow::get_required_instance_extensions(uint32_t* count) { + static char const* extensions[]{VK_KHR_SURFACE_EXTENSION_NAME, + VK_KHR_DISPLAY_EXTENSION_NAME, + VK_EXT_ACQUIRE_XLIB_DISPLAY_EXTENSION_NAME, + VK_EXT_DIRECT_MODE_DISPLAY_EXTENSION_NAME}; - *count = sizeof(extensions) / sizeof(extensions[0]); - return extensions; + *count = sizeof(extensions) / sizeof(extensions[0]); + return extensions; } -const char **ExclusiveWindow::get_required_device_extensions(uint32_t *count) { - static char const *extensions[]{VK_KHR_SWAPCHAIN_EXTENSION_NAME}; +const char** ExclusiveWindow::get_required_device_extensions(uint32_t* count) { + static char const* extensions[]{VK_KHR_SWAPCHAIN_EXTENSION_NAME}; - *count = sizeof(extensions) / sizeof(extensions[0]); - return extensions; + *count = sizeof(extensions) / sizeof(extensions[0]); + return extensions; } -void ExclusiveWindow::get_framebuffer_size(uint32_t *width, uint32_t *height) { - *width = impl_->width_; - *height = impl_->height_; +uint32_t ExclusiveWindow::select_device(vk::Instance instance, + const std::vector& physical_devices) { + std::string first_display; + uint32_t first_device_index; + for (uint32_t index = 0; index < physical_devices.size(); ++index) { + const std::vector display_properties = + physical_devices[index].getDisplayPropertiesKHR(); + for (auto&& displayProperty : display_properties) { + if (std::string(displayProperty.displayName).find(impl_->display_name_) != + std::string::npos) { + return index; + } + + if (first_display.empty()) { + first_display = displayProperty.displayName; + first_device_index = index; + } + } + } + if (first_display.empty()) { + throw std::runtime_error("No device with a connected display found"); + } + HOLOSCAN_LOG_WARN("Display \"{}\" not found, using the first available display \"{}\" instead", + impl_->display_name_.c_str(), + first_display.c_str()); + return first_device_index; } -VkSurfaceKHR ExclusiveWindow::create_surface(VkPhysicalDevice physical_device, - VkInstance instance) { - uint32_t display_count = 0; - NVVK_CHECK(vkGetPhysicalDeviceDisplayPropertiesKHR(physical_device, &display_count, nullptr)); - std::vector display_properties(display_count); - NVVK_CHECK(vkGetPhysicalDeviceDisplayPropertiesKHR(physical_device, &display_count, - display_properties.data())); - - // pick the display - LOGI("____________________\n"); - LOGI("Available displays :\n"); - VkDisplayPropertiesKHR selected_display = display_properties[0]; - bool found_display = false; - for (auto &&displayProperty : display_properties) { - LOGI("%s\n", displayProperty.displayName); - if (std::string(displayProperty.displayName).find(impl_->display_name_) - != std::string::npos) { - selected_display = displayProperty; - found_display = true; - break; - } - } - LOGI("\n"); +void ExclusiveWindow::get_framebuffer_size(uint32_t* width, uint32_t* height) { + *width = impl_->width_; + *height = impl_->height_; +} - if (!found_display) { - LOGW("Display \"%s\" not found, using the first available display instead\n", - impl_->display_name_.c_str()); +vk::SurfaceKHR ExclusiveWindow::create_surface(vk::PhysicalDevice physical_device, + vk::Instance instance) { + const std::vector display_properties = + physical_device.getDisplayPropertiesKHR(); + + // pick the display + HOLOSCAN_LOG_INFO("____________________"); + HOLOSCAN_LOG_INFO("Available displays :"); + vk::DisplayPropertiesKHR selected_display = display_properties[0]; + for (auto&& displayProperty : display_properties) { + HOLOSCAN_LOG_INFO("{}", displayProperty.displayName); + if (std::string(displayProperty.displayName).find(impl_->display_name_) != + std::string::npos) { + selected_display = displayProperty; } - LOGI("Using display \"%s\"\n", selected_display.displayName); - - const VkDisplayKHR display = selected_display.display; - - // If the X11 server is running, acquire permission from the X-Server to directly - // access the display in Vulkan - impl_->dpy_ = XOpenDisplay(NULL); - if (impl_->dpy_) { - const PFN_vkAcquireXlibDisplayEXT vkAcquireXlibDisplayEXT = - PFN_vkAcquireXlibDisplayEXT(vkGetInstanceProcAddr(instance, "vkAcquireXlibDisplayEXT")); - if (!vkAcquireXlibDisplayEXT) { - throw std::runtime_error("Could not get proc address of vkAcquireXlibDisplayEXT"); - } - LOGI("X server is running, trying to acquire display\n"); - VkResult result = vkAcquireXlibDisplayEXT(physical_device, impl_->dpy_, display); - if (result < 0) { - nvvk::checkResult(result); - throw std::runtime_error("Failed to acquire display from X-Server."); - } + } + HOLOSCAN_LOG_INFO(""); + HOLOSCAN_LOG_INFO("Using display \"{}\"", selected_display.displayName); + + const vk::DisplayKHR display = selected_display.display; + + // If the X11 server is running, acquire permission from the X-Server to directly + // access the display in Vulkan + impl_->dpy_ = XOpenDisplay(NULL); + if (impl_->dpy_) { + const PFN_vkAcquireXlibDisplayEXT vkAcquireXlibDisplayEXT = + PFN_vkAcquireXlibDisplayEXT(vkGetInstanceProcAddr(instance, "vkAcquireXlibDisplayEXT")); + if (!vkAcquireXlibDisplayEXT) { + throw std::runtime_error("Could not get proc address of vkAcquireXlibDisplayEXT"); } - - // pick highest available resolution - uint32_t mode_count = 0; - NVVK_CHECK(vkGetDisplayModePropertiesKHR(physical_device, display, &mode_count, nullptr)); - std::vector modes(mode_count); - NVVK_CHECK(vkGetDisplayModePropertiesKHR(physical_device, display, &mode_count, modes.data())); - VkDisplayModePropertiesKHR mode_properties = modes[0]; - // find the mode - for (const auto &m : modes) { - if (((impl_->desired_width_ > 0) && (m.parameters.visibleRegion.width >= - impl_->desired_width_)) && - ((impl_->desired_height_ > 0) && (m.parameters.visibleRegion.height >= - impl_->desired_height_)) && - ((impl_->desired_refresh_rate_ > 0) && (m.parameters.refreshRate >= - impl_->desired_refresh_rate_))) { - mode_properties = m; - } + HOLOSCAN_LOG_INFO("X server is running, trying to acquire display"); + VkResult result = vkAcquireXlibDisplayEXT(physical_device, impl_->dpy_, display); + if (result < 0) { + nvvk::checkResult(result); + throw std::runtime_error("Failed to acquire display from X-Server."); } - - if (((impl_->desired_width_ > 0) && (mode_properties.parameters.visibleRegion.width - != impl_->desired_width_)) || - ((impl_->desired_height_ > 0) && (mode_properties.parameters.visibleRegion.height - != impl_->desired_height_)) || + } + + // pick highest available resolution + const std::vector modes = + physical_device.getDisplayModePropertiesKHR(display); + vk::DisplayModePropertiesKHR mode_properties = modes[0]; + // find the mode + for (const auto& m : modes) { + if (((impl_->desired_width_ > 0) && + (m.parameters.visibleRegion.width >= impl_->desired_width_)) && + ((impl_->desired_height_ > 0) && + (m.parameters.visibleRegion.height >= impl_->desired_height_)) && ((impl_->desired_refresh_rate_ > 0) && - (mode_properties.parameters.refreshRate != impl_->desired_refresh_rate_))) { - LOGW("Did not find a display mode with the desired properties %dx%d %.3f Hz\n", - impl_->desired_width_, - impl_->desired_height_, static_cast(impl_->desired_refresh_rate_) / 1000.f); - } - LOGW("Using display mode %dx%d %.3f Hz\n", mode_properties.parameters.visibleRegion.width, - mode_properties.parameters.visibleRegion.height, - static_cast(mode_properties.parameters.refreshRate) / 1000.f); - - impl_->width_ = mode_properties.parameters.visibleRegion.width; - impl_->height_ = mode_properties.parameters.visibleRegion.height; - impl_->refresh_rate_ = mode_properties.parameters.refreshRate; - - // pick first compatible plane - uint32_t plane_count = 0; - NVVK_CHECK(vkGetPhysicalDeviceDisplayPlanePropertiesKHR(physical_device, &plane_count, - nullptr)); - std::vector planes(plane_count); - NVVK_CHECK(vkGetPhysicalDeviceDisplayPlanePropertiesKHR(physical_device, &plane_count, - planes.data())); - uint32_t plane_index; - bool found_plane = false; - for (uint32_t i = 0; i < planes.size(); ++i) { - auto p = planes[i]; - - // skip planes bound to different display - if (p.currentDisplay && (p.currentDisplay != display)) { - continue; - } - - uint32_t display_count = 0; - NVVK_CHECK(vkGetDisplayPlaneSupportedDisplaysKHR(physical_device, i, &display_count, - nullptr)); - std::vector displays(display_count); - NVVK_CHECK(vkGetDisplayPlaneSupportedDisplaysKHR(physical_device, i, &display_count, - displays.data())); - for (auto &d : displays) { - if (d == display) { - found_plane = true; - plane_index = i; - break; - } - } - - if (found_plane) { - break; - } + (m.parameters.refreshRate >= impl_->desired_refresh_rate_))) { + mode_properties = m; } - - if (!found_plane) { - throw std::runtime_error("Could not find a compatible display plane!"); + } + + if (((impl_->desired_width_ > 0) && + (mode_properties.parameters.visibleRegion.width != impl_->desired_width_)) || + ((impl_->desired_height_ > 0) && + (mode_properties.parameters.visibleRegion.height != impl_->desired_height_)) || + ((impl_->desired_refresh_rate_ > 0) && + (mode_properties.parameters.refreshRate != impl_->desired_refresh_rate_))) { + HOLOSCAN_LOG_WARN("Did not find a display mode with the desired properties {}x{} {:.3f} Hz", + impl_->desired_width_, + impl_->desired_height_, + static_cast(impl_->desired_refresh_rate_) / 1000.f); + } + HOLOSCAN_LOG_INFO("Using display mode {}x{} {:.3f} Hz", + mode_properties.parameters.visibleRegion.width, + mode_properties.parameters.visibleRegion.height, + static_cast(mode_properties.parameters.refreshRate) / 1000.f); + + impl_->width_ = mode_properties.parameters.visibleRegion.width; + impl_->height_ = mode_properties.parameters.visibleRegion.height; + impl_->refresh_rate_ = mode_properties.parameters.refreshRate; + + // pick first compatible plane + const std::vector planes = + physical_device.getDisplayPlanePropertiesKHR(); + uint32_t plane_index; + bool found_plane = false; + for (uint32_t i = 0; i < planes.size(); ++i) { + auto p = planes[i]; + + // skip planes bound to different display + if (p.currentDisplay && (p.currentDisplay != display)) { continue; } + + const std::vector displays = + physical_device.getDisplayPlaneSupportedDisplaysKHR(i); + for (auto& d : displays) { + if (d == display) { + found_plane = true; + plane_index = i; + break; + } } - // find alpha mode bit - VkDisplayPlaneCapabilitiesKHR plane_capabilities; - NVVK_CHECK(vkGetDisplayPlaneCapabilitiesKHR(physical_device, mode_properties.displayMode, - plane_index, &plane_capabilities)); - VkDisplayPlaneAlphaFlagBitsKHR selected_alpha_mode = - VkDisplayPlaneAlphaFlagBitsKHR::VK_DISPLAY_PLANE_ALPHA_OPAQUE_BIT_KHR; - const std::array available_alpha_modes{ - VkDisplayPlaneAlphaFlagBitsKHR::VK_DISPLAY_PLANE_ALPHA_OPAQUE_BIT_KHR, - VkDisplayPlaneAlphaFlagBitsKHR::VK_DISPLAY_PLANE_ALPHA_GLOBAL_BIT_KHR, - VkDisplayPlaneAlphaFlagBitsKHR::VK_DISPLAY_PLANE_ALPHA_PER_PIXEL_BIT_KHR, - VkDisplayPlaneAlphaFlagBitsKHR::VK_DISPLAY_PLANE_ALPHA_PER_PIXEL_PREMULTIPLIED_BIT_KHR}; - for (auto &&alpha_mode : available_alpha_modes) { - if (plane_capabilities.supportedAlpha & alpha_mode) { - selected_alpha_mode = alpha_mode; - break; - } + if (found_plane) { break; } + } + + if (!found_plane) { throw std::runtime_error("Could not find a compatible display plane!"); } + + // find alpha mode bit + const vk::DisplayPlaneCapabilitiesKHR plane_capabilities = + physical_device.getDisplayPlaneCapabilitiesKHR(mode_properties.displayMode, plane_index); + vk::DisplayPlaneAlphaFlagBitsKHR selected_alpha_mode = vk::DisplayPlaneAlphaFlagBitsKHR::eOpaque; + const std::array available_alpha_modes{ + vk::DisplayPlaneAlphaFlagBitsKHR::eOpaque, + vk::DisplayPlaneAlphaFlagBitsKHR::eGlobal, + vk::DisplayPlaneAlphaFlagBitsKHR::ePerPixel, + vk::DisplayPlaneAlphaFlagBitsKHR::ePerPixelPremultiplied}; + for (auto&& alpha_mode : available_alpha_modes) { + if (plane_capabilities.supportedAlpha & alpha_mode) { + selected_alpha_mode = alpha_mode; + break; } - - VkDisplaySurfaceCreateInfoKHR surface_create_info - {VK_STRUCTURE_TYPE_DISPLAY_SURFACE_CREATE_INFO_KHR}; - surface_create_info.displayMode = mode_properties.displayMode; - surface_create_info.planeIndex = plane_index; - surface_create_info.planeStackIndex = planes[plane_index].currentStackIndex; - surface_create_info.transform = VkSurfaceTransformFlagBitsKHR:: - VK_SURFACE_TRANSFORM_IDENTITY_BIT_KHR; - surface_create_info.globalAlpha = 1.0f; - surface_create_info.alphaMode = selected_alpha_mode; - surface_create_info.imageExtent = - VkExtent2D{mode_properties.parameters.visibleRegion.width, - mode_properties.parameters.visibleRegion.height}; - - VkSurfaceKHR surface; - NVVK_CHECK(vkCreateDisplayPlaneSurfaceKHR(instance, &surface_create_info, nullptr, &surface)); - - return surface; + } + + vk::DisplaySurfaceCreateInfoKHR surface_create_info; + surface_create_info.displayMode = mode_properties.displayMode; + surface_create_info.planeIndex = plane_index; + surface_create_info.planeStackIndex = planes[plane_index].currentStackIndex; + surface_create_info.transform = vk::SurfaceTransformFlagBitsKHR::eIdentity; + surface_create_info.globalAlpha = 1.0f; + surface_create_info.alphaMode = selected_alpha_mode; + surface_create_info.imageExtent = vk::Extent2D{mode_properties.parameters.visibleRegion.width, + mode_properties.parameters.visibleRegion.height}; + + return instance.createDisplayPlaneSurfaceKHR(surface_create_info); } bool ExclusiveWindow::should_close() { - return false; + return false; } bool ExclusiveWindow::is_minimized() { - return false; + return false; } void ExclusiveWindow::im_gui_new_frame() { - ImGuiIO &io = ImGui::GetIO(); - io.DisplaySize = ImVec2(static_cast(impl_->width_), - static_cast(impl_->height_)); - io.DisplayFramebufferScale = ImVec2(1.f, 1.f); + ImGuiIO& io = ImGui::GetIO(); + io.DisplaySize = ImVec2(static_cast(impl_->width_), static_cast(impl_->height_)); + io.DisplayFramebufferScale = ImVec2(1.f, 1.f); - ImGui::NewFrame(); + ImGui::NewFrame(); } void ExclusiveWindow::begin() {} void ExclusiveWindow::end() {} +float ExclusiveWindow::get_aspect_ratio() { + return float(impl_->width_) / float(impl_->height_); +} + } // namespace holoscan::viz diff --git a/modules/holoviz/src/exclusive_window.hpp b/modules/holoviz/src/exclusive_window.hpp index a0ef0f07..af8a5f41 100644 --- a/modules/holoviz/src/exclusive_window.hpp +++ b/modules/holoviz/src/exclusive_window.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,6 +20,7 @@ #include #include +#include #include "window.hpp" @@ -32,54 +33,58 @@ namespace holoscan::viz { */ class ExclusiveWindow : public Window { public: - /** - * Construct a new exclusive window. - * - * @param display_name name of the display, this can either be the EDID name as displayed - * in the NVIDIA Settings, or the output name used by xrandr, - * if nullptr then the first display is selected. - * @param width desired width, ignored if 0 - * @param height desired height, ignored if 0 - * @param refresh_rate desired refresh rate (number of times the display is refreshed - * each second multiplied by 1000), ignored if 0 - * @param flags init flags - */ - ExclusiveWindow(const char *display_name, uint32_t width, uint32_t height, - uint32_t refresh_rate, InitFlags flags); - - /** - * Delete the standard constructor, always need parameters to construct. - */ - ExclusiveWindow() = delete; - - /** - * Destroy the exclusive window object. - */ - virtual ~ExclusiveWindow(); - - /// holoscan::viz::Window virtual members - ///@{ - void init_im_gui() override; - void setup_callbacks(std::function frame_buffer_size_cb) override; - - const char **get_required_instance_extensions(uint32_t *count) override; - const char **get_required_device_extensions(uint32_t *count) override; - void get_framebuffer_size(uint32_t *width, uint32_t *height) override; - - VkSurfaceKHR create_surface(VkPhysicalDevice physical_device, VkInstance instance) override; - - bool should_close() override; - bool is_minimized() override; - - void im_gui_new_frame() override; - - void begin() override; - void end() override; - ///@} + /** + * Construct a new exclusive window. + * + * @param display_name name of the display, this can either be the EDID name as displayed + * in the NVIDIA Settings, or the output name used by xrandr, + * if nullptr then the first display is selected. + * @param width desired width, ignored if 0 + * @param height desired height, ignored if 0 + * @param refresh_rate desired refresh rate (number of times the display is refreshed + * each second multiplied by 1000), ignored if 0 + * @param flags init flags + */ + ExclusiveWindow(const char* display_name, uint32_t width, uint32_t height, uint32_t refresh_rate, + InitFlags flags); + + /** + * Delete the standard constructor, always need parameters to construct. + */ + ExclusiveWindow() = delete; + + /** + * Destroy the exclusive window object. + */ + virtual ~ExclusiveWindow(); + + /// holoscan::viz::Window virtual members + ///@{ + void init_im_gui() override; + void setup_callbacks(std::function frame_buffer_size_cb) override; + + const char** get_required_instance_extensions(uint32_t* count) override; + const char** get_required_device_extensions(uint32_t* count) override; + uint32_t select_device(vk::Instance instance, + const std::vector& physical_devices) override; + void get_framebuffer_size(uint32_t* width, uint32_t* height) override; + + vk::SurfaceKHR create_surface(vk::PhysicalDevice physical_device, vk::Instance instance) override; + + bool should_close() override; + bool is_minimized() override; + + void im_gui_new_frame() override; + + void begin() override; + void end() override; + + float get_aspect_ratio() override; + ///@} private: - struct Impl; - std::shared_ptr impl_; + struct Impl; + std::shared_ptr impl_; }; } // namespace holoscan::viz diff --git a/modules/holoviz/src/export.map b/modules/holoviz/src/export.map index 410eb566..fd548e1f 100644 --- a/modules/holoviz/src/export.map +++ b/modules/holoviz/src/export.map @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,6 +25,8 @@ "holoscan::viz::SetCudaStream(CUstream_st*)"; + "holoscan::viz::SetFont(char const*, float)"; + "holoscan::viz::WindowShouldClose()"; "holoscan::viz::WindowIsMinimized()"; @@ -37,7 +39,7 @@ "holoscan::viz::ImageCudaDevice(unsigned int, unsigned int, holoscan::viz::ImageFormat, unsigned long long)"; "holoscan::viz::ImageCudaArray(holoscan::viz::ImageFormat, CUarray_st*)"; "holoscan::viz::ImageHost(unsigned int, unsigned int, holoscan::viz::ImageFormat, void const*)"; - "holoscan::viz::LUT(unsigned int, holoscan::viz::ImageFormat, unsigned long, void const*)"; + "holoscan::viz::LUT(unsigned int, holoscan::viz::ImageFormat, unsigned long, void const*, bool)"; "holoscan::viz::BeginImGuiLayer()"; @@ -47,13 +49,15 @@ "holoscan::viz::PointSize(float)"; "holoscan::viz::Primitive(holoscan::viz::PrimitiveTopology, unsigned int, unsigned long, float const*)"; "holoscan::viz::Text(float, float, float, char const*)"; + "holoscan::viz::DepthMap(holoscan::viz::DepthMapRenderMode, unsigned int, unsigned int, holoscan::viz::ImageFormat, unsigned long long, holoscan::viz::ImageFormat, unsigned long long)"; "holoscan::viz::LayerOpacity(float)"; "holoscan::viz::LayerPriority(int)"; + "holoscan::viz::LayerViewCamera(holoscan::viz::Vector3f*, holoscan::viz::Vector3f*, holoscan::viz::Vector3f*)"; "holoscan::viz::EndLayer()"; - "holoscan::viz::ReadFramebuffer(holoscan::viz::ImageFormat, unsigned long, unsigned long long)"; + "holoscan::viz::ReadFramebuffer(holoscan::viz::ImageFormat, unsigned int, unsigned int, unsigned long, unsigned long long)"; }; local: *; diff --git a/gxf_extensions/visualizer_tool_tracking/fonts/Roboto-Bold.ttf b/modules/holoviz/src/fonts/Roboto-Bold.ttf similarity index 100% rename from gxf_extensions/visualizer_tool_tracking/fonts/Roboto-Bold.ttf rename to modules/holoviz/src/fonts/Roboto-Bold.ttf diff --git a/modules/holoviz/src/glfw_window.cpp b/modules/holoviz/src/glfw_window.cpp index e8e44bf9..e78eac7b 100644 --- a/modules/holoviz/src/glfw_window.cpp +++ b/modules/holoviz/src/glfw_window.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,144 +20,262 @@ #define GLFW_INCLUDE_NONE #define GLFW_INCLUDE_VULKAN #include -#include #include +#include #include +#include #include +#include +#include #include namespace holoscan::viz { -static void glfw_error_callback(int error, const char *description) { - std::cerr << "Glfw Error " << error << ": " << description << std::endl; +static void glfw_error_callback(int error, const char* description) { + std::cerr << "Glfw Error " << error << ": " << description << std::endl; } struct GLFWWindow::Impl { public: - Impl() { - glfwSetErrorCallback(glfw_error_callback); + Impl() { + glfwSetErrorCallback(glfw_error_callback); + + if (glfwInit() == GLFW_FALSE) { throw std::runtime_error("Failed to initialize glfw"); } - if (glfwInit() == GLFW_FALSE) { - throw std::runtime_error("Failed to initialize glfw"); - } + glfwWindowHint(GLFW_CLIENT_API, GLFW_NO_API); - glfwWindowHint(GLFW_CLIENT_API, GLFW_NO_API); + if (!glfwVulkanSupported()) { throw std::runtime_error("Vulkan is not supported"); } + } - if (!glfwVulkanSupported()) { - throw std::runtime_error("Vulkan is not supported"); - } + ~Impl() { + if (intern_window_ && window_) { + // GLFW is not switching back to the original code when just destroying the window, + // have to set the window monitor explicitly to NULL before destroy to switch back + // to the original mode. + glfwSetWindowMonitor(window_, NULL, 0, 0, 640, 480, GLFW_DONT_CARE); + glfwDestroyWindow(window_); } - ~Impl() { - if (intern_window_ && window_) { - // GLFW is not switching back to the original code when just destroying the window, - // have to set the window monitor explicitly to NULL before destroy to switch back - // to the original mode. - glfwSetWindowMonitor(window_, NULL, 0, 0, 640, 480, GLFW_DONT_CARE); - glfwDestroyWindow(window_); - } + glfwTerminate(); + } - glfwTerminate(); - } + static void frame_buffer_size_cb(GLFWwindow* window, int width, int height); + static void key_cb(GLFWwindow* window, int key, int scancode, int action, int mods); + static void cursor_pos_cb(GLFWwindow* window, double x, double y); + static void mouse_button_cb(GLFWwindow* window, int button, int action, int mods); + static void scroll_cb(GLFWwindow* window, double x, double y); + + GLFWwindow* window_ = nullptr; + bool intern_window_ = false; + + std::function frame_buffer_size_cb_; + GLFWframebuffersizefun prev_frame_buffer_size_cb_ = nullptr; - static void key_cb(GLFWwindow *window, int key, int scancode, int action, int mods); - static void frame_buffer_size_cb(GLFWwindow *window, int width, int height); + GLFWkeyfun prev_key_cb_ = nullptr; + GLFWcursorposfun prev_cursor_pos_cb_ = nullptr; + GLFWmousebuttonfun prev_mouse_button_cb_ = nullptr; + GLFWscrollfun prev_scroll_cb_ = nullptr; - GLFWwindow *window_ = nullptr; - bool intern_window_ = false; + nvh::CameraManipulator::Inputs inputs_; ///< Mouse button pressed + nvh::Stopwatch timer_; ///< measure time from frame to frame to base camera movement on - std::function frame_buffer_size_cb_; + uint32_t width_; + uint32_t height_; }; -GLFWWindow::GLFWWindow(GLFWwindow *window) - : impl_(new Impl) { - impl_->window_ = window; +GLFWWindow::GLFWWindow(GLFWwindow* window) : impl_(new Impl) { + impl_->window_ = window; + + // set the user pointer to the implementation class to be used in callbacks, fail if the provided + // window already has the user pointer set. + if (glfwGetWindowUserPointer(impl_->window_) != nullptr) { + throw std::runtime_error("GLFW window user pointer already set"); + } + glfwSetWindowUserPointer(impl_->window_, impl_.get()); + + // set framebuffer size with initial window size + int width, height; + glfwGetWindowSize(window, &width, &height); + impl_->frame_buffer_size_cb(window, width, height); + + // setup camera + CameraManip.setLookat( + nvmath::vec3f(0.f, 0.f, 1.f), nvmath::vec3f(0.f, 0.f, 0.f), nvmath::vec3f(0.f, 1.f, 0.f)); } -GLFWWindow::GLFWWindow(uint32_t width, uint32_t height, const char *title, InitFlags flags) +GLFWWindow::GLFWWindow(uint32_t width, uint32_t height, const char* title, InitFlags flags) : impl_(new Impl) { - impl_->window_ = - glfwCreateWindow(width, height, title, (flags & InitFlags::FULLSCREEN) ? - glfwGetPrimaryMonitor() : NULL, NULL); - if (!impl_->window_) { - throw std::runtime_error("Failed to create glfw window"); - } + impl_->window_ = glfwCreateWindow( + width, height, title, (flags & InitFlags::FULLSCREEN) ? glfwGetPrimaryMonitor() : NULL, NULL); + if (!impl_->window_) { throw std::runtime_error("Failed to create glfw window"); } + + impl_->intern_window_ = true; - impl_->intern_window_ = true; + // set the user pointer to the implementation class to be used in callbacks + glfwSetWindowUserPointer(impl_->window_, impl_.get()); + + // set framebuffer size with initial window size + impl_->frame_buffer_size_cb(impl_->window_, width, height); + + // setup camera + CameraManip.setLookat( + nvmath::vec3f(0.f, 0.f, 1.f), nvmath::vec3f(0.f, 0.f, 0.f), nvmath::vec3f(0.f, 1.f, 0.f)); } GLFWWindow::~GLFWWindow() {} void GLFWWindow::init_im_gui() { - ImGui_ImplGlfw_InitForVulkan(impl_->window_, true); + ImGui_ImplGlfw_InitForVulkan(impl_->window_, true); } -void GLFWWindow::Impl::key_cb(GLFWwindow *window, int key, int scancode, int action, int mods) { - if ((action != GLFW_RELEASE) && (key == GLFW_KEY_ESCAPE)) { - glfwSetWindowShouldClose(window, 1); - } +void GLFWWindow::Impl::frame_buffer_size_cb(GLFWwindow* window, int width, int height) { + GLFWWindow::Impl* const impl = static_cast(glfwGetWindowUserPointer(window)); + + if (impl->prev_frame_buffer_size_cb_) { impl->prev_frame_buffer_size_cb_(window, width, height); } + + if (impl->frame_buffer_size_cb_) { impl->frame_buffer_size_cb_(width, height); } + + impl->width_ = width; + impl->height_ = height; + CameraManip.setWindowSize(width, height); } -void GLFWWindow::Impl::frame_buffer_size_cb(GLFWwindow *window, int width, int height) { - static_cast(glfwGetWindowUserPointer(window))-> - frame_buffer_size_cb_(width, height); +void GLFWWindow::Impl::key_cb(GLFWwindow* window, int key, int scancode, int action, int mods) { + GLFWWindow::Impl* const impl = static_cast(glfwGetWindowUserPointer(window)); + + if (impl->prev_key_cb_) { impl->prev_key_cb_(window, key, scancode, action, mods); } + + const bool pressed = action != GLFW_RELEASE; + + if (pressed && (key == GLFW_KEY_ESCAPE)) { glfwSetWindowShouldClose(window, 1); } + + // Keeping track of the modifiers + impl->inputs_.ctrl = + pressed & ((key == GLFW_KEY_LEFT_CONTROL) || (key == GLFW_KEY_RIGHT_CONTROL)); + impl->inputs_.shift = pressed & ((key == GLFW_KEY_LEFT_SHIFT) || (key == GLFW_KEY_RIGHT_SHIFT)); + impl->inputs_.alt = pressed & ((key == GLFW_KEY_LEFT_ALT) || (key == GLFW_KEY_RIGHT_ALT)); } -void GLFWWindow::setup_callbacks(std::function frame_buffer_size_cb_) { - impl_->frame_buffer_size_cb_ = frame_buffer_size_cb_; +void GLFWWindow::Impl::mouse_button_cb(GLFWwindow* window, int button, int action, int mods) { + GLFWWindow::Impl* const impl = static_cast(glfwGetWindowUserPointer(window)); - glfwSetWindowUserPointer(impl_->window_, impl_.get()); - glfwSetFramebufferSizeCallback(impl_->window_, &GLFWWindow::Impl::frame_buffer_size_cb); - glfwSetKeyCallback(impl_->window_, &GLFWWindow::Impl::key_cb); + if (impl->prev_mouse_button_cb_) { impl->prev_mouse_button_cb_(window, button, action, mods); } + + double x, y; + glfwGetCursorPos(impl->window_, &x, &y); + CameraManip.setMousePosition(static_cast(x), static_cast(y)); + + impl->inputs_.lmb = (button == GLFW_MOUSE_BUTTON_LEFT) && (action == GLFW_PRESS); + impl->inputs_.mmb = (button == GLFW_MOUSE_BUTTON_MIDDLE) && (action == GLFW_PRESS); + impl->inputs_.rmb = (button == GLFW_MOUSE_BUTTON_RIGHT) && (action == GLFW_PRESS); } -const char **GLFWWindow::get_required_instance_extensions(uint32_t *count) { - return glfwGetRequiredInstanceExtensions(count); +void GLFWWindow::Impl::cursor_pos_cb(GLFWwindow* window, double x, double y) { + GLFWWindow::Impl* const impl = static_cast(glfwGetWindowUserPointer(window)); + + if (impl->prev_cursor_pos_cb_) { impl->prev_cursor_pos_cb_(window, x, y); } + + // Allow camera movement only when not editing + if ((ImGui::GetCurrentContext() != nullptr) && ImGui::GetIO().WantCaptureMouse) { return; } + + if (impl->inputs_.lmb || impl->inputs_.rmb || impl->inputs_.mmb) { + CameraManip.mouseMove(static_cast(x), static_cast(y), impl->inputs_); + } } -const char **GLFWWindow::get_required_device_extensions(uint32_t *count) { - static char const *extensions[]{VK_KHR_SWAPCHAIN_EXTENSION_NAME}; +void GLFWWindow::Impl::scroll_cb(GLFWwindow* window, double x, double y) { + GLFWWindow::Impl* const impl = static_cast(glfwGetWindowUserPointer(window)); + + if (impl->prev_scroll_cb_) { impl->prev_scroll_cb_(window, x, y); } - *count = sizeof(extensions) / sizeof(extensions[0]); - return extensions; + // Allow camera movement only when not editing + if ((ImGui::GetCurrentContext() != nullptr) && ImGui::GetIO().WantCaptureMouse) { return; } + + CameraManip.wheel(y > 0.0 ? 1 : -1, impl->inputs_); +} + +void GLFWWindow::setup_callbacks(std::function frame_buffer_size_cb_) { + impl_->frame_buffer_size_cb_ = frame_buffer_size_cb_; + + impl_->prev_frame_buffer_size_cb_ = + glfwSetFramebufferSizeCallback(impl_->window_, &GLFWWindow::Impl::frame_buffer_size_cb); + impl_->prev_mouse_button_cb_ = + glfwSetMouseButtonCallback(impl_->window_, &GLFWWindow::Impl::mouse_button_cb); + impl_->prev_scroll_cb_ = glfwSetScrollCallback(impl_->window_, &GLFWWindow::Impl::scroll_cb); + impl_->prev_cursor_pos_cb_ = + glfwSetCursorPosCallback(impl_->window_, &GLFWWindow::Impl::cursor_pos_cb); + impl_->prev_key_cb_ = glfwSetKeyCallback(impl_->window_, &GLFWWindow::Impl::key_cb); } -void GLFWWindow::get_framebuffer_size(uint32_t *width, uint32_t *height) { - glfwGetFramebufferSize(impl_->window_, reinterpret_cast(width), - reinterpret_cast(height)); +const char** GLFWWindow::get_required_instance_extensions(uint32_t* count) { + return glfwGetRequiredInstanceExtensions(count); } -VkSurfaceKHR GLFWWindow::create_surface(VkPhysicalDevice physical_device, VkInstance instance) { - VkSurfaceKHR surface; - NVVK_CHECK(glfwCreateWindowSurface(instance, impl_->window_, nullptr, &surface)); - return surface; +const char** GLFWWindow::get_required_device_extensions(uint32_t* count) { + static char const* extensions[]{VK_KHR_SWAPCHAIN_EXTENSION_NAME}; + + *count = sizeof(extensions) / sizeof(extensions[0]); + return extensions; +} + +uint32_t GLFWWindow::select_device(vk::Instance instance, + const std::vector& physical_devices) { + // select the first device which has presentation support + for (uint32_t index = 0; index < physical_devices.size(); ++index) { + if (glfwGetPhysicalDevicePresentationSupport(instance, physical_devices[index], 0) == + GLFW_TRUE) { + return index; + } + } + throw std::runtime_error("No device with presentation support found"); +} + +void GLFWWindow::get_framebuffer_size(uint32_t* width, uint32_t* height) { + glfwGetFramebufferSize( + impl_->window_, reinterpret_cast(width), reinterpret_cast(height)); +} + +vk::SurfaceKHR GLFWWindow::create_surface(vk::PhysicalDevice physical_device, + vk::Instance instance) { + VkSurfaceKHR surface; + const vk::Result result = + vk::Result(glfwCreateWindowSurface(instance, impl_->window_, nullptr, &surface)); + vk::resultCheck(result, "Failed to create glfw window surface"); + return surface; } bool GLFWWindow::should_close() { - return (glfwWindowShouldClose(impl_->window_) != 0); + return (glfwWindowShouldClose(impl_->window_) != 0); } bool GLFWWindow::is_minimized() { - int w, h; - glfwGetWindowSize(impl_->window_, &w, &h); - bool minimized(w == 0 || h == 0); - if (minimized) { - usleep(50); - } - return minimized; + int w, h; + glfwGetWindowSize(impl_->window_, &w, &h); + bool minimized(w == 0 || h == 0); + if (minimized) { usleep(50); } + return minimized; } void GLFWWindow::im_gui_new_frame() { - ImGui_ImplGlfw_NewFrame(); - ImGui::NewFrame(); + ImGui_ImplGlfw_NewFrame(); + ImGui::NewFrame(); } void GLFWWindow::begin() { - glfwPollEvents(); + glfwPollEvents(); } void GLFWWindow::end() {} +void GLFWWindow::get_view_matrix(nvmath::mat4f* view_matrix) { + *view_matrix = nvmath::perspectiveVK(CameraManip.getFov(), 1.f /*aspectRatio*/, 0.1f, 1000.0f) * + CameraManip.getMatrix(); +} + +float GLFWWindow::get_aspect_ratio() { + return float(impl_->width_) / float(impl_->height_); +} + } // namespace holoscan::viz diff --git a/modules/holoviz/src/glfw_window.hpp b/modules/holoviz/src/glfw_window.hpp index 3c343e12..7bb15bcd 100644 --- a/modules/holoviz/src/glfw_window.hpp +++ b/modules/holoviz/src/glfw_window.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,9 +20,10 @@ #include #include +#include -#include "window.hpp" #include "holoviz/init_flags.hpp" +#include "window.hpp" typedef struct GLFWwindow GLFWwindow; @@ -33,55 +34,61 @@ namespace holoscan::viz { */ class GLFWWindow : public Window { public: - /** - * Construct a new GLFWWindow object with an existing GLFWwindow object. - * - * @param window existing GLFWwindow window - */ - explicit GLFWWindow(GLFWwindow *window); - - /** - * Construct a new GLFWWindow object of a given size. - * - * @param width, height window size - * @param title window tile - * @param flags init flags - */ - GLFWWindow(uint32_t width, uint32_t height, const char *title, InitFlags flags); - - /** - * Delete the standard constructor, always need parameters to construct. - */ - GLFWWindow() = delete; - - /** - * Destroy the GLFWWindow object. - */ - virtual ~GLFWWindow(); - - /// holoscan::viz::Window virtual members - ///@{ - void init_im_gui() override; - void setup_callbacks(std::function frame_buffer_size_cb) override; - - const char **get_required_instance_extensions(uint32_t *count) override; - const char **get_required_device_extensions(uint32_t *count) override; - void get_framebuffer_size(uint32_t *width, uint32_t *height) override; - - VkSurfaceKHR create_surface(VkPhysicalDevice physical_device, VkInstance instance) override; - - bool should_close() override; - bool is_minimized() override; - - void im_gui_new_frame() override; - - void begin() override; - void end() override; - ///@} + /** + * Construct a new GLFWWindow object with an existing GLFWwindow object. + * + * @param window existing GLFWwindow window + */ + explicit GLFWWindow(GLFWwindow* window); + + /** + * Construct a new GLFWWindow object of a given size. + * + * @param width, height window size + * @param title window tile + * @param flags init flags + */ + GLFWWindow(uint32_t width, uint32_t height, const char* title, InitFlags flags); + + /** + * Delete the standard constructor, always need parameters to construct. + */ + GLFWWindow() = delete; + + /** + * Destroy the GLFWWindow object. + */ + virtual ~GLFWWindow(); + + /// holoscan::viz::Window virtual members + ///@{ + void init_im_gui() override; + void setup_callbacks(std::function frame_buffer_size_cb) override; + + const char** get_required_instance_extensions(uint32_t* count) override; + const char** get_required_device_extensions(uint32_t* count) override; + uint32_t select_device(vk::Instance instance, + const std::vector& physical_devices) override; + void get_framebuffer_size(uint32_t* width, uint32_t* height) override; + + vk::SurfaceKHR create_surface(vk::PhysicalDevice physical_device, vk::Instance instance) override; + + bool should_close() override; + bool is_minimized() override; + + void im_gui_new_frame() override; + + void begin() override; + void end() override; + + void get_view_matrix(nvmath::mat4f* view_matrix) override; + + float get_aspect_ratio() override; + ///@} private: - struct Impl; - std::shared_ptr impl_; + struct Impl; + std::shared_ptr impl_; }; } // namespace holoscan::viz diff --git a/modules/holoviz/src/headless_window.cpp b/modules/holoviz/src/headless_window.cpp index ac067c5b..aa30d705 100644 --- a/modules/holoviz/src/headless_window.cpp +++ b/modules/holoviz/src/headless_window.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,63 +25,72 @@ namespace holoscan::viz { * HeadlessWindow implementation details */ struct HeadlessWindow::Impl { - uint32_t width_ = 0; - uint32_t height_ = 0; + uint32_t width_ = 0; + uint32_t height_ = 0; }; HeadlessWindow::~HeadlessWindow() {} -HeadlessWindow::HeadlessWindow(uint32_t width, uint32_t height, InitFlags flags) - : impl_(new Impl) { - impl_->width_ = width; - impl_->height_ = height; +HeadlessWindow::HeadlessWindow(uint32_t width, uint32_t height, InitFlags flags) : impl_(new Impl) { + impl_->width_ = width; + impl_->height_ = height; } void HeadlessWindow::init_im_gui() {} -void HeadlessWindow::setup_callbacks(std::function - frame_buffer_size_cb) {} +void HeadlessWindow::setup_callbacks( + std::function frame_buffer_size_cb) {} -const char **HeadlessWindow::get_required_instance_extensions(uint32_t *count) { - static char const *extensions[]{}; +const char** HeadlessWindow::get_required_instance_extensions(uint32_t* count) { + static char const* extensions[]{}; - *count = sizeof(extensions) / sizeof(extensions[0]); - return extensions; + *count = sizeof(extensions) / sizeof(extensions[0]); + return extensions; } -const char **HeadlessWindow::get_required_device_extensions(uint32_t *count) { - *count = 0; - return nullptr; +const char** HeadlessWindow::get_required_device_extensions(uint32_t* count) { + *count = 0; + return nullptr; } -void HeadlessWindow::get_framebuffer_size(uint32_t *width, uint32_t *height) { - *width = impl_->width_; - *height = impl_->height_; +uint32_t HeadlessWindow::select_device(vk::Instance instance, + const std::vector& physical_devices) { + // headless can be on any device so select the first one + return 0; } -VkSurfaceKHR HeadlessWindow::create_surface(VkPhysicalDevice physical_device, VkInstance instance) { - return VK_NULL_HANDLE; +void HeadlessWindow::get_framebuffer_size(uint32_t* width, uint32_t* height) { + *width = impl_->width_; + *height = impl_->height_; +} + +vk::SurfaceKHR HeadlessWindow::create_surface(vk::PhysicalDevice physical_device, + vk::Instance instance) { + return VK_NULL_HANDLE; } bool HeadlessWindow::should_close() { - return false; + return false; } bool HeadlessWindow::is_minimized() { - return false; + return false; } void HeadlessWindow::im_gui_new_frame() { - ImGuiIO &io = ImGui::GetIO(); - io.DisplaySize = ImVec2(static_cast(impl_->width_), - static_cast(impl_->height_)); - io.DisplayFramebufferScale = ImVec2(1.f, 1.f); + ImGuiIO& io = ImGui::GetIO(); + io.DisplaySize = ImVec2(static_cast(impl_->width_), static_cast(impl_->height_)); + io.DisplayFramebufferScale = ImVec2(1.f, 1.f); - ImGui::NewFrame(); + ImGui::NewFrame(); } void HeadlessWindow::begin() {} void HeadlessWindow::end() {} +float HeadlessWindow::get_aspect_ratio() { + return float(impl_->width_) / float(impl_->height_); +} + } // namespace holoscan::viz diff --git a/modules/holoviz/src/headless_window.hpp b/modules/holoviz/src/headless_window.hpp index 72c61f2b..6928e51c 100644 --- a/modules/holoviz/src/headless_window.hpp +++ b/modules/holoviz/src/headless_window.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,9 +20,10 @@ #include #include +#include -#include "window.hpp" #include "holoviz/init_flags.hpp" +#include "window.hpp" namespace holoscan::viz { @@ -31,48 +32,52 @@ namespace holoscan::viz { */ class HeadlessWindow : public Window { public: - /** - * Construct a new headless window. - * - * @param width desired width, ignored if 0 - * @param height desired height, ignored if 0 - * @param flags init flags - */ - HeadlessWindow(uint32_t width, uint32_t height, InitFlags flags); + /** + * Construct a new headless window. + * + * @param width desired width, ignored if 0 + * @param height desired height, ignored if 0 + * @param flags init flags + */ + HeadlessWindow(uint32_t width, uint32_t height, InitFlags flags); + + /** + * Delete the standard constructor, always need parameters to construct. + */ + HeadlessWindow() = delete; - /** - * Delete the standard constructor, always need parameters to construct. - */ - HeadlessWindow() = delete; + /** + * Destroy the headless window object. + */ + virtual ~HeadlessWindow(); - /** - * Destroy the headless window object. - */ - virtual ~HeadlessWindow(); + /// holoscan::viz::Window virtual members + ///@{ + void init_im_gui() override; + void setup_callbacks(std::function frame_buffer_size_cb) override; - /// holoscan::viz::Window virtual members - ///@{ - void init_im_gui() override; - void setup_callbacks(std::function frame_buffer_size_cb) override; + const char** get_required_instance_extensions(uint32_t* count) override; + const char** get_required_device_extensions(uint32_t* count) override; + uint32_t select_device(vk::Instance instance, + const std::vector& physical_devices) override; + void get_framebuffer_size(uint32_t* width, uint32_t* height) override; - const char **get_required_instance_extensions(uint32_t *count) override; - const char **get_required_device_extensions(uint32_t *count) override; - void get_framebuffer_size(uint32_t *width, uint32_t *height) override; + vk::SurfaceKHR create_surface(vk::PhysicalDevice physical_device, vk::Instance instance) override; - VkSurfaceKHR create_surface(VkPhysicalDevice physical_device, VkInstance instance) override; + bool should_close() override; + bool is_minimized() override; - bool should_close() override; - bool is_minimized() override; + void im_gui_new_frame() override; - void im_gui_new_frame() override; + void begin() override; + void end() override; - void begin() override; - void end() override; - ///@} + float get_aspect_ratio() override; + ///@} private: - struct Impl; - std::shared_ptr impl_; + struct Impl; + std::shared_ptr impl_; }; } // namespace holoscan::viz diff --git a/modules/holoviz/src/holoviz.cpp b/modules/holoviz/src/holoviz.cpp index b84a0625..c07d4dc8 100644 --- a/modules/holoviz/src/holoviz.cpp +++ b/modules/holoviz/src/holoviz.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,118 +22,128 @@ #include #include "context.hpp" -#include "window.hpp" #include "layers/geometry_layer.hpp" #include "layers/image_layer.hpp" - +#include "window.hpp" namespace holoscan::viz { -void Init(GLFWwindow *window, InitFlags flags) { - Context::get().init(window, flags); +void Init(GLFWwindow* window, InitFlags flags) { + Context::get().init(window, flags); } -void Init(uint32_t width, uint32_t height, const char *title, InitFlags flags) { - Context::get().init(width, height, title, flags); +void Init(uint32_t width, uint32_t height, const char* title, InitFlags flags) { + Context::get().init(width, height, title, flags); } -void Init(const char *displayName, uint32_t width, uint32_t height, uint32_t refreshRate, - InitFlags flags) { - Context::get().init(displayName, width, height, refreshRate, flags); +void Init(const char* displayName, uint32_t width, uint32_t height, uint32_t refreshRate, + InitFlags flags) { + Context::get().init(displayName, width, height, refreshRate, flags); } -void ImGuiSetCurrentContext(ImGuiContext *context) { - Context::get().im_gui_set_current_context(context); +void ImGuiSetCurrentContext(ImGuiContext* context) { + Context::get().im_gui_set_current_context(context); } void SetCudaStream(CUstream stream) { - Context::get().set_cuda_stream(stream); + Context::get().set_cuda_stream(stream); +} + +void SetFont(const char* path, float size_in_pixels) { + Context::get().set_font(path, size_in_pixels); } bool WindowShouldClose() { - return Context::get().get_window()->should_close(); + return Context::get().get_window()->should_close(); } bool WindowIsMinimized() { - return Context::get().get_window()->is_minimized(); + return Context::get().get_window()->is_minimized(); } void Shutdown() { - Context::get().shutdown(); + Context::get().shutdown(); } void Begin() { - Context::get().begin(); + Context::get().begin(); } void End() { - Context::get().end(); + Context::get().end(); } void BeginImageLayer() { - Context::get().begin_image_layer(); + Context::get().begin_image_layer(); } void ImageCudaDevice(uint32_t w, uint32_t h, ImageFormat fmt, CUdeviceptr device_ptr) { - Context::get().get_active_image_layer()->image_cuda_device(w, h, fmt, device_ptr); + Context::get().get_active_image_layer()->image_cuda_device(w, h, fmt, device_ptr); } void ImageCudaArray(ImageFormat fmt, CUarray array) { - Context::get().get_active_image_layer()->image_cuda_array(fmt, array); + Context::get().get_active_image_layer()->image_cuda_array(fmt, array); } -void ImageHost(uint32_t w, uint32_t h, ImageFormat fmt, const void *data) { - Context::get().get_active_image_layer()->image_host(w, h, fmt, data); +void ImageHost(uint32_t w, uint32_t h, ImageFormat fmt, const void* data) { + Context::get().get_active_image_layer()->image_host(w, h, fmt, data); } -void LUT(uint32_t size, ImageFormat fmt, size_t data_size, const void *data) { - Context::get().get_active_image_layer()->lut(size, fmt, data_size, data); +void LUT(uint32_t size, ImageFormat fmt, size_t data_size, const void* data, bool normalized) { + Context::get().get_active_image_layer()->lut(size, fmt, data_size, data, normalized); } void BeginImGuiLayer() { - Context::get().begin_im_gui_layer(); + Context::get().begin_im_gui_layer(); } void BeginGeometryLayer() { - Context::get().begin_geometry_layer(); + Context::get().begin_geometry_layer(); } void Color(float r, float g, float b, float a) { - Context::get().get_active_geometry_layer()->color(r, g, b, a); + Context::get().get_active_geometry_layer()->color(r, g, b, a); } void LineWidth(float width) { - Context::get().get_active_geometry_layer()->line_width(width); + Context::get().get_active_geometry_layer()->line_width(width); } void PointSize(float size) { - Context::get().get_active_geometry_layer()->point_size(size); + Context::get().get_active_geometry_layer()->point_size(size); } -void Text(float x, float y, float size, const char *text) { - Context::get().get_active_geometry_layer()->text(x, y, size, text); +void Text(float x, float y, float size, const char* text) { + Context::get().get_active_geometry_layer()->text(x, y, size, text); } void Primitive(PrimitiveTopology topology, uint32_t primitive_count, size_t data_size, - const float *data) { - Context::get().get_active_geometry_layer()->primitive(topology, primitive_count, - data_size, data); + const float* data) { + Context::get().get_active_geometry_layer()->primitive(topology, primitive_count, data_size, data); +} + +void DepthMap(DepthMapRenderMode render_mode, uint32_t width, uint32_t height, + ImageFormat depth_fmt, CUdeviceptr depth_device_ptr, ImageFormat color_fmt, + CUdeviceptr color_device_ptr) { + Context::get().get_active_geometry_layer()->depth_map( + render_mode, width, height, depth_fmt, depth_device_ptr, color_fmt, color_device_ptr); } void LayerOpacity(float opacity) { - Context::get().get_active_layer()->set_opacity(opacity); + Context::get().get_active_layer()->set_opacity(opacity); } void LayerPriority(int32_t priority) { - Context::get().get_active_layer()->set_priority(priority); + Context::get().get_active_layer()->set_priority(priority); } void EndLayer() { - Context::get().end_layer(); + Context::get().end_layer(); } -void ReadFramebuffer(ImageFormat fmt, size_t buffer_size, CUdeviceptr device_ptr) { - Context::get().read_framebuffer(fmt, buffer_size, device_ptr); +void ReadFramebuffer(ImageFormat fmt, uint32_t width, uint32_t height, size_t buffer_size, + CUdeviceptr device_ptr) { + Context::get().read_framebuffer(fmt, width, height, buffer_size, device_ptr); } } // namespace holoscan::viz diff --git a/modules/holoviz/src/holoviz/depth_map_render_mode.hpp b/modules/holoviz/src/holoviz/depth_map_render_mode.hpp new file mode 100644 index 00000000..758acef6 --- /dev/null +++ b/modules/holoviz/src/holoviz/depth_map_render_mode.hpp @@ -0,0 +1,36 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HOLOSCAN_VIZ_HOLOVIZ_DEPTH_MAP_RENDER_MODER_HPP +#define HOLOSCAN_VIZ_HOLOVIZ_DEPTH_MAP_RENDER_MODER_HPP + +#include + +namespace holoscan::viz { + +/** + * Depth map render mode. + */ +enum class DepthMapRenderMode { + POINTS, ///< render points + LINES, ///< render lines + TRIANGLES ///< render triangles +}; + +} // namespace holoscan::viz + +#endif /* HOLOSCAN_VIZ_HOLOVIZ_DEPTH_MAP_RENDER_MODER_HPP */ diff --git a/modules/holoviz/src/holoviz/holoviz.hpp b/modules/holoviz/src/holoviz/holoviz.hpp index c8d02b97..d5a3feb2 100644 --- a/modules/holoviz/src/holoviz/holoviz.hpp +++ b/modules/holoviz/src/holoviz/holoviz.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef HOLOSCAN_VIZ_HOLOVIZ_HOLOVIZ_HPP -#define HOLOSCAN_VIZ_HOLOVIZ_HOLOVIZ_HPP +#ifndef MODULES_HOLOVIZ_SRC_HOLOVIZ_HOLOVIZ_HPP +#define MODULES_HOLOVIZ_SRC_HOLOVIZ_HOLOVIZ_HPP /** * \file @@ -40,13 +40,13 @@ * \section Usage * * The code below creates a window and displays an image. - * First Holoviz needs to be initialized. This is done by calling clara::viz::Init(). + * First Holoviz needs to be initialized. This is done by calling holoscan::viz::Init(). * * The elements to display are defined in the render loop, termination of the loop is checked with - * clara::viz::WindowShouldClose(). + * holoscan::viz::WindowShouldClose(). * - * The definition of the displayed content starts with clara::viz::Begin() and ends with - * clara::viz::End(). clara::viz::End() starts the rendering and displays the rendered result. + * The definition of the displayed content starts with holoscan::viz::Begin() and ends with + * holoscan::viz::End(). holoscan::viz::End() starts the rendering and displays the rendered result. * * Finally Holoviz is shutdown with viz::Shutdown(). * @@ -72,6 +72,7 @@ viz::Shutdown(); #include +#include "holoviz/depth_map_render_mode.hpp" #include "holoviz/image_format.hpp" #include "holoviz/init_flags.hpp" #include "holoviz/primitive_topology.hpp" @@ -80,8 +81,7 @@ viz::Shutdown(); typedef struct GLFWwindow GLFWwindow; struct ImGuiContext; -namespace holoscan::viz -{ +namespace holoscan::viz { /** * Initialize Holoviz using an existing GLFW window. @@ -89,7 +89,7 @@ namespace holoscan::viz * @param window GLFW window * @param flags init flags */ -void Init(GLFWwindow *window, InitFlags flags = InitFlags::NONE); +void Init(GLFWwindow* window, InitFlags flags = InitFlags::NONE); /** * Initialize Holoviz. @@ -101,15 +101,16 @@ void Init(GLFWwindow *window, InitFlags flags = InitFlags::NONE); * @param title window title * @param flags init flags */ -void Init(uint32_t width, uint32_t height, const char *title, InitFlags flags = InitFlags::NONE); +void Init(uint32_t width, uint32_t height, const char* title, InitFlags flags = InitFlags::NONE); /** * Initialize Holoviz to use a display in exclusive mode. * * Setup: * - when multiple displays are connected: - * The display to be used in exclusive mode needs to be disabled in the NVIDIA Settings. Open the `X Server Display Configuration` - * tab, select the display and under `Configuration` select `Disabled`. Press `Apply`. + * The display to be used in exclusive mode needs to be disabled in the NVIDIA Settings. Open the + * `X Server Display Configuration` tab, select the display and under `Configuration` select + * `Disabled`. Press `Apply`. * - when a single display is connected: * SSH into the machine, stop the X server with `sudo systemctl stop display-manager`. * @@ -122,8 +123,8 @@ void Init(uint32_t width, uint32_t height, const char *title, InitFlags flags = * each second multiplied by 1000), ignored if 0 * @param flags init flags */ -void Init(const char *displayName, uint32_t width = 0, uint32_t height = 0, uint32_t refreshRate = 0, - InitFlags flags = InitFlags::NONE); +void Init(const char* displayName, uint32_t width = 0, uint32_t height = 0, + uint32_t refreshRate = 0, InitFlags flags = InitFlags::NONE); /** * If using ImGui, create a context and pass it to Holoviz, do this before calling viz::Init(). @@ -138,7 +139,17 @@ void Init(const char *displayName, uint32_t width = 0, uint32_t height = 0, uint * * @param context ImGui context */ -void ImGuiSetCurrentContext(ImGuiContext *context); +void ImGuiSetCurrentContext(ImGuiContext* context); + +/** + * Set the font used to render text, do this before calling viz::Init(). + * + * The font is converted to bitmaps, these bitmaps are scaled to the final size when rendering. + * + * @param path path to TTF font file + * @param size_in_pixels size of the font bitmaps + */ +void SetFont(const char* path, float size_in_pixels); /** * Set the Cuda stream used by Holoviz for Cuda operations. @@ -219,27 +230,36 @@ void ImageCudaArray(ImageFormat fmt, CUarray array); * @param fmt image format * @param data host memory pointer */ -void ImageHost(uint32_t width, uint32_t height, ImageFormat fmt, const void *data); +void ImageHost(uint32_t width, uint32_t height, ImageFormat fmt, const void* data); /** * Defines the lookup table for this image layer. * * If a lookup table is used the image format has to be a single channel integer or * float format (e.g. ::ImageFormat::R8_UINT, ::ImageFormat::R16_UINT, ::ImageFormat::R32_UINT, - * ::ImageFormat::R32_SFLOAT). + * ::ImageFormat::R8_UNORM, ::ImageFormat::R16_UNORM, ::ImageFormat::R32_SFLOAT). * - * The function processed is as follow + * If normalized is 'true' the function processed is as follow + * @code{.cpp} + * out = lut[clamp(in, 0.0, 1.0)] + * @endcode + * Input image values are clamped to the range of the lookup table size: `[0.0, 1.0[`. + * + * If normalized is 'false' the function processed is as follow * @code{.cpp} * out = lut[clamp(in, 0, size)] * @endcode - * Input image values are clamped to the range of the lookup table size: `[0, size[`. + * Input image values are clamped to the range of the lookup table size: `[0.0, size[`. * * @param size size of the lookup table in elements * @param fmt lookup table color format * @param data_size size of the lookup table data in bytes * @param data host memory pointer to lookup table data + * @param normalized if true then the range of the lookup table is '[0.0, 1.0[', else it is + * `[0.0, size[` */ -void LUT(uint32_t size, ImageFormat fmt, size_t data_size, const void *data); +void LUT(uint32_t size, ImageFormat fmt, size_t data_size, const void* data, + bool normalized = false); /** * Start a ImGUI layer. @@ -286,7 +306,8 @@ void PointSize(float size); * @param data pointer to data, the format and size of the array depends on the * primitive count and topology */ -void Primitive(PrimitiveTopology topology, uint32_t primitive_count, size_t data_size, const float *data); +void Primitive(PrimitiveTopology topology, uint32_t primitive_count, size_t data_size, + const float* data); /** * Draw text. * @@ -295,7 +316,34 @@ void Primitive(PrimitiveTopology topology, uint32_t primitive_count, size_t data * @param size font size * @param text text to draw */ -void Text(float x, float y, float size, const char *text); +void Text(float x, float y, float size, const char* text); + +/** + * Render a depth map. + * + * Depth maps are rectangular 2D arrays where each element represents a depth value. The data is + * rendered as a 3D object using points, lines or triangles. + * Additionally a 2D array with a color value for each point in the grid can be specified. + * + * Depth maps are rendered in 3D and support camera movement. + * The camera is operated using the mouse. + * - Orbit (LMB) + * - Pan (LMB + CTRL | MMB) + * - Dolly (LMB + SHIFT | RMB | Mouse wheel) + * - Look Around (LMB + ALT | LMB + CTRL + SHIFT) + * - Zoom (Mouse wheel + SHIFT) + * + * @param render_mode depth map render mode + * @param width width of the depth map + * @param height height of the depth map + * @param depth_fmt format of the depth map data (has to be ImageFormat::R8_UNORM) + * @param depth_device_ptr Cuda device memory pointer holding the depth data + * @param color_fmt format of the color data (has to be ImageFormat::R8G8B8A8_UNORM) + * @param color_device_ptr Cuda device memory pointer holding the color data (optional) + */ +void DepthMap(DepthMapRenderMode render_mode, uint32_t width, uint32_t height, + ImageFormat depth_fmt, CUdeviceptr depth_device_ptr, ImageFormat color_fmt, + CUdeviceptr color_device_ptr); /** * Set the layer opacity. @@ -326,12 +374,15 @@ void EndLayer(); * * Can only be called outside of Begin()/End(). * - * @param fmt image format, currently only R8G8B8A8_UNORM is supported. - * @param buffer_size size of the storage buffer in bytes - * @param device_ptr pointer to Cuda device memory to store the framebuffer into + * @param fmt image format, currently only R8G8B8A8_UNORM is supported + * @param width, height width and height of the region to read back, will be limited to the + * framebuffer size if the framebuffer is smaller than that + * @param buffer_size size of the storage buffer in bytes + * @param device_ptr pointer to Cuda device memory to store the framebuffer into */ -void ReadFramebuffer(ImageFormat fmt, size_t buffer_size, CUdeviceptr device_ptr); +void ReadFramebuffer(ImageFormat fmt, uint32_t width, uint32_t height, size_t buffer_size, + CUdeviceptr device_ptr); -} // namespace holoscan::viz +} // namespace holoscan::viz -#endif /* HOLOSCAN_VIZ_HOLOVIZ_HOLOVIZ_HPP */ +#endif /* MODULES_HOLOVIZ_SRC_HOLOVIZ_HOLOVIZ_HPP */ diff --git a/modules/holoviz/src/holoviz/image_format.hpp b/modules/holoviz/src/holoviz/image_format.hpp index 93513d29..434dae7c 100644 --- a/modules/holoviz/src/holoviz/image_format.hpp +++ b/modules/holoviz/src/holoviz/image_format.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -40,50 +40,54 @@ namespace holoscan::viz { * component (if one exists) is a regular unsigned normalized value */ enum class ImageFormat { - R8_UINT, ///< specifies a one-component, - /// 8-bit unsigned integer format that has a single 8-bit R component - R16_UINT, ///< specifies a one-component, - /// 16-bit unsigned integer format that has a single 16-bit R component - R16_SFLOAT, ///< specifies a one-component, - /// 16-bit signed floating-point format that has a single 16-bit R component - R32_UINT, ///< specifies a one-component, - /// 16-bit unsigned integer format that has a single 16-bit R component - R32_SFLOAT, ///< specifies a one-component, - /// 32-bit signed floating-point format that has a single 32-bit R component + R8_UINT, ///< specifies a one-component, + /// 8-bit unsigned integer format that has a single 8-bit R component + R16_UINT, ///< specifies a one-component, + /// 16-bit unsigned integer format that has a single 16-bit R component + R16_SFLOAT, ///< specifies a one-component, + /// 16-bit signed floating-point format that has a single 16-bit R component + R32_UINT, ///< specifies a one-component, + /// 16-bit unsigned integer format that has a single 16-bit R component + R32_SFLOAT, ///< specifies a one-component, + /// 32-bit signed floating-point format that has a single 32-bit R component - R8G8B8_UNORM, ///< specifies a three-component, - /// 24-bit unsigned normalized format that has an 8-bit R component in byte 0, - /// an 8-bit G component in byte 1, and an 8-bit B component in byte 2 - B8G8R8_UNORM, ///< specifies a three-component, - /// 24-bit unsigned normalized format that has an 8-bit B component in byte 0, - /// an 8-bit G component in byte 1, and an 8-bit R component in byte 2 + R8G8B8_UNORM, ///< specifies a three-component, + /// 24-bit unsigned normalized format that has an 8-bit R component in byte 0, + /// an 8-bit G component in byte 1, and an 8-bit B component in byte 2 + B8G8R8_UNORM, ///< specifies a three-component, + /// 24-bit unsigned normalized format that has an 8-bit B component in byte 0, + /// an 8-bit G component in byte 1, and an 8-bit R component in byte 2 - R8G8B8A8_UNORM, ///< specifies a four-component, - /// 32-bit unsigned normalized format that has an 8-bit R component in byte 0, - /// an 8-bit G component in byte 1, an 8-bit B component in byte 2, - /// and an 8-bit A component in byte 3 - B8G8R8A8_UNORM, ///< specifies a four-component, - /// 32-bit unsigned normalized format that has an 8-bit B component in byte 0, - /// an 8-bit G component in byte 1, an 8-bit R component in byte 2, - /// and an 8-bit A component in byte 3 - R16G16B16A16_UNORM, ///< specifies a four-component, - /// 64-bit unsigned normalized format that has - /// a 16-bit R component in bytes 0..1, - /// a 16-bit G component in bytes 2..3, - /// a 16-bit B component in bytes 4..5, - /// and a 16-bit A component in bytes 6..7 - R16G16B16A16_SFLOAT, ///< specifies a four-component, - /// 64-bit signed floating-point format that has - /// a 16-bit R component in bytes 0..1, - /// a 16-bit G component in bytes 2..3, - /// a 16-bit B component in bytes 4..5, - /// and a 16-bit A component in bytes 6..7 - R32G32B32A32_SFLOAT ///< specifies a four-component, - /// 128-bit signed floating-point format that has - /// a 32-bit R component in bytes 0..3, - /// a 32-bit G component in bytes 4..7, - /// a 32-bit B component in bytes 8..11, - /// and a 32-bit A component in bytes 12..15 + R8G8B8A8_UNORM, ///< specifies a four-component, + /// 32-bit unsigned normalized format that has an 8-bit R component in byte 0, + /// an 8-bit G component in byte 1, an 8-bit B component in byte 2, + /// and an 8-bit A component in byte 3 + B8G8R8A8_UNORM, ///< specifies a four-component, + /// 32-bit unsigned normalized format that has an 8-bit B component in byte 0, + /// an 8-bit G component in byte 1, an 8-bit R component in byte 2, + /// and an 8-bit A component in byte 3 + R16G16B16A16_UNORM, ///< specifies a four-component, + /// 64-bit unsigned normalized format that has + /// a 16-bit R component in bytes 0..1, + /// a 16-bit G component in bytes 2..3, + /// a 16-bit B component in bytes 4..5, + /// and a 16-bit A component in bytes 6..7 + R16G16B16A16_SFLOAT, ///< specifies a four-component, + /// 64-bit signed floating-point format that has + /// a 16-bit R component in bytes 0..1, + /// a 16-bit G component in bytes 2..3, + /// a 16-bit B component in bytes 4..5, + /// and a 16-bit A component in bytes 6..7 + R32G32B32A32_SFLOAT, ///< specifies a four-component, + /// 128-bit signed floating-point format that has + /// a 32-bit R component in bytes 0..3, + /// a 32-bit G component in bytes 4..7, + /// a 32-bit B component in bytes 8..11, + /// and a 32-bit A component in bytes 12..15 + R8_UNORM, ///< specifies a one-component, + /// 8-bit unsigned normalized format that has a single 8-bit R component. + R16_UNORM, ///< specifies a one-component, + /// 16-bit unsigned normalized format that has a single 8-bit R component. }; } // namespace holoscan::viz diff --git a/modules/holoviz/src/holoviz/init_flags.hpp b/modules/holoviz/src/holoviz/init_flags.hpp index d84fb9fc..318c11a8 100644 --- a/modules/holoviz/src/holoviz/init_flags.hpp +++ b/modules/holoviz/src/holoviz/init_flags.hpp @@ -24,9 +24,9 @@ namespace holoscan::viz { /// Flags passed to the init function typedef enum { - NONE = 0x00000000, ///< none - FULLSCREEN = 0x00000001, ///< switch the app to full screen mode - HEADLESS = 0x00000002 ///< run in headless mode + NONE = 0x00000000, ///< none + FULLSCREEN = 0x00000001, ///< switch the app to full screen mode + HEADLESS = 0x00000002 ///< run in headless mode } InitFlags; } // namespace holoscan::viz diff --git a/modules/holoviz/src/holoviz/primitive_topology.hpp b/modules/holoviz/src/holoviz/primitive_topology.hpp index 3e60806a..6e6c038e 100644 --- a/modules/holoviz/src/holoviz/primitive_topology.hpp +++ b/modules/holoviz/src/holoviz/primitive_topology.hpp @@ -24,18 +24,18 @@ namespace holoscan::viz { * Primitive topology */ enum class PrimitiveTopology { - POINT_LIST, ///< point primitives, one coordinate (x, y) per primitive - LINE_LIST, ///< line primitives, two coordinates (x0, y0) and (x1, y1) per primitive - LINE_STRIP, ///< line strip primitive, a line primitive i is defined by - /// each coordinate (xi, yi) and the following (xi+1, yi+1) - TRIANGLE_LIST, ///< triangle primitive, three coordinates - /// (x0, y0), (x1, y1) and (x2, y2) per primitive - CROSS_LIST, ///< cross primitive, a cross is defined by the center coordinate - /// and the size (xi, yi, si) - RECTANGLE_LIST, ///< axis aligned rectangle primitive, - /// each rectangle is defined by two coordinates (xi, yi) and (xi+1, yi+1) - OVAL_LIST, ///< oval primitive, an oval primitive is defined by the center coordinate - /// and the axis sizes (xi, yi, sxi, syi) + POINT_LIST, ///< point primitives, one coordinate (x, y) per primitive + LINE_LIST, ///< line primitives, two coordinates (x0, y0) and (x1, y1) per primitive + LINE_STRIP, ///< line strip primitive, a line primitive i is defined by + /// each coordinate (xi, yi) and the following (xi+1, yi+1) + TRIANGLE_LIST, ///< triangle primitive, three coordinates + /// (x0, y0), (x1, y1) and (x2, y2) per primitive + CROSS_LIST, ///< cross primitive, a cross is defined by the center coordinate + /// and the size (xi, yi, si) + RECTANGLE_LIST, ///< axis aligned rectangle primitive, + /// each rectangle is defined by two coordinates (xi, yi) and (xi+1, yi+1) + OVAL_LIST, ///< oval primitive, an oval primitive is defined by the center coordinate + /// and the axis sizes (xi, yi, sxi, syi) }; } // namespace holoscan::viz diff --git a/modules/holoviz/src/layers/geometry_layer.cpp b/modules/holoviz/src/layers/geometry_layer.cpp index 34e25ce1..1ca3e7f6 100644 --- a/modules/holoviz/src/layers/geometry_layer.cpp +++ b/modules/holoviz/src/layers/geometry_layer.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -27,7 +28,9 @@ #include #include -#include "../vulkan/vulkan.hpp" +#include "../context.hpp" +#include "../cuda/cuda_service.hpp" +#include "../vulkan/vulkan_app.hpp" namespace holoscan::viz { @@ -36,362 +39,692 @@ constexpr uint32_t CIRCLE_SEGMENTS = 32; class Attributes { public: - Attributes() - : color_({1.f, 1.f, 1.f, 1.f}) - , line_width_(1.f) - , point_size_(1.f) { - } + Attributes() : color_({1.f, 1.f, 1.f, 1.f}), line_width_(1.f), point_size_(1.f) {} - bool operator==(const Attributes &rhs) const { - return ((color_ == rhs.color_) && (line_width_ == rhs.line_width_) && - (point_size_ == rhs.point_size_)); - } + bool operator==(const Attributes& rhs) const { + return ((color_ == rhs.color_) && (line_width_ == rhs.line_width_) && + (point_size_ == rhs.point_size_)); + } - std::array color_; - float line_width_; - float point_size_; + std::array color_; + float line_width_; + float point_size_; }; class Primitive { public: - Primitive(const Attributes &attributes, PrimitiveTopology topology, uint32_t primitive_count, - size_t data_size, const float *data, uint32_t vertex_offset, - std::vector &vertex_counts, VkPrimitiveTopology vk_topology) - : attributes_(attributes) - , topology_(topology) - , primitive_count_(primitive_count) - , vertex_offset_(vertex_offset) - , vertex_counts_(vertex_counts) - , vk_topology_(vk_topology) { - data_.assign(data, data + data_size); - } - Primitive() = delete; - - bool operator==(const Primitive &rhs) const { - return ((attributes_ == rhs.attributes_) && (topology_ == rhs.topology_) && - (primitive_count_ == rhs.primitive_count_) && (data_ == rhs.data_)); - } - - const Attributes attributes_; - - const PrimitiveTopology topology_; - const uint32_t primitive_count_; - std::vector data_; - - // internal state - const uint32_t vertex_offset_; - const std::vector vertex_counts_; - const VkPrimitiveTopology vk_topology_; + Primitive(const Attributes& attributes, PrimitiveTopology topology, uint32_t primitive_count, + size_t data_size, const float* data, uint32_t vertex_offset, + std::vector& vertex_counts, vk::PrimitiveTopology vk_topology) + : attributes_(attributes), + topology_(topology), + primitive_count_(primitive_count), + vertex_offset_(vertex_offset), + vertex_counts_(vertex_counts), + vk_topology_(vk_topology) { + data_.assign(data, data + data_size); + } + Primitive() = delete; + + bool operator==(const Primitive& rhs) const { + return ((attributes_ == rhs.attributes_) && (topology_ == rhs.topology_) && + (primitive_count_ == rhs.primitive_count_) && (data_ == rhs.data_)); + } + + const Attributes attributes_; + + const PrimitiveTopology topology_; + const uint32_t primitive_count_; + std::vector data_; + + // internal state + const uint32_t vertex_offset_; + const std::vector vertex_counts_; + const vk::PrimitiveTopology vk_topology_; }; class Text { public: - Text(const Attributes &attributes, float x, float y, float size, const char *text) - : attributes_(attributes) - , x_(x) - , y_(y) - , size_(size) - , text_(text) { - } - - bool operator==(const Text &rhs) const { - return ((attributes_ == rhs.attributes_) && (x_ == rhs.x_) && (y_ == rhs.y_) - && (size_ == rhs.size_) && (text_ == rhs.text_)); - } + Text(const Attributes& attributes, float x, float y, float size, const char* text) + : attributes_(attributes), x_(x), y_(y), size_(size), text_(text) {} + + bool operator==(const Text& rhs) const { + return ((attributes_ == rhs.attributes_) && (x_ == rhs.x_) && (y_ == rhs.y_) && + (size_ == rhs.size_) && (text_ == rhs.text_)); + } + + const Attributes attributes_; + const float x_; + const float y_; + const float size_; + const std::string text_; +}; - const Attributes attributes_; - const float x_; - const float y_; - const float size_; - const std::string text_; +class DepthMap { + public: + DepthMap(const Attributes& attributes, DepthMapRenderMode render_mode, uint32_t width, + uint32_t height, CUdeviceptr depth_device_ptr, CUdeviceptr color_device_ptr, + CUstream cuda_stream) + : attributes_(attributes), + render_mode_(render_mode), + width_(width), + height_(height), + depth_device_ptr_(depth_device_ptr), + color_device_ptr_(color_device_ptr), + cuda_stream_(cuda_stream) {} + + bool operator==(const DepthMap& rhs) const { + // check attributes, render mode, width and height. Skip *_device_ptr_ since that data + // will be uploaded each frame. The *_device_ptr_ is updated in the can_be_reused() function + // below. + return ((attributes_ == rhs.attributes_) && (render_mode_ == rhs.render_mode_) && + (width_ == rhs.width_) && (height_ == rhs.height_)); + } + + const Attributes attributes_; + const DepthMapRenderMode render_mode_; + const uint32_t width_; + const uint32_t height_; + CUdeviceptr color_device_ptr_; + CUdeviceptr depth_device_ptr_; + CUstream cuda_stream_; + + // internal state + uint32_t index_count_ = 0; + uint32_t index_offset_ = 0; + uint32_t vertex_offset_ = 0; }; class GeometryLayer::Impl { public: - bool can_be_reused(Impl &other) const { - return ((vertex_count_ == other.vertex_count_) && (primitives_ == other.primitives_) && - (texts_ == other.texts_)); + bool can_be_reused(Impl& other) const { + if ((vertex_count_ == other.vertex_count_) && (primitives_ == other.primitives_) && + (texts_ == other.texts_) && (depth_maps_ == other.depth_maps_)) { + // update the Cuda device pointers and the cuda stream. + // Data will be uploaded when drawing regardless if the layer is reused or not + /// @todo this should be made explicit, first check if the layer can be reused and then + /// update the reused layer with these properties below which don't prevent reusing + auto it = other.depth_maps_.begin(); + for (auto&& depth_map : depth_maps_) { + it->depth_device_ptr_ = depth_map.depth_device_ptr_; + it->color_device_ptr_ = depth_map.color_device_ptr_; + it->cuda_stream_ = depth_map.cuda_stream_; + } + return true; } + return false; + } + + Attributes attributes_; - Attributes attributes_; + std::list primitives_; + std::list texts_; + std::list depth_maps_; - std::list primitives_; - std::list texts_; + // internal state + Vulkan* vulkan_ = nullptr; - // internal state - Vulkan *vulkan_ = nullptr; + float aspect_ratio_ = 1.f; - size_t vertex_count_ = 0; - Vulkan::Buffer *vertex_buffer_ = nullptr; + size_t vertex_count_ = 0; + Vulkan::Buffer* vertex_buffer_ = nullptr; - std::unique_ptr text_draw_list_; - Vulkan::Buffer *text_vertex_buffer_ = nullptr; - Vulkan::Buffer *text_index_buffer_ = nullptr; + std::unique_ptr text_draw_list_; + Vulkan::Buffer* text_vertex_buffer_ = nullptr; + Vulkan::Buffer* text_index_buffer_ = nullptr; + + size_t depth_map_vertex_count_ = 0; + Vulkan::Buffer* depth_map_vertex_buffer_ = nullptr; + Vulkan::Buffer* depth_map_index_buffer_ = nullptr; + Vulkan::Buffer* depth_map_color_buffer_ = nullptr; }; -GeometryLayer::GeometryLayer() - : Layer(Type::Geometry) - , impl_(new GeometryLayer::Impl) { -} +GeometryLayer::GeometryLayer() : Layer(Type::Geometry), impl_(new GeometryLayer::Impl) {} GeometryLayer::~GeometryLayer() { - if (impl_->vulkan_) { - if (impl_->vertex_buffer_) { - impl_->vulkan_->destroy_buffer(impl_->vertex_buffer_); - } - if (impl_->text_vertex_buffer_) { - impl_->vulkan_->destroy_buffer(impl_->text_vertex_buffer_); - } - if (impl_->text_index_buffer_) { - impl_->vulkan_->destroy_buffer(impl_->text_index_buffer_); - } + if (impl_->vulkan_) { + if (impl_->vertex_buffer_) { impl_->vulkan_->destroy_buffer(impl_->vertex_buffer_); } + if (impl_->text_vertex_buffer_) { impl_->vulkan_->destroy_buffer(impl_->text_vertex_buffer_); } + if (impl_->text_index_buffer_) { impl_->vulkan_->destroy_buffer(impl_->text_index_buffer_); } + if (impl_->depth_map_vertex_buffer_) { + impl_->vulkan_->destroy_buffer(impl_->depth_map_vertex_buffer_); + } + if (impl_->depth_map_index_buffer_) { + impl_->vulkan_->destroy_buffer(impl_->depth_map_index_buffer_); + } + if (impl_->depth_map_color_buffer_) { + impl_->vulkan_->destroy_buffer(impl_->depth_map_color_buffer_); } + } } void GeometryLayer::color(float r, float g, float b, float a) { - impl_->attributes_.color_[0] = r; - impl_->attributes_.color_[1] = g; - impl_->attributes_.color_[2] = b; - impl_->attributes_.color_[3] = a; + impl_->attributes_.color_[0] = r; + impl_->attributes_.color_[1] = g; + impl_->attributes_.color_[2] = b; + impl_->attributes_.color_[3] = a; } void GeometryLayer::line_width(float width) { - impl_->attributes_.line_width_ = width; + impl_->attributes_.line_width_ = width; } void GeometryLayer::point_size(float size) { - impl_->attributes_.point_size_ = size; + impl_->attributes_.point_size_ = size; } void GeometryLayer::primitive(PrimitiveTopology topology, uint32_t primitive_count, - size_t data_size, const float *data) { - if (primitive_count == 0) { - throw std::invalid_argument("primitive_count should not be zero"); - } - if (data_size == 0) { - throw std::invalid_argument("data_size should not be zero"); - } - if (data == nullptr) { - throw std::invalid_argument("data should not be nullptr"); - } - - uint32_t required_data_size; - std::vector vertex_counts; - VkPrimitiveTopology vkTopology; - switch (topology) { + size_t data_size, const float* data) { + if (primitive_count == 0) { throw std::invalid_argument("primitive_count should not be zero"); } + if (data_size == 0) { throw std::invalid_argument("data_size should not be zero"); } + if (data == nullptr) { throw std::invalid_argument("data should not be nullptr"); } + + uint32_t required_data_size; + std::vector vertex_counts; + vk::PrimitiveTopology vkTopology; + switch (topology) { case PrimitiveTopology::POINT_LIST: - required_data_size = primitive_count * 2; - vertex_counts.push_back(required_data_size / 2); - vkTopology = VK_PRIMITIVE_TOPOLOGY_POINT_LIST; - break; + required_data_size = primitive_count * 2; + vertex_counts.push_back(required_data_size / 2); + vkTopology = vk::PrimitiveTopology::ePointList; + break; case PrimitiveTopology::LINE_LIST: - required_data_size = primitive_count * 2 * 2; - vertex_counts.push_back(required_data_size / 2); - vkTopology = VK_PRIMITIVE_TOPOLOGY_LINE_LIST; - break; + required_data_size = primitive_count * 2 * 2; + vertex_counts.push_back(required_data_size / 2); + vkTopology = vk::PrimitiveTopology::eLineList; + break; case PrimitiveTopology::LINE_STRIP: - required_data_size = 2 + primitive_count * 2; - vertex_counts.push_back(required_data_size / 2); - vkTopology = VK_PRIMITIVE_TOPOLOGY_LINE_STRIP; - break; + required_data_size = 2 + primitive_count * 2; + vertex_counts.push_back(required_data_size / 2); + vkTopology = vk::PrimitiveTopology::eLineStrip; + break; case PrimitiveTopology::TRIANGLE_LIST: - required_data_size = primitive_count * 3 * 2; - vertex_counts.push_back(required_data_size / 2); - vkTopology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; - break; + required_data_size = primitive_count * 3 * 2; + vertex_counts.push_back(required_data_size / 2); + vkTopology = vk::PrimitiveTopology::eTriangleList; + break; case PrimitiveTopology::CROSS_LIST: - required_data_size = primitive_count * 3; - vertex_counts.push_back(primitive_count * 4); - vkTopology = VK_PRIMITIVE_TOPOLOGY_LINE_LIST; - break; + required_data_size = primitive_count * 3; + vertex_counts.push_back(primitive_count * 4); + vkTopology = vk::PrimitiveTopology::eLineList; + break; case PrimitiveTopology::RECTANGLE_LIST: - required_data_size = primitive_count * 2 * 2; - for (uint32_t i = 0; i < primitive_count; ++i) { - vertex_counts.push_back(5); - } - vkTopology = VK_PRIMITIVE_TOPOLOGY_LINE_STRIP; - break; + required_data_size = primitive_count * 2 * 2; + for (uint32_t i = 0; i < primitive_count; ++i) { vertex_counts.push_back(5); } + vkTopology = vk::PrimitiveTopology::eLineStrip; + break; case PrimitiveTopology::OVAL_LIST: - required_data_size = primitive_count * 4; - for (uint32_t i = 0; i < primitive_count; ++i) { - vertex_counts.push_back(CIRCLE_SEGMENTS + 1); - } - vkTopology = VK_PRIMITIVE_TOPOLOGY_LINE_STRIP; - break; - } - - if (data_size < required_data_size) { - std::stringstream buf; - buf << "Required data array size is " << required_data_size << " but only " << data_size << - " where specified"; - throw std::runtime_error(buf.str().c_str()); - } + required_data_size = primitive_count * 4; + for (uint32_t i = 0; i < primitive_count; ++i) { + vertex_counts.push_back(CIRCLE_SEGMENTS + 1); + } + vkTopology = vk::PrimitiveTopology::eLineStrip; + break; + } + + if (data_size < required_data_size) { + std::stringstream buf; + buf << "Required data array size is " << required_data_size << " but only " << data_size + << " where specified"; + throw std::runtime_error(buf.str().c_str()); + } + + impl_->primitives_.emplace_back(impl_->attributes_, + topology, + primitive_count, + data_size, + data, + impl_->vertex_count_, + vertex_counts, + vkTopology); + + for (auto&& vertex_count : vertex_counts) { impl_->vertex_count_ += vertex_count; } +} - impl_->primitives_.emplace_back(impl_->attributes_, topology, primitive_count, data_size, data, - impl_->vertex_count_, vertex_counts, vkTopology); +void GeometryLayer::text(float x, float y, float size, const char* text) { + if (size == 0) { throw std::invalid_argument("size should not be zero"); } + if (text == nullptr) { throw std::invalid_argument("text should not be nullptr"); } - for (auto &&vertex_count : vertex_counts) { - impl_->vertex_count_ += vertex_count; - } + impl_->texts_.emplace_back(impl_->attributes_, x, y, size, text); } -void GeometryLayer::text(float x, float y, float size, const char *text) { - if (size == 0) { - throw std::invalid_argument("size should not be zero"); - } - if (text == nullptr) { - throw std::invalid_argument("text should not be nullptr"); - } - - impl_->texts_.emplace_back(impl_->attributes_, x, y, size, text); +void GeometryLayer::depth_map(DepthMapRenderMode render_mode, uint32_t width, uint32_t height, + ImageFormat depth_fmt, CUdeviceptr depth_device_ptr, + ImageFormat color_fmt, CUdeviceptr color_device_ptr) { + if ((width == 0) || (height == 0)) { + throw std::invalid_argument("width or height should not be zero"); + } + if (depth_fmt != ImageFormat::R8_UNORM) { + throw std::invalid_argument("The depth format should be ImageFormat::R8_UNORM"); + } + if (depth_device_ptr == 0) { + throw std::invalid_argument("The depth device pointer should not be 0"); + } + if (color_device_ptr && (color_fmt != ImageFormat::R8G8B8A8_UNORM)) { + throw std::invalid_argument("The color format should be ImageFormat::R8G8B8A8_UNORM"); + } + + impl_->depth_maps_.emplace_back(impl_->attributes_, + render_mode, + width, + height, + depth_device_ptr, + color_device_ptr, + Context::get().get_cuda_stream()); } -bool GeometryLayer::can_be_reused(Layer &other) const { - return Layer::can_be_reused(other) && impl_->can_be_reused(*static_cast - (other).impl_.get()); +bool GeometryLayer::can_be_reused(Layer& other) const { + return Layer::can_be_reused(other) && + impl_->can_be_reused(*static_cast(other).impl_.get()); } -void GeometryLayer::end(Vulkan *vulkan) { - if (!impl_->primitives_.empty()) { - if (!impl_->vertex_buffer_) { - /// @todo need to remember Vulkan instance for destroying buffer, - /// destroy should probably be handled by Vulkan class - impl_->vulkan_ = vulkan; - - // setup the vertex buffer - std::vector vertices; - vertices.reserve(impl_->vertex_count_ * 2); - - for (auto &&primitive : impl_->primitives_) { - switch (primitive.topology_) { - case PrimitiveTopology::POINT_LIST: - case PrimitiveTopology::LINE_LIST: - case PrimitiveTopology::LINE_STRIP: - case PrimitiveTopology::TRIANGLE_LIST: - // just copy - vertices.insert(vertices.end(), primitive.data_.begin(), primitive.data_.end()); - break; - case PrimitiveTopology::CROSS_LIST: - // generate crosses - for (uint32_t index = 0; index < primitive.primitive_count_; ++index) { - const float x = primitive.data_[index * 3 + 0]; - const float y = primitive.data_[index * 3 + 1]; - const float s = primitive.data_[index * 3 + 2] * 0.5f; - vertices.insert(vertices.end(), {x - s, y, x + s, y, x, y - s, x, y + s}); - } - break; - case PrimitiveTopology::RECTANGLE_LIST: - // generate rectangles - for (uint32_t index = 0; index < primitive.primitive_count_; ++index) { - const float x0 = primitive.data_[index * 4 + 0]; - const float y0 = primitive.data_[index * 4 + 1]; - const float x1 = primitive.data_[index * 4 + 2]; - const float y1 = primitive.data_[index * 4 + 3]; - vertices.insert(vertices.end(), {x0, y0, x1, y0, x1, y1, x0, y1, x0, y0}); - } - break; - case PrimitiveTopology::OVAL_LIST: - for (uint32_t index = 0; index < primitive.primitive_count_; ++index) { - const float x = primitive.data_[index * 4 + 0]; - const float y = primitive.data_[index * 4 + 1]; - const float rx = primitive.data_[index * 4 + 2] * 0.5f; - const float ry = primitive.data_[index * 4 + 3] * 0.5f; - for (uint32_t segment = 0; segment <= CIRCLE_SEGMENTS; ++segment) { - const float rad = (2.f * M_PI) / CIRCLE_SEGMENTS * segment; - const float px = x + std::cos(rad) * rx; - const float py = y + std::sin(rad) * ry; - vertices.insert(vertices.end(), {px, py}); - } - } - break; - } - } +void GeometryLayer::end(Vulkan* vulkan) { + // if the aspect ratio changed, re-create the text and primitive buffers because the generated + // vertex positions depend on the aspect ratio + if (impl_->aspect_ratio_ != vulkan->get_window()->get_aspect_ratio()) { + impl_->aspect_ratio_ = vulkan->get_window()->get_aspect_ratio(); - impl_->vertex_buffer_ = vulkan->create_buffer(vertices.size() * sizeof(float), - vertices.data(), - VK_BUFFER_USAGE_VERTEX_BUFFER_BIT); - } + impl_->text_draw_list_.reset(); + if (impl_->text_vertex_buffer_) { + impl_->vulkan_->destroy_buffer(impl_->text_vertex_buffer_); + impl_->text_vertex_buffer_ = nullptr; + } + if (impl_->text_index_buffer_) { + impl_->vulkan_->destroy_buffer(impl_->text_index_buffer_); + impl_->text_index_buffer_ = nullptr; } - if (!impl_->texts_.empty()) { - if (!impl_->text_draw_list_) { - impl_->text_draw_list_.reset(new ImDrawList(ImGui::GetDrawListSharedData())); - impl_->text_draw_list_->_ResetForNewFrame(); - - // ImGui is using integer coordinates for the text position, we use the 0...1 range. - // Therefore generate vertices in larger scale and scale them down afterwards. - const float scale = 16384.f; - const ImVec4 clip_rect(0.f, 0.f, scale, scale); - - for (auto &&text : impl_->texts_) { - const ImU32 color = - ImGui::ColorConvertFloat4ToU32(ImVec4(text.attributes_.color_[0], - text.attributes_.color_[1], - text.attributes_.color_[2], - text.attributes_.color_[3])); - ImGui::GetFont()->RenderText(impl_->text_draw_list_.get(), text.size_ * scale, - ImVec2(text.x_ * scale, text.y_ * scale), color, - clip_rect, text.text_.c_str(), - text.text_.c_str() + text.text_.size()); + // only crosses depend on the aspect ratio + bool has_crosses = false; + for (auto&& primitive : impl_->primitives_) { + if (primitive.topology_ == PrimitiveTopology::CROSS_LIST) { + has_crosses = true; + break; + } + } + if (has_crosses) { + if (impl_->vertex_buffer_) { + impl_->vulkan_->destroy_buffer(impl_->vertex_buffer_); + impl_->vertex_buffer_ = nullptr; + } + } + } + + if (!impl_->primitives_.empty()) { + if (!impl_->vertex_buffer_) { + /// @todo need to remember Vulkan instance for destroying buffer, + /// destroy should probably be handled by Vulkan class + impl_->vulkan_ = vulkan; + + // setup the vertex buffer + std::vector vertices; + vertices.reserve(impl_->vertex_count_ * 3); + + for (auto&& primitive : impl_->primitives_) { + switch (primitive.topology_) { + case PrimitiveTopology::POINT_LIST: + case PrimitiveTopology::LINE_LIST: + case PrimitiveTopology::LINE_STRIP: + case PrimitiveTopology::TRIANGLE_LIST: + // just copy + for (uint32_t index = 0; index < primitive.data_.size() / 2; ++index) { + vertices.insert( + vertices.end(), + {primitive.data_[index * 2 + 0], primitive.data_[index * 2 + 1], 0.f}); + } + break; + case PrimitiveTopology::CROSS_LIST: + // generate crosses + for (uint32_t index = 0; index < primitive.primitive_count_; ++index) { + const float x = primitive.data_[index * 3 + 0]; + const float y = primitive.data_[index * 3 + 1]; + const float sy = primitive.data_[index * 3 + 2] * 0.5f; + const float sx = sy / impl_->aspect_ratio_; + vertices.insert(vertices.end(), + {x - sx, y, 0.f, x + sx, y, 0.f, x, y - sy, 0.f, x, y + sy, 0.f}); } + break; + case PrimitiveTopology::RECTANGLE_LIST: + // generate rectangles + for (uint32_t index = 0; index < primitive.primitive_count_; ++index) { + const float x0 = primitive.data_[index * 4 + 0]; + const float y0 = primitive.data_[index * 4 + 1]; + const float x1 = primitive.data_[index * 4 + 2]; + const float y1 = primitive.data_[index * 4 + 3]; + vertices.insert(vertices.end(), + {x0, y0, 0.f, x1, y0, 0.f, x1, y1, 0.f, x0, y1, 0.f, x0, y0, 0.f}); + } + break; + case PrimitiveTopology::OVAL_LIST: + for (uint32_t index = 0; index < primitive.primitive_count_; ++index) { + const float x = primitive.data_[index * 4 + 0]; + const float y = primitive.data_[index * 4 + 1]; + const float rx = primitive.data_[index * 4 + 2] * 0.5f; + const float ry = primitive.data_[index * 4 + 3] * 0.5f; + for (uint32_t segment = 0; segment <= CIRCLE_SEGMENTS; ++segment) { + const float rad = (2.f * M_PI) / CIRCLE_SEGMENTS * segment; + const float px = x + std::cos(rad) * rx; + const float py = y + std::sin(rad) * ry; + vertices.insert(vertices.end(), {px, py, 0.f}); + } + } + break; + } + } - // text might be completely out of clip rectangle, - // if this is the case no vertices had been generated - if (impl_->text_draw_list_->VtxBuffer.size() != 0) { - // scale back vertex data - const float inv_scale = 1.f / scale; - ImDrawVert *vertex = impl_->text_draw_list_->VtxBuffer.Data; - ImDrawVert *vertex_end = vertex + impl_->text_draw_list_->VtxBuffer.size(); - while (vertex < vertex_end) { - vertex->pos.x *= inv_scale; - vertex->pos.y *= inv_scale; - ++vertex; + impl_->vertex_buffer_ = vulkan->create_buffer( + vertices.size() * sizeof(float), vertices.data(), vk::BufferUsageFlagBits::eVertexBuffer); + } + } + + if (!impl_->texts_.empty()) { + if (!impl_->text_draw_list_) { + impl_->text_draw_list_.reset(new ImDrawList(ImGui::GetDrawListSharedData())); + impl_->text_draw_list_->_ResetForNewFrame(); + + // ImGui is using integer coordinates for the text position, we use the 0...1 range. + // Therefore generate vertices in larger scale and scale them down afterwards. + const float scale = 16384.f; + const ImVec4 clip_rect(0.f, 0.f, scale * std::max(1.f, impl_->aspect_ratio_), scale); + const float inv_scale = 1.f / scale; + + ImDrawVert *vertex_base = nullptr, *vertex = nullptr; + for (auto&& text : impl_->texts_) { + const ImU32 color = ImGui::ColorConvertFloat4ToU32(ImVec4(text.attributes_.color_[0], + text.attributes_.color_[1], + text.attributes_.color_[2], + text.attributes_.color_[3])); + ImGui::GetFont()->RenderText(impl_->text_draw_list_.get(), + text.size_ * scale, + ImVec2(text.x_ * scale, text.y_ * scale), + color, + clip_rect, + text.text_.c_str(), + text.text_.c_str() + text.text_.size()); + + // scale back vertex data + if (vertex_base != impl_->text_draw_list_->VtxBuffer.Data) { + const size_t offset = vertex - vertex_base; + vertex_base = impl_->text_draw_list_->VtxBuffer.Data; + vertex = vertex_base + offset; + } + while (vertex < impl_->text_draw_list_->_VtxWritePtr) { + vertex->pos.x = + (vertex->pos.x * inv_scale - text.x_) * (1.f / impl_->aspect_ratio_) + text.x_; + vertex->pos.y *= inv_scale; + ++vertex; + } + } + + // text might be completely out of clip rectangle, + // if this is the case no vertices had been generated + if (impl_->text_draw_list_->VtxBuffer.size() != 0) { + /// @todo need to remember Vulkan instance for destroying buffer, destroy should + // probably be handled by Vulkan class + impl_->vulkan_ = vulkan; + + impl_->text_vertex_buffer_ = + vulkan->create_buffer(impl_->text_draw_list_->VtxBuffer.size() * sizeof(ImDrawVert), + impl_->text_draw_list_->VtxBuffer.Data, + vk::BufferUsageFlagBits::eVertexBuffer); + impl_->text_index_buffer_ = + vulkan->create_buffer(impl_->text_draw_list_->IdxBuffer.size() * sizeof(ImDrawIdx), + impl_->text_draw_list_->IdxBuffer.Data, + vk::BufferUsageFlagBits::eIndexBuffer); + } else { + impl_->text_draw_list_.reset(); + } + } + } + + if (!impl_->depth_maps_.empty()) { + // allocate vertex buffer + if (!impl_->depth_map_vertex_buffer_) { + /// @todo need to remember Vulkan instance for destroying buffer, destroy should probably be + /// handled by Vulkan class + impl_->vulkan_ = vulkan; + + // calculate the index count needed + size_t index_count = 0; + bool has_color_buffer = false; + impl_->depth_map_vertex_count_ = 0; + for (auto&& depth_map : impl_->depth_maps_) { + switch (depth_map.render_mode_) { + case DepthMapRenderMode::POINTS: + // points don't need indices + depth_map.index_count_ = 0; + break; + case DepthMapRenderMode::LINES: + // each point in the depth map needs two lines (each line has two indices) except the + // last column and row which needs one line only + depth_map.index_count_ = ((depth_map.width_ - 1) * (depth_map.height_ - 1) * 2 + + (depth_map.width_ - 1) + (depth_map.height_ - 1)) * + 2; + break; + case DepthMapRenderMode::TRIANGLES: + // need two triangle (each triangle has three indices) for each full quad of depth + // points + depth_map.index_count_ = (depth_map.width_ - 1) * (depth_map.height_ - 1) * 2 * 3; + break; + default: + throw std::runtime_error("Unhandled render mode"); + } + index_count += depth_map.index_count_; + has_color_buffer |= (depth_map.color_device_ptr_ != 0); + impl_->depth_map_vertex_count_ += depth_map.width_ * depth_map.height_; + } + + if (index_count) { + // generate index data + std::unique_ptr index_data(new uint32_t[index_count]); + + uint32_t* dst = index_data.get(); + for (auto&& depth_map : impl_->depth_maps_) { + depth_map.index_offset_ = dst - index_data.get(); + + switch (depth_map.render_mode_) { + case DepthMapRenderMode::LINES: + for (uint32_t y = 0; y < depth_map.height_ - 1; ++y) { + for (uint32_t x = 0; x < depth_map.width_ - 1; ++x) { + const uint32_t i = (y * depth_map.width_) + x; + dst[0] = i; + dst[1] = i + 1; + dst[2] = i; + dst[3] = i + depth_map.width_; + dst += 4; } - - /// @todo need to remember Vulkan instance for destroying buffer, destroy should - // probably be handled by Vulkan class - impl_->vulkan_ = vulkan; - - impl_->text_vertex_buffer_ = - vulkan->create_buffer(impl_->text_draw_list_-> - VtxBuffer.size() * sizeof(ImDrawVert), - impl_->text_draw_list_->VtxBuffer.Data, - VK_BUFFER_USAGE_VERTEX_BUFFER_BIT); - impl_->text_index_buffer_ = - vulkan->create_buffer(impl_->text_draw_list_-> - IdxBuffer.size() * sizeof(ImDrawIdx), - impl_->text_draw_list_->IdxBuffer.Data, - VK_BUFFER_USAGE_INDEX_BUFFER_BIT); - } else { - impl_->text_draw_list_.reset(); - } + // last column + const uint32_t i = (y * depth_map.width_) + depth_map.width_ - 1; + dst[0] = i; + dst[1] = i + depth_map.width_; + dst += 2; + } + // last row + for (uint32_t x = 0; x < depth_map.width_ - 1; ++x) { + const uint32_t i = ((depth_map.height_ - 1) * depth_map.width_) + x; + dst[0] = i; + dst[1] = i + 1; + dst += 2; + } + break; + case DepthMapRenderMode::TRIANGLES: + for (uint32_t y = 0; y < depth_map.height_ - 1; ++y) { + for (uint32_t x = 0; x < depth_map.width_ - 1; ++x) { + const uint32_t i = (y * depth_map.width_) + x; + dst[0] = i; + dst[1] = i + 1; + dst[2] = i + depth_map.width_; + dst[3] = i + 1; + dst[4] = i + depth_map.width_ + 1; + dst[5] = i + depth_map.width_; + dst += 6; + } + } + break; + default: + throw std::runtime_error("Unhandled render mode"); + } } + if ((dst - index_data.get()) != index_count) { + throw std::runtime_error("Index count mismatch."); + } + impl_->depth_map_index_buffer_ = + vulkan->create_buffer(index_count * sizeof(uint32_t), + index_data.get(), + vk::BufferUsageFlagBits::eIndexBuffer); + } + + if (has_color_buffer) { + impl_->depth_map_color_buffer_ = vulkan->create_buffer_for_cuda_interop( + impl_->depth_map_vertex_count_ * 4 * sizeof(uint8_t), + vk::BufferUsageFlagBits::eVertexBuffer); + } + + impl_->depth_map_vertex_buffer_ = + vulkan->create_buffer(impl_->depth_map_vertex_count_ * 3 * sizeof(float), + nullptr, + vk::BufferUsageFlagBits::eVertexBuffer); } -} -void GeometryLayer::render(Vulkan *vulkan) { - // draw geometry primitives - for (auto &&primitive : impl_->primitives_) { - uint32_t vertex_offset = primitive.vertex_offset_; - for (auto &&vertex_count : primitive.vertex_counts_) { - vulkan->draw(primitive.vk_topology_, vertex_count, vertex_offset, - impl_->vertex_buffer_, get_opacity(), - primitive.attributes_.color_, primitive.attributes_.point_size_, - primitive.attributes_.line_width_); - vertex_offset += vertex_count; + // generate and upload vertex data + std::unique_ptr vertex_data(new float[impl_->depth_map_vertex_count_ * 3]); + float* dst = vertex_data.get(); + + const CudaService::ScopedPush cuda_context = CudaService::get().PushContext(); + + for (auto&& depth_map : impl_->depth_maps_) { + depth_map.vertex_offset_ = (dst - vertex_data.get()) / 3; + + // download the depth map data and create 3D position data for each depth value + /// @todo this is not optimal. It would be faster to use a vertex or geometry shader which + /// directly fetches the depth data and generates primitive (geometry shader) or emits + /// the 3D position vertex (vertex shader) + std::unique_ptr host_data(new uint8_t[depth_map.width_ * depth_map.height_]); + CudaCheck(cuMemcpyDtoHAsync(reinterpret_cast(host_data.get()), + depth_map.depth_device_ptr_, + depth_map.width_ * depth_map.height_ * sizeof(uint8_t), + depth_map.cuda_stream_)); + CudaCheck(cuStreamSynchronize(depth_map.cuda_stream_)); + + const uint8_t* src = host_data.get(); + const float inv_width = 1.f / float(depth_map.width_); + const float inv_height = 1.f / float(depth_map.height_); + for (uint32_t y = 0; y < depth_map.height_; ++y) { + for (uint32_t x = 0; x < depth_map.width_; ++x) { + dst[0] = float(x) * inv_width - 0.5f; + dst[1] = float(y) * inv_height - 0.5f; + dst[2] = float(src[0]) / 255.f; + src += 1; + dst += 3; } + } } - // draw text - if (impl_->text_draw_list_) { - for (int i = 0; i < impl_->text_draw_list_->CmdBuffer.size(); ++i) { - const ImDrawCmd *pcmd = &impl_->text_draw_list_->CmdBuffer[i]; - vulkan->draw_indexed(reinterpret_cast(ImGui::GetIO().Fonts->TexID), - impl_->text_vertex_buffer_, impl_->text_index_buffer_, - (sizeof(ImDrawIdx) == 2) ? - VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32, pcmd->ElemCount, - pcmd->IdxOffset, pcmd->VtxOffset, get_opacity()); - } + if ((dst - vertex_data.get()) != impl_->depth_map_vertex_count_ * 3) { + throw std::runtime_error("Vertex data count mismatch."); + } + + // upload vertex data + vulkan->upload_to_buffer(impl_->depth_map_vertex_count_ * 3 * sizeof(float), + vertex_data.get(), + impl_->depth_map_vertex_buffer_); + + // upload color data + size_t offset = 0; + for (auto&& depth_map : impl_->depth_maps_) { + const size_t size = depth_map.width_ * depth_map.height_ * sizeof(uint8_t) * 4; + if (depth_map.color_device_ptr_) { + vulkan->upload_to_buffer(size, + depth_map.color_device_ptr_, + impl_->depth_map_color_buffer_, + offset, + depth_map.cuda_stream_); + } + offset += size; + } + } +} + +void GeometryLayer::render(Vulkan* vulkan) { + // setup the view matrix in a way that geometry coordinates are in the range [0...1] + nvmath::mat4f view_matrix; + view_matrix.identity(); + view_matrix.scale({2.f, 2.f, 1.f}); + view_matrix.translate({-.5f, -.5f, 0.f}); + + // draw geometry primitives + for (auto&& primitive : impl_->primitives_) { + uint32_t vertex_offset = primitive.vertex_offset_; + for (auto&& vertex_count : primitive.vertex_counts_) { + vulkan->draw(primitive.vk_topology_, + vertex_count, + vertex_offset, + {impl_->vertex_buffer_}, + get_opacity(), + primitive.attributes_.color_, + primitive.attributes_.point_size_, + primitive.attributes_.line_width_, + view_matrix); + vertex_offset += vertex_count; + } + } + + // draw text + if (impl_->text_draw_list_) { + for (int i = 0; i < impl_->text_draw_list_->CmdBuffer.size(); ++i) { + const ImDrawCmd* pcmd = &impl_->text_draw_list_->CmdBuffer[i]; + vulkan->draw_text_indexed( + vk::DescriptorSet(reinterpret_cast(ImGui::GetIO().Fonts->TexID)), + impl_->text_vertex_buffer_, + impl_->text_index_buffer_, + (sizeof(ImDrawIdx) == 2) ? vk::IndexType::eUint16 : vk::IndexType::eUint32, + pcmd->ElemCount, + pcmd->IdxOffset, + pcmd->VtxOffset, + get_opacity(), + view_matrix); + } + } + + // draw depth maps + if (!impl_->depth_maps_.empty()) { + vulkan->get_window()->get_view_matrix(&view_matrix); + + for (auto&& depth_map : impl_->depth_maps_) { + std::vector vertex_buffers; + vertex_buffers.push_back(impl_->depth_map_vertex_buffer_); + if (depth_map.color_device_ptr_) { vertex_buffers.push_back(impl_->depth_map_color_buffer_); } + + if ((depth_map.render_mode_ == DepthMapRenderMode::LINES) || + (depth_map.render_mode_ == DepthMapRenderMode::TRIANGLES)) { + vulkan->draw_indexed((depth_map.render_mode_ == DepthMapRenderMode::LINES) + ? vk::PrimitiveTopology::eLineList + : vk::PrimitiveTopology::eTriangleList, + vertex_buffers, + impl_->depth_map_index_buffer_, + vk::IndexType::eUint32, + depth_map.index_count_, + depth_map.index_offset_, + depth_map.vertex_offset_, + get_opacity(), + depth_map.attributes_.color_, + depth_map.attributes_.point_size_, + depth_map.attributes_.line_width_, + view_matrix); + } else if (depth_map.render_mode_ == DepthMapRenderMode::POINTS) { + vulkan->draw(vk::PrimitiveTopology::ePointList, + depth_map.width_ * depth_map.height_, + depth_map.vertex_offset_, + vertex_buffers, + get_opacity(), + depth_map.attributes_.color_, + depth_map.attributes_.point_size_, + depth_map.attributes_.line_width_, + view_matrix); + } else { + throw std::runtime_error("Unhandled depth render mode."); + } } + } } } // namespace holoscan::viz diff --git a/modules/holoviz/src/layers/geometry_layer.hpp b/modules/holoviz/src/layers/geometry_layer.hpp index 3a7c7afa..4c7091b0 100644 --- a/modules/holoviz/src/layers/geometry_layer.hpp +++ b/modules/holoviz/src/layers/geometry_layer.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,14 +15,18 @@ * limitations under the License. */ -#ifndef HOLOSCAN_VIZ_LAYERS_GEOMETRY_LAYER_HPP -#define HOLOSCAN_VIZ_LAYERS_GEOMETRY_LAYER_HPP +#ifndef MODULES_HOLOVIZ_SRC_LAYERS_GEOMETRY_LAYER_HPP +#define MODULES_HOLOVIZ_SRC_LAYERS_GEOMETRY_LAYER_HPP + +#include #include #include #include "layer.hpp" +#include "../holoviz/depth_map_render_mode.hpp" +#include "../holoviz/image_format.hpp" #include "../holoviz/primitive_topology.hpp" namespace holoscan::viz { @@ -32,71 +36,86 @@ namespace holoscan::viz { */ class GeometryLayer : public Layer { public: - /** - * Construct a new GeometryLayer object. - */ - GeometryLayer(); - - /** - * Destroy the GeometryLayer object. - */ - ~GeometryLayer(); - - /** - * Set the color for following geometry. - * - * @param r,g,b,a RGBA color. Default (1.0, 1.0, 1.0, 1.0). - */ - void color(float r, float g, float b, float a); - - /** - * Set the line width for geometry made of lines. - * - * @param width line width in pixels. Default 1.0. - */ - void line_width(float width); - - /** - * Set the point size for geometry made of points. - * - * @param size point size in pixels. Default 1.0. - */ - void point_size(float size); - - /** - * Draw a geometric primitive. - * - * @param topology primitive topology - * @param primitive_count primitive count - * @param data_size size of the data array in floats - * @param data pointer to data, the format and size of the array - * depends on the primitive count and topology - */ - void primitive(PrimitiveTopology topology, uint32_t primitive_count, size_t data_size, - const float *data); - - /** - * Draw text. - * - * @param x x coordinate - * @param y y coordinate - * @param size font size - * @param text text to draw - */ - void text(float x, float y, float size, const char *text); - - /// holoscan::viz::Layer virtual members - ///@{ - bool can_be_reused(Layer &other) const override; - void end(Vulkan *vulkan) override; - void render(Vulkan *vulkan) override; - ///@} + /** + * Construct a new GeometryLayer object. + */ + GeometryLayer(); + + /** + * Destroy the GeometryLayer object. + */ + ~GeometryLayer(); + + /** + * Set the color for following geometry. + * + * @param r,g,b,a RGBA color. Default (1.0, 1.0, 1.0, 1.0). + */ + void color(float r, float g, float b, float a); + + /** + * Set the line width for geometry made of lines. + * + * @param width line width in pixels. Default 1.0. + */ + void line_width(float width); + + /** + * Set the point size for geometry made of points. + * + * @param size point size in pixels. Default 1.0. + */ + void point_size(float size); + + /** + * Draw a geometric primitive. + * + * @param topology primitive topology + * @param primitive_count primitive count + * @param data_size size of the data array in floats + * @param data pointer to data, the format and size of the array + * depends on the primitive count and topology + */ + void primitive(PrimitiveTopology topology, uint32_t primitive_count, size_t data_size, + const float* data); + + /** + * Draw text. + * + * @param x x coordinate + * @param y y coordinate + * @param size font size + * @param text text to draw + */ + void text(float x, float y, float size, const char* text); + + /** + * Render a depth map. + * + * @param render_mode depth map render mode + * @param width width of the depth map + * @param height height of the depth map + * @param depth_fmt format of the depth map data (has to be ImageFormat::R8_UNORM) + * @param depth_device_ptr Cuda device memory pointer holding the depth data + * @param color_fmt format of the color data (has to be ImageFormat::R8G8B8A8_UNORM) + * @param color_device_ptr Cuda device memory pointer holding the color data (optional) + */ + void depth_map(DepthMapRenderMode render_mode, uint32_t width, uint32_t height, + ImageFormat depth_fmt, CUdeviceptr depth_device_ptr, ImageFormat color_fmt, + CUdeviceptr color_device_ptr); + + /// holoscan::viz::Layer virtual members + ///@{ + bool can_be_reused(Layer& other) const override; + void end(Vulkan* vulkan) override; + void render(Vulkan* vulkan) override; + ///@} private: - struct Impl; - std::shared_ptr impl_; + struct Impl; + std::shared_ptr impl_; }; } // namespace holoscan::viz -#endif /* HOLOSCAN_VIZ_LAYERS_GEOMETRY_LAYER_HPP */ +#endif /* MODULES_HOLOVIZ_SRC_LAYERS_GEOMETRY_LAYER_HPP */ diff --git a/modules/holoviz/src/layers/im_gui_layer.cpp b/modules/holoviz/src/layers/im_gui_layer.cpp index 30fc8544..46017a2a 100644 --- a/modules/holoviz/src/layers/im_gui_layer.cpp +++ b/modules/holoviz/src/layers/im_gui_layer.cpp @@ -20,52 +20,48 @@ #include #include -#include "../vulkan/vulkan.hpp" +#include "../vulkan/vulkan_app.hpp" namespace holoscan::viz { struct ImGuiLayer::Impl { - bool pushed_style_ = false; + bool pushed_style_ = false; }; -ImGuiLayer::ImGuiLayer() - : Layer(Type::ImGui) - , impl_(new ImGuiLayer::Impl) { - ImGui::PushStyleVar(ImGuiStyleVar_Alpha, get_opacity()); - impl_->pushed_style_ = true; +ImGuiLayer::ImGuiLayer() : Layer(Type::ImGui), impl_(new ImGuiLayer::Impl) { + ImGui::PushStyleVar(ImGuiStyleVar_Alpha, get_opacity()); + impl_->pushed_style_ = true; } ImGuiLayer::~ImGuiLayer() { - if (impl_->pushed_style_) { - ImGui::PopStyleVar(); - } + if (impl_->pushed_style_) { ImGui::PopStyleVar(); } } void ImGuiLayer::set_opacity(float opacity) { - // call the base class - Layer::set_opacity(opacity); + // call the base class + Layer::set_opacity(opacity); - // set the opacity - if (impl_->pushed_style_) { - ImGui::PopStyleVar(); - impl_->pushed_style_ = false; - } - ImGui::PushStyleVar(ImGuiStyleVar_Alpha, get_opacity()); - impl_->pushed_style_ = true; + // set the opacity + if (impl_->pushed_style_) { + ImGui::PopStyleVar(); + impl_->pushed_style_ = false; + } + ImGui::PushStyleVar(ImGuiStyleVar_Alpha, get_opacity()); + impl_->pushed_style_ = true; } -void ImGuiLayer::render(Vulkan *vulkan) { - if (impl_->pushed_style_) { - ImGui::PopStyleVar(); - impl_->pushed_style_ = false; - } +void ImGuiLayer::render(Vulkan* vulkan) { + if (impl_->pushed_style_) { + ImGui::PopStyleVar(); + impl_->pushed_style_ = false; + } - /// @todo we can't renderer multiple ImGui layers, figure out - /// a way to handle that (store DrawData returned by GetDrawData()?) + /// @todo we can't renderer multiple ImGui layers, figure out + /// a way to handle that (store DrawData returned by GetDrawData()?) - // Render UI - ImGui::Render(); - ImGui_ImplVulkan_RenderDrawData(ImGui::GetDrawData(), vulkan->get_command_buffer()); + // Render UI + ImGui::Render(); + ImGui_ImplVulkan_RenderDrawData(ImGui::GetDrawData(), vulkan->get_command_buffer()); } } // namespace holoscan::viz diff --git a/modules/holoviz/src/layers/im_gui_layer.hpp b/modules/holoviz/src/layers/im_gui_layer.hpp index de02088b..37048726 100644 --- a/modules/holoviz/src/layers/im_gui_layer.hpp +++ b/modules/holoviz/src/layers/im_gui_layer.hpp @@ -30,25 +30,25 @@ namespace holoscan::viz { */ class ImGuiLayer : public Layer { public: - /** - * Construct a new ImGuiLayer object. - */ - ImGuiLayer(); - - /** - * Destroy the ImGuiLayer object. - */ - ~ImGuiLayer(); - - /// holoscan::viz::Layer virtual members - ///@{ - void set_opacity(float opacity) override; - void render(Vulkan *vulkan) override; - ///@} + /** + * Construct a new ImGuiLayer object. + */ + ImGuiLayer(); + + /** + * Destroy the ImGuiLayer object. + */ + ~ImGuiLayer(); + + /// holoscan::viz::Layer virtual members + ///@{ + void set_opacity(float opacity) override; + void render(Vulkan* vulkan) override; + ///@} private: - struct Impl; - std::shared_ptr impl_; + struct Impl; + std::shared_ptr impl_; }; } // namespace holoscan::viz diff --git a/modules/holoviz/src/layers/image_layer.cpp b/modules/holoviz/src/layers/image_layer.cpp index 1cc73f4f..1877acf6 100644 --- a/modules/holoviz/src/layers/image_layer.cpp +++ b/modules/holoviz/src/layers/image_layer.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,164 +23,163 @@ #include #include "../context.hpp" -#include "../vulkan/vulkan.hpp" +#include "../vulkan/vulkan_app.hpp" namespace holoscan::viz { struct ImageLayer::Impl { - bool can_be_reused(Impl &other) const { - // we can reuse if the format/size and LUT match and - // if we did not switch from host to device memory and vice versa - if ((format_ == other.format_) && (width_ == other.width_) && (height_ == other.height_) && - (lut_size_ == other.lut_size_) && (lut_format_ == other.lut_format_) && - (lut_data_ == other.lut_data_) && ((host_ptr_ != nullptr) == - (other.device_ptr_ == 0)) && ((device_ptr_ != 0) == (other.host_ptr_ == nullptr))) { - // update the host pointer, Cuda device pointer and the cuda stream. - // Data will be uploaded when drawing regardless is the layer is reused or not - /// @todo this should be made explicit, first check if the layer can be reused and then - /// update the reused layer with these properties below which don't prevent reusing - other.host_ptr_ = host_ptr_; - other.device_ptr_ = device_ptr_; - other.cuda_stream_ = cuda_stream_; - return true; - } - - return false; + bool can_be_reused(Impl& other) const { + // we can reuse if the format/size and LUT match and + // if we did not switch from host to device memory and vice versa + if ((format_ == other.format_) && (width_ == other.width_) && (height_ == other.height_) && + (lut_size_ == other.lut_size_) && (lut_format_ == other.lut_format_) && + (lut_data_ == other.lut_data_) && (lut_normalized_ == other.lut_normalized_) && + ((host_ptr_ != nullptr) == (other.device_ptr_ == 0)) && + ((device_ptr_ != 0) == (other.host_ptr_ == nullptr))) { + // update the host pointer, Cuda device pointer and the cuda stream. + // Data will be uploaded when drawing regardless if the layer is reused or not + /// @todo this should be made explicit, first check if the layer can be reused and then + /// update the reused layer with these properties below which don't prevent reusing + other.host_ptr_ = host_ptr_; + other.device_ptr_ = device_ptr_; + other.cuda_stream_ = cuda_stream_; + return true; } - // user provided state - ImageFormat format_ = ImageFormat(-1); - uint32_t width_ = 0; - uint32_t height_ = 0; - CUdeviceptr device_ptr_ = 0; - const void *host_ptr_ = nullptr; - CUstream cuda_stream_ = 0; - - uint32_t lut_size_ = 0; - ImageFormat lut_format_ = ImageFormat(-1); - std::vector lut_data_; - - // internal state - Vulkan *vulkan_ = nullptr; - Vulkan::Texture *texture_ = nullptr; - Vulkan::Texture *lut_texture_ = nullptr; + return false; + } + + // user provided state + ImageFormat format_ = ImageFormat(-1); + uint32_t width_ = 0; + uint32_t height_ = 0; + CUdeviceptr device_ptr_ = 0; + const void* host_ptr_ = nullptr; + CUstream cuda_stream_ = 0; + + uint32_t lut_size_ = 0; + ImageFormat lut_format_ = ImageFormat(-1); + std::vector lut_data_; + bool lut_normalized_ = false; + + // internal state + Vulkan* vulkan_ = nullptr; + Vulkan::Texture* texture_ = nullptr; + Vulkan::Texture* lut_texture_ = nullptr; }; -ImageLayer::ImageLayer() - : Layer(Type::Image) - , impl_(new ImageLayer::Impl) { -} +ImageLayer::ImageLayer() : Layer(Type::Image), impl_(new ImageLayer::Impl) {} ImageLayer::~ImageLayer() { - if (impl_->vulkan_) { - if (impl_->texture_) { - impl_->vulkan_->destroy_texture(impl_->texture_); - } - if (impl_->lut_texture_) { - impl_->vulkan_->destroy_texture(impl_->lut_texture_); - } - } + if (impl_->vulkan_) { + if (impl_->texture_) { impl_->vulkan_->destroy_texture(impl_->texture_); } + if (impl_->lut_texture_) { impl_->vulkan_->destroy_texture(impl_->lut_texture_); } + } } void ImageLayer::image_cuda_device(uint32_t width, uint32_t height, ImageFormat fmt, - CUdeviceptr device_ptr) { - if (impl_->host_ptr_) { - throw std::runtime_error("Can't simultaneously specify device and host image for a layer."); - } - impl_->width_ = width; - impl_->height_ = height; - impl_->format_ = fmt; - impl_->device_ptr_ = device_ptr; - impl_->cuda_stream_ = Context::get().get_cuda_stream(); + CUdeviceptr device_ptr) { + if (impl_->host_ptr_) { + throw std::runtime_error("Can't simultaneously specify device and host image for a layer."); + } + impl_->width_ = width; + impl_->height_ = height; + impl_->format_ = fmt; + impl_->device_ptr_ = device_ptr; + impl_->cuda_stream_ = Context::get().get_cuda_stream(); } void ImageLayer::image_cuda_array(ImageFormat fmt, CUarray array) { - throw std::runtime_error("Not implemented"); + throw std::runtime_error("Not implemented"); } -void ImageLayer::image_host(uint32_t width, uint32_t height, ImageFormat fmt, const void *data) { - if (impl_->device_ptr_) { - throw std::runtime_error("Can't simultaneously specify device and host image for a layer."); - } - impl_->width_ = width; - impl_->height_ = height; - impl_->format_ = fmt; - impl_->host_ptr_ = data; +void ImageLayer::image_host(uint32_t width, uint32_t height, ImageFormat fmt, const void* data) { + if (impl_->device_ptr_) { + throw std::runtime_error("Can't simultaneously specify device and host image for a layer."); + } + impl_->width_ = width; + impl_->height_ = height; + impl_->format_ = fmt; + impl_->host_ptr_ = data; } -void ImageLayer::lut(uint32_t size, ImageFormat fmt, size_t data_size, const void *data) { - impl_->lut_size_ = size; - impl_->lut_format_ = fmt; - impl_->lut_data_.assign(reinterpret_cast(data), - reinterpret_cast(data) + data_size); +void ImageLayer::lut(uint32_t size, ImageFormat fmt, size_t data_size, const void* data, + bool normalized) { + impl_->lut_size_ = size; + impl_->lut_format_ = fmt; + impl_->lut_data_.assign(reinterpret_cast(data), + reinterpret_cast(data) + data_size); + impl_->lut_normalized_ = normalized; } -bool ImageLayer::can_be_reused(Layer &other) const { - return Layer::can_be_reused(other) && impl_->can_be_reused(*static_cast - (other).impl_.get()); +bool ImageLayer::can_be_reused(Layer& other) const { + return Layer::can_be_reused(other) && + impl_->can_be_reused(*static_cast(other).impl_.get()); } -void ImageLayer::end(Vulkan *vulkan) { - if (impl_->device_ptr_) { - // check if this is a reused layer, in this case - // we just have to upload the data to the texture - if (!impl_->texture_) { - /// @todo need to remember Vulkan instance for destroying texture, - /// destroy should probably be handled by Vulkan class - impl_->vulkan_ = vulkan; - - // check if we have a lut, if yes, the texture needs to - // be nearest sampled since it has index values - const bool has_lut = !impl_->lut_data_.empty(); - - // create a texture to which we can upload from CUDA - impl_->texture_ = vulkan->create_texture_for_cuda_upload(impl_->width_, impl_->height_, - impl_->format_, has_lut ? VK_FILTER_NEAREST : VK_FILTER_LINEAR); - - if (has_lut) { - // create LUT texture - impl_->lut_texture_ = - vulkan->create_texture(impl_->lut_size_, 1, impl_->lut_format_, - impl_->lut_data_.size(), impl_->lut_data_.data(), - VK_FILTER_NEAREST, false /*normalized*/); - } - } - - vulkan->upload_to_texture(impl_->device_ptr_, impl_->texture_, impl_->cuda_stream_); - } else if (impl_->host_ptr_) { - // check if this is a reused layer, - // in this case we just have to upload the data to the texture - if (!impl_->texture_) { - /// @todo need to remember Vulkan instance for destroying texture, - /// destroy should probably be handled by Vulkan class - impl_->vulkan_ = vulkan; - - // check if we have a lut, if yes, the texture needs to be - // nearest sampled since it has index values - const bool has_lut = !impl_->lut_data_.empty(); - - // create a texture to which we can upload from CUDA - impl_->texture_ = vulkan->create_texture(impl_->width_, impl_->height_, impl_->format_, - 0, nullptr, - has_lut ? VK_FILTER_NEAREST : VK_FILTER_LINEAR); - - if (has_lut) { - // create LUT texture - impl_->lut_texture_ = - vulkan->create_texture(impl_->lut_size_, 1, impl_->lut_format_, - impl_->lut_data_.size(), impl_->lut_data_.data(), - VK_FILTER_NEAREST, false /*normalized*/); - } - } - vulkan->upload_to_texture(impl_->host_ptr_, impl_->texture_); +void ImageLayer::end(Vulkan* vulkan) { + if (impl_->device_ptr_) { + // check if this is a reused layer, in this case + // we just have to upload the data to the texture + if (!impl_->texture_) { + /// @todo need to remember Vulkan instance for destroying texture, + /// destroy should probably be handled by Vulkan class + impl_->vulkan_ = vulkan; + + // check if we have a lut, if yes, the texture needs to + // be nearest sampled since it has index values + const bool has_lut = !impl_->lut_data_.empty(); + + // create a texture to which we can upload from CUDA + impl_->texture_ = vulkan->create_texture_for_cuda_interop( + impl_->width_, + impl_->height_, + impl_->format_, + has_lut ? vk::Filter::eNearest : vk::Filter::eLinear); + } + vulkan->upload_to_texture(impl_->device_ptr_, impl_->texture_, impl_->cuda_stream_); + } else if (impl_->host_ptr_) { + // check if this is a reused layer, + // in this case we just have to upload the data to the texture + if (!impl_->texture_) { + /// @todo need to remember Vulkan instance for destroying texture, + /// destroy should probably be handled by Vulkan class + impl_->vulkan_ = vulkan; + + // check if we have a lut, if yes, the texture needs to be + // nearest sampled since it has index values + const bool has_lut = !impl_->lut_data_.empty(); + + // create a texture to which we can upload from CUDA + impl_->texture_ = + vulkan->create_texture(impl_->width_, + impl_->height_, + impl_->format_, + 0, + nullptr, + has_lut ? vk::Filter::eNearest : vk::Filter::eLinear); } + vulkan->upload_to_texture(impl_->host_ptr_, impl_->texture_); + } + + if (!impl_->lut_data_.empty() && !impl_->lut_texture_) { + // create LUT texture + impl_->lut_texture_ = + vulkan->create_texture(impl_->lut_size_, + 1, + impl_->lut_format_, + impl_->lut_data_.size(), + impl_->lut_data_.data(), + impl_->lut_normalized_ ? vk::Filter::eLinear : vk::Filter::eNearest, + impl_->lut_normalized_); + } } -void ImageLayer::render(Vulkan *vulkan) { - if (impl_->texture_) { - // draw - vulkan->draw_texture(impl_->texture_, impl_->lut_texture_, get_opacity()); - } +void ImageLayer::render(Vulkan* vulkan) { + if (impl_->texture_) { + // draw + vulkan->draw_texture(impl_->texture_, impl_->lut_texture_, get_opacity()); + } } } // namespace holoscan::viz diff --git a/modules/holoviz/src/layers/image_layer.hpp b/modules/holoviz/src/layers/image_layer.hpp index bf91dbea..824154aa 100644 --- a/modules/holoviz/src/layers/image_layer.hpp +++ b/modules/holoviz/src/layers/image_layer.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -27,7 +27,6 @@ #include "layer.hpp" - namespace holoscan::viz { /** @@ -35,79 +34,86 @@ namespace holoscan::viz { */ class ImageLayer : public Layer { public: - /** - * Construct a new ImageLayer object. - */ - ImageLayer(); - - /** - * Destroy the ImageLayer object. - */ - ~ImageLayer(); - - /** - * Defines the image data for this layer, source is Cuda device memory. - * - * If the image has a alpha value it's multiplied with the layer opacity. - * - * @param width width of the image - * @param height height of the image - * @param fmt image format - * @param device_ptr Cuda device memory pointer - */ - void image_cuda_device(uint32_t width, uint32_t height, ImageFormat fmt, - CUdeviceptr device_ptr); - - /** - * Defines the image data for this layer, source is a Cuda array. - * - * If the image has a alpha value it's multiplied with the layer opacity. - * - * @param fmt image format - * @param array Cuda array - */ - void image_cuda_array(ImageFormat fmt, CUarray array); - - /** - * Defines the image data for this layer, source is a Cuda array. - * - * If the image has a alpha value it's multiplied with the layer opacity. - * - * @param fmt image format - * @param array Cuda array - */ - void image_host(uint32_t width, uint32_t height, ImageFormat fmt, const void *data); - - /** - * Defines the lookup table for this image layer. - * - * If a lookup table is used the image format has to be a single channel integer or - * float format (e.g. ::ImageFormat::R8_UINT, ::ImageFormat::R16_UINT, ::ImageFormat::R32_UINT, - * ::ImageFormat::R32_SFLOAT). - * - * The function processed is as follow - * @code{.cpp} - * out = lut[clamp(in, 0, size)] - * @endcode - * Input image values are clamped to the range of the lookup table size: `[0, size[`. - * - * @param size size of the lookup table in elements - * @param fmt lookup table color format - * @param data_size size of the lookup table data in bytes - * @param data host memory pointer to lookup table data - */ - void lut(uint32_t size, ImageFormat fmt, size_t data_size, const void *data); - - /// holoscan::viz::Layer virtual members - ///@{ - bool can_be_reused(Layer &other) const override; - void end(Vulkan *vulkan) override; - void render(Vulkan *vulkan) override; - ///@} + /** + * Construct a new ImageLayer object. + */ + ImageLayer(); + + /** + * Destroy the ImageLayer object. + */ + ~ImageLayer(); + + /** + * Defines the image data for this layer, source is Cuda device memory. + * + * If the image has a alpha value it's multiplied with the layer opacity. + * + * @param width width of the image + * @param height height of the image + * @param fmt image format + * @param device_ptr Cuda device memory pointer + */ + void image_cuda_device(uint32_t width, uint32_t height, ImageFormat fmt, CUdeviceptr device_ptr); + + /** + * Defines the image data for this layer, source is a Cuda array. + * + * If the image has a alpha value it's multiplied with the layer opacity. + * + * @param fmt image format + * @param array Cuda array + */ + void image_cuda_array(ImageFormat fmt, CUarray array); + + /** + * Defines the image data for this layer, source is a Cuda array. + * + * If the image has a alpha value it's multiplied with the layer opacity. + * + * @param fmt image format + * @param array Cuda array + */ + void image_host(uint32_t width, uint32_t height, ImageFormat fmt, const void* data); + + /** + * Defines the lookup table for this image layer. + * + * If a lookup table is used the image format has to be a single channel integer or + * float format (e.g. ::ImageFormat::R8_UINT, ::ImageFormat::R16_UINT, ::ImageFormat::R32_UINT, + * ::ImageFormat::R8_UNORM, ::ImageFormat::R16_UNORM, ::ImageFormat::R32_SFLOAT). + * + * If normalized is 'true' the function processed is as follow + * @code{.cpp} + * out = lut[clamp(in, 0.0, 1.0)] + * @endcode + * Input image values are clamped to the range of the lookup table size: `[0.0, 1.0[`. + * + * If normalized is 'false' the function processed is as follow + * @code{.cpp} + * out = lut[clamp(in, 0, size)] + * @endcode + * Input image values are clamped to the range of the lookup table size: `[0.0, size[`. + * + * @param size size of the lookup table in elements + * @param fmt lookup table color format + * @param data_size size of the lookup table data in bytes + * @param data host memory pointer to lookup table data + * @param normalized if true then the range of the lookup table is '[0.0, 1.0[', else it is + * `[0.0, size[` + */ + void lut(uint32_t size, ImageFormat fmt, size_t data_size, const void* data, bool normalized); + + /// holoscan::viz::Layer virtual members + ///@{ + bool can_be_reused(Layer& other) const override; + void end(Vulkan* vulkan) override; + void render(Vulkan* vulkan) override; + ///@} private: - struct Impl; - std::shared_ptr impl_; + struct Impl; + std::shared_ptr impl_; }; } // namespace holoscan::viz diff --git a/modules/holoviz/src/layers/layer.cpp b/modules/holoviz/src/layers/layer.cpp index 27b818f2..bf700819 100644 --- a/modules/holoviz/src/layers/layer.cpp +++ b/modules/holoviz/src/layers/layer.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,39 +20,36 @@ namespace holoscan::viz { struct Layer::Impl { - int32_t priority_ = 0; - float opacity_ = 1.f; + int32_t priority_ = 0; + float opacity_ = 1.f; }; -Layer::Layer(Type type) - : type_(type) - , impl_(new Impl) { -} +Layer::Layer(Type type) : type_(type), impl_(new Impl) {} Layer::~Layer() {} Layer::Type Layer::get_type() const { - return type_; + return type_; } -bool Layer::can_be_reused(Layer &other) const { - return (type_ == other.type_); +bool Layer::can_be_reused(Layer& other) const { + return (type_ == other.type_); } int32_t Layer::get_priority() const { - return impl_->priority_; + return impl_->priority_; } void Layer::set_priority(int32_t priority) { - impl_->priority_ = priority; + impl_->priority_ = priority; } float Layer::get_opacity() const { - return impl_->opacity_; + return impl_->opacity_; } void Layer::set_opacity(float opacity) { - impl_->opacity_ = opacity; + impl_->opacity_ = opacity; } } // namespace holoscan::viz diff --git a/modules/holoviz/src/layers/layer.hpp b/modules/holoviz/src/layers/layer.hpp index d07fb3e2..2a977ef0 100644 --- a/modules/holoviz/src/layers/layer.hpp +++ b/modules/holoviz/src/layers/layer.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,84 +30,84 @@ class Vulkan; */ class Layer { public: - /** - * Layer types - */ - enum class Type { - Image, ///< image layer - Geometry, ///< geometry layer - ImGui ///< ImBui layer - }; - - /** - * Construct a new Layer object. - * - * @param type layer type - */ - explicit Layer(Type type); - Layer() = delete; - - /** - * Destroy the Layer object. - */ - virtual ~Layer(); - - /** - * @returns the layer type - */ - Type get_type() const; - - /** - * @returns the layer priority - */ - int32_t get_priority() const; - - /** - * Set the layer priority. - * - * @param priority new layer priority - */ - void set_priority(int32_t priority); - - /** - * @returns the layer opacity - */ - float get_opacity() const; - - /** - * Set the layer opacity. - * - * @param opacity new layer opacity - */ - virtual void set_opacity(float opacity); - - /** - * Checks if a layer can be reused (properties have to match). - * - * @param other layer which is to be checked for re-usability - */ - virtual bool can_be_reused(Layer &other) const; - - /** - * End layer construction. Upload data. - * - * @param vulkan vulkan instance to use for updating data - */ - virtual void end(Vulkan *vulkan) {} - - /** - * Render the layer. - * - * @param vulkan vulkan instance to use for drawing - */ - virtual void render(Vulkan *vulkan) = 0; + /** + * Layer types + */ + enum class Type { + Image, ///< image layer + Geometry, ///< geometry layer + ImGui ///< ImBui layer + }; + + /** + * Construct a new Layer object. + * + * @param type layer type + */ + explicit Layer(Type type); + Layer() = delete; + + /** + * Destroy the Layer object. + */ + virtual ~Layer(); + + /** + * @returns the layer type + */ + Type get_type() const; + + /** + * @returns the layer priority + */ + int32_t get_priority() const; + + /** + * Set the layer priority. + * + * @param priority new layer priority + */ + void set_priority(int32_t priority); + + /** + * @returns the layer opacity + */ + float get_opacity() const; + + /** + * Set the layer opacity. + * + * @param opacity new layer opacity + */ + virtual void set_opacity(float opacity); + + /** + * Checks if a layer can be reused (properties have to match). + * + * @param other layer which is to be checked for re-usability + */ + virtual bool can_be_reused(Layer& other) const; + + /** + * End layer construction. Upload data. + * + * @param vulkan vulkan instance to use for updating data + */ + virtual void end(Vulkan* vulkan) {} + + /** + * Render the layer. + * + * @param vulkan vulkan instance to use for drawing + */ + virtual void render(Vulkan* vulkan) = 0; protected: - const Type type_; ///< layer type + const Type type_; ///< layer type private: - struct Impl; - std::shared_ptr impl_; + struct Impl; + std::shared_ptr impl_; }; } // namespace holoscan::viz diff --git a/modules/holoviz/src/util/non_copyable.hpp b/modules/holoviz/src/util/non_copyable.hpp index 94d004f0..e6b893a6 100644 --- a/modules/holoviz/src/util/non_copyable.hpp +++ b/modules/holoviz/src/util/non_copyable.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef HOLOSCAN_VIZ_UTIL_NONCOPYABLE_HPP -#define HOLOSCAN_VIZ_UTIL_NONCOPYABLE_HPP +#ifndef HOLOVIZ_SRC_UTIL_NON_COPYABLE_HPP +#define HOLOVIZ_SRC_UTIL_NON_COPYABLE_HPP namespace holoscan::viz { @@ -25,13 +25,13 @@ namespace holoscan::viz { */ class NonCopyable { protected: - constexpr NonCopyable() = default; - virtual ~NonCopyable() = default; + constexpr NonCopyable() = default; + virtual ~NonCopyable() = default; - NonCopyable(const NonCopyable &) = delete; - NonCopyable &operator=(const NonCopyable &) = delete; + NonCopyable(const NonCopyable&) = delete; + NonCopyable& operator=(const NonCopyable&) = delete; }; } // namespace holoscan::viz -#endif /* HOLOSCAN_VIZ_UTIL_NONCOPYABLE_HPP */ +#endif /* HOLOVIZ_SRC_UTIL_NON_COPYABLE_HPP */ diff --git a/modules/holoviz/src/util/unique_value.hpp b/modules/holoviz/src/util/unique_value.hpp index d162c976..26858f6f 100644 --- a/modules/holoviz/src/util/unique_value.hpp +++ b/modules/holoviz/src/util/unique_value.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef HOLOSCAN_VIZ_UTIL_UNIQUE_VALUE_HPP -#define HOLOSCAN_VIZ_UTIL_UNIQUE_VALUE_HPP +#ifndef HOLOVIZ_SRC_UTIL_UNIQUE_VALUE_HPP +#define HOLOVIZ_SRC_UTIL_UNIQUE_VALUE_HPP #include #include @@ -35,124 +35,100 @@ namespace holoscan::viz { * @tparam TF signature of the function to be called * @tparam F function to be called */ -template +template class UniqueValue : public NonCopyable { public: - /** - * Construct - */ - UniqueValue() - : value_(T()) { - } - /** - * Construct from value - * - * @param value initial value - */ - explicit UniqueValue(T value) - : value_(value) { - } - - /** - * Move constructor - * - * @param other the object to transfer ownership from - */ - UniqueValue(UniqueValue &&other) noexcept - : value_(other.release()) { - } - - ~UniqueValue() { - reset(); - } - - /** - * Release the value - * - * @returns value - */ - T release() noexcept { - T value = value_; - value_ = T(); - return value; - } - - /** - * Reset with new value. Previous will be destroyed. - * - * @param value new value - */ - void reset(T value = T()) noexcept { - T old_value = value_; - value_ = value; - if (old_value != T()) { - F(old_value); - } - } - - /** - * Swap - */ - void swap(UniqueValue &other) noexcept { - std::swap(value_, other.value_); - } - - /** - * Move assignment operator - * - * @param other the object to transfer ownership from - */ - UniqueValue &operator=(UniqueValue &&other) noexcept { - reset(other.release()); - return *this; - } - - /** - * @return the value - */ - T get() const noexcept { - return value_; - } - - /** - * @returns true if the value is set - */ - explicit operator bool() const noexcept { - return (value_ != T()); - } - - /** - * @returns reference to value - */ - T &operator*() const { - return value_; - } - - /** - * @returns value - */ - T operator->() const noexcept { - return value_; - } - - /** - * @returns true if equal - */ - bool operator==(const UniqueValue &other) const { - return (value_ == other.value_); - } - - /** - * @returns true if not equal - */ - bool operator!=(const UniqueValue &other) const { - return !(operator==(other)); - } + /** + * Construct + */ + UniqueValue() : value_(T()) {} + /** + * Construct from value + * + * @param value initial value + */ + explicit UniqueValue(T value) : value_(value) {} + + /** + * Move constructor + * + * @param other the object to transfer ownership from + */ + UniqueValue(UniqueValue&& other) noexcept : value_(other.release()) {} + + ~UniqueValue() { reset(); } + + /** + * Release the value + * + * @returns value + */ + T release() noexcept { + T value = value_; + value_ = T(); + return value; + } + + /** + * Reset with new value. Previous will be destroyed. + * + * @param value new value + */ + void reset(T value = T()) noexcept { + T old_value = value_; + value_ = value; + if (old_value != T()) { F(old_value); } + } + + /** + * Swap + */ + void swap(UniqueValue& other) noexcept { std::swap(value_, other.value_); } + + /** + * Move assignment operator + * + * @param other the object to transfer ownership from + */ + UniqueValue& operator=(UniqueValue&& other) noexcept { + reset(other.release()); + return *this; + } + + /** + * @return the value + */ + T get() const noexcept { return value_; } + + /** + * @returns true if the value is set + */ + explicit operator bool() const noexcept { return (value_ != T()); } + + /** + * @returns reference to value + */ + T& operator*() const { return value_; } + + /** + * @returns value + */ + T operator->() const noexcept { return value_; } + + /** + * @returns true if equal + */ + bool operator==(const UniqueValue& other) const { return (value_ == other.value_); } + + /** + * @returns true if not equal + */ + bool operator!=(const UniqueValue& other) const { return !(operator==(other)); } private: - T value_; + T value_; }; } // namespace holoscan::viz -#endif /* HOLOSCAN_VIZ_UTIL_UNIQUE_VALUE_HPP */ +#endif /* HOLOVIZ_SRC_UTIL_UNIQUE_VALUE_HPP */ diff --git a/modules/holoviz/src/vulkan/framebuffer_sequence.cpp b/modules/holoviz/src/vulkan/framebuffer_sequence.cpp index a27ba295..90e775e9 100644 --- a/modules/holoviz/src/vulkan/framebuffer_sequence.cpp +++ b/modules/holoviz/src/vulkan/framebuffer_sequence.cpp @@ -23,192 +23,167 @@ #include #include - namespace holoscan::viz { -bool FramebufferSequence::init(nvvk::ResourceAllocator *alloc, const nvvk::Context &vkctx, - VkQueue queue, VkSurfaceKHR surface) { - color_format_ = VkFormat::VK_FORMAT_R8G8B8A8_UNORM; - - if (surface) { - // pick a preferred format or use the first available one - uint32_t surface_format_count; - NVVK_CHECK( - vkGetPhysicalDeviceSurfaceFormatsKHR(vkctx.m_physicalDevice, surface, - &surface_format_count, nullptr)); - std::vector surfaceFormats(surface_format_count); - NVVK_CHECK(vkGetPhysicalDeviceSurfaceFormatsKHR(vkctx.m_physicalDevice, surface, - &surface_format_count, surfaceFormats.data())); - - bool found = false; - for (auto &f : surfaceFormats) { - if (color_format_ == f.format) { - found = true; - break; - } - } - if (!found) { - color_format_ = surfaceFormats[0].format; - } - - swap_chain_.reset(new nvvk::SwapChain()); - if (!swap_chain_->init(vkctx.m_device, vkctx.m_physicalDevice, queue, - vkctx.m_queueGCT.familyIndex, surface, color_format_, - VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT)) - return false; - - // the color format might have changed when creating the swap chain - color_format_ = swap_chain_->getFormat(); - } else { - device_ = vkctx.m_device; - queue_family_index_ = vkctx.m_queueGCT.familyIndex; - - image_count_ = 3; - - alloc_ = alloc; - - // create semaphores - semaphores_.resize(image_count_); - const VkSemaphoreCreateInfo sem_create_info = {VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO}; - for (uint32_t i = 0; i < image_count_; ++i) { - if (NVVK_CHECK(vkCreateSemaphore(device_, &sem_create_info, nullptr, &semaphores_[i]))) - return false; - } +FramebufferSequence::~FramebufferSequence() { + if (swap_chain_) { + swap_chain_->deinit(); + swap_chain_.reset(); + } else { + for (auto&& color_texture : color_textures_) { alloc_->destroy(color_texture); } + semaphores_.clear(); + } +} + +void FramebufferSequence::init(nvvk::ResourceAllocator* alloc, const vk::Device& device, + const vk::PhysicalDevice& physical_device, vk::Queue queue, + uint32_t queue_family_index, vk::SurfaceKHR surface) { + color_format_ = vk::Format::eR8G8B8A8Unorm; + + if (surface) { + // pick a preferred format or use the first available one + const std::vector surfaceFormats = + physical_device.getSurfaceFormatsKHR(surface); + + bool found = false; + for (auto& f : surfaceFormats) { + if (color_format_ == f.format) { + found = true; + break; + } + } + if (!found) { color_format_ = surfaceFormats[0].format; } + + swap_chain_.reset(new nvvk::SwapChain()); + if (!swap_chain_->init(device, + physical_device, + queue, + queue_family_index, + surface, + VkFormat(color_format_), + VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT)) { + throw std::runtime_error("Failed to init swap chain."); } - return true; -} + // the color format might have changed when creating the swap chain + color_format_ = vk::Format(swap_chain_->getFormat()); + } else { + device_ = device; + queue_family_index_ = queue_family_index; -void FramebufferSequence::deinit() { - if (swap_chain_) { - swap_chain_->deinit(); - swap_chain_.reset(); - } else { - for (auto &&color_texture : color_textures_) { - alloc_->destroy(color_texture); - } - for (auto &&semaphore : semaphores_) { - vkDestroySemaphore(device_, semaphore, nullptr); - } + image_count_ = 3; + + alloc_ = alloc; + + // create semaphores + for (uint32_t i = 0; i < image_count_; ++i) { + semaphores_.push_back(device.createSemaphoreUnique({})); } + } } -bool FramebufferSequence::update(uint32_t width, uint32_t height, VkExtent2D *dimensions) { - if (swap_chain_) { - // vkCreateSwapchainKHR() randomly fails with VK_ERROR_INITIALIZATION_FAILED on driver 510 - // https://nvbugswb.nvidia.com/NvBugs5/SWBug.aspx?bugid=3612509&cmtNo= - // Workaround: retry several times. - uint32_t retries = 3; - while (retries) { - if (swap_chain_->update(static_cast (width), static_cast (height), dimensions)) - break; - - --retries; - if (retries == 0) { - return false; - } - } - - image_count_ = swap_chain_->getImageCount(); - } else { - for (auto &&color_texture : color_textures_) { - alloc_->destroy(color_texture); - } - color_textures_.clear(); - - color_textures_.resize(image_count_); - for (uint32_t i = 0; i < image_count_; ++i) { - const VkImageCreateInfo color_create_info = - nvvk::makeImage2DCreateInfo(VkExtent2D{width, height}, color_format_, - VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | - VK_IMAGE_USAGE_TRANSFER_SRC_BIT); - const nvvk::Image image = alloc_->createImage(color_create_info); - - const VkImageViewCreateInfo image_view_info = - nvvk::makeImageViewCreateInfo(image.image, color_create_info); - - color_textures_[i] = alloc_->createTexture(image, image_view_info); - { - nvvk::CommandPool cmd_buf_get(device_, queue_family_index_); - VkCommandBuffer cmd_buf = cmd_buf_get.createCommandBuffer(); - - nvvk::cmdBarrierImageLayout(cmd_buf, color_textures_[i].image, - VK_IMAGE_LAYOUT_UNDEFINED, - VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); - - cmd_buf_get.submitAndWait(cmd_buf); - } - color_textures_[i].descriptor.imageLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; - } +void FramebufferSequence::update(uint32_t width, uint32_t height, vk::Extent2D* dimensions) { + if (swap_chain_) { + // vkCreateSwapchainKHR() randomly fails with VK_ERROR_INITIALIZATION_FAILED on driver 510 + // https://nvbugswb.nvidia.com/NvBugs5/SWBug.aspx?bugid=3612509&cmtNo= + // Workaround: retry several times. + uint32_t retries = 3; + while (retries) { + if (swap_chain_->update(static_cast(width), + static_cast(height), + reinterpret_cast(dimensions))) { + break; + } + + --retries; + if (retries == 0) { throw std::runtime_error("Failed to update swap chain."); } + } + + image_count_ = swap_chain_->getImageCount(); + } else { + for (auto&& color_texture : color_textures_) { alloc_->destroy(color_texture); } + color_textures_.clear(); + + color_textures_.resize(image_count_); + for (uint32_t i = 0; i < image_count_; ++i) { + const vk::ImageCreateInfo color_create_info = nvvk::makeImage2DCreateInfo( + vk::Extent2D{width, height}, + color_format_, + vk::ImageUsageFlagBits::eColorAttachment | vk::ImageUsageFlagBits::eTransferSrc); + const nvvk::Image image = alloc_->createImage(color_create_info); + + const vk::ImageViewCreateInfo image_view_info = + nvvk::makeImageViewCreateInfo(image.image, color_create_info); + + color_textures_[i] = alloc_->createTexture(image, image_view_info); + { + nvvk::CommandPool cmd_buf_get(device_, queue_family_index_); + vk::CommandBuffer cmd_buf = cmd_buf_get.createCommandBuffer(); + + nvvk::cmdBarrierImageLayout(cmd_buf, + color_textures_[i].image, + VK_IMAGE_LAYOUT_UNDEFINED, + VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL); + + cmd_buf_get.submitAndWait(cmd_buf); + } + color_textures_[i].descriptor.imageLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; } - return true; + } } -bool FramebufferSequence::acquire() { - if (swap_chain_) { - return swap_chain_->acquire(); +void FramebufferSequence::acquire() { + if (swap_chain_) { + if (!swap_chain_->acquire()) { + throw std::runtime_error("Failed to acquire next swap chain image."); } + } - current_image_ = (current_image_ + 1) % image_count_; - return true; + current_image_ = (current_image_ + 1) % image_count_; } uint32_t FramebufferSequence::get_active_image_index() const { - if (swap_chain_) { - return swap_chain_->getActiveImageIndex(); - } + if (swap_chain_) { return swap_chain_->getActiveImageIndex(); } - return current_image_; + return current_image_; } -VkImageView FramebufferSequence::get_image_view(uint32_t i) const { - if (swap_chain_) { - return swap_chain_->getImageView(i); - } +vk::ImageView FramebufferSequence::get_image_view(uint32_t i) const { + if (swap_chain_) { return swap_chain_->getImageView(i); } - if (i >= color_textures_.size()) { - throw std::runtime_error("Invalid image view index"); - } + if (i >= color_textures_.size()) { throw std::runtime_error("Invalid image view index"); } - return color_textures_[i].descriptor.imageView; + return color_textures_[i].descriptor.imageView; } -void FramebufferSequence::present(VkQueue queue) { - if (swap_chain_) { - swap_chain_->present(queue); - } else { - active_semaphore_ = semaphores_[current_image_]; - } +void FramebufferSequence::present(vk::Queue queue) { + if (swap_chain_) { + swap_chain_->present(queue); + } else { + active_semaphore_ = semaphores_[current_image_].get(); + } } -VkSemaphore FramebufferSequence::get_active_read_semaphore() const { - if (swap_chain_) { - return swap_chain_->getActiveReadSemaphore(); - } +vk::Semaphore FramebufferSequence::get_active_read_semaphore() const { + if (swap_chain_) { return swap_chain_->getActiveReadSemaphore(); } - return active_semaphore_; + return active_semaphore_; } -VkSemaphore FramebufferSequence::get_active_written_semaphore() const { - if (swap_chain_) { - return swap_chain_->getActiveWrittenSemaphore(); - } +vk::Semaphore FramebufferSequence::get_active_written_semaphore() const { + if (swap_chain_) { return swap_chain_->getActiveWrittenSemaphore(); } - return semaphores_[current_image_]; + return semaphores_[current_image_].get(); } -VkImage FramebufferSequence::get_active_image() const { - if (swap_chain_) { - return swap_chain_->getActiveImage(); - } +vk::Image FramebufferSequence::get_active_image() const { + if (swap_chain_) { return swap_chain_->getActiveImage(); } - return color_textures_[current_image_].image; + return color_textures_[current_image_].image; } -void FramebufferSequence::cmd_update_barriers(VkCommandBuffer cmd) const { - if (swap_chain_) { - swap_chain_->cmdUpdateBarriers(cmd); - } +void FramebufferSequence::cmd_update_barriers(vk::CommandBuffer cmd) const { + if (swap_chain_) { swap_chain_->cmdUpdateBarriers(cmd); } } } // namespace holoscan::viz diff --git a/modules/holoviz/src/vulkan/framebuffer_sequence.hpp b/modules/holoviz/src/vulkan/framebuffer_sequence.hpp index 2c814ed4..b059b014 100644 --- a/modules/holoviz/src/vulkan/framebuffer_sequence.hpp +++ b/modules/holoviz/src/vulkan/framebuffer_sequence.hpp @@ -22,6 +22,8 @@ #include #include +#include + #include #include #include @@ -33,114 +35,118 @@ namespace holoscan::viz { */ class FramebufferSequence { public: - /** - * Initialize. - * - * @param alloc resource allocator to use - * @param vkctx context - * @param queue queue - * @param surface surface, if set then use a swap chain to present to a display, - * else render offscreen - */ - bool init(nvvk::ResourceAllocator *alloc, const nvvk::Context &vkctx, - VkQueue queue, VkSurfaceKHR surface); - - // triggers queue/device wait idle - void deinit(); - - /** - * Update the framebuffer to the given size. - * - * @param width new framebuffer width - * @param height new framebuffer height - * @param [out] dimensions actual dimensions, which may differ from the requested (optional) - */ - bool update(uint32_t width, uint32_t height, VkExtent2D *dimensions = nullptr); - - /** - * Acquire the next image to render to - */ - bool acquire(); - - /** - * @returns the framebuffer color format/ - */ - VkFormat get_format() const { - return color_format_; - } - - /** - * @returns the image view of a color buffer - */ - VkImageView get_image_view(uint32_t i) const; - - /** - * @returns the color buffer count - */ - uint32_t get_image_count() const { - return image_count_; - } - - /** - * @returns the index of the current active color buffer - */ - uint32_t get_active_image_index() const; - - /** - * Present on provided queue. After this call the active color buffer is switched to the next one. - * - * @param queue queue to present on - */ - void present(VkQueue queue); - - /** - * Get the active read semaphore. For offscreen rendering this is identical to the written semaphore - * of the previous frame, else it's the semaphore used to acquire the swap queue image. - * - * @returns active read semaphore - */ - VkSemaphore get_active_read_semaphore() const; - - /** - * Get the active write semaphore. - * - * @returns active write semaphore - */ - VkSemaphore get_active_written_semaphore() const; - - /** - * Get the active image. - * - * @returns active image - */ - VkImage get_active_image() const; - - /** - * Do a vkCmdPipelineBarrier for VK_IMAGE_LAYOUT_UNDEFINED to VK_IMAGE_LAYOUT_PRESENT_SRC_KHR. - * Must apply resource transitions after update calls. - * - * @param cmd command buffer to use - */ - void cmd_update_barriers(VkCommandBuffer cmd) const; + /** + * Destroy the framebuffer sequence object + */ + ~FramebufferSequence(); + + /** + * Initialize. + * + * @param alloc resource allocator to use + * @param vkctx device + * @param physical_device physical device + * @param queue queue + * @param queue_family_index queue family index + * @param surface surface, if set then use a swap chain to present to a display, else render + * offscreen + */ + void init(nvvk::ResourceAllocator* alloc, const vk::Device& device, + const vk::PhysicalDevice& physical_device, vk::Queue queue, uint32_t queue_family_index, + vk::SurfaceKHR surface); + + /** + * Update the framebuffer to the given size. + * + * @param width new framebuffer width + * @param height new framebuffer height + * @param [out] dimensions actual dimensions, which may differ from the requested (optional) + */ + void update(uint32_t width, uint32_t height, vk::Extent2D* dimensions = nullptr); + + /** + * Acquire the next image to render to + */ + void acquire(); + + /** + * @returns the framebuffer color format/ + */ + vk::Format get_format() const { return color_format_; } + + /** + * @returns the image view of a color buffer + */ + vk::ImageView get_image_view(uint32_t i) const; + + /** + * @returns the color buffer count + */ + uint32_t get_image_count() const { return image_count_; } + + /** + * @returns the index of the current active color buffer + */ + uint32_t get_active_image_index() const; + + /** + * Present on provided queue. After this call the active color buffer is switched to the next + * one. + * + * @param queue queue to present on + */ + void present(vk::Queue queue); + + /** + * Get the active read semaphore. For offscreen rendering this is identical to the written + * semaphore of the previous frame, else it's the semaphore used to acquire the swap queue + * image. + * + * @returns active read semaphore + */ + vk::Semaphore get_active_read_semaphore() const; + + /** + * Get the active write semaphore. + * + * @returns active write semaphore + */ + vk::Semaphore get_active_written_semaphore() const; + + /** + * Get the active image. + * + * @returns active image + */ + vk::Image get_active_image() const; + + /** + * Do a vkCmdPipelineBarrier for VK_IMAGE_LAYOUT_UNDEFINED to VK_IMAGE_LAYOUT_PRESENT_SRC_KHR. + * Must apply resource transitions after update calls. + * + * @param cmd command buffer to use + */ + void cmd_update_barriers(vk::CommandBuffer cmd) const; private: - VkDevice device_; - uint32_t queue_family_index_; - nvvk::ResourceAllocator *alloc_ = nullptr; ///< Allocator for color buffers + vk::Device device_; + uint32_t queue_family_index_; + nvvk::ResourceAllocator* alloc_ = nullptr; ///< Allocator for color buffers - VkFormat color_format_; + vk::Format color_format_; - uint32_t image_count_; + uint32_t image_count_; - /// the swap chain is used if no surface is set - std::unique_ptr swap_chain_; + /// the swap chain is used if no surface is set + std::unique_ptr swap_chain_; - // members used when rendering offscreen - uint32_t current_image_ = 0; - VkSemaphore active_semaphore_ = VK_NULL_HANDLE; ///< the active semaphore for the current image + // members used when rendering offscreen + uint32_t current_image_ = 0; + // the active semaphore for the current image + vk::Semaphore active_semaphore_ = VK_NULL_HANDLE; - std::vector color_textures_; // color buffers when rendering offscreen - std::vector semaphores_; + std::vector color_textures_; // color buffers when rendering offscreen + std::vector semaphores_; }; } // namespace holoscan::viz diff --git a/gxf_extensions/visualizer_tool_tracking/glsl/video_frame.frag b/modules/holoviz/src/vulkan/shaders/geometry_color_shader.glsl.vert similarity index 64% rename from gxf_extensions/visualizer_tool_tracking/glsl/video_frame.frag rename to modules/holoviz/src/vulkan/shaders/geometry_color_shader.glsl.vert index 0b214df1..1b000af6 100644 --- a/gxf_extensions/visualizer_tool_tracking/glsl/video_frame.frag +++ b/modules/holoviz/src/vulkan/shaders/geometry_color_shader.glsl.vert @@ -1,5 +1,3 @@ -#version 450 - /* * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 @@ -17,10 +15,25 @@ * limitations under the License. */ -in vec2 tex_coords; -layout(binding = 0) uniform sampler2D background; -layout(location = 0) out vec4 out_color; +#version 450 + +// incoming +layout(location = 0) in vec3 i_position; +layout(location = 1) in vec4 i_color; + +// outgoing +layout(location = 0) out vec4 o_color; + +layout(push_constant) uniform constants +{ + mat4x4 matrix; + float pointSize; +} pushConstants; + +void main() +{ + gl_PointSize = pushConstants.pointSize; + o_color = i_color; -void main() { - out_color = texture2D(background, tex_coords); -}; + gl_Position = pushConstants.matrix * vec4(i_position, 1.0); +} \ No newline at end of file diff --git a/modules/holoviz/src/vulkan/shaders/geometry_shader.glsl.vert b/modules/holoviz/src/vulkan/shaders/geometry_shader.glsl.vert index 875f775b..7a5afa63 100644 --- a/modules/holoviz/src/vulkan/shaders/geometry_shader.glsl.vert +++ b/modules/holoviz/src/vulkan/shaders/geometry_shader.glsl.vert @@ -18,7 +18,7 @@ #version 450 // incoming -layout(location = 0) in vec2 i_position; +layout(location = 0) in vec3 i_position; // outgoing layout(location = 0) out vec4 o_color; @@ -38,5 +38,5 @@ void main() gl_PointSize = pushConstants.pointSize; o_color = vec4(pushConstants.colorRed, pushConstants.colorGreen, pushConstants.colorBlue, pushConstants.colorAlpha); - gl_Position = pushConstants.matrix * vec4(i_position, 0.0, 1.0); + gl_Position = pushConstants.matrix * vec4(i_position, 1.0); } \ No newline at end of file diff --git a/modules/holoviz/src/vulkan/shaders/geometry_text_shader.glsl.frag b/modules/holoviz/src/vulkan/shaders/geometry_text_shader.glsl.frag index 4bb7cea9..bcc04956 100644 --- a/modules/holoviz/src/vulkan/shaders/geometry_text_shader.glsl.frag +++ b/modules/holoviz/src/vulkan/shaders/geometry_text_shader.glsl.frag @@ -32,7 +32,7 @@ layout(binding = 0) uniform sampler2D texSampler; layout(push_constant) uniform constants { - layout(offset = 16 * 4) float opacity; + layout(offset = 21 * 4) float opacity; } pushConstants; diff --git a/modules/holoviz/src/vulkan/shaders/geometry_text_shader.glsl.vert b/modules/holoviz/src/vulkan/shaders/geometry_text_shader.glsl.vert index 5a896282..93cd436c 100644 --- a/modules/holoviz/src/vulkan/shaders/geometry_text_shader.glsl.vert +++ b/modules/holoviz/src/vulkan/shaders/geometry_text_shader.glsl.vert @@ -18,7 +18,7 @@ #version 450 // incoming -layout(location = 0) in vec2 i_position; +layout(location = 0) in vec3 i_position; layout(location = 1) in vec2 i_texCoord; layout(location = 2) in vec4 i_color; @@ -39,5 +39,5 @@ void main() Out.color = i_color; Out.texCoord = i_texCoord; - gl_Position = pushConstants.matrix * vec4(i_position, 0.0, 1.0); + gl_Position = pushConstants.matrix * vec4(i_position, 1.0); } \ No newline at end of file diff --git a/modules/holoviz/src/vulkan/shaders/image_lut_float_shader.glsl.frag b/modules/holoviz/src/vulkan/shaders/image_lut_float_shader.glsl.frag index 4301b0ca..f441b28a 100644 --- a/modules/holoviz/src/vulkan/shaders/image_lut_float_shader.glsl.frag +++ b/modules/holoviz/src/vulkan/shaders/image_lut_float_shader.glsl.frag @@ -29,7 +29,7 @@ layout(binding = 1) uniform sampler2D texSamplerLUT; layout(push_constant) uniform constants { - float opacity; + layout(offset = 21 * 4) float opacity; } pushConstants; void main() diff --git a/modules/holoviz/src/vulkan/shaders/image_lut_uint_shader.glsl.frag b/modules/holoviz/src/vulkan/shaders/image_lut_uint_shader.glsl.frag index 86710ebf..2836cadc 100644 --- a/modules/holoviz/src/vulkan/shaders/image_lut_uint_shader.glsl.frag +++ b/modules/holoviz/src/vulkan/shaders/image_lut_uint_shader.glsl.frag @@ -29,7 +29,7 @@ layout(binding = 1) uniform sampler2D texSamplerLUT; layout(push_constant) uniform constants { - float opacity; + layout(offset = 21 * 4) float opacity; } pushConstants; void main() diff --git a/modules/holoviz/src/vulkan/shaders/image_shader.glsl.frag b/modules/holoviz/src/vulkan/shaders/image_shader.glsl.frag index 6adf4637..f25832da 100644 --- a/modules/holoviz/src/vulkan/shaders/image_shader.glsl.frag +++ b/modules/holoviz/src/vulkan/shaders/image_shader.glsl.frag @@ -28,7 +28,7 @@ layout(binding = 0) uniform sampler2D texSampler; layout(push_constant) uniform constants { - float opacity; + layout(offset = 21 * 4) float opacity; } pushConstants; void main() diff --git a/modules/holoviz/src/vulkan/shaders/image_shader.glsl.vert b/modules/holoviz/src/vulkan/shaders/image_shader.glsl.vert index 3bcf29c9..238a1075 100644 --- a/modules/holoviz/src/vulkan/shaders/image_shader.glsl.vert +++ b/modules/holoviz/src/vulkan/shaders/image_shader.glsl.vert @@ -23,8 +23,18 @@ layout(location = 0) in vec2 i_position; // outgoing layout(location = 0) out vec2 o_texCoord; +layout(push_constant) uniform constants +{ + mat4x4 matrix; + float pointSize; + float colorRed; + float colorGreen; + float colorBlue; + float colorAlpha; +} pushConstants; + void main() { - gl_Position = vec4(i_position, 0.0, 1.0); + gl_Position = pushConstants.matrix * vec4(i_position, 0.0, 1.0); o_texCoord = (i_position + vec2(1.f)) * vec2(0.5f); } \ No newline at end of file diff --git a/modules/holoviz/src/vulkan/vulkan.cpp b/modules/holoviz/src/vulkan/vulkan.cpp deleted file mode 100644 index 01536cc0..00000000 --- a/modules/holoviz/src/vulkan/vulkan.cpp +++ /dev/null @@ -1,2099 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "vulkan.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include "../cuda/convert.hpp" -#include "../cuda/cuda_service.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../layers/layer.hpp" -#include "framebuffer_sequence.hpp" - -namespace holoscan::viz { - -struct PushConstantFragment { - float opacity; -}; - -struct PushConstantVertex { - nvmath::mat4f matrix; - float point_size; - std::array color; -}; - -struct PushConstantTextVertex { - nvmath::mat4f matrix; -}; - -struct Vulkan::Texture { - Texture(uint32_t width, uint32_t height, ImageFormat format, nvvk::ResourceAllocator *alloc) - : width_(width) - , height_(height) - , format_(format) - , alloc_(alloc) { - } - - const uint32_t width_; - const uint32_t height_; - const ImageFormat format_; - nvvk::ResourceAllocator *const alloc_; - - enum class State { - UNKNOWN, - UPLOADED, - RENDERED - }; - State state_ = State::UNKNOWN; - - nvvk::Texture texture_{}; - UniqueCUexternalMemory external_mem_; - UniqueCUmipmappedArray mipmap_; - - /// this semaphore is used to synchronize uploading and rendering, it's signaled by Cuda - /// on upload and waited on by vulkan on rendering - VkSemaphore upload_semaphore_ = nullptr; - UniqueCUexternalSemaphore cuda_upload_semaphore_; - - /// this semaphore is used to synchronize rendering and uploading, it's signaled by Vulkan - /// on rendering and waited on by Cuda on uploading - VkSemaphore render_semaphore_ = nullptr; - UniqueCUexternalSemaphore cuda_render_semaphore_; - - VkFence fence_ = nullptr; ///< last usage of the texture, need to sync before destroying memory -}; - -struct Vulkan::Buffer { - Buffer(size_t size, nvvk::ResourceAllocator *alloc) - : size_(size) - , alloc_(alloc) { - } - - const size_t size_; - nvvk::ResourceAllocator *const alloc_; - - nvvk::Buffer buffer_{}; - - VkFence fence_ = nullptr; ///< last usage of the buffer, need to sync before destroying memory -}; - -class Vulkan::Impl { - public: - Impl() = default; - virtual ~Impl(); - - void setup(Window *window); - - void begin_transfer_pass(); - void end_transfer_pass(); - void begin_render_pass(); - void end_render_pass(); - void cleanup_transfer_jobs(); - - void prepare_frame(); - void submit_frame(); - uint32_t get_active_image_index() const { - return fb_sequence_.get_active_image_index(); - } - const std::vector &get_command_buffers() { - return command_buffers_; - } - - Texture *create_texture_for_cuda_upload(uint32_t width, uint32_t height, - ImageFormat format, VkFilter filter, - bool normalized); - Texture *create_texture(uint32_t width, uint32_t height, ImageFormat format, - size_t data_size, const void *data, - VkFilter filter, bool normalized); - void destroy_texture(Texture *texture); - - void upload_to_texture(CUdeviceptr device_ptr, Texture *texture, CUstream stream); - void upload_to_texture(const void *host_ptr, Texture *texture); - - void draw_texture(Texture *texture, Texture *lut, float opacity); - - Buffer *create_buffer(size_t data_size, VkBufferUsageFlags usage, const void *data = nullptr); - - void destroy_buffer(Buffer *buffer); - - void draw(VkPrimitiveTopology topology, uint32_t count, uint32_t first, Buffer *buffer, - float opacity, const std::array &color, float point_size, float line_width); - - void draw_indexed(VkDescriptorSet desc_set, Buffer *vertex_buffer, Buffer *index_buffer, - VkIndexType index_type, uint32_t index_count, uint32_t first_index, - uint32_t vertex_offset, float opacity); - - void read_framebuffer(ImageFormat fmt, size_t buffer_size, CUdeviceptr device_ptr, - CUstream stream); - - private: - void init_im_gui(); - bool create_framebuffer_sequence(); - void create_depth_buffer(); - void create_render_pass(); - - /** - * Create all the framebuffers in which the image will be rendered - * - Swapchain need to be created before calling this - */ - void create_frame_buffers(); - - /** - * Callback when the window is resized - * - Destroy allocated frames, then rebuild them with the new size - */ - void on_framebuffer_size(int w, int h); - - VkCommandBuffer create_temp_cmd_buffer(); - void submit_temp_cmd_buffer(VkCommandBuffer cmdBuffer); - - uint32_t get_memory_type(uint32_t typeBits, const VkMemoryPropertyFlags &properties) const; - - VkPipeline create_pipeline(VkPipelineLayout pipeline_layout, const uint32_t *vertex_shader, - size_t vertex_shader_size, const uint32_t *fragment_shader, - size_t fragment_shader_size, VkPrimitiveTopology topology, - const std::vector dynamic_state = {}, - bool imgui_attrib_desc = false); - UniqueCUexternalSemaphore import_semaphore_to_cuda(VkSemaphore semaphore); - - Window *window_ = nullptr; - - // Vulkan low level - VkSurfaceKHR surface_ = VK_NULL_HANDLE; - VkQueue queue_ = VK_NULL_HANDLE; - VkCommandPool cmd_pool_ = VK_NULL_HANDLE; - VkDescriptorPool im_gui_desc_pool_ = VK_NULL_HANDLE; - - // Drawing/Surface - FramebufferSequence fb_sequence_; - std::vector framebuffers_; // All framebuffers, - // correspond to the Swapchain - std::vector command_buffers_; // Command buffer per nb element in Swapchain - std::vector wait_fences_; // Fences per nb element in Swapchain - VkImage depth_image_ = VK_NULL_HANDLE; // Depth/Stencil - VkDeviceMemory depth_memory_ = VK_NULL_HANDLE; // Depth/Stencil - VkImageView depth_view_ = VK_NULL_HANDLE; // Depth/Stencil - VkRenderPass render_pass_ = VK_NULL_HANDLE; // Base render pass - VkExtent2D size_{0, 0}; // Size of the window - VkPipelineCache pipeline_cache_ = VK_NULL_HANDLE; // Cache for pipeline/shaders - - // Depth buffer format - VkFormat depth_format_{VK_FORMAT_UNDEFINED}; - - // allocators - nvvk::ResourceAllocatorDma alloc_; ///< Allocator for buffer, images, - /// acceleration structures - nvvk::ExportResourceAllocator export_alloc_; ///< Allocator for allocations - /// which can be exported - - nvvk::Context vk_ctx_{}; - - nvvk::BatchSubmission batch_submission_; - - nvvk::CommandPool transfer_cmd_pool_; - struct TransferJob { - VkCommandBuffer cmd_buffer_ = nullptr; - VkSemaphore semaphore_ = nullptr; - VkFence fence_ = nullptr; - VkFence frame_fence_ = nullptr; - }; - std::list transfer_jobs_; - - nvvk::Buffer vertex_buffer_{}; - nvvk::Buffer index_buffer_{}; - - VkPipelineLayout image_pipeline_layout_ = nullptr; - VkPipelineLayout image_lut_pipeline_layout_ = nullptr; - VkPipelineLayout geometry_pipeline_layout_ = nullptr; - VkPipelineLayout geometry_text_pipeline_layout_ = nullptr; - - const uint32_t bindings_offset_texture_ = 0; - const uint32_t bindings_offset_texture_lut_ = 1; - - nvvk::DescriptorSetBindings desc_set_layout_bind_; - VkDescriptorSetLayout desc_set_layout_ = nullptr; - - nvvk::DescriptorSetBindings desc_set_layout_bind_lut_; - VkDescriptorSetLayout desc_set_layout_lut_ = nullptr; - - nvvk::DescriptorSetBindings desc_set_layout_bind_text_; - VkDescriptorSetLayout desc_set_layout_text_ = nullptr; - VkDescriptorPool desc_pool_text_ = nullptr; - VkDescriptorSet desc_set_text_ = nullptr; - VkSampler sampler_text_ = nullptr; - - VkPipeline image_pipeline_ = nullptr; - VkPipeline image_lut_uint_pipeline_ = nullptr; - VkPipeline image_lut_float_pipeline_ = nullptr; - - VkPipeline geometry_point_pipeline_ = nullptr; - VkPipeline geometry_line_pipeline_ = nullptr; - VkPipeline geometry_line_strip_pipeline_ = nullptr; - VkPipeline geometry_triangle_pipeline_ = nullptr; - VkPipeline geometry_text_pipeline_ = nullptr; - - ImGuiContext *im_gui_context_ = nullptr; -}; - -Vulkan::Impl::~Impl() { - if (vk_ctx_.m_device) { - NVVK_CHECK(vkDeviceWaitIdle(vk_ctx_.m_device)); - - cleanup_transfer_jobs(); - - vkDestroyDescriptorSetLayout(vk_ctx_.m_device, desc_set_layout_lut_, nullptr); - vkDestroyDescriptorSetLayout(vk_ctx_.m_device, desc_set_layout_, nullptr); - - vkDestroyDescriptorSetLayout(vk_ctx_.m_device, desc_set_layout_text_, nullptr); - vkDestroyDescriptorPool(vk_ctx_.m_device, desc_pool_text_, nullptr); - alloc_.releaseSampler(sampler_text_); - - vkDestroyPipeline(vk_ctx_.m_device, geometry_text_pipeline_, nullptr); - geometry_text_pipeline_ = nullptr; - vkDestroyPipeline(vk_ctx_.m_device, geometry_triangle_pipeline_, nullptr); - geometry_triangle_pipeline_ = nullptr; - vkDestroyPipeline(vk_ctx_.m_device, geometry_line_strip_pipeline_, nullptr); - geometry_line_strip_pipeline_ = nullptr; - vkDestroyPipeline(vk_ctx_.m_device, geometry_line_pipeline_, nullptr); - geometry_line_pipeline_ = nullptr; - vkDestroyPipeline(vk_ctx_.m_device, geometry_point_pipeline_, nullptr); - geometry_point_pipeline_ = nullptr; - vkDestroyPipeline(vk_ctx_.m_device, image_lut_float_pipeline_, nullptr); - image_lut_float_pipeline_ = nullptr; - vkDestroyPipeline(vk_ctx_.m_device, image_lut_uint_pipeline_, nullptr); - image_lut_uint_pipeline_ = nullptr; - vkDestroyPipeline(vk_ctx_.m_device, image_pipeline_, nullptr); - image_pipeline_ = nullptr; - - vkDestroyPipelineLayout(vk_ctx_.m_device, geometry_text_pipeline_layout_, nullptr); - geometry_text_pipeline_layout_ = nullptr; - vkDestroyPipelineLayout(vk_ctx_.m_device, geometry_pipeline_layout_, nullptr); - geometry_pipeline_layout_ = nullptr; - vkDestroyPipelineLayout(vk_ctx_.m_device, image_lut_pipeline_layout_, nullptr); - image_lut_pipeline_layout_ = nullptr; - vkDestroyPipelineLayout(vk_ctx_.m_device, image_pipeline_layout_, nullptr); - image_pipeline_layout_ = nullptr; - - alloc_.destroy(index_buffer_); - alloc_.destroy(vertex_buffer_); - - transfer_cmd_pool_.deinit(); - - if (ImGui::GetCurrentContext() != nullptr) { - ImGui_ImplVulkan_Shutdown(); - if (im_gui_context_) { - ImGui::DestroyContext(im_gui_context_); - } - } - - vkDestroyRenderPass(vk_ctx_.m_device, render_pass_, nullptr); - - vkDestroyImageView(vk_ctx_.m_device, depth_view_, nullptr); - vkDestroyImage(vk_ctx_.m_device, depth_image_, nullptr); - vkFreeMemory(vk_ctx_.m_device, depth_memory_, nullptr); - vkDestroyPipelineCache(vk_ctx_.m_device, pipeline_cache_, nullptr); - - for (uint32_t i = 0; i < fb_sequence_.get_image_count(); i++) { - vkDestroyFence(vk_ctx_.m_device, wait_fences_[i], nullptr); - - vkDestroyFramebuffer(vk_ctx_.m_device, framebuffers_[i], nullptr); - - vkFreeCommandBuffers(vk_ctx_.m_device, cmd_pool_, 1, &command_buffers_[i]); - } - fb_sequence_.deinit(); - vkDestroyDescriptorPool(vk_ctx_.m_device, im_gui_desc_pool_, nullptr); - vkDestroyCommandPool(vk_ctx_.m_device, cmd_pool_, nullptr); - - if (surface_) { - vkDestroySurfaceKHR(vk_ctx_.m_instance, surface_, nullptr); - } - - export_alloc_.deinit(); - alloc_.deinit(); - } - - vk_ctx_.deinit(); -} - -void Vulkan::Impl::setup(Window *window) { - window_ = window; - -#ifdef NDEBUG - nvvk::ContextCreateInfo context_info; -#else - nvvk::ContextCreateInfo context_info(true /*bUseValidation*/); -#endif - - context_info.setVersion(1, 2); // Using Vulkan 1.2 - - // Requesting Vulkan extensions and layers - uint32_t count{0}; - const char **req_extensions = window_->get_required_instance_extensions(&count); - for (uint32_t ext_id = 0; ext_id < count; ext_id++) - context_info.addInstanceExtension(req_extensions[ext_id]); - - // Allow debug names - context_info.addInstanceExtension(VK_EXT_DEBUG_UTILS_EXTENSION_NAME, true); - context_info.addInstanceExtension(VK_KHR_EXTERNAL_MEMORY_CAPABILITIES_EXTENSION_NAME); - - req_extensions = window_->get_required_device_extensions(&count); - for (uint32_t ext_id = 0; ext_id < count; ext_id++) - context_info.addDeviceExtension(req_extensions[ext_id]); - - context_info.addDeviceExtension(VK_KHR_EXTERNAL_MEMORY_EXTENSION_NAME); - context_info.addDeviceExtension(VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME); - context_info.addDeviceExtension(VK_KHR_EXTERNAL_SEMAPHORE_EXTENSION_NAME); - context_info.addDeviceExtension(VK_KHR_EXTERNAL_SEMAPHORE_FD_EXTENSION_NAME); - context_info.addDeviceExtension(VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME); - - // Creating Vulkan base application - if (!vk_ctx_.initInstance(context_info)) { - throw std::runtime_error("Failed to create the Vulkan instance."); - } - - // Find all compatible devices - const std::vector compatible_devices = vk_ctx_.getCompatibleDevices(context_info); - if (compatible_devices.empty()) { - throw std::runtime_error("No Vulkan capable GPU present."); - } - // Use a compatible device - vk_ctx_.initDevice(compatible_devices[0], context_info); - - // Find the most suitable depth format - { - const VkFormatFeatureFlagBits feature = VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT; - for (const auto &f : {VK_FORMAT_D24_UNORM_S8_UINT, VK_FORMAT_D32_SFLOAT_S8_UINT, - VK_FORMAT_D16_UNORM_S8_UINT}) { - VkFormatProperties formatProp{VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2}; - vkGetPhysicalDeviceFormatProperties(vk_ctx_.m_physicalDevice, f, &formatProp); - if ((formatProp.optimalTilingFeatures & feature) == feature) { - depth_format_ = f; - break; - } - } - if (depth_format_ == VK_FORMAT_UNDEFINED) { - throw std::runtime_error("Could not find a suitable depth format."); - } - } - - // create a surface, headless windows don't have a surface - surface_ = window_->create_surface(vk_ctx_.m_physicalDevice, vk_ctx_.m_instance); - if (surface_) { - if (!vk_ctx_.setGCTQueueWithPresent(surface_)) { - throw std::runtime_error("Surface not supported by queue"); - } - } - - vkGetDeviceQueue(vk_ctx_.m_device, vk_ctx_.m_queueGCT.familyIndex, 0, &queue_); - - VkCommandPoolCreateInfo pool_reate_info{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; - pool_reate_info.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; - NVVK_CHECK(vkCreateCommandPool(vk_ctx_.m_device, &pool_reate_info, nullptr, &cmd_pool_)); - - VkPipelineCacheCreateInfo pipeline_cache_info{VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO}; - NVVK_CHECK(vkCreatePipelineCache(vk_ctx_.m_device, &pipeline_cache_info, - nullptr, &pipeline_cache_)); - - alloc_.init(vk_ctx_.m_instance, vk_ctx_.m_device, vk_ctx_.m_physicalDevice); - export_alloc_.init(vk_ctx_.m_device, vk_ctx_.m_physicalDevice, alloc_.getMemoryAllocator()); - - create_framebuffer_sequence(); - create_depth_buffer(); - create_render_pass(); - create_frame_buffers(); - - // init batch submission - batch_submission_.init(vk_ctx_.m_queueGCT); - - // init command pool - transfer_cmd_pool_.init(vk_ctx_.m_device, vk_ctx_.m_queueT.familyIndex); - - // allocate the vertex and index buffer for the image draw pass - { - nvvk::CommandPool cmd_buf_get(vk_ctx_.m_device, vk_ctx_.m_queueGCT.familyIndex); - VkCommandBuffer cmd_buf = cmd_buf_get.createCommandBuffer(); - - const std::vector vertices{-1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f}; - vertex_buffer_ = alloc_.createBuffer(cmd_buf, vertices, VK_BUFFER_USAGE_VERTEX_BUFFER_BIT); - const std::vector indices{0, 2, 1, 2, 0, 3}; - index_buffer_ = alloc_.createBuffer(cmd_buf, indices, VK_BUFFER_USAGE_INDEX_BUFFER_BIT); - - cmd_buf_get.submitAndWait(cmd_buf); - alloc_.finalizeAndReleaseStaging(); - } - - // create the descriptor sets - desc_set_layout_bind_.addBinding(bindings_offset_texture_, - VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1, - VK_SHADER_STAGE_FRAGMENT_BIT); - desc_set_layout_ = - desc_set_layout_bind_.createLayout(vk_ctx_.m_device, - VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR); - - desc_set_layout_bind_lut_.addBinding(bindings_offset_texture_, - VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1, - VK_SHADER_STAGE_FRAGMENT_BIT); - desc_set_layout_bind_lut_.addBinding(bindings_offset_texture_lut_, - VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1, - VK_SHADER_STAGE_FRAGMENT_BIT); - desc_set_layout_lut_ = desc_set_layout_bind_lut_.createLayout( - vk_ctx_.m_device, VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR); - - { - VkSamplerCreateInfo info{VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO}; - info.magFilter = VK_FILTER_LINEAR; - info.minFilter = VK_FILTER_LINEAR; - info.mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR; - info.addressModeU = VK_SAMPLER_ADDRESS_MODE_REPEAT; - info.addressModeV = VK_SAMPLER_ADDRESS_MODE_REPEAT; - info.addressModeW = VK_SAMPLER_ADDRESS_MODE_REPEAT; - info.minLod = -1000; - info.maxLod = 1000; - info.maxAnisotropy = 1.0f; - sampler_text_ = alloc_.acquireSampler(info); - } - desc_set_layout_bind_text_.addBinding(bindings_offset_texture_, - VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1, - VK_SHADER_STAGE_FRAGMENT_BIT, &sampler_text_); - desc_set_layout_text_ = desc_set_layout_bind_text_.createLayout(vk_ctx_.m_device); - desc_pool_text_ = desc_set_layout_bind_text_.createPool(vk_ctx_.m_device); - desc_set_text_ = nvvk::allocateDescriptorSet(vk_ctx_.m_device, - desc_pool_text_, - desc_set_layout_text_); - - // create the pipeline layout for images - { - // Push constants - VkPushConstantRange push_constant_ranges{}; - push_constant_ranges.stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT; - push_constant_ranges.offset = 0; - push_constant_ranges.size = sizeof(PushConstantFragment); - - // Creating the Pipeline Layout - VkPipelineLayoutCreateInfo create_info{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; - create_info.setLayoutCount = 1; - create_info.pSetLayouts = &desc_set_layout_; - create_info.pushConstantRangeCount = 1; - create_info.pPushConstantRanges = &push_constant_ranges; - NVVK_CHECK(vkCreatePipelineLayout(vk_ctx_.m_device, &create_info, - nullptr, &image_pipeline_layout_)); - } - - // Create the Pipeline - image_pipeline_ = create_pipeline( - image_pipeline_layout_, image_shader_glsl_vert, - sizeof(image_shader_glsl_vert) / sizeof(image_shader_glsl_vert[0]), image_shader_glsl_frag, - sizeof(image_shader_glsl_frag) / sizeof(image_shader_glsl_frag[0]), - VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST); - - // create the pipeline layout for images with lut - { - // Push constants - VkPushConstantRange push_constant_ranges{}; - push_constant_ranges.stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT; - push_constant_ranges.offset = 0; - push_constant_ranges.size = sizeof(PushConstantFragment); - - // Creating the Pipeline Layout - VkPipelineLayoutCreateInfo create_info{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; - create_info.setLayoutCount = 1; - create_info.pSetLayouts = &desc_set_layout_lut_; - create_info.pushConstantRangeCount = 1; - create_info.pPushConstantRanges = &push_constant_ranges; - NVVK_CHECK(vkCreatePipelineLayout(vk_ctx_.m_device, &create_info, - nullptr, &image_lut_pipeline_layout_)); - } - - image_lut_uint_pipeline_ = create_pipeline( - image_lut_pipeline_layout_, image_shader_glsl_vert, - sizeof(image_shader_glsl_vert) / sizeof(image_shader_glsl_vert[0]), - image_lut_uint_shader_glsl_frag, - sizeof(image_lut_uint_shader_glsl_frag) / sizeof(image_lut_uint_shader_glsl_frag[0]), - VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST); - - image_lut_float_pipeline_ = create_pipeline( - image_lut_pipeline_layout_, image_shader_glsl_vert, - sizeof(image_shader_glsl_vert) / sizeof(image_shader_glsl_vert[0]), - image_lut_float_shader_glsl_frag, - sizeof(image_lut_float_shader_glsl_frag) / sizeof(image_lut_float_shader_glsl_frag[0]), - VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST); - - // create the pipeline layout for geometry - { - // Push constants - VkPushConstantRange push_constant_ranges[2]{}; - push_constant_ranges[0].stageFlags = VK_SHADER_STAGE_VERTEX_BIT; - push_constant_ranges[0].offset = 0; - push_constant_ranges[0].size = sizeof(PushConstantVertex); - push_constant_ranges[1].stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT; - push_constant_ranges[1].offset = sizeof(PushConstantVertex); - push_constant_ranges[1].size = sizeof(PushConstantFragment); - - VkPipelineLayoutCreateInfo create_info{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; - create_info.pushConstantRangeCount = 2; - create_info.pPushConstantRanges = push_constant_ranges; - NVVK_CHECK(vkCreatePipelineLayout(vk_ctx_.m_device, &create_info, - nullptr, &geometry_pipeline_layout_)); - } - - geometry_point_pipeline_ = create_pipeline( - geometry_pipeline_layout_, geometry_shader_glsl_vert, - sizeof(geometry_shader_glsl_vert) / sizeof(geometry_shader_glsl_vert[0]), - geometry_shader_glsl_frag, - sizeof(geometry_shader_glsl_frag) / sizeof(geometry_shader_glsl_frag[0]), - VK_PRIMITIVE_TOPOLOGY_POINT_LIST); - geometry_line_pipeline_ = create_pipeline(geometry_pipeline_layout_, - geometry_shader_glsl_vert, - sizeof(geometry_shader_glsl_vert) / - sizeof(geometry_shader_glsl_vert[0]), - geometry_shader_glsl_frag, - sizeof(geometry_shader_glsl_frag) / - sizeof(geometry_shader_glsl_frag[0]), - VK_PRIMITIVE_TOPOLOGY_LINE_LIST, - {VK_DYNAMIC_STATE_LINE_WIDTH}); - geometry_line_strip_pipeline_ = create_pipeline( - geometry_pipeline_layout_, geometry_shader_glsl_vert, - sizeof(geometry_shader_glsl_vert) / sizeof(geometry_shader_glsl_vert[0]), - geometry_shader_glsl_frag, - sizeof(geometry_shader_glsl_frag) / sizeof(geometry_shader_glsl_frag[0]), - VK_PRIMITIVE_TOPOLOGY_LINE_STRIP, - {VK_DYNAMIC_STATE_LINE_WIDTH}); - geometry_triangle_pipeline_ = create_pipeline( - geometry_pipeline_layout_, geometry_shader_glsl_vert, - sizeof(geometry_shader_glsl_vert) / sizeof(geometry_shader_glsl_vert[0]), - geometry_shader_glsl_frag, - sizeof(geometry_shader_glsl_frag) / sizeof(geometry_shader_glsl_frag[0]), - VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST); - - // create the pipeline layout for text geometry - { - // Push constants - VkPushConstantRange push_constant_ranges[2]{}; - push_constant_ranges[0].stageFlags = VK_SHADER_STAGE_VERTEX_BIT; - push_constant_ranges[0].offset = 0; - push_constant_ranges[0].size = sizeof(PushConstantTextVertex); - push_constant_ranges[1].stageFlags = VK_SHADER_STAGE_FRAGMENT_BIT; - push_constant_ranges[1].offset = sizeof(PushConstantTextVertex); - push_constant_ranges[1].size = sizeof(PushConstantFragment); - - VkPipelineLayoutCreateInfo create_info{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; - create_info.setLayoutCount = 1; - create_info.pSetLayouts = &desc_set_layout_text_; - create_info.pushConstantRangeCount = 2; - create_info.pPushConstantRanges = push_constant_ranges; - NVVK_CHECK(vkCreatePipelineLayout(vk_ctx_.m_device, &create_info, - nullptr, &geometry_text_pipeline_layout_)); - } - - geometry_text_pipeline_ = - create_pipeline(geometry_text_pipeline_layout_, geometry_text_shader_glsl_vert, - sizeof(geometry_text_shader_glsl_vert) / - sizeof(geometry_text_shader_glsl_vert[0]), - geometry_text_shader_glsl_frag, - sizeof(geometry_text_shader_glsl_frag) / - sizeof(geometry_text_shader_glsl_frag[0]), - VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, {}, true); - - // ImGui initialization - init_im_gui(); - window_->setup_callbacks([this](int width, int height) - { this->on_framebuffer_size(width, height); }); - window_->init_im_gui(); -} - -void Vulkan::Impl::init_im_gui() { - // if the app did not specify a context, create our own - if (!ImGui::GetCurrentContext()) { - im_gui_context_ = ImGui::CreateContext(); - } - - ImGuiIO &io = ImGui::GetIO(); - io.IniFilename = nullptr; // Avoiding the INI file - io.LogFilename = nullptr; - io.ConfigFlags |= ImGuiConfigFlags_NavEnableKeyboard; // Enable Keyboard Controls - io.ConfigFlags |= ImGuiConfigFlags_DockingEnable; // Enable Docking - - std::vector pool_size{{VK_DESCRIPTOR_TYPE_SAMPLER, 1}, - {VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1}}; - VkDescriptorPoolCreateInfo pool_info{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; - pool_info.maxSets = 2; - pool_info.poolSizeCount = 2; - pool_info.pPoolSizes = pool_size.data(); - NVVK_CHECK(vkCreateDescriptorPool(vk_ctx_.m_device, &pool_info, nullptr, &im_gui_desc_pool_)); - - // Setup Platform/Renderer back ends - ImGui_ImplVulkan_InitInfo init_info{}; - init_info.Instance = vk_ctx_.m_instance; - init_info.PhysicalDevice = vk_ctx_.m_physicalDevice; - init_info.Device = vk_ctx_.m_device; - init_info.QueueFamily = vk_ctx_.m_queueGCT.familyIndex; - init_info.Queue = queue_; - init_info.PipelineCache = VK_NULL_HANDLE; - init_info.DescriptorPool = im_gui_desc_pool_; - init_info.Subpass = 0; - init_info.MinImageCount = 2; - init_info.ImageCount = static_cast(fb_sequence_.get_image_count()); - init_info.MSAASamples = VK_SAMPLE_COUNT_1_BIT; - init_info.CheckVkResultFn = nullptr; - init_info.Allocator = nullptr; - - ImGui_ImplVulkan_Init(&init_info, render_pass_); - - // Upload Fonts - VkCommandBuffer cmd_buf = create_temp_cmd_buffer(); - ImGui_ImplVulkan_CreateFontsTexture(cmd_buf); - submit_temp_cmd_buffer(cmd_buf); - - // set the default font - ImGui::SetCurrentFont(ImGui::GetDefaultFont()); -} - -void Vulkan::Impl::begin_transfer_pass() { - // create a new transfer job and a command buffer - transfer_jobs_.emplace_back(); - TransferJob &transfer_job = transfer_jobs_.back(); - - transfer_job.cmd_buffer_ = transfer_cmd_pool_.createCommandBuffer(); -} - -void Vulkan::Impl::end_transfer_pass() { - if (transfer_jobs_.empty() || (transfer_jobs_.back().fence_ != nullptr)) { - throw std::runtime_error("Not in transfer pass."); - } - - TransferJob &transfer_job = transfer_jobs_.back(); - - // end the command buffer for this job - NVVK_CHECK(vkEndCommandBuffer(transfer_job.cmd_buffer_)); - - // create the fence and semaphore needed for submission - VkSemaphoreCreateInfo semaphore_create_info{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO}; - NVVK_CHECK(vkCreateSemaphore(vk_ctx_.m_device, &semaphore_create_info, - nullptr, &transfer_job.semaphore_)); - VkFenceCreateInfo fence_create_info{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; - NVVK_CHECK(vkCreateFence(vk_ctx_.m_device, &fence_create_info, nullptr, &transfer_job.fence_)); - - // finalize the staging job for later cleanup of resources - // associates all current staging resources with the transfer fence - alloc_.finalizeStaging(transfer_job.fence_); - - // submit staged transfers - VkSubmitInfo submit_info = nvvk::makeSubmitInfo(1, &transfer_job.cmd_buffer_, 1, - &transfer_job.semaphore_); - NVVK_CHECK(vkQueueSubmit(vk_ctx_.m_queueT.queue, 1, &submit_info, transfer_job.fence_)); - - // next graphics submission must wait for transfer completion - batch_submission_.enqueueWait(transfer_job.semaphore_, VK_PIPELINE_STAGE_TRANSFER_BIT); -} - -void Vulkan::Impl::begin_render_pass() { - // Acquire the next image - prepare_frame(); - - // Get the command buffer for the frame. - // There are n command buffers equal to the number of in-flight frames. - const uint32_t cur_frame = get_active_image_index(); - const VkCommandBuffer cmd_buf = command_buffers_[cur_frame]; - - VkCommandBufferBeginInfo begin_info{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; - begin_info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; - NVVK_CHECK(vkBeginCommandBuffer(cmd_buf, &begin_info)); - - // Clearing values - std::array clear_values{}; - clear_values[0].color = {{0.f, 0.f, 0.f, 1.f}}; - clear_values[1].depthStencil = {1.0f, 0}; - - // Begin rendering - VkRenderPassBeginInfo render_pass_begin_info{VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO}; - render_pass_begin_info.clearValueCount = 2; - render_pass_begin_info.pClearValues = clear_values.data(); - render_pass_begin_info.renderPass = render_pass_; - render_pass_begin_info.framebuffer = framebuffers_[cur_frame]; - render_pass_begin_info.renderArea = {{0, 0}, size_}; - vkCmdBeginRenderPass(cmd_buf, &render_pass_begin_info, VK_SUBPASS_CONTENTS_INLINE); - - // set the dynamic viewport - VkViewport viewport{0.0f, 0.0f, static_cast(size_.width), - static_cast(size_.height), 0.0f, 1.0f}; - vkCmdSetViewport(cmd_buf, 0, 1, &viewport); - - VkRect2D scissor{{0, 0}, {size_.width, size_.height}}; - vkCmdSetScissor(cmd_buf, 0, 1, &scissor); -} - -void Vulkan::Impl::end_render_pass() { - const VkCommandBuffer cmd_buf = command_buffers_[get_active_image_index()]; - - // End rendering - vkCmdEndRenderPass(cmd_buf); - - // Submit for display - NVVK_CHECK(vkEndCommandBuffer(cmd_buf)); - submit_frame(); -} - -void Vulkan::Impl::cleanup_transfer_jobs() { - for (auto it = transfer_jobs_.begin(); it != transfer_jobs_.end();) { - if (it->fence_) { - // check if the upload fence was triggered, that means the copy has completed - // and cmd buffer can be destroyed - const VkResult result = vkGetFenceStatus(vk_ctx_.m_device, it->fence_); - if (result == VK_SUCCESS) { - transfer_cmd_pool_.destroy(it->cmd_buffer_); - it->cmd_buffer_ = nullptr; - - // before destroying the fence release all staging buffers using that fence - alloc_.releaseStaging(); - - vkDestroyFence(vk_ctx_.m_device, it->fence_, nullptr); - it->fence_ = nullptr; - } else if (result != VK_NOT_READY) { - NVVK_CHECK(result); - } - } - - if (!it->fence_) { - if (it->frame_fence_) { - // check if the frame fence was triggered, that means the job can be destroyed - const VkResult result = vkGetFenceStatus(vk_ctx_.m_device, it->frame_fence_); - if (result == VK_SUCCESS) { - vkDestroySemaphore(vk_ctx_.m_device, it->semaphore_, nullptr); - it->semaphore_ = nullptr; - /// @todo instead of allocating and destroying semaphore and fences, move to - /// unused list and reuse - /// (call 'NVVK_CHECK(vkResetFences(vk_ctx_.m_device, 1, - /// &it->fence_));' to reuse) - it = transfer_jobs_.erase(it); - continue; - } else if (result != VK_NOT_READY) { - NVVK_CHECK(result); - } - } else { - // this is a stale transfer buffer (no end_transfer_pass()?), remove it - it = transfer_jobs_.erase(it); - continue; - } - } - ++it; - } -} - -void Vulkan::Impl::prepare_frame() { - if (!transfer_jobs_.empty() && (transfer_jobs_.back().fence_ == nullptr)) { - throw std::runtime_error("Transfer pass is active!"); - } - - // Acquire the next image from the framebuffer sequence - if (!fb_sequence_.acquire()) { - throw std::runtime_error("Failed to acquire next framebuffer sequence image."); - } - - // Use a fence to wait until the command buffer has finished execution before using it again - const uint32_t image_index = get_active_image_index(); - - VkResult result{VK_SUCCESS}; - do { - result = vkWaitForFences(vk_ctx_.m_device, 1, &wait_fences_[image_index], - VK_TRUE, 1'000'000); - } while (result == VK_TIMEOUT); - - if (result != VK_SUCCESS) { - // This allows Aftermath to do things and exit below - usleep(1000); - NVVK_CHECK(result); - exit(-1); - } - - // reset the fence to be re-used - NVVK_CHECK(vkResetFences(vk_ctx_.m_device, 1, &wait_fences_[image_index])); - - // if there is a pending transfer job assign the frame fence of the frame - // which is about to be rendered. - if (!transfer_jobs_.empty() && !transfer_jobs_.back().frame_fence_) { - transfer_jobs_.back().frame_fence_ = wait_fences_[image_index]; - } - - // try to free previous transfer jobs - cleanup_transfer_jobs(); -} - -void Vulkan::Impl::submit_frame() { - const uint32_t image_index = get_active_image_index(); - - batch_submission_.enqueue(command_buffers_[image_index]); - - // wait for the previous frame's semaphore - if (fb_sequence_.get_active_read_semaphore() != VK_NULL_HANDLE) { - batch_submission_.enqueueWait(fb_sequence_.get_active_read_semaphore(), - VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT); - } - // and signal this frames semaphore on completion - batch_submission_.enqueueSignal(fb_sequence_.get_active_written_semaphore()); - - NVVK_CHECK(batch_submission_.execute(wait_fences_[image_index], 0b0000'0001)); - - // Presenting frame - fb_sequence_.present(queue_); -} - -bool Vulkan::Impl::create_framebuffer_sequence() { - window_->get_framebuffer_size(&size_.width, &size_.height); - - if (!fb_sequence_.init(&alloc_, vk_ctx_, queue_, surface_)) { - return false; - } - - if (!fb_sequence_.update(size_.width, size_.height, &size_)) { - return false; - } - - // Create Synchronization Primitives - VkFenceCreateInfo fence_create_info{VK_STRUCTURE_TYPE_FENCE_CREATE_INFO}; - fence_create_info.flags = VK_FENCE_CREATE_SIGNALED_BIT; - wait_fences_.resize(fb_sequence_.get_image_count()); - for (auto &fence : wait_fences_) { - NVVK_CHECK(vkCreateFence(vk_ctx_.m_device, &fence_create_info, nullptr, &fence)); - } - - // Command buffers store a reference to the frame buffer inside their render pass info - // so for static usage without having to rebuild them each frame, we use one per frame buffer - VkCommandBufferAllocateInfo allocate_info{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; - allocate_info.commandPool = cmd_pool_; - allocate_info.commandBufferCount = fb_sequence_.get_image_count(); - allocate_info.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; - command_buffers_.resize(fb_sequence_.get_image_count()); - NVVK_CHECK(vkAllocateCommandBuffers(vk_ctx_.m_device, &allocate_info, command_buffers_.data())); - - const VkCommandBuffer cmd_buffer = create_temp_cmd_buffer(); - fb_sequence_.cmd_update_barriers(cmd_buffer); - submit_temp_cmd_buffer(cmd_buffer); - -#ifdef _DEBUG - for (size_t i = 0; i < command_buffers_.size(); i++) { - std::string name = std::string("AppBase") + std::to_string(i); - - VkDebugUtilsObjectNameInfoEXT name_info{VK_STRUCTURE_TYPE_DEBUG_UTILS_OBJECT_NAME_INFO_EXT}; - name_info.objectHandle = (uint64_t)command_buffers_[i]; - name_info.objectType = VK_OBJECT_TYPE_COMMAND_BUFFER; - name_info.pObjectName = name.c_str(); - NVVK_CHECK(vkSetDebugUtilsObjectNameEXT(vk_ctx_.m_device, &name_info)); - } -#endif // _DEBUG - - return true; -} - -void Vulkan::Impl::create_depth_buffer() { - if (depth_view_) { - vkDestroyImageView(vk_ctx_.m_device, depth_view_, nullptr); - } - - if (depth_image_) { - vkDestroyImage(vk_ctx_.m_device, depth_image_, nullptr); - } - - if (depth_memory_) { - vkFreeMemory(vk_ctx_.m_device, depth_memory_, nullptr); - } - - // Depth information - const VkImageAspectFlags aspect = VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT; - VkImageCreateInfo depth_stencil_create_info{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; - depth_stencil_create_info.imageType = VK_IMAGE_TYPE_2D; - depth_stencil_create_info.extent = VkExtent3D{size_.width, size_.height, 1}; - depth_stencil_create_info.format = depth_format_; - depth_stencil_create_info.mipLevels = 1; - depth_stencil_create_info.arrayLayers = 1; - depth_stencil_create_info.samples = VK_SAMPLE_COUNT_1_BIT; - depth_stencil_create_info.usage = VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT | - VK_IMAGE_USAGE_TRANSFER_SRC_BIT; - // Create the depth image - NVVK_CHECK(vkCreateImage(vk_ctx_.m_device, &depth_stencil_create_info, nullptr, &depth_image_)); - -#ifdef _DEBUG - std::string name = std::string("AppBaseDepth"); - VkDebugUtilsObjectNameInfoEXT name_info{VK_STRUCTURE_TYPE_DEBUG_UTILS_OBJECT_NAME_INFO_EXT}; - name_info.objectHandle = (uint64_t)depth_image_; - name_info.objectType = VK_OBJECT_TYPE_IMAGE; - name_info.pObjectName = R"(AppBase)"; - NVVK_CHECK(vkSetDebugUtilsObjectNameEXT(vk_ctx_.m_device, &name_info)); -#endif // _DEBUG - - // Allocate the memory - VkMemoryRequirements mem_reqs; - vkGetImageMemoryRequirements(vk_ctx_.m_device, depth_image_, &mem_reqs); - VkMemoryAllocateInfo memAllocInfo{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; - memAllocInfo.allocationSize = mem_reqs.size; - memAllocInfo.memoryTypeIndex = get_memory_type(mem_reqs.memoryTypeBits, - VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - NVVK_CHECK(vkAllocateMemory(vk_ctx_.m_device, &memAllocInfo, nullptr, &depth_memory_)); - - // Bind image and memory - NVVK_CHECK(vkBindImageMemory(vk_ctx_.m_device, depth_image_, depth_memory_, 0)); - - const VkCommandBuffer cmd_buffer = create_temp_cmd_buffer(); - - // Put barrier on top, Put barrier inside setup command buffer - VkImageSubresourceRange subresource_range{}; - subresource_range.aspectMask = aspect; - subresource_range.levelCount = 1; - subresource_range.layerCount = 1; - VkImageMemoryBarrier image_memory_barrier{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; - image_memory_barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; - image_memory_barrier.newLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; - image_memory_barrier.image = depth_image_; - image_memory_barrier.subresourceRange = subresource_range; - image_memory_barrier.srcAccessMask = VkAccessFlags(); - image_memory_barrier.dstAccessMask = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; - const VkPipelineStageFlags src_stage_mask = VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT; - const VkPipelineStageFlags destStageMask = VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT; - - vkCmdPipelineBarrier(cmd_buffer, src_stage_mask, destStageMask, VK_FALSE, 0, nullptr, - 0, nullptr, 1, - &image_memory_barrier); - submit_temp_cmd_buffer(cmd_buffer); - - // Setting up the view - VkImageViewCreateInfo depth_stencil_view{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; - depth_stencil_view.viewType = VK_IMAGE_VIEW_TYPE_2D; - depth_stencil_view.format = depth_format_; - depth_stencil_view.subresourceRange = subresource_range; - depth_stencil_view.image = depth_image_; - NVVK_CHECK(vkCreateImageView(vk_ctx_.m_device, &depth_stencil_view, nullptr, &depth_view_)); -} - -void Vulkan::Impl::create_render_pass() { - if (render_pass_) { - vkDestroyRenderPass(vk_ctx_.m_device, render_pass_, nullptr); - } - - std::array attachments{}; - // Color attachment - attachments[0].format = fb_sequence_.get_format(); - attachments[0].loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR; - attachments[0].finalLayout = surface_ ? VK_IMAGE_LAYOUT_PRESENT_SRC_KHR : - VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; - attachments[0].samples = VK_SAMPLE_COUNT_1_BIT; - - // Depth attachment - attachments[1].format = depth_format_; - attachments[1].loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR; - attachments[1].stencilLoadOp = VK_ATTACHMENT_LOAD_OP_CLEAR; - attachments[1].finalLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL; - attachments[1].samples = VK_SAMPLE_COUNT_1_BIT; - - // One color, one depth - const VkAttachmentReference color_reference{0, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL}; - const VkAttachmentReference depth_reference{1, - VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL}; - - std::array subpass_dependencies{}; - // Transition from final to initial - // (VK_SUBPASS_EXTERNAL refers to all commands executed outside of the actual renderpass) - subpass_dependencies[0].srcSubpass = VK_SUBPASS_EXTERNAL; - subpass_dependencies[0].dstSubpass = 0; - subpass_dependencies[0].srcStageMask = VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT; - subpass_dependencies[0].dstStageMask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT; - subpass_dependencies[0].srcAccessMask = VK_ACCESS_MEMORY_READ_BIT; - subpass_dependencies[0].dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | - VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; - subpass_dependencies[0].dependencyFlags = VK_DEPENDENCY_BY_REGION_BIT; - - VkSubpassDescription subpass_description{}; - subpass_description.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS; - subpass_description.colorAttachmentCount = 1; - subpass_description.pColorAttachments = &color_reference; - subpass_description.pDepthStencilAttachment = &depth_reference; - - VkRenderPassCreateInfo render_pass_info{VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO}; - render_pass_info.attachmentCount = static_cast(attachments.size()); - render_pass_info.pAttachments = attachments.data(); - render_pass_info.subpassCount = 1; - render_pass_info.pSubpasses = &subpass_description; - render_pass_info.dependencyCount = static_cast(subpass_dependencies.size()); - render_pass_info.pDependencies = subpass_dependencies.data(); - - NVVK_CHECK(vkCreateRenderPass(vk_ctx_.m_device, &render_pass_info, nullptr, &render_pass_)); - -#ifdef _DEBUG - VkDebugUtilsObjectNameInfoEXT name_info{VK_STRUCTURE_TYPE_DEBUG_UTILS_OBJECT_NAME_INFO_EXT}; - name_info.objectHandle = (uint64_t)render_pass_; - name_info.objectType = VK_OBJECT_TYPE_RENDER_PASS; - name_info.pObjectName = R"(AppBaseVk)"; - NVVK_CHECK(vkSetDebugUtilsObjectNameEXT(vk_ctx_.m_device, &name_info)); -#endif // _DEBUG -} - -void Vulkan::Impl::create_frame_buffers() { - // Recreate the frame buffers - for (auto framebuffer : framebuffers_) { - vkDestroyFramebuffer(vk_ctx_.m_device, framebuffer, nullptr); - } - - // Array of attachment (color, depth) - std::array attachments{}; - - // Create frame buffers for every swap chain image - VkFramebufferCreateInfo framebuffer_create_info{VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO}; - framebuffer_create_info.renderPass = render_pass_; - framebuffer_create_info.attachmentCount = 2; - framebuffer_create_info.width = size_.width; - framebuffer_create_info.height = size_.height; - framebuffer_create_info.layers = 1; - framebuffer_create_info.pAttachments = attachments.data(); - - // Create frame buffers for every swap chain image - framebuffers_.resize(fb_sequence_.get_image_count()); - for (uint32_t i = 0; i < fb_sequence_.get_image_count(); i++) { - attachments[0] = fb_sequence_.get_image_view(i); - attachments[1] = depth_view_; - NVVK_CHECK(vkCreateFramebuffer(vk_ctx_.m_device, &framebuffer_create_info, nullptr, - &framebuffers_[i])); - } - -#ifdef _DEBUG - for (size_t i = 0; i < framebuffers_.size(); i++) { - std::string name = std::string("AppBase") + std::to_string(i); - VkDebugUtilsObjectNameInfoEXT name_info{VK_STRUCTURE_TYPE_DEBUG_UTILS_OBJECT_NAME_INFO_EXT}; - name_info.objectHandle = (uint64_t)framebuffers_[i]; - name_info.objectType = VK_OBJECT_TYPE_FRAMEBUFFER; - name_info.pObjectName = name.c_str(); - NVVK_CHECK(vkSetDebugUtilsObjectNameEXT(vk_ctx_.m_device, &name_info)); - } -#endif // _DEBUG -} - -void Vulkan::Impl::on_framebuffer_size(int w, int h) { - if ((w == 0) || (h == 0)) { - return; - } - - // Update imgui - if (ImGui::GetCurrentContext() != nullptr) { - auto &imgui_io = ImGui::GetIO(); - imgui_io.DisplaySize = ImVec2(static_cast(w), static_cast(h)); - } - - // Wait to finish what is currently drawing - NVVK_CHECK(vkDeviceWaitIdle(vk_ctx_.m_device)); - NVVK_CHECK(vkQueueWaitIdle(queue_)); - - // Request new swapchain image size - fb_sequence_.update(w, h, &size_); { - const VkCommandBuffer cmd_buffer = create_temp_cmd_buffer(); - fb_sequence_.cmd_update_barriers(cmd_buffer); // Make them presentable - submit_temp_cmd_buffer(cmd_buffer); - } - - if ((size_.width != w) || (size_.height != h)) { - LOGW("Requested size (%d, %d) is different from created size (%u, %u) ", w, h, - size_.width, size_.height); - } - - // Recreating other resources - create_depth_buffer(); - create_frame_buffers(); -} - -VkCommandBuffer Vulkan::Impl::create_temp_cmd_buffer() { - // Create an image barrier to change the layout from undefined to DepthStencilAttachmentOptimal - VkCommandBufferAllocateInfo allocate_info{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO}; - allocate_info.commandBufferCount = 1; - allocate_info.commandPool = cmd_pool_; - allocate_info.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; - VkCommandBuffer cmd_buffer; - NVVK_CHECK(vkAllocateCommandBuffers(vk_ctx_.m_device, &allocate_info, &cmd_buffer)); - - VkCommandBufferBeginInfo begin_info{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; - begin_info.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; - NVVK_CHECK(vkBeginCommandBuffer(cmd_buffer, &begin_info)); - return cmd_buffer; -} - -void Vulkan::Impl::submit_temp_cmd_buffer(VkCommandBuffer cmdBuffer) { - NVVK_CHECK(vkEndCommandBuffer(cmdBuffer)); - - VkSubmitInfo submit_info{VK_STRUCTURE_TYPE_SUBMIT_INFO}; - submit_info.commandBufferCount = 1; - submit_info.pCommandBuffers = &cmdBuffer; - NVVK_CHECK(vkQueueSubmit(queue_, 1, &submit_info, {})); - NVVK_CHECK(vkQueueWaitIdle(queue_)); - vkFreeCommandBuffers(vk_ctx_.m_device, cmd_pool_, 1, &cmdBuffer); -} - -uint32_t Vulkan::Impl::get_memory_type(uint32_t typeBits, - const VkMemoryPropertyFlags &properties) const { - VkPhysicalDeviceMemoryProperties memory_properties; - vkGetPhysicalDeviceMemoryProperties(vk_ctx_.m_physicalDevice, &memory_properties); - - for (uint32_t i = 0; i < memory_properties.memoryTypeCount; i++) { - if (((typeBits & (1 << i)) > 0) && - (memory_properties.memoryTypes[i].propertyFlags & properties) == properties) - return i; - } - std::string err = "Unable to find memory type " + std::to_string(properties); - LOGE(err.c_str()); - return ~0u; -} - -VkPipeline Vulkan::Impl::create_pipeline(VkPipelineLayout pipeline_layout, - const uint32_t *vertex_shader, - size_t vertex_shader_size, const uint32_t *fragment_shader, - size_t fragment_shader_size, VkPrimitiveTopology topology, - const std::vector dynamic_state, - bool imgui_attrib_desc) { - nvvk::GraphicsPipelineGeneratorCombined gpb(vk_ctx_.m_device, pipeline_layout, render_pass_); - gpb.depthStencilState.depthTestEnable = true; - - std::vector code; - code.assign(vertex_shader, vertex_shader + vertex_shader_size); - gpb.addShader(code, VK_SHADER_STAGE_VERTEX_BIT); - code.assign(fragment_shader, fragment_shader + fragment_shader_size); - gpb.addShader(code, VK_SHADER_STAGE_FRAGMENT_BIT); - - gpb.inputAssemblyState.topology = topology; - - if (imgui_attrib_desc) { - gpb.addBindingDescription({0, sizeof(ImDrawVert), VK_VERTEX_INPUT_RATE_VERTEX}); - gpb.addAttributeDescriptions({{0, 0, VK_FORMAT_R32G32_SFLOAT, - IM_OFFSETOF(ImDrawVert, pos)}}); - gpb.addAttributeDescriptions({{1, 0, VK_FORMAT_R32G32_SFLOAT, - IM_OFFSETOF(ImDrawVert, uv)}}); - gpb.addAttributeDescriptions({{2, 0, VK_FORMAT_R8G8B8A8_UNORM, - IM_OFFSETOF(ImDrawVert, col)}}); - } else { - gpb.addBindingDescription({0, sizeof(float) * 2, VK_VERTEX_INPUT_RATE_VERTEX}); - gpb.addAttributeDescriptions({{0, 0, VK_FORMAT_R32G32_SFLOAT, 0}}); - } - - // disable culling - gpb.rasterizationState.cullMode = VK_CULL_MODE_NONE; - - // enable blending - VkPipelineColorBlendAttachmentState color_blend_attachment_state{}; - color_blend_attachment_state.colorWriteMask = - VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT - | VK_COLOR_COMPONENT_A_BIT; - color_blend_attachment_state.blendEnable = VK_TRUE; - color_blend_attachment_state.srcColorBlendFactor = VK_BLEND_FACTOR_SRC_ALPHA; - color_blend_attachment_state.dstColorBlendFactor = VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA; - color_blend_attachment_state.colorBlendOp = VK_BLEND_OP_ADD; - color_blend_attachment_state.srcAlphaBlendFactor = VK_BLEND_FACTOR_ONE; - color_blend_attachment_state.dstAlphaBlendFactor = VK_BLEND_FACTOR_ZERO; - color_blend_attachment_state.alphaBlendOp = VK_BLEND_OP_ADD; - // remove the default blend attachment state - gpb.clearBlendAttachmentStates(); - gpb.addBlendAttachmentState(color_blend_attachment_state); - - for (auto &&state : dynamic_state) { - gpb.addDynamicStateEnable(state); - } - - return gpb.createPipeline(); -} - -static void format_info(ImageFormat format, uint32_t *src_channels, uint32_t *dst_channels, - uint32_t *component_size) { - switch (format) { - case ImageFormat::R8_UINT: - *src_channels = *dst_channels = 1u; - *component_size = sizeof(uint8_t); - break; - case ImageFormat::R16_UINT: - *src_channels = *dst_channels = 1u; - *component_size = sizeof(uint16_t); - break; - case ImageFormat::R16_SFLOAT: - *src_channels = *dst_channels = 1u; - *component_size = sizeof(uint16_t); - break; - case ImageFormat::R32_UINT: - *src_channels = *dst_channels = 1u; - *component_size = sizeof(uint32_t); - break; - case ImageFormat::R32_SFLOAT: - *src_channels = *dst_channels = 1u; - *component_size = sizeof(float); - break; - case ImageFormat::R8G8B8_UNORM: - *src_channels = 3u; - *dst_channels = 4u; - *component_size = sizeof(uint8_t); - break; - case ImageFormat::B8G8R8_UNORM: - *src_channels = 3u; - *dst_channels = 4u; - *component_size = sizeof(uint8_t); - break; - case ImageFormat::R8G8B8A8_UNORM: - *src_channels = *dst_channels = 4u; - *component_size = sizeof(uint8_t); - break; - case ImageFormat::B8G8R8A8_UNORM: - *src_channels = *dst_channels = 4u; - *component_size = sizeof(uint8_t); - break; - case ImageFormat::R16G16B16A16_UNORM: - *src_channels = *dst_channels = 4u; - *component_size = sizeof(uint16_t); - break; - case ImageFormat::R16G16B16A16_SFLOAT: - *src_channels = *dst_channels = 4u; - *component_size = sizeof(uint16_t); - break; - case ImageFormat::R32G32B32A32_SFLOAT: - *src_channels = *dst_channels = 4u; - *component_size = sizeof(uint32_t); - break; - default: - throw std::runtime_error("Unhandled image format."); - } -} - -static VkFormat to_vulkan_format(ImageFormat format) { - VkFormat vk_format; - - switch (format) { - case ImageFormat::R8_UINT: - vk_format = VK_FORMAT_R8_UINT; - break; - case ImageFormat::R16_UINT: - vk_format = VK_FORMAT_R16_UINT; - break; - case ImageFormat::R16_SFLOAT: - vk_format = VK_FORMAT_R16_SFLOAT; - break; - case ImageFormat::R32_UINT: - vk_format = VK_FORMAT_R32_UINT; - break; - case ImageFormat::R32_SFLOAT: - vk_format = VK_FORMAT_R32_SFLOAT; - break; - case ImageFormat::R8G8B8_UNORM: - vk_format = VK_FORMAT_R8G8B8A8_UNORM; - break; - case ImageFormat::B8G8R8_UNORM: - vk_format = VK_FORMAT_B8G8R8A8_UNORM; - break; - case ImageFormat::R8G8B8A8_UNORM: - vk_format = VK_FORMAT_R8G8B8A8_UNORM; - break; - case ImageFormat::B8G8R8A8_UNORM: - vk_format = VK_FORMAT_B8G8R8A8_UNORM; - break; - case ImageFormat::R16G16B16A16_UNORM: - vk_format = VK_FORMAT_R16G16B16A16_UNORM; - break; - case ImageFormat::R16G16B16A16_SFLOAT: - vk_format = VK_FORMAT_R16G16B16A16_SFLOAT; - break; - case ImageFormat::R32G32B32A32_SFLOAT: - vk_format = VK_FORMAT_R32G32B32A32_SFLOAT; - break; - default: - throw std::runtime_error("Unhandled image format."); - } - - return vk_format; -} - -UniqueCUexternalSemaphore Vulkan::Impl::import_semaphore_to_cuda(VkSemaphore semaphore) { - VkSemaphoreGetFdInfoKHR semaphore_get_fd_info{VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR}; - semaphore_get_fd_info.semaphore = semaphore; - semaphore_get_fd_info.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT; - - UniqueValue file_handle; - file_handle.reset([this, &semaphore_get_fd_info] { - int handle; - NVVK_CHECK(vkGetSemaphoreFdKHR(vk_ctx_.m_device, &semaphore_get_fd_info, &handle)); - return handle; - }()); - - CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC semaphore_handle_desc{}; - semaphore_handle_desc.type = CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD; - semaphore_handle_desc.handle.fd = file_handle.get(); - - UniqueCUexternalSemaphore cuda_semaphore; - cuda_semaphore.reset([&semaphore_handle_desc] { - CUexternalSemaphore ext_semaphore; - CudaCheck(cuImportExternalSemaphore(&ext_semaphore, &semaphore_handle_desc)); - return ext_semaphore; - }()); - - // don't need to close the file handle if it had been successfully imported - file_handle.release(); - - return cuda_semaphore; -} - -Vulkan::Texture *Vulkan::Impl::create_texture_for_cuda_upload(uint32_t width, uint32_t height, - ImageFormat format, - VkFilter filter, bool normalized) { - if (transfer_jobs_.empty() || (transfer_jobs_.back().fence_ != nullptr)) { - throw std::runtime_error( - "Transfer command buffer not set. Calls to create_texture_for_cuda_upload() " - "need to be enclosed by " - "begin_transfer_pass() and " - "end_transfer_pass()"); - } - - const VkFormat vk_format = to_vulkan_format(format); - uint32_t src_channels, dst_channels, component_size; - format_info(format, &src_channels, &dst_channels, &component_size); - - // create the image - const VkImageCreateInfo image_create_info = nvvk::makeImage2DCreateInfo( - VkExtent2D{width, height}, vk_format); - // the VkExternalMemoryImageCreateInfoKHR struct is appended by nvvk::ExportResourceAllocator - const nvvk::Image image = export_alloc_.createImage(image_create_info, - VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); - - // create the texture - std::unique_ptr texture = std::make_unique(width, height, format, - &export_alloc_); - - // create the vulkan texture - VkSamplerCreateInfo sampler_create_info{VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO}; - sampler_create_info.minFilter = filter; - sampler_create_info.magFilter = filter; - sampler_create_info.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST; - sampler_create_info.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; - sampler_create_info.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; - sampler_create_info.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; - sampler_create_info.maxLod = normalized ? FLT_MAX : 0; - sampler_create_info.unnormalizedCoordinates = normalized ? VK_FALSE : VK_TRUE; - - const VkImageViewCreateInfo image_view_info = nvvk::makeImageViewCreateInfo(image.image, - image_create_info); - texture->texture_ = export_alloc_.createTexture(image, image_view_info, sampler_create_info); - - // create the semaphores, one for upload and one for rendering - VkSemaphoreCreateInfo semaphore_create_info{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO}; - VkExportSemaphoreCreateInfoKHR export_semaphore_create_info - {VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO_KHR}; - export_semaphore_create_info.handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT; - semaphore_create_info.pNext = &export_semaphore_create_info; - NVVK_CHECK(vkCreateSemaphore(vk_ctx_.m_device, &semaphore_create_info, nullptr, - &texture->upload_semaphore_)); - NVVK_CHECK(vkCreateSemaphore(vk_ctx_.m_device, &semaphore_create_info, nullptr, - &texture->render_semaphore_)); - - { - const CudaService::ScopedPush cuda_context = CudaService::get().PushContext(); - - // import memory to CUDA - const nvvk::MemAllocator::MemInfo memInfo = export_alloc_ - .getMemoryAllocator()->getMemoryInfo(image.memHandle); - - VkMemoryGetFdInfoKHR memory_get_fd_info{VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR}; - memory_get_fd_info.memory = memInfo.memory; - memory_get_fd_info.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT; - UniqueValue file_handle; - file_handle.reset([this, &memory_get_fd_info] { - int handle; - NVVK_CHECK(vkGetMemoryFdKHR(vk_ctx_.m_device, &memory_get_fd_info, &handle)); - return handle; - }()); - - CUDA_EXTERNAL_MEMORY_HANDLE_DESC memmory_handle_desc{}; - memmory_handle_desc.type = CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD; - memmory_handle_desc.handle.fd = file_handle.get(); - memmory_handle_desc.size = memInfo.offset + memInfo.size; - - texture->external_mem_.reset([&memmory_handle_desc] { - CUexternalMemory external_mem; - CudaCheck(cuImportExternalMemory(&external_mem, &memmory_handle_desc)); - return external_mem; - }()); - // don't need to close the file handle if it had been successfully imported - file_handle.release(); - - CUDA_EXTERNAL_MEMORY_MIPMAPPED_ARRAY_DESC mipmapped_array_desc{}; - mipmapped_array_desc.arrayDesc.Width = width; - mipmapped_array_desc.arrayDesc.Height = height; - mipmapped_array_desc.arrayDesc.Depth = 0; - switch (component_size) { - case 1: - mipmapped_array_desc.arrayDesc.Format = CU_AD_FORMAT_UNSIGNED_INT8; - break; - case 2: - mipmapped_array_desc.arrayDesc.Format = CU_AD_FORMAT_UNSIGNED_INT16; - break; - case 4: - mipmapped_array_desc.arrayDesc.Format = CU_AD_FORMAT_UNSIGNED_INT32; - break; - default: - throw std::runtime_error("Unhandled component size"); - } - mipmapped_array_desc.arrayDesc.NumChannels = dst_channels; - mipmapped_array_desc.arrayDesc.Flags = CUDA_ARRAY3D_SURFACE_LDST; - - mipmapped_array_desc.numLevels = 1; - mipmapped_array_desc.offset = memInfo.offset; - - texture->mipmap_.reset([external_mem = texture->external_mem_.get(), - &mipmapped_array_desc] { - CUmipmappedArray mipmaped_array; - CudaCheck(cuExternalMemoryGetMappedMipmappedArray(&mipmaped_array, external_mem, - &mipmapped_array_desc)); - return mipmaped_array; - }()); - - // import the semaphores to Cuda - texture->cuda_upload_semaphore_ = import_semaphore_to_cuda(texture->upload_semaphore_); - texture->cuda_render_semaphore_ = import_semaphore_to_cuda(texture->render_semaphore_); - } - - // transition to shader layout - nvvk::cmdBarrierImageLayout(transfer_jobs_.back().cmd_buffer_, texture->texture_.image, - VK_IMAGE_LAYOUT_UNDEFINED, - VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL); - - return texture.release(); -} - -Vulkan::Texture *Vulkan::Impl::create_texture(uint32_t width, uint32_t height, ImageFormat format, - size_t data_size, const void *data, - VkFilter filter, bool normalized) { - if (transfer_jobs_.empty() || (transfer_jobs_.back().fence_ != nullptr)) { - throw std::runtime_error( - "Transfer command buffer not set. Calls to create_texture() need to be enclosed by " - "begin_transfer_pass() and end_transfer_pass()"); - } - - const VkFormat vk_format = to_vulkan_format(format); - uint32_t src_channels, dst_channels, component_size; - format_info(format, &src_channels, &dst_channels, &component_size); - - if (data && (data_size != width * height * src_channels * component_size)) { - throw std::runtime_error("The size of the data array is wrong"); - } - - const VkImageCreateInfo image_create_info = nvvk::makeImage2DCreateInfo( - VkExtent2D{width, height}, vk_format); - const nvvk::Image image = alloc_.createImage(transfer_jobs_.back().cmd_buffer_, data_size, data, - image_create_info); - - // create the texture - std::unique_ptr texture = std::make_unique(width, height, format, &alloc_); - - // create the Vulkan texture - VkSamplerCreateInfo sampler_create_info{VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO}; - sampler_create_info.minFilter = filter; - sampler_create_info.magFilter = filter; - sampler_create_info.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST; - sampler_create_info.addressModeU = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; - sampler_create_info.addressModeV = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; - sampler_create_info.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; - sampler_create_info.maxLod = normalized ? FLT_MAX : 0; - sampler_create_info.unnormalizedCoordinates = normalized ? VK_FALSE : VK_TRUE; - - const VkImageViewCreateInfo image_view_info = nvvk::makeImageViewCreateInfo(image.image, - image_create_info); - texture->texture_ = alloc_.createTexture(image, image_view_info, - sampler_create_info); - - // transition to shader layout - /// @todo I don't know if this is defined. Should the old layout be - /// VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, like it would be if we uploaded using Vulkan? - nvvk::cmdBarrierImageLayout(transfer_jobs_.back().cmd_buffer_, image.image, - image_create_info.initialLayout, - VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL); - - return texture.release(); -} - -void Vulkan::Impl::destroy_texture(Texture *texture) { - if (texture->fence_) { - // if the texture had been tagged with a fence, wait for it before freeing the memory - NVVK_CHECK(vkWaitForFences(vk_ctx_.m_device, 1, &texture->fence_, VK_TRUE, 1000000)); - } - - // check if this texture had been imported to CUDA - if (texture->mipmap_ || texture->external_mem_) { - const CudaService::ScopedPush cuda_context = CudaService::get().PushContext(); - - texture->mipmap_.reset(); - texture->external_mem_.reset(); - texture->cuda_upload_semaphore_.reset(); - texture->cuda_render_semaphore_.reset(); - } - - if (texture->upload_semaphore_) { - vkDestroySemaphore(vk_ctx_.m_device, texture->upload_semaphore_, nullptr); - } - if (texture->render_semaphore_) { - vkDestroySemaphore(vk_ctx_.m_device, texture->render_semaphore_, nullptr); - } - - texture->alloc_->destroy(texture->texture_); - - delete texture; -} - -void Vulkan::Impl::upload_to_texture(CUdeviceptr device_ptr, Texture *texture, CUstream stream) { - if (!texture->mipmap_) { - throw std::runtime_error("Texture had not been imported to CUDA, can't upload data."); - } - - const CudaService::ScopedPush cuda_context = CudaService::get().PushContext(); - - if (texture->state_ == Texture::State::RENDERED) { - // if the texture had been used in rendering, wait for Vulkan to finish it's work - CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS ext_wait_params{}; - const CUexternalSemaphore external_semaphore = texture->cuda_render_semaphore_.get(); - CudaCheck(cuWaitExternalSemaphoresAsync(&external_semaphore, &ext_wait_params, 1, stream)); - texture->state_ = Texture::State::UNKNOWN; - } - - CUarray array; - CudaCheck(cuMipmappedArrayGetLevel(&array, texture->mipmap_.get(), 0)); - - uint32_t src_channels, dst_channels, component_size; - format_info(texture->format_, &src_channels, &dst_channels, &component_size); - const uint32_t src_pitch = texture->width_ * src_channels * component_size; - - if (src_channels != dst_channels) { - // three channel texture data is not hardware natively supported, convert to four channel - if ((src_channels != 3) || (dst_channels != 4) || (component_size != 1)) { - throw std::runtime_error("Unhandled conversion."); - } - ConvertR8G8B8ToR8G8B8A8(texture->width_, texture->height_, device_ptr, src_pitch, - array, stream); - } else { - // else just copy - CUDA_MEMCPY2D memcpy_2d{}; - memcpy_2d.srcMemoryType = CU_MEMORYTYPE_DEVICE; - memcpy_2d.srcDevice = device_ptr; - memcpy_2d.srcPitch = src_pitch; - memcpy_2d.dstMemoryType = CU_MEMORYTYPE_ARRAY; - memcpy_2d.dstArray = array; - memcpy_2d.WidthInBytes = texture->width_ * dst_channels * component_size; - memcpy_2d.Height = texture->height_; - CudaCheck(cuMemcpy2DAsync(&memcpy_2d, stream)); - } - - // signal the semaphore - CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS ext_signal_params{}; - const CUexternalSemaphore external_semaphore = texture->cuda_upload_semaphore_.get(); - CudaCheck(cuSignalExternalSemaphoresAsync(&external_semaphore, &ext_signal_params, 1, stream)); - - // the texture is now in uploaded state, Vulkan needs to wait for it - texture->state_ = Texture::State::UPLOADED; -} - -void Vulkan::Impl::upload_to_texture(const void *host_ptr, Texture *texture) { - if (transfer_jobs_.empty() || (transfer_jobs_.back().fence_ != nullptr)) { - throw std::runtime_error( - "Transfer command buffer not set. Calls to upload_to_texture() need to be enclosed by " - "begin_transfer_pass() and end_transfer_pass()"); - } - - if ((texture->state_ != Texture::State::RENDERED) && - (texture->state_ != Texture::State::UNKNOWN)) { - throw std::runtime_error("When uploading to texture, the texture should be in rendered " - "or unknown state"); - } - - const VkCommandBuffer cmd_buf = transfer_jobs_.back().cmd_buffer_; - - // Copy buffer to image - VkImageSubresourceRange subresource_range{}; - subresource_range.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - subresource_range.baseArrayLayer = 0; - subresource_range.baseMipLevel = 0; - subresource_range.layerCount = 1; - subresource_range.levelCount = 1; - - nvvk::cmdBarrierImageLayout(cmd_buf, texture->texture_.image, VK_IMAGE_LAYOUT_UNDEFINED, - VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, subresource_range); - - VkOffset3D offset = {0}; - VkImageSubresourceLayers subresource = {0}; - subresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - subresource.layerCount = 1; - - uint32_t src_channels, dst_channels, component_size; - format_info(texture->format_, &src_channels, &dst_channels, &component_size); - - const VkDeviceSize data_size = texture->width_ * texture->height_ * dst_channels * - component_size; - void *mapping = alloc_.getStaging()->cmdToImage(cmd_buf, texture->texture_.image, - offset, - VkExtent3D{texture->width_, - texture->height_, 1}, - subresource, - data_size, nullptr); - - if (src_channels != dst_channels) { - // three channel texture data is not hardware natively supported, convert to four channel - if ((src_channels != 3) || (dst_channels != 4) || (component_size != 1)) { - throw std::runtime_error("Unhandled conversion."); - } - const uint8_t *src = reinterpret_cast(host_ptr); - uint32_t *dst = reinterpret_cast(mapping); - for (uint32_t y = 0; y < texture->height_; ++y) { - for (uint32_t x = 0; x < texture->width_; ++x) { - const uint8_t data[4]{src[0], src[1], src[2], 0xFF}; - *dst = *reinterpret_cast(&data); - src += 3; - ++dst; - } - } - } else { - memcpy(mapping, host_ptr, data_size); - } - - // Setting final image layout - nvvk::cmdBarrierImageLayout(cmd_buf, texture->texture_.image, - VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, - VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL); - - // no need to set the texture state here, the transfer command buffer submission is - // always synchronized to the render command buffer submission. -} - -Vulkan::Buffer *Vulkan::Impl::create_buffer(size_t data_size, VkBufferUsageFlags usage, - const void *data) { - std::unique_ptr buffer(new Buffer(data_size, &alloc_)); - if (data) { - if (transfer_jobs_.empty() || (transfer_jobs_.back().fence_ != nullptr)) { - throw std::runtime_error( - "Transfer command buffer not set. Calls to create_buffer() with data need to be " - "enclosed by begin_transfer_pass() and end_transfer_pass()"); - } - - buffer->buffer_ = - alloc_.createBuffer(transfer_jobs_.back().cmd_buffer_, - static_cast(data_size), data, usage); - } else { - buffer->buffer_ = alloc_.createBuffer(static_cast(data_size), usage); - } - - return buffer.release(); -} - -void Vulkan::Impl::destroy_buffer(Buffer *buffer) { - if (buffer->fence_) { - // if the buffer had been tagged with a fence, wait for it before freeing the memory - NVVK_CHECK(vkWaitForFences(vk_ctx_.m_device, 1, &buffer->fence_, VK_TRUE, 100'000'000)); - } - - buffer->alloc_->destroy(buffer->buffer_); - delete buffer; -} - -void Vulkan::Impl::draw_texture(Texture *texture, Texture *lut, float opacity) { - const VkCommandBuffer cmd_buf = command_buffers_[get_active_image_index()]; - - if (texture->state_ == Texture::State::UPLOADED) { - // enqueue the semaphore signalled by Cuda to be waited on by rendering - batch_submission_.enqueueWait(texture->upload_semaphore_, - VK_PIPELINE_STAGE_ALL_COMMANDS_BIT); - // also signal the render semapore which will be waited on by Cuda - batch_submission_.enqueueSignal(texture->render_semaphore_); - texture->state_ = Texture::State::RENDERED; - } - - VkPipeline pipeline; - VkPipelineLayout pipeline_layout; - - // update descriptor sets - std::vector writes; - if (lut) { - if (lut->state_ == Texture::State::UPLOADED) { - // enqueue the semaphore signalled by Cuda to be waited on by rendering - batch_submission_.enqueueWait(lut->upload_semaphore_, - VK_PIPELINE_STAGE_ALL_COMMANDS_BIT); - // also signal the render semapore which will be waited on by Cuda - batch_submission_.enqueueSignal(lut->render_semaphore_); - lut->state_ = Texture::State::RENDERED; - } - - if ((texture->format_ == ImageFormat::R8_UINT) || - (texture->format_ == ImageFormat::R16_UINT) || - (texture->format_ == ImageFormat::R32_UINT)) { - pipeline = image_lut_uint_pipeline_; - } else { - pipeline = image_lut_float_pipeline_; - } - - pipeline_layout = image_lut_pipeline_layout_; - - writes.emplace_back( - desc_set_layout_bind_lut_.makeWrite(nullptr, bindings_offset_texture_, - &texture->texture_.descriptor)); - writes.emplace_back( - desc_set_layout_bind_lut_.makeWrite(nullptr, bindings_offset_texture_lut_, - &lut->texture_.descriptor)); - } else { - pipeline = image_pipeline_; - pipeline_layout = image_pipeline_layout_; - - writes.emplace_back( - desc_set_layout_bind_.makeWrite(nullptr, bindings_offset_texture_, - &texture->texture_.descriptor)); - } - - vkCmdBindPipeline(cmd_buf, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); - - vkCmdPushDescriptorSetKHR(cmd_buf, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline_layout, 0, - static_cast(writes.size()), writes.data()); - - // push the constants - PushConstantFragment push_constants; - push_constants.opacity = opacity; - vkCmdPushConstants(cmd_buf, pipeline_layout, VK_SHADER_STAGE_FRAGMENT_BIT, 0, - sizeof(PushConstantFragment), &push_constants); - - // bind the buffers - VkDeviceSize offset{0}; - vkCmdBindVertexBuffers(cmd_buf, 0, 1, &vertex_buffer_.buffer, &offset); - vkCmdBindIndexBuffer(cmd_buf, index_buffer_.buffer, 0, VK_INDEX_TYPE_UINT16); - - // draw - vkCmdDrawIndexed(cmd_buf, 6, 1, 0, 0, 0); - - // tag the texture and lut with the current fence - texture->fence_ = wait_fences_[get_active_image_index()]; - if (lut) { - lut->fence_ = wait_fences_[get_active_image_index()]; - } -} - -void Vulkan::Impl::draw(VkPrimitiveTopology topology, uint32_t count, uint32_t first, - Buffer *buffer, float opacity, const std::array &color, - float point_size, float line_width) { - const VkCommandBuffer cmd_buf = command_buffers_[get_active_image_index()]; - - switch (topology) { - case VkPrimitiveTopology::VK_PRIMITIVE_TOPOLOGY_POINT_LIST: - vkCmdBindPipeline(cmd_buf, VK_PIPELINE_BIND_POINT_GRAPHICS, geometry_point_pipeline_); - break; - case VkPrimitiveTopology::VK_PRIMITIVE_TOPOLOGY_LINE_LIST: - vkCmdBindPipeline(cmd_buf, VK_PIPELINE_BIND_POINT_GRAPHICS, geometry_line_pipeline_); - vkCmdSetLineWidth(cmd_buf, line_width); - break; - case VkPrimitiveTopology::VK_PRIMITIVE_TOPOLOGY_LINE_STRIP: - vkCmdBindPipeline(cmd_buf, VK_PIPELINE_BIND_POINT_GRAPHICS, geometry_line_strip_pipeline_); - vkCmdSetLineWidth(cmd_buf, line_width); - break; - case VkPrimitiveTopology::VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST: - vkCmdBindPipeline(cmd_buf, VK_PIPELINE_BIND_POINT_GRAPHICS, geometry_triangle_pipeline_); - break; - default: - throw std::runtime_error("Unhandled primitive type"); - } - - // push the constants - PushConstantFragment push_constants_fragment; - push_constants_fragment.opacity = opacity; - vkCmdPushConstants(cmd_buf, geometry_pipeline_layout_, VK_SHADER_STAGE_FRAGMENT_BIT, - sizeof(PushConstantVertex), sizeof(PushConstantFragment), - &push_constants_fragment); - - PushConstantVertex push_constant_vertex; - push_constant_vertex.matrix.identity(); - push_constant_vertex.matrix.scale({2.f, 2.f, 1.f}); - push_constant_vertex.matrix.translate({-.5f, -.5f, 0.f}); - push_constant_vertex.point_size = point_size; - push_constant_vertex.color = color; - vkCmdPushConstants(cmd_buf, geometry_pipeline_layout_, VK_SHADER_STAGE_VERTEX_BIT, 0, - sizeof(PushConstantVertex), &push_constant_vertex); - - VkDeviceSize offset{0}; - vkCmdBindVertexBuffers(cmd_buf, 0, 1, &buffer->buffer_.buffer, &offset); - - // draw - vkCmdDraw(cmd_buf, count, 1, first, 0); - - // tag the buffer with the current fence - buffer->fence_ = wait_fences_[get_active_image_index()]; -} - -void Vulkan::Impl::draw_indexed(VkDescriptorSet desc_set, Buffer *vertex_buffer, - Buffer *index_buffer, VkIndexType index_type, - uint32_t index_count, uint32_t first_index, - uint32_t vertex_offset, float opacity) { - const VkCommandBuffer cmd_buf = command_buffers_[get_active_image_index()]; - - vkCmdBindPipeline(cmd_buf, VK_PIPELINE_BIND_POINT_GRAPHICS, geometry_text_pipeline_); - - vkCmdBindDescriptorSets(cmd_buf, VK_PIPELINE_BIND_POINT_GRAPHICS, - geometry_text_pipeline_layout_, 0, 1, &desc_set, - 0, nullptr); - - // push the constants - PushConstantFragment push_constants; - push_constants.opacity = opacity; - vkCmdPushConstants(cmd_buf, geometry_text_pipeline_layout_, VK_SHADER_STAGE_FRAGMENT_BIT, - sizeof(PushConstantTextVertex), sizeof(PushConstantFragment), - &push_constants); - - PushConstantTextVertex push_constant_vertex; - push_constant_vertex.matrix.identity(); - push_constant_vertex.matrix.scale({2.f, 2.f, 1.f}); - push_constant_vertex.matrix.translate({-.5f, -.5f, 0.f}); - vkCmdPushConstants(cmd_buf, geometry_text_pipeline_layout_, VK_SHADER_STAGE_VERTEX_BIT, 0, - sizeof(PushConstantTextVertex), &push_constant_vertex); - - // bind the buffers - VkDeviceSize offset{0}; - vkCmdBindVertexBuffers(cmd_buf, 0, 1, &vertex_buffer->buffer_.buffer, &offset); - vkCmdBindIndexBuffer(cmd_buf, index_buffer->buffer_.buffer, 0, index_type); - - // draw - vkCmdDrawIndexed(cmd_buf, index_count, 1, first_index, vertex_offset, 0); - - // tag the buffers with the current fence - vertex_buffer->fence_ = index_buffer->fence_ = wait_fences_[get_active_image_index()]; -} - -void Vulkan::Impl::read_framebuffer(ImageFormat fmt, size_t buffer_size, - CUdeviceptr device_ptr, CUstream stream) { - if (fmt != ImageFormat::R8G8B8A8_UNORM) { - throw std::runtime_error("Unsupported image format, supported formats: R8G8B8A8_UNORM."); - } - - const VkFormat vk_format = to_vulkan_format(fmt); - uint32_t src_channels, dst_channels, component_size; - format_info(fmt, &src_channels, &dst_channels, &component_size); - - const size_t data_size = size_.width * size_.height * dst_channels * component_size; - if (buffer_size < data_size) { - throw std::runtime_error("The size of the buffer is too small"); - } - - /// @todo need to wait for frame, use semaphores to sync - batch_submission_.waitIdle(); - - // create a command buffer - nvvk::CommandPool cmd_buf_get(vk_ctx_.m_device, vk_ctx_.m_queueGCT.familyIndex); - VkCommandBuffer cmd_buf = cmd_buf_get.createCommandBuffer(); - - // Make the image layout VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL to copy to buffer - VkImageSubresourceRange subresource_range{VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; - nvvk::cmdBarrierImageLayout(cmd_buf, fb_sequence_.get_active_image(), - surface_ ? VK_IMAGE_LAYOUT_PRESENT_SRC_KHR : - VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, - VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, subresource_range); - - // allocate the buffer - /// @todo keep the buffer and the mapping to Cuda to avoid allocations - Vulkan::Buffer *const transfer_buffer = create_buffer(data_size, - VK_BUFFER_USAGE_TRANSFER_DST_BIT); - - // Copy the image to the buffer - VkBufferImageCopy copy_region{}; - copy_region.imageSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - copy_region.imageSubresource.layerCount = 1; - copy_region.imageExtent.width = size_.width; - copy_region.imageExtent.height = size_.height; - copy_region.imageExtent.depth = 1; - vkCmdCopyImageToBuffer(cmd_buf, fb_sequence_.get_active_image(), - VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, - transfer_buffer->buffer_.buffer, 1, ©_region); - - // Put back the image as it was - nvvk::cmdBarrierImageLayout(cmd_buf, fb_sequence_.get_active_image(), - VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, - surface_ ? VK_IMAGE_LAYOUT_PRESENT_SRC_KHR : - VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, - subresource_range); - /// @todo avoid wait, use semaphore to sync - cmd_buf_get.submitAndWait(cmd_buf); - - { - const CudaService::ScopedPush cuda_context = CudaService::get().PushContext(); - - // import memory to CUDA - const nvvk::MemAllocator::MemInfo memInfo = - export_alloc_.getMemoryAllocator()->getMemoryInfo(transfer_buffer->buffer_.memHandle); - - VkMemoryGetFdInfoKHR memory_get_fd_info{VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR}; - memory_get_fd_info.memory = memInfo.memory; - memory_get_fd_info.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT; - UniqueValue file_handle; - file_handle.reset([this, &memory_get_fd_info] { - int handle; - NVVK_CHECK(vkGetMemoryFdKHR(vk_ctx_.m_device, &memory_get_fd_info, &handle)); - return handle; - }()); - - CUDA_EXTERNAL_MEMORY_HANDLE_DESC memmory_handle_desc{}; - memmory_handle_desc.type = CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD; - memmory_handle_desc.handle.fd = file_handle.get(); - memmory_handle_desc.size = memInfo.offset + memInfo.size; - - UniqueCUexternalMemory external_mem; - external_mem.reset([&memmory_handle_desc] { - CUexternalMemory external_mem; - CudaCheck(cuImportExternalMemory(&external_mem, &memmory_handle_desc)); - return external_mem; - }()); - // don't need to close the file handle if it had been successfully imported - file_handle.release(); - - CUDA_EXTERNAL_MEMORY_BUFFER_DESC buffer_desc{}; - buffer_desc.size = data_size; - buffer_desc.offset = memInfo.offset; - - UniqueCUdeviceptr transfer_mem; - transfer_mem.reset([&external_mem, &buffer_desc] { - CUdeviceptr device_ptr; - CudaCheck(cuExternalMemoryGetMappedBuffer(&device_ptr, external_mem.get(), - &buffer_desc)); - return device_ptr; - }()); - - if (fb_sequence_.get_format() == VkFormat::VK_FORMAT_B8G8R8A8_UNORM) { - // convert from B8G8R8A8 to R8G8B8A8 - ConvertB8G8R8A8ToR8G8B8A8(size_.width, size_.height, transfer_mem.get(), - size_.width * dst_channels * component_size, device_ptr, - size_.width * dst_channels * component_size, stream); - } else if (fb_sequence_.get_format() == VkFormat::VK_FORMAT_R8G8B8A8_UNORM) { - CUDA_MEMCPY2D memcpy2d{}; - memcpy2d.srcMemoryType = CU_MEMORYTYPE_DEVICE; - memcpy2d.srcDevice = transfer_mem.get(); - memcpy2d.srcPitch = size_.width * dst_channels * component_size; - memcpy2d.dstMemoryType = CU_MEMORYTYPE_DEVICE; - memcpy2d.dstDevice = device_ptr; - memcpy2d.dstPitch = size_.width * dst_channels * component_size; - memcpy2d.WidthInBytes = size_.width * dst_channels * component_size; - memcpy2d.Height = size_.height; - CudaCheck(cuMemcpy2DAsync(&memcpy2d, stream)); - } else { - throw std::runtime_error("Unhandled framebuffer format."); - } - } - - destroy_buffer(transfer_buffer); -} - -Vulkan::Vulkan() - : impl_(new Vulkan::Impl) { -} - -Vulkan::~Vulkan() {} - -void Vulkan::setup(Window *window) { - impl_->setup(window); -} - -void Vulkan::begin_transfer_pass() { - impl_->begin_transfer_pass(); -} - -void Vulkan::end_transfer_pass() { - impl_->end_transfer_pass(); -} - -void Vulkan::begin_render_pass() { - impl_->begin_render_pass(); -} - -void Vulkan::end_render_pass() { - impl_->end_render_pass(); -} - -VkCommandBuffer Vulkan::get_command_buffer() { - return impl_->get_command_buffers()[impl_->get_active_image_index()]; -} - -Vulkan::Texture *Vulkan::create_texture_for_cuda_upload(uint32_t width, uint32_t height, - ImageFormat format, - VkFilter filter, bool normalized) { - return impl_->create_texture_for_cuda_upload(width, height, format, filter, normalized); -} - -Vulkan::Texture *Vulkan::create_texture(uint32_t width, uint32_t height, ImageFormat format, - size_t data_size, const void *data, - VkFilter filter, bool normalized) { - return impl_->create_texture(width, height, format, data_size, data, filter, normalized); -} - -void Vulkan::destroy_texture(Texture *texture) { - impl_->destroy_texture(texture); -} - -void Vulkan::upload_to_texture(CUdeviceptr device_ptr, Texture *texture, CUstream stream) { - impl_->upload_to_texture(device_ptr, texture, stream); -} - -void Vulkan::upload_to_texture(const void *host_ptr, Texture *texture) { - impl_->upload_to_texture(host_ptr, texture); -} - -void Vulkan::draw_texture(Texture *texture, Texture *lut, float opacity) { - impl_->draw_texture(texture, lut, opacity); -} - -Vulkan::Buffer *Vulkan::create_buffer(size_t data_size, const void *data, - VkBufferUsageFlags usage) { - return impl_->create_buffer(data_size, usage, data); -} - -void Vulkan::destroy_buffer(Buffer *buffer) { - impl_->destroy_buffer(buffer); -} - -void Vulkan::draw(VkPrimitiveTopology topology, uint32_t count, uint32_t first, Buffer *buffer, - float opacity, const std::array &color, - float point_size, float line_width) { - impl_->draw(topology, count, first, buffer, opacity, color, point_size, line_width); -} - -void Vulkan::draw_indexed(VkDescriptorSet desc_set, Buffer *vertex_buffer, Buffer *index_buffer, - VkIndexType index_type, uint32_t index_count, uint32_t first_index, - uint32_t vertex_offset, float opacity) { - impl_->draw_indexed(desc_set, vertex_buffer, index_buffer, index_type, index_count, - first_index, vertex_offset, opacity); -} - -void Vulkan::read_framebuffer(ImageFormat fmt, size_t buffer_size, - CUdeviceptr buffer, CUstream stream) { - impl_->read_framebuffer(fmt, buffer_size, buffer, stream); -} - -} // namespace holoscan::viz diff --git a/modules/holoviz/src/vulkan/vulkan.hpp b/modules/holoviz/src/vulkan/vulkan.hpp deleted file mode 100644 index 1eb60902..00000000 --- a/modules/holoviz/src/vulkan/vulkan.hpp +++ /dev/null @@ -1,224 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef HOLOSCAN_VIZ_VULKAN_VULKAN_HPP -#define HOLOSCAN_VIZ_VULKAN_VULKAN_HPP - -#include -#include - -#include -#include -#include -#include - -#include "../holoviz/image_format.hpp" -#include "../window.hpp" - -namespace holoscan::viz { - -class Layer; - -/** - * The class is responsible for all operations regarding Vulkan. - */ -class Vulkan { - public: - /** - * Construct a new Vulkan object. - */ - Vulkan(); - - /** - * Destroy the Vulkan object. - */ - ~Vulkan(); - - struct Texture; ///< texture object - struct Buffer; ///< buffer object - - /** - * Setup Vulkan using the given window. - * - * @param window window to use - */ - void setup(Window *window); - - /** - * Begin the transfer pass. This creates a transfer job and a command buffer. - */ - void begin_transfer_pass(); - - /** - * End the transfer pass. This ends and submits the transfer command buffer. - * It's an error to call end_transfer_pass() - * without begin_transfer_pass(). - */ - void end_transfer_pass(); - - /** - * Begin the render pass. This acquires the next image to render to - * and sets up the render command buffer. - */ - void begin_render_pass(); - - /** - * End the render pass. Submits the render command buffer. - */ - void end_render_pass(); - - /** - * Get the command buffer for the current frame. - * - * @return VkCommandBuffer - */ - VkCommandBuffer get_command_buffer(); - - /** - * Create a texture to be used for upload from Cuda data, see ::UploadToTexture. - * Destroy with ::DestroyTexture. - * - * @param width, height size - * @param format texture format - * @param filter texture filter - * @param normalized if true, then texture coordinates are normalize (0...1), - * else (0...width, 0...height) - * @return created texture object - */ - Texture *create_texture_for_cuda_upload(uint32_t width, uint32_t height, ImageFormat format, - VkFilter filter = VK_FILTER_LINEAR, - bool normalized = true); - - /** - * Create a Texture using host data. Destroy with ::DestroyTexture. - * - * @param width, height size - * @param format texture format - * @param data_size data size in bytes - * @param data texture data - * @param filter texture filter - * @param normalized if true, then texture coordinates are normalize (0...1), - * else (0...width, 0...height) - * @return created texture object - */ - Texture *create_texture(uint32_t width, uint32_t height, ImageFormat format, size_t data_size, - const void *data, VkFilter filter = VK_FILTER_LINEAR, - bool normalized = true); - - /** - * Destroy a texture created with ::CreateTextureForCudaUpload or ::CreateTexture. - * - * @param texture texture to destroy - */ - void destroy_texture(Texture *texture); - - /** - * Upload data from Cuda device memory to a texture created with ::CreateTextureForCudaUpload - * - * @param device_ptr Cuda device memory - * @param texture texture to be updated - * @param stream Cuda stream - */ - void upload_to_texture(CUdeviceptr device_ptr, Texture *texture, CUstream stream = 0); - - /** - * Upload data from host memory to a texture created with ::CreateTexture - * - * @param host_ptr data to upload in host memory - * @param texture texture to be updated - */ - void upload_to_texture(const void *host_ptr, Texture *texture); - - /** - * Draw a texture with an optional color lookup table. - * - * @param texture texture to draw - * @param lut lookup table, can be nullptr - * @param opacity opacity, 0.0 is transparent, 1.0 is opaque - */ - void draw_texture(Texture *texture, Texture *lut, float opacity); - - /** - * Create a vertex or index buffer and initialize with data. Destroy with ::DestroyBuffer. - * - * @param data_size size of the buffer in bytes - * @param data host size data to initialize buffer with. - * @param usage buffer usage - * @return created buffer - */ - Buffer *create_buffer(size_t data_size, const void *data, VkBufferUsageFlags usage); - - /** - * Destroy a buffer created with ::CreateBuffer. - * - * @param buffer buffer to destroy - */ - void destroy_buffer(Buffer *buffer); - - /** - * Draw geometry. - * - * @param topology topology - * @param count vertex count - * @param first first vertex - * @param buffer vertex buffer - * @param opacity opacity, 0.0 is transparent, 1.0 is opaque - * @param color color - * @param point_size point size - * @param line_width line width - */ - void draw(VkPrimitiveTopology topology, uint32_t count, uint32_t first, Buffer *buffer, - float opacity, const std::array &color, - float point_size, float line_width); - - /** - * Draw indexed triangle list geometry. Used to draw ImGui draw list for text drawing. - * - * @param desc_set descriptor set for texture atlas - * @param vertex_buffer vertex buffer - * @param index_buffer index buffer - * @param index_type index type - * @param index_count index count - * @param first_index first index - * @param vertex_offset vertex offset - * @param opacity opacity, 0.0 is transparent, 1.0 is opaque - */ - void draw_indexed(VkDescriptorSet desc_set, Buffer *vertex_buffer, Buffer *index_buffer, - VkIndexType index_type, uint32_t index_count, uint32_t first_index, - uint32_t vertex_offset, float opacity); - - /** - * Read the framebuffer and store it to cuda device memory. - * - * Can only be called outside of Begin()/End(). - * - * @param fmt image format, currently only R8G8B8A8_UNORM is supported. - * @param buffer_size size of the storage buffer in bytes - * @param buffer pointer to Cuda device memory to store the framebuffer into - * @param stream Cuda stream - */ - void read_framebuffer(ImageFormat fmt, size_t buffer_size, CUdeviceptr buffer, - CUstream stream = 0); - - private: - struct Impl; - std::shared_ptr impl_; -}; - -} // namespace holoscan::viz - -#endif /* HOLOSCAN_VIZ_VULKAN_VULKAN_HPP */ diff --git a/modules/holoviz/src/vulkan/vulkan_app.cpp b/modules/holoviz/src/vulkan/vulkan_app.cpp new file mode 100644 index 00000000..579cd134 --- /dev/null +++ b/modules/holoviz/src/vulkan/vulkan_app.cpp @@ -0,0 +1,2526 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "vulkan_app.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "../cuda/convert.hpp" +#include "../cuda/cuda_service.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../layers/layer.hpp" +#include "framebuffer_sequence.hpp" + +VULKAN_HPP_DEFAULT_DISPATCH_LOADER_DYNAMIC_STORAGE + +namespace holoscan::viz { + +struct PushConstantFragment { + float opacity; +}; + +struct PushConstantVertex { + nvmath::mat4f matrix; + float point_size; + std::array color; +}; + +struct Vulkan::Texture { + Texture(uint32_t width, uint32_t height, ImageFormat format, nvvk::ResourceAllocator* alloc) + : width_(width), height_(height), format_(format), alloc_(alloc) {} + + const uint32_t width_; + const uint32_t height_; + const ImageFormat format_; + nvvk::ResourceAllocator* const alloc_; + + enum class State { UNKNOWN, UPLOADED, RENDERED }; + State state_ = State::UNKNOWN; + + nvvk::Texture texture_{}; + UniqueCUexternalMemory external_mem_; + UniqueCUmipmappedArray mipmap_; + + /// this semaphore is used to synchronize uploading and rendering, it's signaled by Cuda on + /// upload and waited on by vulkan on rendering + vk::UniqueSemaphore upload_semaphore_; + UniqueCUexternalSemaphore cuda_upload_semaphore_; + + /// this semaphore is used to synchronize rendering and uploading, it's signaled by Vulkan on + /// rendering and waited on by Cuda on uploading + vk::UniqueSemaphore render_semaphore_; + UniqueCUexternalSemaphore cuda_render_semaphore_; + + /// last usage of the texture, need to sync before destroying memory + vk::Fence fence_ = nullptr; +}; + +struct Vulkan::Buffer { + Buffer(size_t size, nvvk::ResourceAllocator* alloc) : size_(size), alloc_(alloc) {} + + const size_t size_; + nvvk::ResourceAllocator* const alloc_; + + enum class State { UNKNOWN, UPLOADED, RENDERED }; + State state_ = State::UNKNOWN; + + nvvk::Buffer buffer_{}; + UniqueCUexternalMemory external_mem_; + UniqueCUdeviceptr device_ptr_; + + /// this semaphore is used to synchronize uploading and rendering, it's signaled by Cuda on + /// upload and waited on by vulkan on rendering + vk::UniqueSemaphore upload_semaphore_; + UniqueCUexternalSemaphore cuda_upload_semaphore_; + + /// this semaphore is used to synchronize rendering and uploading, it's signaled by Vulkan on + /// rendering and waited on by Cuda on uploading + vk::UniqueSemaphore render_semaphore_; + UniqueCUexternalSemaphore cuda_render_semaphore_; + + /// last usage of the buffer, need to sync before destroying memory + vk::Fence fence_ = nullptr; +}; + +class Vulkan::Impl { + public: + Impl() = default; + virtual ~Impl(); + + void setup(Window* window, const std::string& font_path, float font_size_in_pixels); + + Window* get_window() const; + + void begin_transfer_pass(); + void end_transfer_pass(); + void begin_render_pass(); + void end_render_pass(); + void cleanup_transfer_jobs(); + + void prepare_frame(); + void submit_frame(); + uint32_t get_active_image_index() const { return fb_sequence_.get_active_image_index(); } + const std::vector& get_command_buffers() { return command_buffers_; } + + Texture* create_texture_for_cuda_interop(uint32_t width, uint32_t height, ImageFormat format, + vk::Filter filter, bool normalized); + Texture* create_texture(uint32_t width, uint32_t height, ImageFormat format, size_t data_size, + const void* data, vk::Filter filter, bool normalized); + void destroy_texture(Texture* texture); + + void upload_to_texture(CUdeviceptr device_ptr, Texture* texture, CUstream stream); + void upload_to_texture(const void* host_ptr, Texture* texture); + + Buffer* create_buffer_for_cuda_interop(size_t data_size, vk::BufferUsageFlags usage); + Buffer* create_buffer(size_t data_size, vk::BufferUsageFlags usage, const void* data = nullptr); + + void upload_to_buffer(size_t data_size, CUdeviceptr device_ptr, Buffer* buffer, size_t dst_offset, + CUstream stream); + void upload_to_buffer(size_t data_size, const void* data, const Buffer* buffer); + + void destroy_buffer(Buffer* buffer); + + void draw_texture(Texture* texture, Texture* lut, float opacity, + const nvmath::mat4f& view_matrix); + + void draw(vk::PrimitiveTopology topology, uint32_t count, uint32_t first, + const std::vector& vertex_buffers, float opacity, + const std::array& color, float point_size, float line_width, + const nvmath::mat4f& view_matrix); + + void draw_text_indexed(vk::DescriptorSet desc_set, Buffer* vertex_buffer, Buffer* index_buffer, + vk::IndexType index_type, uint32_t index_count, uint32_t first_index, + uint32_t vertex_offset, float opacity, const nvmath::mat4f& view_matrix); + + void draw_indexed(vk::Pipeline pipeline, vk::PipelineLayout pipeline_layout, + vk::DescriptorSet desc_set, const std::vector& vertex_buffers, + Buffer* index_buffer, vk::IndexType index_type, uint32_t index_count, + uint32_t first_index, uint32_t vertex_offset, float opacity, + const std::array& color, float point_size, float line_width, + const nvmath::mat4f& view_matrix); + + void draw_indexed(vk::PrimitiveTopology topology, const std::vector& vertex_buffers, + Buffer* index_buffer, vk::IndexType index_type, uint32_t index_count, + uint32_t first_index, uint32_t vertex_offset, float opacity, + const std::array& color, float point_size, float line_width, + const nvmath::mat4f& view_matrix); + + void read_framebuffer(ImageFormat fmt, uint32_t width, uint32_t height, size_t buffer_size, + CUdeviceptr device_ptr, CUstream stream); + + private: + void init_im_gui(const std::string& font_path, float font_size_in_pixels); + void create_framebuffer_sequence(); + void create_depth_buffer(); + void create_render_pass(); + + /** + * Create all the framebuffers in which the image will be rendered + * - Swapchain need to be created before calling this + */ + void create_frame_buffers(); + + /** + * Callback when the window is resized + * - Destroy allocated frames, then rebuild them with the new size + */ + void on_framebuffer_size(int w, int h); + + vk::CommandBuffer create_temp_cmd_buffer(); + void submit_temp_cmd_buffer(vk::CommandBuffer cmd_buffer); + + uint32_t get_memory_type(uint32_t typeBits, const vk::MemoryPropertyFlags& properties) const; + + vk::UniquePipeline create_pipeline( + vk::PipelineLayout pipeline_layout, const uint32_t* vertex_shader, size_t vertex_shader_size, + const uint32_t* fragment_shader, size_t fragment_shader_size, vk::PrimitiveTopology topology, + const std::vector& dynamic_states, + const std::vector& binding_descriptions, + const std::vector& attribute_descriptions); + UniqueCUexternalSemaphore import_semaphore_to_cuda(vk::Semaphore semaphore); + + Window* window_ = nullptr; + + /** + * NVVK objects don't use destructors but init()/deinit(). To maintain the destructor calling + * sequence with Vulkan HPP objects store all NVVK objects in this struct and deinit() on + * destructor. + */ + class NvvkObjects { + public: + ~NvvkObjects() { + try { + if (index_buffer_.buffer) { alloc_.destroy(index_buffer_); } + if (vertex_buffer_.buffer) { alloc_.destroy(vertex_buffer_); } + if (transfer_cmd_pool_initialized_) { transfer_cmd_pool_.deinit(); } + if (export_alloc_initialized_) { export_alloc_.deinit(); } + if (alloc_initialized_) { alloc_.deinit(); } + vk_ctx_.deinit(); + } catch (const std::exception& e) { + HOLOSCAN_LOG_ERROR("NvvkObjects destructor failed with {}", e.what()); + } + } + + nvvk::Context vk_ctx_; + + // allocators + /// Allocator for buffer, images, acceleration structures + nvvk::ResourceAllocatorDma alloc_; + bool alloc_initialized_ = false; + /// Allocator for allocations which can be exported + nvvk::ExportResourceAllocator export_alloc_; + bool export_alloc_initialized_ = false; + + nvvk::BatchSubmission batch_submission_; + + nvvk::CommandPool transfer_cmd_pool_; + bool transfer_cmd_pool_initialized_ = false; + + nvvk::Buffer vertex_buffer_{}; + nvvk::Buffer index_buffer_{}; + } nvvk_; + + // Vulkan low level + vk::Instance instance_; + vk::Device device_; + vk::UniqueSurfaceKHR surface_; + vk::PhysicalDevice physical_device_; + vk::Queue queue_gct_; + vk::Queue queue_t_; + vk::UniqueCommandPool cmd_pool_; + vk::UniqueDescriptorPool im_gui_desc_pool_; + + /// Drawing/Surface + FramebufferSequence fb_sequence_; + /// All framebuffers, correspond to the Swapchain + std::vector framebuffers_; + /// Command buffer per nb element in Swapchain + std::vector command_buffers_; + /// Fences per nb element in Swapchain + std::vector wait_fences_; + /// Depth/Stencil + vk::UniqueImage depth_image_; + /// Depth/Stencil + vk::UniqueDeviceMemory depth_memory_; + /// Depth/Stencil + vk::UniqueImageView depth_view_; + /// Base render pass + vk::UniqueRenderPass render_pass_; + /// Size of the window + vk::Extent2D size_{0, 0}; + /// Cache for pipeline/shaders + vk::UniquePipelineCache pipeline_cache_; + + /// Depth buffer format + vk::Format depth_format_{vk::Format::eUndefined}; + + class TransferJob { + public: + vk::CommandBuffer cmd_buffer_ = nullptr; + vk::UniqueSemaphore semaphore_; + vk::UniqueFence fence_; + vk::Fence frame_fence_ = nullptr; + }; + std::list transfer_jobs_; + + vk::UniquePipelineLayout image_pipeline_layout_; + vk::UniquePipelineLayout image_lut_pipeline_layout_; + vk::UniquePipelineLayout geometry_pipeline_layout_; + vk::UniquePipelineLayout geometry_text_pipeline_layout_; + + const uint32_t bindings_offset_texture_ = 0; + const uint32_t bindings_offset_texture_lut_ = 1; + + nvvk::DescriptorSetBindings desc_set_layout_bind_; + vk::UniqueDescriptorSetLayout desc_set_layout_; + + nvvk::DescriptorSetBindings desc_set_layout_bind_lut_; + vk::UniqueDescriptorSetLayout desc_set_layout_lut_; + + nvvk::DescriptorSetBindings desc_set_layout_bind_text_; + vk::UniqueDescriptorSetLayout desc_set_layout_text_; + vk::UniqueDescriptorPool desc_pool_text_; + vk::DescriptorSet desc_set_text_; + vk::Sampler sampler_text_; + + vk::UniquePipeline image_pipeline_; + vk::UniquePipeline image_lut_uint_pipeline_; + vk::UniquePipeline image_lut_float_pipeline_; + + vk::UniquePipeline geometry_point_pipeline_; + vk::UniquePipeline geometry_line_pipeline_; + vk::UniquePipeline geometry_line_strip_pipeline_; + vk::UniquePipeline geometry_triangle_pipeline_; + + vk::UniquePipeline geometry_point_color_pipeline_; + vk::UniquePipeline geometry_line_color_pipeline_; + vk::UniquePipeline geometry_line_strip_color_pipeline_; + vk::UniquePipeline geometry_triangle_color_pipeline_; + + vk::UniquePipeline geometry_text_pipeline_; + + ImGuiContext* im_gui_context_ = nullptr; +}; + +Vulkan::Impl::~Impl() { + try { + if (device_) { + device_.waitIdle(); + + cleanup_transfer_jobs(); + + nvvk_.alloc_.releaseSampler(sampler_text_); + + if (ImGui::GetCurrentContext() != nullptr) { + ImGui_ImplVulkan_Shutdown(); + if (im_gui_context_) { ImGui::DestroyContext(im_gui_context_); } + } + } + } catch (const std::exception& e) { + HOLOSCAN_LOG_ERROR("Vulkan::Impl destructor failed with {}", e.what()); + } +} + +void Vulkan::Impl::setup(Window* window, const std::string& font_path, float font_size_in_pixels) { + window_ = window; + + // Initialize instance independent function pointers + { + vk::DynamicLoader dl; + PFN_vkGetInstanceProcAddr vkGetInstanceProcAddr = + dl.getProcAddress("vkGetInstanceProcAddr"); + VULKAN_HPP_DEFAULT_DISPATCHER.init(vkGetInstanceProcAddr); + } + +#ifdef NDEBUG + nvvk::ContextCreateInfo context_info; +#else + nvvk::ContextCreateInfo context_info(true /*bUseValidation*/); +#endif + + context_info.setVersion(1, 2); // Using Vulkan 1.2 + + // Requesting Vulkan extensions and layers + uint32_t count{0}; + const char** req_extensions = window_->get_required_instance_extensions(&count); + for (uint32_t ext_id = 0; ext_id < count; ext_id++) + context_info.addInstanceExtension(req_extensions[ext_id]); + + // Allow debug names + context_info.addInstanceExtension(VK_EXT_DEBUG_UTILS_EXTENSION_NAME, true); + context_info.addInstanceExtension(VK_KHR_EXTERNAL_MEMORY_CAPABILITIES_EXTENSION_NAME); + + req_extensions = window_->get_required_device_extensions(&count); + for (uint32_t ext_id = 0; ext_id < count; ext_id++) + context_info.addDeviceExtension(req_extensions[ext_id]); + + context_info.addDeviceExtension(VK_KHR_EXTERNAL_MEMORY_EXTENSION_NAME); + context_info.addDeviceExtension(VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME); + context_info.addDeviceExtension(VK_KHR_EXTERNAL_SEMAPHORE_EXTENSION_NAME); + context_info.addDeviceExtension(VK_KHR_EXTERNAL_SEMAPHORE_FD_EXTENSION_NAME); + context_info.addDeviceExtension(VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME); + + // Creating Vulkan base application + if (!nvvk_.vk_ctx_.initInstance(context_info)) { + throw std::runtime_error("Failed to create the Vulkan instance."); + } + instance_ = nvvk_.vk_ctx_.m_instance; + + // Initialize instance specific function pointers + VULKAN_HPP_DEFAULT_DISPATCHER.init(instance_); + + // Find all compatible devices + const std::vector compatible_devices = nvvk_.vk_ctx_.getCompatibleDevices(context_info); + if (compatible_devices.empty()) { throw std::runtime_error("No Vulkan capable GPU present."); } + + // Build a list of compatible physical devices + const std::vector physical_devices = instance_.enumeratePhysicalDevices(); + std::vector compatible_physical_devices; + for (auto&& compatible_device : compatible_devices) { + compatible_physical_devices.push_back(vk::PhysicalDevice(physical_devices[compatible_device])); + } + + // Let the window select the device to use (e.g. the one connected to the display if we opened + // a visible windows) + const uint32_t device_index = window_->select_device(instance_, compatible_physical_devices); + + // Finally initialize the device + nvvk_.vk_ctx_.initDevice(device_index, context_info); + device_ = nvvk_.vk_ctx_.m_device; + physical_device_ = nvvk_.vk_ctx_.m_physicalDevice; + + // Initialize device-specific function pointers function pointers + VULKAN_HPP_DEFAULT_DISPATCHER.init(device_); + + // Find the most suitable depth format + { + const vk::FormatFeatureFlagBits feature = vk::FormatFeatureFlagBits::eDepthStencilAttachment; + for (const auto& f : + {vk::Format::eD24UnormS8Uint, vk::Format::eD32SfloatS8Uint, vk::Format::eD16UnormS8Uint}) { + vk::FormatProperties format_prop; + physical_device_.getFormatProperties(f, &format_prop); + if ((format_prop.optimalTilingFeatures & feature) == feature) { + depth_format_ = f; + break; + } + } + if (depth_format_ == vk::Format::eUndefined) { + throw std::runtime_error("Could not find a suitable depth format."); + } + } + + // create a surface, headless windows don't have a surface + surface_ = vk::UniqueSurfaceKHR(window_->create_surface(physical_device_, instance_), instance_); + if (surface_) { + if (!nvvk_.vk_ctx_.setGCTQueueWithPresent(surface_.get())) { + throw std::runtime_error("Surface not supported by queue"); + } + } + + queue_gct_ = device_.getQueue(nvvk_.vk_ctx_.m_queueGCT.familyIndex, 0); + queue_t_ = device_.getQueue(nvvk_.vk_ctx_.m_queueT.familyIndex, 0); + cmd_pool_ = device_.createCommandPoolUnique({vk::CommandPoolCreateFlagBits::eResetCommandBuffer}); + pipeline_cache_ = device_.createPipelineCacheUnique({}); + + nvvk_.alloc_.init(instance_, device_, physical_device_); + nvvk_.alloc_initialized_ = true; + nvvk_.export_alloc_.init(device_, physical_device_, nvvk_.alloc_.getMemoryAllocator()); + nvvk_.export_alloc_initialized_ = true; + + create_framebuffer_sequence(); + create_depth_buffer(); + create_render_pass(); + create_frame_buffers(); + + // init batch submission + nvvk_.batch_submission_.init(nvvk_.vk_ctx_.m_queueGCT); + + // init command pool + nvvk_.transfer_cmd_pool_.init(device_, nvvk_.vk_ctx_.m_queueT.familyIndex); + nvvk_.transfer_cmd_pool_initialized_ = true; + + // allocate the vertex and index buffer for the image draw pass + { + nvvk::CommandPool cmd_buf_get(device_, nvvk_.vk_ctx_.m_queueGCT.familyIndex); + vk::CommandBuffer cmd_buf = cmd_buf_get.createCommandBuffer(); + + const std::vector vertices{ + -1.0f, -1.0f, 0.f, 1.0f, -1.0f, 0.f, 1.0f, 1.0f, 0.f, -1.0f, 1.0f, 0.f}; + nvvk_.vertex_buffer_ = + nvvk_.alloc_.createBuffer(cmd_buf, vertices, vk::BufferUsageFlagBits::eVertexBuffer); + const std::vector indices{0, 2, 1, 2, 0, 3}; + nvvk_.index_buffer_ = + nvvk_.alloc_.createBuffer(cmd_buf, indices, vk::BufferUsageFlagBits::eIndexBuffer); + + cmd_buf_get.submitAndWait(cmd_buf); + nvvk_.alloc_.finalizeAndReleaseStaging(); + } + + // create the descriptor sets + desc_set_layout_bind_.addBinding(bindings_offset_texture_, + VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + 1, + VK_SHADER_STAGE_FRAGMENT_BIT); + desc_set_layout_ = vk::UniqueDescriptorSetLayout( + desc_set_layout_bind_.createLayout(device_, + VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR), + device_); + + desc_set_layout_bind_lut_.addBinding(bindings_offset_texture_, + VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + 1, + VK_SHADER_STAGE_FRAGMENT_BIT); + desc_set_layout_bind_lut_.addBinding(bindings_offset_texture_lut_, + VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + 1, + VK_SHADER_STAGE_FRAGMENT_BIT); + desc_set_layout_lut_ = vk::UniqueDescriptorSetLayout( + desc_set_layout_bind_lut_.createLayout( + device_, VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR), + device_); + + { + VkSamplerCreateInfo info{VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO}; + info.magFilter = VK_FILTER_LINEAR; + info.minFilter = VK_FILTER_LINEAR; + info.mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR; + info.addressModeU = VK_SAMPLER_ADDRESS_MODE_REPEAT; + info.addressModeV = VK_SAMPLER_ADDRESS_MODE_REPEAT; + info.addressModeW = VK_SAMPLER_ADDRESS_MODE_REPEAT; + info.minLod = -1000; + info.maxLod = 1000; + info.maxAnisotropy = 1.0f; + sampler_text_ = nvvk_.alloc_.acquireSampler(info); + } + desc_set_layout_bind_text_.addBinding(bindings_offset_texture_, + vk::DescriptorType::eCombinedImageSampler, + 1, + vk::ShaderStageFlagBits::eFragment, + &sampler_text_); + desc_set_layout_text_ = + vk::UniqueDescriptorSetLayout(desc_set_layout_bind_text_.createLayout(device_), device_); + desc_pool_text_ = + vk::UniqueDescriptorPool(desc_set_layout_bind_text_.createPool(device_), device_); + desc_set_text_ = + nvvk::allocateDescriptorSet(device_, desc_pool_text_.get(), desc_set_layout_text_.get()); + + // Push constants + vk::PushConstantRange push_constant_ranges[2]; + push_constant_ranges[0].stageFlags = vk::ShaderStageFlagBits::eVertex; + push_constant_ranges[0].offset = 0; + push_constant_ranges[0].size = sizeof(PushConstantVertex); + push_constant_ranges[1].stageFlags = vk::ShaderStageFlagBits::eFragment; + push_constant_ranges[1].offset = sizeof(PushConstantVertex); + push_constant_ranges[1].size = sizeof(PushConstantFragment); + + // create the pipeline layout for images + { + // Creating the Pipeline Layout + vk::PipelineLayoutCreateInfo create_info; + create_info.setLayoutCount = 1; + create_info.pSetLayouts = &desc_set_layout_.get(); + create_info.pushConstantRangeCount = 2; + create_info.pPushConstantRanges = push_constant_ranges; + image_pipeline_layout_ = device_.createPipelineLayoutUnique(create_info); + } + + const std::vector binding_description_float_3{ + {0, sizeof(float) * 3, vk::VertexInputRate::eVertex}}; + const std::vector attribute_description_float_3{ + {0, 0, vk::Format::eR32G32B32Sfloat, 0}}; + + // Create the Pipeline + image_pipeline_ = + create_pipeline(image_pipeline_layout_.get(), + image_shader_glsl_vert, + sizeof(image_shader_glsl_vert) / sizeof(image_shader_glsl_vert[0]), + image_shader_glsl_frag, + sizeof(image_shader_glsl_frag) / sizeof(image_shader_glsl_frag[0]), + vk::PrimitiveTopology::eTriangleList, + {}, + binding_description_float_3, + attribute_description_float_3); + + // create the pipeline layout for images with lut + { + // Creating the Pipeline Layout + vk::PipelineLayoutCreateInfo create_info; + create_info.setLayoutCount = 1; + create_info.pSetLayouts = &desc_set_layout_lut_.get(); + create_info.pushConstantRangeCount = 2; + create_info.pPushConstantRanges = push_constant_ranges; + image_lut_pipeline_layout_ = device_.createPipelineLayoutUnique(create_info); + } + + image_lut_uint_pipeline_ = create_pipeline( + image_lut_pipeline_layout_.get(), + image_shader_glsl_vert, + sizeof(image_shader_glsl_vert) / sizeof(image_shader_glsl_vert[0]), + image_lut_uint_shader_glsl_frag, + sizeof(image_lut_uint_shader_glsl_frag) / sizeof(image_lut_uint_shader_glsl_frag[0]), + vk::PrimitiveTopology::eTriangleList, + {}, + binding_description_float_3, + attribute_description_float_3); + + image_lut_float_pipeline_ = create_pipeline( + image_lut_pipeline_layout_.get(), + image_shader_glsl_vert, + sizeof(image_shader_glsl_vert) / sizeof(image_shader_glsl_vert[0]), + image_lut_float_shader_glsl_frag, + sizeof(image_lut_float_shader_glsl_frag) / sizeof(image_lut_float_shader_glsl_frag[0]), + vk::PrimitiveTopology::eTriangleList, + {}, + binding_description_float_3, + attribute_description_float_3); + + // create the pipeline layout for geometry + { + vk::PipelineLayoutCreateInfo create_info; + create_info.pushConstantRangeCount = 2; + create_info.pPushConstantRanges = push_constant_ranges; + geometry_pipeline_layout_ = device_.createPipelineLayoutUnique(create_info); + } + + geometry_point_pipeline_ = + create_pipeline(geometry_pipeline_layout_.get(), + geometry_shader_glsl_vert, + sizeof(geometry_shader_glsl_vert) / sizeof(geometry_shader_glsl_vert[0]), + geometry_shader_glsl_frag, + sizeof(geometry_shader_glsl_frag) / sizeof(geometry_shader_glsl_frag[0]), + vk::PrimitiveTopology::ePointList, + {}, + binding_description_float_3, + attribute_description_float_3); + geometry_line_pipeline_ = + create_pipeline(geometry_pipeline_layout_.get(), + geometry_shader_glsl_vert, + sizeof(geometry_shader_glsl_vert) / sizeof(geometry_shader_glsl_vert[0]), + geometry_shader_glsl_frag, + sizeof(geometry_shader_glsl_frag) / sizeof(geometry_shader_glsl_frag[0]), + vk::PrimitiveTopology::eLineList, + {vk::DynamicState::eLineWidth}, + binding_description_float_3, + attribute_description_float_3); + geometry_line_strip_pipeline_ = + create_pipeline(geometry_pipeline_layout_.get(), + geometry_shader_glsl_vert, + sizeof(geometry_shader_glsl_vert) / sizeof(geometry_shader_glsl_vert[0]), + geometry_shader_glsl_frag, + sizeof(geometry_shader_glsl_frag) / sizeof(geometry_shader_glsl_frag[0]), + vk::PrimitiveTopology::eLineStrip, + {vk::DynamicState::eLineWidth}, + binding_description_float_3, + attribute_description_float_3); + geometry_triangle_pipeline_ = + create_pipeline(geometry_pipeline_layout_.get(), + geometry_shader_glsl_vert, + sizeof(geometry_shader_glsl_vert) / sizeof(geometry_shader_glsl_vert[0]), + geometry_shader_glsl_frag, + sizeof(geometry_shader_glsl_frag) / sizeof(geometry_shader_glsl_frag[0]), + vk::PrimitiveTopology::eTriangleList, + {}, + binding_description_float_3, + attribute_description_float_3); + + const std::vector binding_description_float_3_uint8_4{ + {0, sizeof(float) * 3, vk::VertexInputRate::eVertex}, + {1, sizeof(uint8_t) * 4, vk::VertexInputRate::eVertex}}; + const std::vector attribute_description_float_3_uint8_4{ + {0, 0, vk::Format::eR32G32B32Sfloat, 0}, {1, 1, vk::Format::eR8G8B8A8Unorm, 0}}; + + geometry_point_color_pipeline_ = create_pipeline( + geometry_pipeline_layout_.get(), + geometry_color_shader_glsl_vert, + sizeof(geometry_color_shader_glsl_vert) / sizeof(geometry_color_shader_glsl_vert[0]), + geometry_shader_glsl_frag, + sizeof(geometry_shader_glsl_frag) / sizeof(geometry_shader_glsl_frag[0]), + vk::PrimitiveTopology::ePointList, + {}, + binding_description_float_3_uint8_4, + attribute_description_float_3_uint8_4); + geometry_line_color_pipeline_ = create_pipeline( + geometry_pipeline_layout_.get(), + geometry_color_shader_glsl_vert, + sizeof(geometry_color_shader_glsl_vert) / sizeof(geometry_color_shader_glsl_vert[0]), + geometry_shader_glsl_frag, + sizeof(geometry_shader_glsl_frag) / sizeof(geometry_shader_glsl_frag[0]), + vk::PrimitiveTopology::eLineList, + {vk::DynamicState::eLineWidth}, + binding_description_float_3_uint8_4, + attribute_description_float_3_uint8_4); + geometry_line_strip_color_pipeline_ = create_pipeline( + geometry_pipeline_layout_.get(), + geometry_color_shader_glsl_vert, + sizeof(geometry_color_shader_glsl_vert) / sizeof(geometry_color_shader_glsl_vert[0]), + geometry_shader_glsl_frag, + sizeof(geometry_shader_glsl_frag) / sizeof(geometry_shader_glsl_frag[0]), + vk::PrimitiveTopology::eLineStrip, + {vk::DynamicState::eLineWidth}, + binding_description_float_3_uint8_4, + attribute_description_float_3_uint8_4); + geometry_triangle_color_pipeline_ = create_pipeline( + geometry_pipeline_layout_.get(), + geometry_color_shader_glsl_vert, + sizeof(geometry_color_shader_glsl_vert) / sizeof(geometry_color_shader_glsl_vert[0]), + geometry_shader_glsl_frag, + sizeof(geometry_shader_glsl_frag) / sizeof(geometry_shader_glsl_frag[0]), + vk::PrimitiveTopology::eTriangleList, + {}, + binding_description_float_3_uint8_4, + attribute_description_float_3_uint8_4); + + // create the pipeline layout for text geometry + { + vk::PipelineLayoutCreateInfo create_info; + create_info.setLayoutCount = 1; + create_info.pSetLayouts = &desc_set_layout_text_.get(); + create_info.pushConstantRangeCount = 2; + create_info.pPushConstantRanges = push_constant_ranges; + geometry_text_pipeline_layout_ = device_.createPipelineLayoutUnique(create_info); + } + + const std::vector binding_description_imgui{ + {0, sizeof(ImDrawVert), vk::VertexInputRate::eVertex}}; + const std::vector attribute_description_imgui{ + {0, 0, vk::Format::eR32G32Sfloat, IM_OFFSETOF(ImDrawVert, pos)}, + {1, 0, vk::Format::eR32G32Sfloat, IM_OFFSETOF(ImDrawVert, uv)}, + {2, 0, vk::Format::eR8G8B8A8Unorm, IM_OFFSETOF(ImDrawVert, col)}}; + + geometry_text_pipeline_ = create_pipeline( + geometry_text_pipeline_layout_.get(), + geometry_text_shader_glsl_vert, + sizeof(geometry_text_shader_glsl_vert) / sizeof(geometry_text_shader_glsl_vert[0]), + geometry_text_shader_glsl_frag, + sizeof(geometry_text_shader_glsl_frag) / sizeof(geometry_text_shader_glsl_frag[0]), + vk::PrimitiveTopology::eTriangleList, + {}, + binding_description_imgui, + attribute_description_imgui); + + // ImGui initialization + init_im_gui(font_path, font_size_in_pixels); + window_->setup_callbacks( + [this](int width, int height) { this->on_framebuffer_size(width, height); }); + window_->init_im_gui(); +} + +Window* Vulkan::Impl::get_window() const { + return window_; +} + +void Vulkan::Impl::init_im_gui(const std::string& font_path, float font_size_in_pixels) { + // if the app did not specify a context, create our own + if (!ImGui::GetCurrentContext()) { im_gui_context_ = ImGui::CreateContext(); } + + ImGuiIO& io = ImGui::GetIO(); + io.IniFilename = nullptr; // Avoiding the INI file + io.LogFilename = nullptr; + io.ConfigFlags |= ImGuiConfigFlags_NavEnableKeyboard; // Enable Keyboard Controls + io.ConfigFlags |= ImGuiConfigFlags_DockingEnable; // Enable Docking + + std::vector pool_size{{vk::DescriptorType::eSampler, 1}, + {vk::DescriptorType::eCombinedImageSampler, 1}}; + vk::DescriptorPoolCreateInfo pool_info; + pool_info.maxSets = 2; + pool_info.poolSizeCount = 2; + pool_info.pPoolSizes = pool_size.data(); + im_gui_desc_pool_ = device_.createDescriptorPoolUnique(pool_info); + + // Setup Platform/Renderer back ends + ImGui_ImplVulkan_InitInfo init_info{}; + init_info.Instance = instance_; + init_info.PhysicalDevice = physical_device_; + init_info.Device = device_; + init_info.QueueFamily = nvvk_.vk_ctx_.m_queueGCT.familyIndex; + init_info.Queue = queue_gct_; + init_info.PipelineCache = VK_NULL_HANDLE; + init_info.DescriptorPool = im_gui_desc_pool_.get(); + init_info.Subpass = 0; + init_info.MinImageCount = 2; + init_info.ImageCount = static_cast(fb_sequence_.get_image_count()); + init_info.MSAASamples = VK_SAMPLE_COUNT_1_BIT; + init_info.CheckVkResultFn = nullptr; + init_info.Allocator = nullptr; + + if (!ImGui_ImplVulkan_Init(&init_info, render_pass_.get())) { + throw std::runtime_error("Failed to initialize ImGui vulkan backend."); + } + + // set the font, if the user provided a font path, use this, else use the default font + ImFont* font = nullptr; + if (!font_path.empty()) { + // ImGui asserts in debug when the file does not exist resulting in program termination, + // therefore first check if the file is there. + if (std::filesystem::exists(font_path)) { + font = io.Fonts->AddFontFromFileTTF(font_path.c_str(), font_size_in_pixels); + } + if (!font) { + const std::string err = "Failed to load font " + font_path; + throw std::runtime_error(err.c_str()); + } + } else { + // by default the font data will be deleted by ImGui, since the font data is a static array + // avoid this + ImFontConfig font_config; + font_config.FontDataOwnedByAtlas = false; + // add the Roboto Bold fond as the default font + font_size_in_pixels = 25.f; + font = io.Fonts->AddFontFromMemoryTTF( + roboto_bold_ttf, sizeof(roboto_bold_ttf), font_size_in_pixels, &font_config); + if (!font) { throw std::runtime_error("Failed to add default font."); } + } + + // the size of the ImGui default font is 13 pixels, set the global font scale so that the + // GUI text has the same size as with the default font. + io.FontGlobalScale = 13.f / font_size_in_pixels; + + // build the font atlast + if (!io.Fonts->Build()) { throw std::runtime_error("Failed to build font atlas."); } + ImGui::SetCurrentFont(font); + + // Upload Fonts + vk::CommandBuffer cmd_buf = create_temp_cmd_buffer(); + if (!ImGui_ImplVulkan_CreateFontsTexture(VkCommandBuffer(cmd_buf))) { + throw std::runtime_error("Failed to create fonts texture."); + } + submit_temp_cmd_buffer(cmd_buf); +} + +void Vulkan::Impl::begin_transfer_pass() { + // create a new transfer job and a command buffer + transfer_jobs_.emplace_back(); + TransferJob& transfer_job = transfer_jobs_.back(); + + transfer_job.cmd_buffer_ = nvvk_.transfer_cmd_pool_.createCommandBuffer(); +} + +void Vulkan::Impl::end_transfer_pass() { + if (transfer_jobs_.empty() || (transfer_jobs_.back().fence_)) { + throw std::runtime_error("Not in transfer pass."); + } + + TransferJob& transfer_job = transfer_jobs_.back(); + + // end the command buffer for this job + transfer_job.cmd_buffer_.end(); + + // create the fence and semaphore needed for submission + transfer_job.semaphore_ = device_.createSemaphoreUnique({}); + transfer_job.fence_ = device_.createFenceUnique({}); + + // finalize the staging job for later cleanup of resources + // associates all current staging resources with the transfer fence + nvvk_.alloc_.finalizeStaging(transfer_job.fence_.get()); + + // submit staged transfers + vk::SubmitInfo submit_info; + submit_info.commandBufferCount = 1; + submit_info.pCommandBuffers = &transfer_job.cmd_buffer_; + submit_info.signalSemaphoreCount = 1; + submit_info.pSignalSemaphores = &transfer_job.semaphore_.get(); + queue_t_.submit(submit_info, transfer_job.fence_.get()); + + // next graphics submission must wait for transfer completion + nvvk_.batch_submission_.enqueueWait(transfer_job.semaphore_.get(), + VK_PIPELINE_STAGE_TRANSFER_BIT); +} + +void Vulkan::Impl::begin_render_pass() { + // Acquire the next image + prepare_frame(); + + // Get the command buffer for the frame. There are n command buffers equal to the number of + // in-flight frames. + const uint32_t cur_frame = get_active_image_index(); + const vk::CommandBuffer cmd_buf = command_buffers_[cur_frame].get(); + + vk::CommandBufferBeginInfo begin_info; + begin_info.flags = vk::CommandBufferUsageFlagBits::eOneTimeSubmit; + cmd_buf.begin(begin_info); + + // Clearing values + std::array clear_values; + clear_values[0].color = vk::ClearColorValue(std::array({0.f, 0.f, 0.f, 1.f})); + clear_values[1].depthStencil = vk::ClearDepthStencilValue(1.0f, 0); + + // Begin rendering + vk::RenderPassBeginInfo render_pass_begin_info; + render_pass_begin_info.clearValueCount = 2; + render_pass_begin_info.pClearValues = clear_values.data(); + render_pass_begin_info.renderPass = render_pass_.get(); + render_pass_begin_info.framebuffer = framebuffers_[cur_frame].get(); + render_pass_begin_info.renderArea = {{0, 0}, size_}; + cmd_buf.beginRenderPass(render_pass_begin_info, vk::SubpassContents::eInline); + + // set the dynamic viewport + vk::Viewport viewport{ + 0.0f, 0.0f, static_cast(size_.width), static_cast(size_.height), 0.0f, 1.0f}; + cmd_buf.setViewport(0, viewport); + + vk::Rect2D scissor{{0, 0}, {size_.width, size_.height}}; + cmd_buf.setScissor(0, scissor); +} + +void Vulkan::Impl::end_render_pass() { + const vk::CommandBuffer cmd_buf = command_buffers_[get_active_image_index()].get(); + + // End rendering + cmd_buf.endRenderPass(); + + // Submit for display + cmd_buf.end(); + submit_frame(); +} + +void Vulkan::Impl::cleanup_transfer_jobs() { + for (auto it = transfer_jobs_.begin(); it != transfer_jobs_.end();) { + if (it->fence_) { + // check if the upload fence was triggered, that means the copy has completed and cmd + // buffer can be destroyed + const vk::Result result = device_.getFenceStatus(it->fence_.get()); + if (result == vk::Result::eSuccess) { + nvvk_.transfer_cmd_pool_.destroy(it->cmd_buffer_); + it->cmd_buffer_ = nullptr; + + // before destroying the fence release all staging buffers using that fence + nvvk_.alloc_.releaseStaging(); + + it->fence_.reset(); + } else if (result != vk::Result::eNotReady) { + vk::resultCheck(result, "Failed to get upload fence status"); + } + } + + if (!it->fence_) { + if (it->frame_fence_) { + // check if the frame fence was triggered, that means the job can be destroyed + const vk::Result result = device_.getFenceStatus(it->frame_fence_); + if (result == vk::Result::eSuccess) { + it->semaphore_.reset(); + /// @todo instead of allocating and destroying semaphore and fences, move to + /// unused list and reuse (call 'device_.resetFences(1, &it->fence_);' to reuse) + it = transfer_jobs_.erase(it); + continue; + } else if (result != vk::Result::eNotReady) { + vk::resultCheck(result, "Failed to get frame fence status"); + } + } else { + // this is a stale transfer buffer (no end_transfer_pass()?), remove it + it = transfer_jobs_.erase(it); + continue; + } + } + ++it; + } +} + +void Vulkan::Impl::prepare_frame() { + if (!transfer_jobs_.empty() && (!transfer_jobs_.back().fence_)) { + throw std::runtime_error("Transfer pass is active!"); + } + + // Acquire the next image from the framebuffer sequence + fb_sequence_.acquire(); + + // Use a fence to wait until the command buffer has finished execution before using it again + const uint32_t image_index = get_active_image_index(); + + vk::Result result{vk::Result::eSuccess}; + do { + result = device_.waitForFences(wait_fences_[image_index].get(), true, 1'000'000); + } while (result == vk::Result::eTimeout); + + if (result != vk::Result::eSuccess) { + // This allows Aftermath to do things and exit below + usleep(1000); + vk::resultCheck(result, "Failed to wait for frame fences"); + exit(-1); + } + + // reset the fence to be re-used + device_.resetFences(wait_fences_[image_index].get()); + + // if there is a pending transfer job assign the frame fence of the frame which is about to be + // rendered. + if (!transfer_jobs_.empty() && !transfer_jobs_.back().frame_fence_) { + transfer_jobs_.back().frame_fence_ = wait_fences_[image_index].get(); + } + + // try to free previous transfer jobs + cleanup_transfer_jobs(); +} + +void Vulkan::Impl::submit_frame() { + const uint32_t image_index = get_active_image_index(); + + nvvk_.batch_submission_.enqueue(command_buffers_[image_index].get()); + + // wait for the previous frame's semaphore + if (fb_sequence_.get_active_read_semaphore()) { + nvvk_.batch_submission_.enqueueWait(fb_sequence_.get_active_read_semaphore(), + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT); + } + // and signal this frames semaphore on completion + nvvk_.batch_submission_.enqueueSignal(fb_sequence_.get_active_written_semaphore()); + + const vk::Result result = + vk::Result(nvvk_.batch_submission_.execute(wait_fences_[image_index].get(), 0b0000'0001)); + vk::resultCheck(result, "Failed to execute bach submission"); + + // Presenting frame + fb_sequence_.present(queue_gct_); +} + +void Vulkan::Impl::create_framebuffer_sequence() { + window_->get_framebuffer_size(&size_.width, &size_.height); + + fb_sequence_.init(&nvvk_.alloc_, + device_, + physical_device_, + queue_gct_, + nvvk_.vk_ctx_.m_queueGCT.familyIndex, + surface_.get()); + + fb_sequence_.update(size_.width, size_.height, &size_); + + // Create Synchronization Primitives + vk::FenceCreateInfo fence_create_info; + fence_create_info.flags = vk::FenceCreateFlagBits::eSignaled; + for (uint32_t index = 0; index < fb_sequence_.get_image_count(); ++index) { + wait_fences_.push_back(device_.createFenceUnique(fence_create_info)); + } + + // Command buffers store a reference to the frame buffer inside their render pass info + // so for static usage without having to rebuild them each frame, we use one per frame buffer + vk::CommandBufferAllocateInfo allocate_info; + allocate_info.commandPool = cmd_pool_.get(); + allocate_info.commandBufferCount = fb_sequence_.get_image_count(); + allocate_info.level = vk::CommandBufferLevel::ePrimary; + command_buffers_ = device_.allocateCommandBuffersUnique(allocate_info); + + const vk::CommandBuffer cmd_buffer = create_temp_cmd_buffer(); + fb_sequence_.cmd_update_barriers(cmd_buffer); + submit_temp_cmd_buffer(cmd_buffer); + +#ifdef _DEBUG + for (size_t i = 0; i < command_buffers_.size(); i++) { + const std::string name = std::string("Holoviz") + std::to_string(i); + vk::DebugUtilsObjectNameInfoEXT name_info; + name_info.objectHandle = (uint64_t)VkCommandBuffer(command_buffers_[i].get()); + name_info.objectType = vk::ObjectType::eCommandBuffer; + name_info.pObjectName = name.c_str(); + device_.setDebugUtilsObjectNameEXT(name_info); + } +#endif // _DEBUG +} + +void Vulkan::Impl::create_depth_buffer() { + depth_view_.reset(); + depth_image_.reset(); + depth_memory_.reset(); + + // Depth information + const vk::ImageAspectFlags aspect = + vk::ImageAspectFlagBits::eDepth | vk::ImageAspectFlagBits::eStencil; + vk::ImageCreateInfo depth_stencil_create_info; + depth_stencil_create_info.imageType = vk::ImageType::e2D; + depth_stencil_create_info.extent = vk::Extent3D{size_.width, size_.height, 1}; + depth_stencil_create_info.format = depth_format_; + depth_stencil_create_info.mipLevels = 1; + depth_stencil_create_info.arrayLayers = 1; + depth_stencil_create_info.samples = vk::SampleCountFlagBits::e1; + depth_stencil_create_info.usage = + vk::ImageUsageFlagBits::eDepthStencilAttachment | vk::ImageUsageFlagBits::eTransferSrc; + // Create the depth image + depth_image_ = device_.createImageUnique(depth_stencil_create_info); + +#ifdef _DEBUG + vk::DebugUtilsObjectNameInfoEXT name_info; + name_info.objectHandle = (uint64_t)VkImage(depth_image_.get()); + name_info.objectType = vk::ObjectType::eImage; + name_info.pObjectName = R"(Holoviz)"; + device_.setDebugUtilsObjectNameEXT(name_info); +#endif // _DEBUG + + // Allocate the memory + const vk::MemoryRequirements mem_reqs = device_.getImageMemoryRequirements(depth_image_.get()); + vk::MemoryAllocateInfo mem_alloc_info; + mem_alloc_info.allocationSize = mem_reqs.size; + mem_alloc_info.memoryTypeIndex = + get_memory_type(mem_reqs.memoryTypeBits, vk::MemoryPropertyFlagBits::eDeviceLocal); + depth_memory_ = device_.allocateMemoryUnique(mem_alloc_info); + + // Bind image and memory + device_.bindImageMemory(depth_image_.get(), depth_memory_.get(), 0); + + const vk::CommandBuffer cmd_buffer = create_temp_cmd_buffer(); + + // Put barrier on top, Put barrier inside setup command buffer + vk::ImageSubresourceRange subresource_range; + subresource_range.aspectMask = aspect; + subresource_range.levelCount = 1; + subresource_range.layerCount = 1; + vk::ImageMemoryBarrier image_memory_barrier; + image_memory_barrier.oldLayout = vk::ImageLayout::eUndefined; + image_memory_barrier.newLayout = vk::ImageLayout::eDepthStencilAttachmentOptimal; + image_memory_barrier.image = depth_image_.get(); + image_memory_barrier.subresourceRange = subresource_range; + image_memory_barrier.srcAccessMask = vk::AccessFlags(); + image_memory_barrier.dstAccessMask = vk::AccessFlagBits::eDepthStencilAttachmentWrite; + const vk::PipelineStageFlags src_stage_mask = vk::PipelineStageFlagBits::eTopOfPipe; + const vk::PipelineStageFlags destStageMask = vk::PipelineStageFlagBits::eEarlyFragmentTests; + + cmd_buffer.pipelineBarrier(src_stage_mask, + destStageMask, + vk::DependencyFlags(), + 0, + nullptr, + 0, + nullptr, + 1, + &image_memory_barrier); + submit_temp_cmd_buffer(cmd_buffer); + + // Setting up the view + vk::ImageViewCreateInfo depth_stencil_view; + depth_stencil_view.viewType = vk::ImageViewType::e2D; + depth_stencil_view.format = depth_format_; + depth_stencil_view.subresourceRange = subresource_range; + depth_stencil_view.image = depth_image_.get(); + depth_view_ = device_.createImageViewUnique(depth_stencil_view); +} + +void Vulkan::Impl::create_render_pass() { + render_pass_.reset(); + + std::array attachments; + // Color attachment + attachments[0].format = fb_sequence_.get_format(); + attachments[0].loadOp = vk::AttachmentLoadOp::eClear; + attachments[0].finalLayout = + surface_ ? vk::ImageLayout::ePresentSrcKHR : vk::ImageLayout::eColorAttachmentOptimal; + attachments[0].samples = vk::SampleCountFlagBits::e1; + + // Depth attachment + attachments[1].format = depth_format_; + attachments[1].loadOp = vk::AttachmentLoadOp::eClear; + attachments[1].stencilLoadOp = vk::AttachmentLoadOp::eClear; + attachments[1].finalLayout = vk::ImageLayout::eDepthStencilAttachmentOptimal; + attachments[1].samples = vk::SampleCountFlagBits::e1; + + // One color, one depth + const vk::AttachmentReference color_reference{0, vk::ImageLayout::eColorAttachmentOptimal}; + const vk::AttachmentReference depth_reference{1, vk::ImageLayout::eDepthStencilAttachmentOptimal}; + + std::array subpass_dependencies; + // Transition from final to initial (VK_SUBPASS_EXTERNAL refers to all commands executed outside + // of the actual renderpass) + subpass_dependencies[0].srcSubpass = VK_SUBPASS_EXTERNAL; + subpass_dependencies[0].dstSubpass = 0; + subpass_dependencies[0].srcStageMask = vk::PipelineStageFlagBits::eBottomOfPipe; + subpass_dependencies[0].dstStageMask = vk::PipelineStageFlagBits::eColorAttachmentOutput; + subpass_dependencies[0].srcAccessMask = vk::AccessFlagBits::eMemoryRead; + subpass_dependencies[0].dstAccessMask = + vk::AccessFlagBits::eColorAttachmentRead | vk::AccessFlagBits::eColorAttachmentWrite; + subpass_dependencies[0].dependencyFlags = vk::DependencyFlagBits::eByRegion; + + vk::SubpassDescription subpass_description; + subpass_description.pipelineBindPoint = vk::PipelineBindPoint::eGraphics; + subpass_description.colorAttachmentCount = 1; + subpass_description.pColorAttachments = &color_reference; + subpass_description.pDepthStencilAttachment = &depth_reference; + + vk::RenderPassCreateInfo render_pass_info; + render_pass_info.attachmentCount = static_cast(attachments.size()); + render_pass_info.pAttachments = attachments.data(); + render_pass_info.subpassCount = 1; + render_pass_info.pSubpasses = &subpass_description; + render_pass_info.dependencyCount = static_cast(subpass_dependencies.size()); + render_pass_info.pDependencies = subpass_dependencies.data(); + + render_pass_ = device_.createRenderPassUnique(render_pass_info); + +#ifdef _DEBUG + vk::DebugUtilsObjectNameInfoEXT name_info; + name_info.objectHandle = (uint64_t)VkRenderPass(render_pass_.get()); + name_info.objectType = vk::ObjectType::eRenderPass; + name_info.pObjectName = R"(Holoviz)"; + device_.setDebugUtilsObjectNameEXT(name_info); +#endif // _DEBUG +} + +void Vulkan::Impl::create_frame_buffers() { + // Recreate the frame buffers + framebuffers_.clear(); + + // Array of attachment (color, depth) + std::array attachments; + + // Create frame buffers for every swap chain image + vk::FramebufferCreateInfo framebuffer_create_info; + framebuffer_create_info.renderPass = render_pass_.get(); + framebuffer_create_info.attachmentCount = 2; + framebuffer_create_info.width = size_.width; + framebuffer_create_info.height = size_.height; + framebuffer_create_info.layers = 1; + framebuffer_create_info.pAttachments = attachments.data(); + + // Create frame buffers for every swap chain image + for (uint32_t i = 0; i < fb_sequence_.get_image_count(); i++) { + attachments[0] = fb_sequence_.get_image_view(i); + attachments[1] = depth_view_.get(); + framebuffers_.push_back(device_.createFramebufferUnique(framebuffer_create_info)); + } + +#ifdef _DEBUG + for (size_t i = 0; i < framebuffers_.size(); i++) { + const std::string name = std::string("Holoviz") + std::to_string(i); + vk::DebugUtilsObjectNameInfoEXT name_info; + name_info.objectHandle = (uint64_t)VkFramebuffer(framebuffers_[i].get()); + name_info.objectType = vk::ObjectType::eFramebuffer; + name_info.pObjectName = name.c_str(); + device_.setDebugUtilsObjectNameEXT(name_info); + } +#endif // _DEBUG +} + +void Vulkan::Impl::on_framebuffer_size(int w, int h) { + if ((w == 0) || (h == 0)) { return; } + + // Update imgui + if (ImGui::GetCurrentContext() != nullptr) { + auto& imgui_io = ImGui::GetIO(); + imgui_io.DisplaySize = ImVec2(static_cast(w), static_cast(h)); + } + + // Wait to finish what is currently drawing + device_.waitIdle(); + queue_gct_.waitIdle(); + + // Request new swapchain image size + fb_sequence_.update(w, h, &size_); + { + const vk::CommandBuffer cmd_buffer = create_temp_cmd_buffer(); + fb_sequence_.cmd_update_barriers(cmd_buffer); // Make them presentable + submit_temp_cmd_buffer(cmd_buffer); + } + + // Recreating other resources + create_depth_buffer(); + create_frame_buffers(); +} + +vk::CommandBuffer Vulkan::Impl::create_temp_cmd_buffer() { + // Create an image barrier to change the layout from undefined to DepthStencilAttachmentOptimal + vk::CommandBufferAllocateInfo allocate_info; + allocate_info.commandBufferCount = 1; + allocate_info.commandPool = cmd_pool_.get(); + allocate_info.level = vk::CommandBufferLevel::ePrimary; + const vk::CommandBuffer cmd_buffer = device_.allocateCommandBuffers(allocate_info)[0]; + + vk::CommandBufferBeginInfo begin_info; + begin_info.flags = vk::CommandBufferUsageFlagBits::eOneTimeSubmit; + cmd_buffer.begin(begin_info); + return cmd_buffer; +} + +void Vulkan::Impl::submit_temp_cmd_buffer(vk::CommandBuffer cmd_buffer) { + cmd_buffer.end(); + + vk::SubmitInfo submit_info; + submit_info.commandBufferCount = 1; + submit_info.pCommandBuffers = &cmd_buffer; + queue_gct_.submit(submit_info); + queue_gct_.waitIdle(); + device_.freeCommandBuffers(cmd_pool_.get(), 1, &cmd_buffer); +} + +uint32_t Vulkan::Impl::get_memory_type(uint32_t typeBits, + const vk::MemoryPropertyFlags& properties) const { + const vk::PhysicalDeviceMemoryProperties memory_properties = + physical_device_.getMemoryProperties(); + + for (uint32_t i = 0; i < memory_properties.memoryTypeCount; i++) { + if (((typeBits & (1 << i)) > 0) && + (memory_properties.memoryTypes[i].propertyFlags & properties) == properties) + return i; + } + std::string err = "Unable to find memory type " + vk::to_string(properties); + HOLOSCAN_LOG_ERROR("{}", err.c_str()); + return ~0u; +} + +vk::UniquePipeline Vulkan::Impl::create_pipeline( + vk::PipelineLayout pipeline_layout, const uint32_t* vertex_shader, size_t vertex_shader_size, + const uint32_t* fragment_shader, size_t fragment_shader_size, vk::PrimitiveTopology topology, + const std::vector& dynamic_states, + const std::vector& binding_descriptions, + const std::vector& attribute_descriptions) { + nvvk::GraphicsPipelineState state; + + state.depthStencilState.depthTestEnable = true; + + state.inputAssemblyState.topology = topology; + + state.addBindingDescriptions(binding_descriptions); + state.addAttributeDescriptions(attribute_descriptions); + + // disable culling + state.rasterizationState.cullMode = vk::CullModeFlagBits::eNone; + + // enable blending + vk::PipelineColorBlendAttachmentState color_blend_attachment_state; + color_blend_attachment_state.colorWriteMask = + vk::ColorComponentFlagBits::eR | vk::ColorComponentFlagBits::eG | + vk::ColorComponentFlagBits::eB | vk::ColorComponentFlagBits::eA; + color_blend_attachment_state.blendEnable = true; + color_blend_attachment_state.srcColorBlendFactor = vk::BlendFactor::eSrcAlpha; + color_blend_attachment_state.dstColorBlendFactor = vk::BlendFactor::eOneMinusSrcAlpha; + color_blend_attachment_state.colorBlendOp = vk::BlendOp::eAdd; + color_blend_attachment_state.srcAlphaBlendFactor = vk::BlendFactor::eOne; + color_blend_attachment_state.dstAlphaBlendFactor = vk::BlendFactor::eZero; + color_blend_attachment_state.alphaBlendOp = vk::BlendOp::eAdd; + // remove the default blend attachment state + state.clearBlendAttachmentStates(); + state.addBlendAttachmentState(color_blend_attachment_state); + + for (auto&& dynamic_state : dynamic_states) { state.addDynamicStateEnable(dynamic_state); } + + state.update(); + + nvvk::GraphicsPipelineGenerator generator(device_, pipeline_layout, render_pass_.get(), state); + + std::vector code; + code.assign(vertex_shader, vertex_shader + vertex_shader_size); + generator.addShader(code, vk::ShaderStageFlagBits::eVertex); + code.assign(fragment_shader, fragment_shader + fragment_shader_size); + generator.addShader(code, vk::ShaderStageFlagBits::eFragment); + + generator.update(); + + return device_.createGraphicsPipelineUnique(pipeline_cache_.get(), generator.createInfo).value; +} + +static void format_info(ImageFormat format, uint32_t* src_channels, uint32_t* dst_channels, + uint32_t* component_size) { + switch (format) { + case ImageFormat::R8_UINT: + case ImageFormat::R8_UNORM: + *src_channels = *dst_channels = 1u; + *component_size = sizeof(uint8_t); + break; + case ImageFormat::R16_UINT: + case ImageFormat::R16_UNORM: + case ImageFormat::R16_SFLOAT: + *src_channels = *dst_channels = 1u; + *component_size = sizeof(uint16_t); + break; + case ImageFormat::R32_UINT: + *src_channels = *dst_channels = 1u; + *component_size = sizeof(uint32_t); + break; + case ImageFormat::R32_SFLOAT: + *src_channels = *dst_channels = 1u; + *component_size = sizeof(float); + break; + case ImageFormat::R8G8B8_UNORM: + *src_channels = 3u; + *dst_channels = 4u; + *component_size = sizeof(uint8_t); + break; + case ImageFormat::B8G8R8_UNORM: + *src_channels = 3u; + *dst_channels = 4u; + *component_size = sizeof(uint8_t); + break; + case ImageFormat::R8G8B8A8_UNORM: + case ImageFormat::B8G8R8A8_UNORM: + *src_channels = *dst_channels = 4u; + *component_size = sizeof(uint8_t); + break; + case ImageFormat::R16G16B16A16_UNORM: + case ImageFormat::R16G16B16A16_SFLOAT: + *src_channels = *dst_channels = 4u; + *component_size = sizeof(uint16_t); + break; + case ImageFormat::R32G32B32A32_SFLOAT: + *src_channels = *dst_channels = 4u; + *component_size = sizeof(uint32_t); + break; + default: + throw std::runtime_error("Unhandled image format."); + } +} + +static vk::Format to_vulkan_format(ImageFormat format) { + vk::Format vk_format; + + switch (format) { + case ImageFormat::R8_UINT: + vk_format = vk::Format::eR8Uint; + break; + case ImageFormat::R8_UNORM: + vk_format = vk::Format::eR8Unorm; + break; + case ImageFormat::R16_UINT: + vk_format = vk::Format::eR16Uint; + break; + case ImageFormat::R16_UNORM: + vk_format = vk::Format::eR16Unorm; + break; + case ImageFormat::R16_SFLOAT: + vk_format = vk::Format::eR16Sfloat; + break; + case ImageFormat::R32_UINT: + vk_format = vk::Format::eR32Uint; + break; + case ImageFormat::R32_SFLOAT: + vk_format = vk::Format::eR32Sfloat; + break; + case ImageFormat::R8G8B8_UNORM: + vk_format = vk::Format::eR8G8B8A8Unorm; + break; + case ImageFormat::B8G8R8_UNORM: + vk_format = vk::Format::eB8G8R8A8Unorm; + break; + case ImageFormat::R8G8B8A8_UNORM: + vk_format = vk::Format::eR8G8B8A8Unorm; + break; + case ImageFormat::B8G8R8A8_UNORM: + vk_format = vk::Format::eB8G8R8A8Unorm; + break; + case ImageFormat::R16G16B16A16_UNORM: + vk_format = vk::Format::eR16G16B16A16Unorm; + break; + case ImageFormat::R16G16B16A16_SFLOAT: + vk_format = vk::Format::eR16G16B16A16Sfloat; + break; + case ImageFormat::R32G32B32A32_SFLOAT: + vk_format = vk::Format::eR32G32B32A32Sfloat; + break; + default: + throw std::runtime_error("Unhandled image format."); + } + + return vk_format; +} + +UniqueCUexternalSemaphore Vulkan::Impl::import_semaphore_to_cuda(vk::Semaphore semaphore) { + vk::SemaphoreGetFdInfoKHR semaphore_get_fd_info; + semaphore_get_fd_info.semaphore = semaphore; + semaphore_get_fd_info.handleType = vk::ExternalSemaphoreHandleTypeFlagBits::eOpaqueFd; + + UniqueValue file_handle; + file_handle.reset(device_.getSemaphoreFdKHR(semaphore_get_fd_info)); + + CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC semaphore_handle_desc{}; + semaphore_handle_desc.type = CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD; + semaphore_handle_desc.handle.fd = file_handle.get(); + + UniqueCUexternalSemaphore cuda_semaphore; + cuda_semaphore.reset([&semaphore_handle_desc] { + CUexternalSemaphore ext_semaphore; + CudaCheck(cuImportExternalSemaphore(&ext_semaphore, &semaphore_handle_desc)); + return ext_semaphore; + }()); + + // don't need to close the file handle if it had been successfully imported + file_handle.release(); + + return cuda_semaphore; +} + +Vulkan::Texture* Vulkan::Impl::create_texture_for_cuda_interop(uint32_t width, uint32_t height, + ImageFormat format, + vk::Filter filter, bool normalized) { + if (transfer_jobs_.empty() || (transfer_jobs_.back().fence_)) { + throw std::runtime_error( + "Transfer command buffer not set. Calls to create_texture_for_cuda_interop() " + "need to be enclosed by " + "begin_transfer_pass() and " + "end_transfer_pass()"); + } + + const vk::Format vk_format = to_vulkan_format(format); + uint32_t src_channels, dst_channels, component_size; + format_info(format, &src_channels, &dst_channels, &component_size); + + // create the image + const vk::ImageCreateInfo image_create_info = + nvvk::makeImage2DCreateInfo(vk::Extent2D{width, height}, vk_format); + // the VkExternalMemoryImageCreateInfoKHR struct is appended by nvvk::ExportResourceAllocator + const nvvk::Image image = + nvvk_.export_alloc_.createImage(image_create_info, vk::MemoryPropertyFlagBits::eDeviceLocal); + + // create the texture + std::unique_ptr texture = + std::make_unique(width, height, format, &nvvk_.export_alloc_); + + // create the vulkan texture + vk::SamplerCreateInfo sampler_create_info; + sampler_create_info.minFilter = filter; + sampler_create_info.magFilter = filter; + sampler_create_info.mipmapMode = vk::SamplerMipmapMode::eNearest; + sampler_create_info.addressModeU = vk::SamplerAddressMode::eClampToEdge; + sampler_create_info.addressModeV = vk::SamplerAddressMode::eClampToEdge; + sampler_create_info.addressModeW = vk::SamplerAddressMode::eClampToEdge; + sampler_create_info.maxLod = normalized ? FLT_MAX : 0; + sampler_create_info.unnormalizedCoordinates = normalized ? false : true; + + const vk::ImageViewCreateInfo image_view_info = + nvvk::makeImageViewCreateInfo(image.image, image_create_info); + texture->texture_ = + nvvk_.export_alloc_.createTexture(image, image_view_info, sampler_create_info); + + // create the semaphores, one for upload and one for rendering + vk::StructureChain chain; + vk::SemaphoreCreateInfo& semaphore_create_info = chain.get(); + chain.get().handleTypes = + vk::ExternalSemaphoreHandleTypeFlagBits::eOpaqueFd; + texture->upload_semaphore_ = device_.createSemaphoreUnique(semaphore_create_info); + texture->render_semaphore_ = device_.createSemaphoreUnique(semaphore_create_info); + + { + const CudaService::ScopedPush cuda_context = CudaService::get().PushContext(); + + // import memory to CUDA + const nvvk::MemAllocator::MemInfo memInfo = + nvvk_.export_alloc_.getMemoryAllocator()->getMemoryInfo(image.memHandle); + + vk::MemoryGetFdInfoKHR memory_get_fd_info; + memory_get_fd_info.memory = memInfo.memory; + memory_get_fd_info.handleType = vk::ExternalMemoryHandleTypeFlagBits::eOpaqueFd; + UniqueValue file_handle; + file_handle.reset(device_.getMemoryFdKHR(memory_get_fd_info)); + + CUDA_EXTERNAL_MEMORY_HANDLE_DESC memory_handle_desc{}; + memory_handle_desc.type = CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD; + memory_handle_desc.handle.fd = file_handle.get(); + memory_handle_desc.size = memInfo.offset + memInfo.size; + + texture->external_mem_.reset([&memory_handle_desc] { + CUexternalMemory external_mem; + CudaCheck(cuImportExternalMemory(&external_mem, &memory_handle_desc)); + return external_mem; + }()); + // don't need to close the file handle if it had been successfully imported + file_handle.release(); + + CUDA_EXTERNAL_MEMORY_MIPMAPPED_ARRAY_DESC mipmapped_array_desc{}; + mipmapped_array_desc.arrayDesc.Width = width; + mipmapped_array_desc.arrayDesc.Height = height; + mipmapped_array_desc.arrayDesc.Depth = 0; + switch (component_size) { + case 1: + mipmapped_array_desc.arrayDesc.Format = CU_AD_FORMAT_UNSIGNED_INT8; + break; + case 2: + mipmapped_array_desc.arrayDesc.Format = CU_AD_FORMAT_UNSIGNED_INT16; + break; + case 4: + mipmapped_array_desc.arrayDesc.Format = CU_AD_FORMAT_UNSIGNED_INT32; + break; + default: + throw std::runtime_error("Unhandled component size"); + } + mipmapped_array_desc.arrayDesc.NumChannels = dst_channels; + mipmapped_array_desc.arrayDesc.Flags = CUDA_ARRAY3D_SURFACE_LDST; + + mipmapped_array_desc.numLevels = 1; + mipmapped_array_desc.offset = memInfo.offset; + + texture->mipmap_.reset([external_mem = texture->external_mem_.get(), &mipmapped_array_desc] { + CUmipmappedArray mipmaped_array; + CudaCheck(cuExternalMemoryGetMappedMipmappedArray( + &mipmaped_array, external_mem, &mipmapped_array_desc)); + return mipmaped_array; + }()); + + // import the semaphores to Cuda + texture->cuda_upload_semaphore_ = import_semaphore_to_cuda(texture->upload_semaphore_.get()); + texture->cuda_render_semaphore_ = import_semaphore_to_cuda(texture->render_semaphore_.get()); + } + + // transition to shader layout + nvvk::cmdBarrierImageLayout(transfer_jobs_.back().cmd_buffer_, + texture->texture_.image, + vk::ImageLayout::eUndefined, + vk::ImageLayout::eShaderReadOnlyOptimal); + + return texture.release(); +} + +Vulkan::Texture* Vulkan::Impl::create_texture(uint32_t width, uint32_t height, ImageFormat format, + size_t data_size, const void* data, vk::Filter filter, + bool normalized) { + if (transfer_jobs_.empty() || (transfer_jobs_.back().fence_)) { + throw std::runtime_error( + "Transfer command buffer not set. Calls to create_texture() need to be enclosed by " + "begin_transfer_pass() and end_transfer_pass()"); + } + + const vk::Format vk_format = to_vulkan_format(format); + uint32_t src_channels, dst_channels, component_size; + format_info(format, &src_channels, &dst_channels, &component_size); + + if (data && (data_size != width * height * src_channels * component_size)) { + throw std::runtime_error("The size of the data array is wrong"); + } + + const vk::ImageCreateInfo image_create_info = + nvvk::makeImage2DCreateInfo(vk::Extent2D{width, height}, vk_format); + const nvvk::Image image = nvvk_.alloc_.createImage( + transfer_jobs_.back().cmd_buffer_, data_size, data, image_create_info); + + // create the texture + std::unique_ptr texture = + std::make_unique(width, height, format, &nvvk_.alloc_); + + // create the Vulkan texture + vk::SamplerCreateInfo sampler_create_info; + sampler_create_info.minFilter = filter; + sampler_create_info.magFilter = filter; + sampler_create_info.mipmapMode = vk::SamplerMipmapMode::eNearest; + sampler_create_info.addressModeU = vk::SamplerAddressMode::eClampToEdge; + sampler_create_info.addressModeV = vk::SamplerAddressMode::eClampToEdge; + sampler_create_info.addressModeW = vk::SamplerAddressMode::eClampToEdge; + sampler_create_info.maxLod = normalized ? FLT_MAX : 0; + sampler_create_info.unnormalizedCoordinates = normalized ? false : true; + + const vk::ImageViewCreateInfo image_view_info = + nvvk::makeImageViewCreateInfo(image.image, image_create_info); + texture->texture_ = nvvk_.alloc_.createTexture(image, image_view_info, sampler_create_info); + + // transition to shader layout + /// @todo I don't know if this is defined. Should the old layout be + /// vk::ImageLayout::eTransferDstOptimal, like it would be if we uploaded using Vulkan? + nvvk::cmdBarrierImageLayout(transfer_jobs_.back().cmd_buffer_, + image.image, + image_create_info.initialLayout, + vk::ImageLayout::eShaderReadOnlyOptimal); + + return texture.release(); +} + +void Vulkan::Impl::destroy_texture(Texture* texture) { + if (texture->fence_) { + // if the texture had been tagged with a fence, wait for it before freeing the memory + const vk::Result result = device_.waitForFences(texture->fence_, true, 100'000'000); + if (result != vk::Result::eSuccess) { + HOLOSCAN_LOG_WARN("Waiting for texture fence failed with {}", vk::to_string(result)); + } + } + + // check if this texture had been imported to CUDA + if (texture->mipmap_ || texture->external_mem_) { + const CudaService::ScopedPush cuda_context = CudaService::get().PushContext(); + + texture->mipmap_.reset(); + texture->external_mem_.reset(); + texture->cuda_upload_semaphore_.reset(); + texture->cuda_render_semaphore_.reset(); + } + + texture->upload_semaphore_.reset(); + texture->render_semaphore_.reset(); + + texture->alloc_->destroy(texture->texture_); + + delete texture; +} + +void Vulkan::Impl::upload_to_texture(CUdeviceptr device_ptr, Texture* texture, CUstream stream) { + if (!texture->mipmap_) { + throw std::runtime_error("Texture had not been imported to CUDA, can't upload data."); + } + + const CudaService::ScopedPush cuda_context = CudaService::get().PushContext(); + + if (texture->state_ == Texture::State::RENDERED) { + // if the texture had been used in rendering, wait for Vulkan to finish its work + CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS ext_wait_params{}; + const CUexternalSemaphore external_semaphore = texture->cuda_render_semaphore_.get(); + CudaCheck(cuWaitExternalSemaphoresAsync(&external_semaphore, &ext_wait_params, 1, stream)); + texture->state_ = Texture::State::UNKNOWN; + } + + CUarray array; + CudaCheck(cuMipmappedArrayGetLevel(&array, texture->mipmap_.get(), 0)); + + uint32_t src_channels, dst_channels, component_size; + format_info(texture->format_, &src_channels, &dst_channels, &component_size); + const uint32_t src_pitch = texture->width_ * src_channels * component_size; + + if (src_channels != dst_channels) { + // three channel texture data is not hardware natively supported, convert to four channel + if ((src_channels != 3) || (dst_channels != 4) || (component_size != 1)) { + throw std::runtime_error("Unhandled conversion."); + } + ConvertR8G8B8ToR8G8B8A8( + texture->width_, texture->height_, device_ptr, src_pitch, array, stream); + } else { + // else just copy + CUDA_MEMCPY2D memcpy_2d{}; + memcpy_2d.srcMemoryType = CU_MEMORYTYPE_DEVICE; + memcpy_2d.srcDevice = device_ptr; + memcpy_2d.srcPitch = src_pitch; + memcpy_2d.dstMemoryType = CU_MEMORYTYPE_ARRAY; + memcpy_2d.dstArray = array; + memcpy_2d.WidthInBytes = texture->width_ * dst_channels * component_size; + memcpy_2d.Height = texture->height_; + CudaCheck(cuMemcpy2DAsync(&memcpy_2d, stream)); + } + + // signal the semaphore + CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS ext_signal_params{}; + const CUexternalSemaphore external_semaphore = texture->cuda_upload_semaphore_.get(); + CudaCheck(cuSignalExternalSemaphoresAsync(&external_semaphore, &ext_signal_params, 1, stream)); + + // the texture is now in uploaded state, Vulkan needs to wait for it + texture->state_ = Texture::State::UPLOADED; +} + +void Vulkan::Impl::upload_to_texture(const void* host_ptr, Texture* texture) { + if (transfer_jobs_.empty() || (transfer_jobs_.back().fence_)) { + throw std::runtime_error( + "Transfer command buffer not set. Calls to upload_to_texture() need to be enclosed by " + "begin_transfer_pass() and end_transfer_pass()"); + } + + if ((texture->state_ != Texture::State::RENDERED) && + (texture->state_ != Texture::State::UNKNOWN)) { + throw std::runtime_error( + "When uploading to texture, the texture should be in rendered " + "or unknown state"); + } + + const vk::CommandBuffer cmd_buf = transfer_jobs_.back().cmd_buffer_; + + // Copy buffer to image + vk::ImageSubresourceRange subresource_range; + subresource_range.aspectMask = vk::ImageAspectFlagBits::eColor; + subresource_range.baseArrayLayer = 0; + subresource_range.baseMipLevel = 0; + subresource_range.layerCount = 1; + subresource_range.levelCount = 1; + + nvvk::cmdBarrierImageLayout(cmd_buf, + texture->texture_.image, + vk::ImageLayout::eUndefined, + vk::ImageLayout::eTransferDstOptimal, + subresource_range); + + vk::Offset3D offset; + vk::ImageSubresourceLayers subresource; + subresource.aspectMask = vk::ImageAspectFlagBits::eColor; + subresource.layerCount = 1; + + uint32_t src_channels, dst_channels, component_size; + format_info(texture->format_, &src_channels, &dst_channels, &component_size); + + const vk::DeviceSize data_size = + texture->width_ * texture->height_ * dst_channels * component_size; + void* mapping = + nvvk_.alloc_.getStaging()->cmdToImage(cmd_buf, + texture->texture_.image, + VkOffset3D(offset), + VkExtent3D{texture->width_, texture->height_, 1}, + VkImageSubresourceLayers(subresource), + data_size, + nullptr); + + if (src_channels != dst_channels) { + // three channel texture data is not hardware natively supported, convert to four channel + if ((src_channels != 3) || (dst_channels != 4) || (component_size != 1)) { + throw std::runtime_error("Unhandled conversion."); + } + const uint8_t* src = reinterpret_cast(host_ptr); + uint32_t* dst = reinterpret_cast(mapping); + for (uint32_t y = 0; y < texture->height_; ++y) { + for (uint32_t x = 0; x < texture->width_; ++x) { + const uint8_t data[4]{src[0], src[1], src[2], 0xFF}; + *dst = *reinterpret_cast(&data); + src += 3; + ++dst; + } + } + } else { + memcpy(mapping, host_ptr, data_size); + } + + // Setting final image layout + nvvk::cmdBarrierImageLayout(cmd_buf, + texture->texture_.image, + vk::ImageLayout::eTransferDstOptimal, + vk::ImageLayout::eShaderReadOnlyOptimal); + + // no need to set the texture state here, the transfer command buffer submission is + // always synchronized to the render command buffer submission. +} + +Vulkan::Buffer* Vulkan::Impl::create_buffer(size_t data_size, vk::BufferUsageFlags usage, + const void* data) { + std::unique_ptr buffer(new Buffer(data_size, &nvvk_.alloc_)); + if (data) { + if (transfer_jobs_.empty() || (transfer_jobs_.back().fence_)) { + throw std::runtime_error( + "Transfer command buffer not set. Calls to create_buffer() with data need to be " + "enclosed by begin_transfer_pass() and end_transfer_pass()"); + } + + buffer->buffer_ = nvvk_.alloc_.createBuffer( + transfer_jobs_.back().cmd_buffer_, static_cast(data_size), data, usage); + } else { + buffer->buffer_ = nvvk_.alloc_.createBuffer( + static_cast(data_size), usage, vk::MemoryPropertyFlagBits::eDeviceLocal); + } + + return buffer.release(); +} + +Vulkan::Buffer* Vulkan::Impl::create_buffer_for_cuda_interop(size_t data_size, + vk::BufferUsageFlags usage) { + Vulkan::Buffer* const buffer = create_buffer(data_size, usage); + + // create the semaphore to synchronize Cuda and Vulkan usage + vk::StructureChain chain; + vk::SemaphoreCreateInfo& semaphore_create_info = chain.get(); + chain.get().handleTypes = + vk::ExternalSemaphoreHandleTypeFlagBits::eOpaqueFd; + buffer->upload_semaphore_ = device_.createSemaphoreUnique(semaphore_create_info); + buffer->render_semaphore_ = device_.createSemaphoreUnique(semaphore_create_info); + + { + const CudaService::ScopedPush cuda_context = CudaService::get().PushContext(); + + // import memory to CUDA + const nvvk::MemAllocator::MemInfo memInfo = + nvvk_.export_alloc_.getMemoryAllocator()->getMemoryInfo(buffer->buffer_.memHandle); + + vk::MemoryGetFdInfoKHR memory_get_fd_info; + memory_get_fd_info.memory = memInfo.memory; + memory_get_fd_info.handleType = vk::ExternalMemoryHandleTypeFlagBits::eOpaqueFd; + UniqueValue file_handle; + file_handle.reset(device_.getMemoryFdKHR(memory_get_fd_info)); + + CUDA_EXTERNAL_MEMORY_HANDLE_DESC memory_handle_desc{}; + memory_handle_desc.type = CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD; + memory_handle_desc.handle.fd = file_handle.get(); + memory_handle_desc.size = memInfo.offset + memInfo.size; + + buffer->external_mem_.reset([&memory_handle_desc] { + CUexternalMemory external_mem; + CudaCheck(cuImportExternalMemory(&external_mem, &memory_handle_desc)); + return external_mem; + }()); + // don't need to close the file handle if it had been successfully imported + file_handle.release(); + + CUDA_EXTERNAL_MEMORY_BUFFER_DESC buffer_desc{}; + buffer_desc.size = data_size; + buffer_desc.offset = memInfo.offset; + + buffer->device_ptr_.reset([external_mem = buffer->external_mem_.get(), &buffer_desc] { + CUdeviceptr device_ptr; + CudaCheck(cuExternalMemoryGetMappedBuffer(&device_ptr, external_mem, &buffer_desc)); + return device_ptr; + }()); + + // import the semaphore to Cuda + buffer->cuda_upload_semaphore_ = import_semaphore_to_cuda(buffer->upload_semaphore_.get()); + buffer->cuda_render_semaphore_ = import_semaphore_to_cuda(buffer->render_semaphore_.get()); + } + + return buffer; +} + +void Vulkan::Impl::upload_to_buffer(size_t data_size, CUdeviceptr device_ptr, Buffer* buffer, + size_t dst_offset, CUstream stream) { + if (transfer_jobs_.empty() || (transfer_jobs_.back().fence_)) { + throw std::runtime_error( + "Transfer command buffer not set. Calls to upload_to_buffer() need to be enclosed by " + "begin_transfer_pass() and end_transfer_pass()"); + } + + if (!buffer->device_ptr_) { + throw std::runtime_error("Buffer had not been imported to CUDA, can't upload data."); + } + +#if 1 + /// @brief @TODO workaround for uploading from device memory: download to host and then upload. + /// When uploading using cuMemcpy the data is corrupted, probably vertex cache. + /// - tried memory barrier before rendering: + /// vk::MemoryBarrier memory_barrier; + /// memory_barrier.srcAccessMask = vk::AccessFlagBits::eMemoryWrite; + /// memory_barrier.dstAccessMask = vk::AccessFlagBits::eVertexAttributeRead; + /// cmd_buf.pipelineBarrier(vk::PipelineStageFlagBits::eTopOfPipe, + /// vk::PipelineStageFlagBits::eVertexInput, + /// vk::DependencyFlags(), + /// memory_barrier, + /// nullptr, + /// nullptr); + /// with added self-dependency subpass + /// // self-dependency + /// subpass_dependencies[1].srcSubpass = 0; + /// subpass_dependencies[1].dstSubpass = 0; + /// subpass_dependencies[1].srcStageMask = vk::PipelineStageFlagBits::eTopOfPipe; + /// subpass_dependencies[1].dstStageMask = vk::PipelineStageFlagBits::eVertexInput; + /// subpass_dependencies[1].srcAccessMask = vk::AccessFlagBits::eMemoryWrite; + /// subpass_dependencies[1].dstAccessMask = vk::AccessFlagBits::eVertexAttributeRead; + /// subpass_dependencies[1].dependencyFlags = vk::DependencyFlags(); + /// - tried cuda and vulkan syncs + std::unique_ptr host_data(new uint8_t[data_size]); + CudaCheck( + cuMemcpyDtoHAsync(reinterpret_cast(host_data.get()), device_ptr, data_size, stream)); + CudaCheck(cuStreamSynchronize(stream)); + upload_to_buffer(data_size, reinterpret_cast(host_data.get()), buffer); +#else + const CudaService::ScopedPush cuda_context = CudaService::get().PushContext(); + + if (buffer->state_ == Buffer::State::RENDERED) { + // if the buffer had been used by Vulkan, wait for Vulkan to finish its work + CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS ext_wait_params{}; + const CUexternalSemaphore external_semaphore = buffer->cuda_render_semaphore_.get(); + CudaCheck(cuWaitExternalSemaphoresAsync(&external_semaphore, &ext_wait_params, 1, stream)); + buffer->state_ = Buffer::State::UNKNOWN; + } + + CudaCheck( + cuMemcpyDtoDAsync(buffer->device_ptr_.get() + dst_offset, device_ptr, data_size, stream)); + + // signal the semaphore + CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS ext_signal_params{}; + const CUexternalSemaphore external_semaphore = buffer->cuda_upload_semaphore_.get(); + CudaCheck(cuSignalExternalSemaphoresAsync(&external_semaphore, &ext_signal_params, 1, stream)); + + // the buffer had now been used by Cuda, Vulkan needs to wait for it + buffer->state_ = Buffer::State::UPLOADED; +#endif +} + +void Vulkan::Impl::upload_to_buffer(size_t data_size, const void* data, const Buffer* buffer) { + if (transfer_jobs_.empty() || (transfer_jobs_.back().fence_)) { + throw std::runtime_error( + "Transfer command buffer not set. Calls to upload_to_buffer() need to be enclosed by " + "begin_transfer_pass() and end_transfer_pass()"); + } + + const vk::CommandBuffer cmd_buf = transfer_jobs_.back().cmd_buffer_; + nvvk::StagingMemoryManager* const staging = nvvk_.alloc_.getStaging(); + + nvvk_.alloc_.getStaging()->cmdToBuffer(cmd_buf, buffer->buffer_.buffer, 0, data_size, data); +} + +void Vulkan::Impl::destroy_buffer(Buffer* buffer) { + if (buffer->fence_) { + // if the buffer had been tagged with a fence, wait for it before freeing the memory + const vk::Result result = device_.waitForFences(buffer->fence_, true, 100'000'000); + if (result != vk::Result::eSuccess) { + HOLOSCAN_LOG_WARN("Waiting for buffer fence failed with {}", vk::to_string(result)); + } + } + // check if this buffer had been imported to CUDA + if (buffer->device_ptr_ || buffer->external_mem_) { + const CudaService::ScopedPush cuda_context = CudaService::get().PushContext(); + + buffer->device_ptr_.reset(); + buffer->external_mem_.reset(); + + buffer->cuda_upload_semaphore_.reset(); + buffer->cuda_render_semaphore_.reset(); + } + + buffer->upload_semaphore_.reset(); + buffer->render_semaphore_.reset(); + + buffer->alloc_->destroy(buffer->buffer_); + delete buffer; +} + +void Vulkan::Impl::draw_texture(Texture* texture, Texture* lut, float opacity, + const nvmath::mat4f& view_matrix) { + const vk::CommandBuffer cmd_buf = command_buffers_[get_active_image_index()].get(); + + if (texture->state_ == Texture::State::UPLOADED) { + // enqueue the semaphore signalled by Cuda to be waited on by rendering + nvvk_.batch_submission_.enqueueWait(texture->upload_semaphore_.get(), + vk::PipelineStageFlagBits::eAllCommands); + // also signal the render semapore which will be waited on by Cuda + nvvk_.batch_submission_.enqueueSignal(texture->render_semaphore_.get()); + texture->state_ = Texture::State::RENDERED; + } + + vk::Pipeline pipeline; + vk::PipelineLayout pipeline_layout; + + // update descriptor sets + std::vector writes; + if (lut) { + if (lut->state_ == Texture::State::UPLOADED) { + // enqueue the semaphore signalled by Cuda to be waited on by rendering + nvvk_.batch_submission_.enqueueWait(lut->upload_semaphore_.get(), + vk::PipelineStageFlagBits::eAllCommands); + // also signal the render semapore which will be waited on by Cuda + nvvk_.batch_submission_.enqueueSignal(lut->render_semaphore_.get()); + lut->state_ = Texture::State::RENDERED; + } + + if ((texture->format_ == ImageFormat::R8_UINT) || (texture->format_ == ImageFormat::R16_UINT) || + (texture->format_ == ImageFormat::R32_UINT)) { + pipeline = image_lut_uint_pipeline_.get(); + } else { + pipeline = image_lut_float_pipeline_.get(); + } + + pipeline_layout = image_lut_pipeline_layout_.get(); + + writes.emplace_back(desc_set_layout_bind_lut_.makeWrite( + nullptr, bindings_offset_texture_, &texture->texture_.descriptor)); + writes.emplace_back(desc_set_layout_bind_lut_.makeWrite( + nullptr, bindings_offset_texture_lut_, &lut->texture_.descriptor)); + } else { + pipeline = image_pipeline_.get(); + pipeline_layout = image_pipeline_layout_.get(); + + writes.emplace_back(desc_set_layout_bind_.makeWrite( + nullptr, bindings_offset_texture_, &texture->texture_.descriptor)); + } + + cmd_buf.bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline); + + cmd_buf.pushDescriptorSetKHR(vk::PipelineBindPoint::eGraphics, + pipeline_layout, + 0, + static_cast(writes.size()), + writes.data()); + + // push the constants + PushConstantFragment push_constants; + push_constants.opacity = opacity; + cmd_buf.pushConstants(pipeline_layout, + vk::ShaderStageFlagBits::eFragment, + sizeof(PushConstantVertex), + sizeof(PushConstantFragment), + &push_constants); + + PushConstantVertex push_constant_vertex; + push_constant_vertex.matrix = view_matrix; + cmd_buf.pushConstants(pipeline_layout, + vk::ShaderStageFlagBits::eVertex, + 0, + sizeof(PushConstantVertex), + &push_constant_vertex); + + // bind the buffers + const vk::DeviceSize offset{0}; + const vk::Buffer buffer(nvvk_.vertex_buffer_.buffer); + cmd_buf.bindVertexBuffers(0, 1, &buffer, &offset); + cmd_buf.bindIndexBuffer(nvvk_.index_buffer_.buffer, 0, vk::IndexType::eUint16); + + // draw + cmd_buf.drawIndexed(6, 1, 0, 0, 0); + + // tag the texture and lut with the current fence + texture->fence_ = wait_fences_[get_active_image_index()].get(); + if (lut) { lut->fence_ = wait_fences_[get_active_image_index()].get(); } +} + +void Vulkan::Impl::draw(vk::PrimitiveTopology topology, uint32_t count, uint32_t first, + const std::vector& vertex_buffers, float opacity, + const std::array& color, float point_size, float line_width, + const nvmath::mat4f& view_matrix) { + const vk::CommandBuffer cmd_buf = command_buffers_[get_active_image_index()].get(); + + switch (topology) { + case vk::PrimitiveTopology::ePointList: + if (vertex_buffers.size() == 1) { + cmd_buf.bindPipeline(vk::PipelineBindPoint::eGraphics, geometry_point_pipeline_.get()); + } else { + cmd_buf.bindPipeline(vk::PipelineBindPoint::eGraphics, + geometry_point_color_pipeline_.get()); + } + break; + case vk::PrimitiveTopology::eLineList: + if (vertex_buffers.size() == 1) { + cmd_buf.bindPipeline(vk::PipelineBindPoint::eGraphics, geometry_line_pipeline_.get()); + } else { + cmd_buf.bindPipeline(vk::PipelineBindPoint::eGraphics, geometry_line_color_pipeline_.get()); + } + cmd_buf.setLineWidth(line_width); + break; + case vk::PrimitiveTopology::eLineStrip: + if (vertex_buffers.size() == 1) { + cmd_buf.bindPipeline(vk::PipelineBindPoint::eGraphics, geometry_line_strip_pipeline_.get()); + } else { + cmd_buf.bindPipeline(vk::PipelineBindPoint::eGraphics, + geometry_line_strip_color_pipeline_.get()); + } + cmd_buf.setLineWidth(line_width); + break; + case vk::PrimitiveTopology::eTriangleList: + if (vertex_buffers.size() == 1) { + cmd_buf.bindPipeline(vk::PipelineBindPoint::eGraphics, geometry_triangle_pipeline_.get()); + } else { + cmd_buf.bindPipeline(vk::PipelineBindPoint::eGraphics, + geometry_triangle_color_pipeline_.get()); + } + break; + default: + throw std::runtime_error("Unhandled primitive type"); + } + + // push the constants + PushConstantFragment push_constants_fragment; + push_constants_fragment.opacity = opacity; + cmd_buf.pushConstants(geometry_pipeline_layout_.get(), + vk::ShaderStageFlagBits::eFragment, + sizeof(PushConstantVertex), + sizeof(PushConstantFragment), + &push_constants_fragment); + + PushConstantVertex push_constant_vertex; + push_constant_vertex.matrix = view_matrix; + push_constant_vertex.point_size = point_size; + push_constant_vertex.color = color; + cmd_buf.pushConstants(geometry_pipeline_layout_.get(), + vk::ShaderStageFlagBits::eVertex, + 0, + sizeof(PushConstantVertex), + &push_constant_vertex); + + // bind the buffers + std::vector offsets(vertex_buffers.size()); + std::vector buffers(vertex_buffers.size()); + for (size_t index = 0; index < vertex_buffers.size(); ++index) { + offsets[index] = 0; + buffers[index] = vertex_buffers[index]->buffer_.buffer; + } + cmd_buf.bindVertexBuffers(0, vertex_buffers.size(), buffers.data(), offsets.data()); + + // draw + cmd_buf.draw(count, 1, first, 0); + + // tag the buffers with the current fence + const vk::Fence fence = wait_fences_[get_active_image_index()].get(); + for (size_t index = 0; index < vertex_buffers.size(); ++index) { + vertex_buffers[index]->fence_ = fence; + } +} + +void Vulkan::Impl::draw_text_indexed(vk::DescriptorSet desc_set, Buffer* vertex_buffer, + Buffer* index_buffer, vk::IndexType index_type, + uint32_t index_count, uint32_t first_index, + uint32_t vertex_offset, float opacity, + const nvmath::mat4f& view_matrix) { + draw_indexed(geometry_text_pipeline_.get(), + geometry_text_pipeline_layout_.get(), + desc_set, + {vertex_buffer}, + index_buffer, + index_type, + index_count, + first_index, + vertex_offset, + opacity, + std::array({1.f, 1.f, 1.f, 1.f}), + 1.f, + 0.f, + view_matrix); +} + +void Vulkan::Impl::draw_indexed(vk::Pipeline pipeline, vk::PipelineLayout pipeline_layout, + vk::DescriptorSet desc_set, + const std::vector& vertex_buffers, Buffer* index_buffer, + vk::IndexType index_type, uint32_t index_count, + uint32_t first_index, uint32_t vertex_offset, float opacity, + const std::array& color, float point_size, + float line_width, const nvmath::mat4f& view_matrix) { + const vk::CommandBuffer cmd_buf = command_buffers_[get_active_image_index()].get(); + + for (auto&& vertex_buffer : vertex_buffers) { + if (vertex_buffer->state_ == Buffer::State::UPLOADED) { + // enqueue the semaphore signalled by Cuda to be waited on by rendering + nvvk_.batch_submission_.enqueueWait(vertex_buffer->upload_semaphore_.get(), + vk::PipelineStageFlagBits::eAllCommands); + // also signal the render semapore which will be waited on by Cuda + nvvk_.batch_submission_.enqueueSignal(vertex_buffer->render_semaphore_.get()); + vertex_buffer->state_ = Buffer::State::RENDERED; + } + } + + if (line_width > 0.f) { cmd_buf.setLineWidth(line_width); } + + if (desc_set) { + cmd_buf.bindDescriptorSets( + vk::PipelineBindPoint::eGraphics, pipeline_layout, 0, 1, &desc_set, 0, nullptr); + } + + cmd_buf.bindPipeline(vk::PipelineBindPoint::eGraphics, pipeline); + + // push the constants + PushConstantFragment push_constants; + push_constants.opacity = opacity; + cmd_buf.pushConstants(pipeline_layout, + vk::ShaderStageFlagBits::eFragment, + sizeof(PushConstantVertex), + sizeof(PushConstantFragment), + &push_constants); + + PushConstantVertex push_constant_vertex; + push_constant_vertex.matrix = view_matrix; + push_constant_vertex.point_size = point_size; + push_constant_vertex.color = color; + cmd_buf.pushConstants(pipeline_layout, + vk::ShaderStageFlagBits::eVertex, + 0, + sizeof(PushConstantVertex), + &push_constant_vertex); + + // bind the buffers + std::vector offsets(vertex_buffers.size()); + std::vector buffers(vertex_buffers.size()); + for (size_t index = 0; index < vertex_buffers.size(); ++index) { + offsets[index] = 0; + buffers[index] = vertex_buffers[index]->buffer_.buffer; + } + cmd_buf.bindVertexBuffers(0, vertex_buffers.size(), buffers.data(), offsets.data()); + cmd_buf.bindIndexBuffer(index_buffer->buffer_.buffer, 0, index_type); + + // draw + cmd_buf.drawIndexed(index_count, 1, first_index, vertex_offset, 0); + + // tag the buffers with the current fence + const vk::Fence fence = wait_fences_[get_active_image_index()].get(); + for (size_t index = 0; index < vertex_buffers.size(); ++index) { + vertex_buffers[index]->fence_ = fence; + } + index_buffer->fence_ = fence; +} + +void Vulkan::Impl::draw_indexed(vk::PrimitiveTopology topology, + const std::vector& vertex_buffers, Buffer* index_buffer, + vk::IndexType index_type, uint32_t index_count, + uint32_t first_index, uint32_t vertex_offset, float opacity, + const std::array& color, float point_size, + float line_width, const nvmath::mat4f& view_matrix) { + vk::Pipeline pipeline; + switch (topology) { + case vk::PrimitiveTopology::ePointList: + if (vertex_buffers.size() == 1) { + pipeline = geometry_point_pipeline_.get(); + } else { + pipeline = geometry_point_color_pipeline_.get(); + } + break; + case vk::PrimitiveTopology::eLineList: + if (vertex_buffers.size() == 1) { + pipeline = geometry_line_pipeline_.get(); + } else { + pipeline = geometry_line_color_pipeline_.get(); + } + break; + case vk::PrimitiveTopology::eTriangleStrip: + if (vertex_buffers.size() == 1) { + pipeline = geometry_line_strip_pipeline_.get(); + } else { + pipeline = geometry_line_strip_color_pipeline_.get(); + } + break; + case vk::PrimitiveTopology::eTriangleList: + if (vertex_buffers.size() == 1) { + pipeline = geometry_triangle_pipeline_.get(); + } else { + pipeline = geometry_triangle_color_pipeline_.get(); + } + break; + default: + throw std::runtime_error("Unhandled primitive type"); + } + draw_indexed(pipeline, + geometry_pipeline_layout_.get(), + nullptr, + vertex_buffers, + index_buffer, + index_type, + index_count, + first_index, + vertex_offset, + opacity, + color, + point_size, + line_width, + view_matrix); +} + +void Vulkan::Impl::read_framebuffer(ImageFormat fmt, uint32_t width, uint32_t height, + size_t buffer_size, CUdeviceptr device_ptr, CUstream stream) { + if (fmt != ImageFormat::R8G8B8A8_UNORM) { + throw std::runtime_error("Unsupported image format, supported formats: R8G8B8A8_UNORM."); + } + + const vk::Format vk_format = to_vulkan_format(fmt); + uint32_t src_channels, dst_channels, component_size; + format_info(fmt, &src_channels, &dst_channels, &component_size); + + // limit size to actual framebuffer size + const uint32_t read_width = std::min(size_.width, width); + const uint32_t read_height = std::min(size_.height, height); + + const size_t data_size = read_width * read_height * dst_channels * component_size; + if (buffer_size < data_size) { throw std::runtime_error("The size of the buffer is too small"); } + + /// @todo need to wait for frame, use semaphores to sync + nvvk_.batch_submission_.waitIdle(); + + // create a command buffer + nvvk::CommandPool cmd_buf_get(device_, nvvk_.vk_ctx_.m_queueGCT.familyIndex); + vk::CommandBuffer cmd_buf = cmd_buf_get.createCommandBuffer(); + + // Make the image layout vk::ImageLayout::eTransferSrcOptimal to copy to buffer + vk::ImageSubresourceRange subresource_range; + subresource_range.aspectMask = vk::ImageAspectFlagBits::eColor; + subresource_range.levelCount = 1; + subresource_range.layerCount = 1; + nvvk::cmdBarrierImageLayout( + cmd_buf, + fb_sequence_.get_active_image(), + surface_ ? vk::ImageLayout::ePresentSrcKHR : vk::ImageLayout::eColorAttachmentOptimal, + vk::ImageLayout::eTransferSrcOptimal, + subresource_range); + + // allocate the buffer + /// @todo keep the buffer and the mapping to Cuda to avoid allocations + const size_t src_data_size = read_width * read_height * src_channels * component_size; + Vulkan::Buffer* const transfer_buffer = + create_buffer_for_cuda_interop(src_data_size, vk::BufferUsageFlagBits::eTransferDst); + + // Copy the image to the buffer + vk::BufferImageCopy copy_region; + copy_region.imageSubresource.aspectMask = vk::ImageAspectFlagBits::eColor; + copy_region.imageSubresource.layerCount = 1; + copy_region.imageExtent.width = read_width; + copy_region.imageExtent.height = read_height; + copy_region.imageExtent.depth = 1; + cmd_buf.copyImageToBuffer(fb_sequence_.get_active_image(), + vk::ImageLayout::eTransferSrcOptimal, + transfer_buffer->buffer_.buffer, + 1, + ©_region); + + // Put back the image as it was + nvvk::cmdBarrierImageLayout( + cmd_buf, + fb_sequence_.get_active_image(), + vk::ImageLayout::eTransferSrcOptimal, + surface_ ? vk::ImageLayout::ePresentSrcKHR : vk::ImageLayout::eColorAttachmentOptimal, + subresource_range); + /// @todo avoid wait, use semaphore to sync + cmd_buf_get.submitAndWait(cmd_buf); + + // copy the buffer to Cuda memory + { + const CudaService::ScopedPush cuda_context = CudaService::get().PushContext(); + + if (fb_sequence_.get_format() == vk::Format::eB8G8R8A8Unorm) { + ConvertB8G8R8A8ToR8G8B8A8(read_width, + read_height, + transfer_buffer->device_ptr_.get(), + read_width * dst_channels * component_size, + device_ptr, + width * dst_channels * component_size, + stream); + } else if (fb_sequence_.get_format() == vk::Format::eR8G8B8A8Unorm) { + CUDA_MEMCPY2D memcpy2d{}; + memcpy2d.srcMemoryType = CU_MEMORYTYPE_DEVICE; + memcpy2d.srcDevice = transfer_buffer->device_ptr_.get(); + memcpy2d.srcPitch = read_width * dst_channels * component_size; + memcpy2d.dstMemoryType = CU_MEMORYTYPE_DEVICE; + memcpy2d.dstDevice = device_ptr; + memcpy2d.dstPitch = width * dst_channels * component_size; + memcpy2d.WidthInBytes = memcpy2d.srcPitch; + memcpy2d.Height = read_height; + CudaCheck(cuMemcpy2DAsync(&memcpy2d, stream)); + } else { + throw std::runtime_error("Unhandled framebuffer format."); + } + } + + destroy_buffer(transfer_buffer); +} + +Vulkan::Vulkan() : impl_(new Vulkan::Impl) {} + +Vulkan::~Vulkan() {} + +void Vulkan::setup(Window* window, const std::string& font_path, float font_size_in_pixels) { + impl_->setup(window, font_path, font_size_in_pixels); +} + +Window* Vulkan::get_window() const { + return impl_->get_window(); +} + +void Vulkan::begin_transfer_pass() { + impl_->begin_transfer_pass(); +} + +void Vulkan::end_transfer_pass() { + impl_->end_transfer_pass(); +} + +void Vulkan::begin_render_pass() { + impl_->begin_render_pass(); +} + +void Vulkan::end_render_pass() { + impl_->end_render_pass(); +} + +vk::CommandBuffer Vulkan::get_command_buffer() { + return impl_->get_command_buffers()[impl_->get_active_image_index()].get(); +} + +Vulkan::Texture* Vulkan::create_texture_for_cuda_interop(uint32_t width, uint32_t height, + ImageFormat format, vk::Filter filter, + bool normalized) { + return impl_->create_texture_for_cuda_interop(width, height, format, filter, normalized); +} + +Vulkan::Texture* Vulkan::create_texture(uint32_t width, uint32_t height, ImageFormat format, + size_t data_size, const void* data, vk::Filter filter, + bool normalized) { + return impl_->create_texture(width, height, format, data_size, data, filter, normalized); +} + +void Vulkan::destroy_texture(Texture* texture) { + impl_->destroy_texture(texture); +} + +void Vulkan::upload_to_texture(CUdeviceptr device_ptr, Texture* texture, CUstream stream) { + impl_->upload_to_texture(device_ptr, texture, stream); +} + +void Vulkan::upload_to_texture(const void* host_ptr, Texture* texture) { + impl_->upload_to_texture(host_ptr, texture); +} + +Vulkan::Buffer* Vulkan::create_buffer(size_t data_size, const void* data, + vk::BufferUsageFlags usage) { + return impl_->create_buffer(data_size, usage, data); +} + +Vulkan::Buffer* Vulkan::create_buffer_for_cuda_interop(size_t data_size, + vk::BufferUsageFlags usage) { + return impl_->create_buffer_for_cuda_interop(data_size, usage); +} + +void Vulkan::upload_to_buffer(size_t data_size, CUdeviceptr device_ptr, Buffer* buffer, + size_t dst_offset, CUstream stream) { + return impl_->upload_to_buffer(data_size, device_ptr, buffer, dst_offset, stream); +} + +void Vulkan::upload_to_buffer(size_t data_size, const void* data, const Buffer* buffer) { + return impl_->upload_to_buffer(data_size, data, buffer); +} + +void Vulkan::destroy_buffer(Buffer* buffer) { + impl_->destroy_buffer(buffer); +} + +void Vulkan::draw_texture(Texture* texture, Texture* lut, float opacity, + const nvmath::mat4f& view_matrix) { + impl_->draw_texture(texture, lut, opacity, view_matrix); +} + +void Vulkan::draw(vk::PrimitiveTopology topology, uint32_t count, uint32_t first, + const std::vector& vertex_buffers, float opacity, + const std::array& color, float point_size, float line_width, + const nvmath::mat4f& view_matrix) { + impl_->draw( + topology, count, first, vertex_buffers, opacity, color, point_size, line_width, view_matrix); +} + +void Vulkan::draw_text_indexed(vk::DescriptorSet desc_set, Buffer* vertex_buffer, + Buffer* index_buffer, vk::IndexType index_type, uint32_t index_count, + uint32_t first_index, uint32_t vertex_offset, float opacity, + const nvmath::mat4f& view_matrix) { + impl_->draw_text_indexed(desc_set, + vertex_buffer, + index_buffer, + index_type, + index_count, + first_index, + vertex_offset, + opacity, + view_matrix); +} + +void Vulkan::draw_indexed(vk::PrimitiveTopology topology, + const std::vector& vertex_buffers, Buffer* index_buffer, + vk::IndexType index_type, uint32_t index_count, uint32_t first_index, + uint32_t vertex_offset, float opacity, const std::array& color, + float point_size, float line_width, const nvmath::mat4f& view_matrix) { + impl_->draw_indexed(topology, + vertex_buffers, + index_buffer, + index_type, + index_count, + first_index, + vertex_offset, + opacity, + color, + point_size, + line_width, + view_matrix); +} + +void Vulkan::read_framebuffer(ImageFormat fmt, uint32_t width, uint32_t height, size_t buffer_size, + CUdeviceptr buffer, CUstream stream) { + impl_->read_framebuffer(fmt, width, height, buffer_size, buffer, stream); +} + +} // namespace holoscan::viz diff --git a/modules/holoviz/src/vulkan/vulkan_app.hpp b/modules/holoviz/src/vulkan/vulkan_app.hpp new file mode 100644 index 00000000..7b353bc7 --- /dev/null +++ b/modules/holoviz/src/vulkan/vulkan_app.hpp @@ -0,0 +1,295 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HOLOSCAN_VIZ_VULKAN_VULKAN_HPP +#define HOLOSCAN_VIZ_VULKAN_VULKAN_HPP + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "../holoviz/image_format.hpp" +#include "../window.hpp" + +namespace holoscan::viz { + +class Layer; + +/** + * The class is responsible for all operations regarding Vulkan. + */ +class Vulkan { + public: + /** + * Construct a new Vulkan object. + */ + Vulkan(); + + /** + * Destroy the Vulkan object. + */ + ~Vulkan(); + + struct Texture; ///< texture object + struct Buffer; ///< buffer object + + /** + * Setup Vulkan using the given window. + * + * @param window window to use + * @param font_path path to font file for text rendering, if not set the default font is used + * @param font_size_in_pixels size of the font bitmaps + */ + void setup(Window* window, const std::string& font_path, float font_size_in_pixels); + + /** + * @return the window used by Vulkan + */ + Window* get_window() const; + + /** + * Begin the transfer pass. This creates a transfer job and a command buffer. + */ + void begin_transfer_pass(); + + /** + * End the transfer pass. This ends and submits the transfer command buffer. + * It's an error to call end_transfer_pass() + * without begin_transfer_pass(). + */ + void end_transfer_pass(); + + /** + * Begin the render pass. This acquires the next image to render to + * and sets up the render command buffer. + */ + void begin_render_pass(); + + /** + * End the render pass. Submits the render command buffer. + */ + void end_render_pass(); + + /** + * Get the command buffer for the current frame. + * + * @return vk::CommandBuffer + */ + vk::CommandBuffer get_command_buffer(); + + /** + * Create a texture to be used for interop with Cuda, see ::upload_to_texture. + * Destroy with ::destroy_texture. + * + * @param width, height size + * @param format texture format + * @param filter texture filter + * @param normalized if true, then texture coordinates are normalize (0...1), + * else (0...width, 0...height) + * @return created texture object + */ + Texture* create_texture_for_cuda_interop(uint32_t width, uint32_t height, ImageFormat format, + vk::Filter filter = vk::Filter::eLinear, + bool normalized = true); + + /** + * Create a Texture using host data. Destroy with ::destroy_texture. + * + * @param width, height size + * @param format texture format + * @param data_size data size in bytes + * @param data texture data + * @param filter texture filter + * @param normalized if true, then texture coordinates are normalize (0...1), + * else (0...width, 0...height) + * @return created texture object + */ + Texture* create_texture(uint32_t width, uint32_t height, ImageFormat format, size_t data_size, + const void* data, vk::Filter filter = vk::Filter::eLinear, + bool normalized = true); + + /** + * Destroy a texture created with ::create_texture_for_cuda_interop or ::create_texture. + * + * @param texture texture to destroy + */ + void destroy_texture(Texture* texture); + + /** + * Upload data from Cuda device memory to a texture created with ::create_texture_for_cuda_interop + * + * @param device_ptr Cuda device memory + * @param texture texture to be updated + * @param stream Cuda stream + */ + void upload_to_texture(CUdeviceptr device_ptr, Texture* texture, CUstream stream = 0); + + /** + * Upload data from host memory to a texture created with ::create_texture + * + * @param host_ptr data to upload in host memory + * @param texture texture to be updated + */ + void upload_to_texture(const void* host_ptr, Texture* texture); + + /** + * Create a vertex or index buffer to be used for interop with Cuda, see ::upload_texture. + * Destroy with ::destroy_buffer. + * + * @param data_size size of the buffer in bytes + * @param usage buffer usage + * @return created buffer + */ + Buffer* create_buffer_for_cuda_interop(size_t data_size, vk::BufferUsageFlags usage); + + /** + * Create a vertex or index buffer and initialize with data. Destroy with ::destroy_buffer. + * + * @param data_size size of the buffer in bytes + * @param data host size data to initialize buffer with or nullptr + * @param usage buffer usage + * @return created buffer + */ + Buffer* create_buffer(size_t data_size, const void* data, vk::BufferUsageFlags usage); + + /** + * Upload data from Cuda device memory to a buffer created with ::create_buffer_for_cuda_interop + * + * @param data_size data size + * @param device_ptr Cuda device memory + * @param buffer buffer to be updated + * @param dst_offset offset in buffer to copy to + * @param stream Cuda stream + */ + void upload_to_buffer(size_t data_size, CUdeviceptr device_ptr, Buffer* buffer, size_t dst_offset, + CUstream stream); + + /** + * Upload data from host memory to a buffer created with ::CreateBuffer + * + * @param data_size data size + * @param data host memory data buffer pointer + * @param buffer buffer to be updated + */ + void upload_to_buffer(size_t data_size, const void* data, const Buffer* buffer); + + /** + * Destroy a buffer created with ::CreateBuffer. + * + * @param buffer buffer to destroy + */ + void destroy_buffer(Buffer* buffer); + + /** + * Draw a texture with an optional color lookup table. + * + * @param texture texture to draw + * @param lut lookup table, can be nullptr + * @param opacity opacity, 0.0 is transparent, 1.0 is opaque + * @param view_matrix view matrix + */ + void draw_texture(Texture* texture, Texture* lut, float opacity, + const nvmath::mat4f& view_matrix = nvmath::mat4f(1)); + + /** + * Draw geometry. + * + * @param topology topology + * @param count vertex count + * @param first first vertex + * @param vertex_buffers vertex buffers + * @param opacity opacity, 0.0 is transparent, 1.0 is opaque + * @param color color + * @param point_size point size + * @param line_width line width + * @param view_matrix view matrix + */ + void draw(vk::PrimitiveTopology topology, uint32_t count, uint32_t first, + const std::vector& vertex_buffers, float opacity, + const std::array& color, float point_size, float line_width, + const nvmath::mat4f& view_matrix = nvmath::mat4f(1)); + + /** + * Draw indexed triangle list geometry. Used to draw ImGui draw list for text drawing. + * + * @param desc_set descriptor set for texture atlas + * @param vertex_buffer vertex buffer + * @param index_buffer index buffer + * @param index_type index type + * @param index_count index count + * @param first_index first index + * @param vertex_offset vertex offset + * @param opacity opacity, 0.0 is transparent, 1.0 is opaque + * @param view_matrix view matrix + */ + void draw_text_indexed(vk::DescriptorSet desc_set, Buffer* vertex_buffer, Buffer* index_buffer, + vk::IndexType index_type, uint32_t index_count, uint32_t first_index, + uint32_t vertex_offset, float opacity, + const nvmath::mat4f& view_matrix = nvmath::mat4f(1)); + + /** + * Draw indexed geometry. + * + * @param topology + * @param vertex_buffers vertex buffers + * @param index_buffer index buffer + * @param index_type index type + * @param index_count index count + * @param first_index first index + * @param vertex_offset vertex offset + * @param opacity opacity, 0.0 is transparent, 1.0 is opaque + * @param color color + * @param point_size point size + * @param line_width line width + * @param view_matrix view matrix + */ + void draw_indexed(vk::PrimitiveTopology topology, const std::vector& vertex_buffers, + Buffer* index_buffer, vk::IndexType index_type, uint32_t index_count, + uint32_t first_index, uint32_t vertex_offset, float opacity, + const std::array& color, float point_size, float line_width, + const nvmath::mat4f& view_matrix = nvmath::mat4f(1)); + + /** + * Read the framebuffer and store it to cuda device memory. + * + * Can only be called outside of Begin()/End(). + * + * @param fmt image format, currently only R8G8B8A8_UNORM is supported. + * @param width, height width and height of the region to read back, will be limited to the + * framebuffer size if the framebuffer is smaller than that + * @param buffer_size size of the storage buffer in bytes + * @param buffer pointer to Cuda device memory to store the framebuffer into + * @param stream Cuda stream + */ + void read_framebuffer(ImageFormat fmt, uint32_t width, uint32_t height, size_t buffer_size, + CUdeviceptr buffer, CUstream stream = 0); + + private: + struct Impl; + std::shared_ptr impl_; +}; + +} // namespace holoscan::viz + +#endif /* HOLOSCAN_VIZ_VULKAN_VULKAN_HPP */ diff --git a/modules/holoviz/src/window.hpp b/modules/holoviz/src/window.hpp index e419d7b3..8e01f090 100644 --- a/modules/holoviz/src/window.hpp +++ b/modules/holoviz/src/window.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,10 +18,13 @@ #ifndef HOLOSCAN_VIZ_WINDOW_HPP #define HOLOSCAN_VIZ_WINDOW_HPP -#include +#include #include #include +#include + +#include namespace holoscan::viz { @@ -30,85 +33,107 @@ namespace holoscan::viz { */ class Window { public: - /** - * Construct a new Window object. - */ - Window() {} - - /** - * Destroy the Window object. - */ - virtual ~Window() {} - - /** - * Initialize ImGui to be used with that window (mouse, keyboard, ...). - */ - virtual void init_im_gui() = 0; - - /** - * Setup call backs. - * - * @param framebuffersize_cb Called when the frame buffer size changes - */ - virtual void setup_callbacks(std::function - frame_buffer_size_cb) = 0; - - /** - * Get the required instance extensions for vulkan. - * - * @param [out] count returns the extension count - * @return array with required extensions - */ - virtual const char **get_required_instance_extensions(uint32_t *count) = 0; - - /** - * Get the required device extensions for vulkan. - * - * @param [out] count returns the extension count - * @return array with required extensions - */ - virtual const char **get_required_device_extensions(uint32_t *count) = 0; - - /** - * Get the current frame buffer size. - * - * @param [out] width, height framebuffer size - */ - virtual void get_framebuffer_size(uint32_t *width, uint32_t *height) = 0; - - /** - * Create a Vulkan surface. - * - * @param physical_device Vulkan device - * @param instance Vulkan instance - * @return vulkan surface - */ - virtual VkSurfaceKHR create_surface(VkPhysicalDevice physical_device, VkInstance instance) = 0; - - /** - * @returns true if the window should be closed - */ - virtual bool should_close() = 0; - - /** - * @returns true if the window is minimized - */ - virtual bool is_minimized() = 0; - - /** - * Start a new ImGui frame. - */ - virtual void im_gui_new_frame() = 0; - - /** - * Start a new frame. - */ - virtual void begin() = 0; - - /** - * End the current frame. - */ - virtual void end() = 0; + /** + * Construct a new Window object. + */ + Window() {} + + /** + * Destroy the Window object. + */ + virtual ~Window() {} + + /** + * Initialize ImGui to be used with that window (mouse, keyboard, ...). + */ + virtual void init_im_gui() = 0; + + /** + * Setup call backs. + * + * @param framebuffersize_cb Called when the frame buffer size changes + */ + virtual void setup_callbacks(std::function frame_buffer_size_cb) = 0; + + /** + * Get the required instance extensions for vulkan. + * + * @param [out] count returns the extension count + * @return array with required extensions + */ + virtual const char** get_required_instance_extensions(uint32_t* count) = 0; + + /** + * Get the required device extensions for vulkan. + * + * @param [out] count returns the extension count + * @return array with required extensions + */ + virtual const char** get_required_device_extensions(uint32_t* count) = 0; + + /** + * Select a device from the given list of supported physical devices. + * + * @param instance instance the devices belong to + * @param physical_devices list of supported physical devices + * @return index of the selected physical device + */ + virtual uint32_t select_device(vk::Instance instance, + const std::vector& physical_devices) = 0; + + /** + * Get the current frame buffer size. + * + * @param [out] width, height framebuffer size + */ + virtual void get_framebuffer_size(uint32_t* width, uint32_t* height) = 0; + + /** + * Create a Vulkan surface. + * + * @param physical_device Vulkan device + * @param instance Vulkan instance + * @return vulkan surface + */ + virtual vk::SurfaceKHR create_surface(vk::PhysicalDevice physical_device, + vk::Instance instance) = 0; + + /** + * @returns true if the window should be closed + */ + virtual bool should_close() = 0; + + /** + * @returns true if the window is minimized + */ + virtual bool is_minimized() = 0; + + /** + * Start a new ImGui frame. + */ + virtual void im_gui_new_frame() = 0; + + /** + * Start a new frame. + */ + virtual void begin() = 0; + + /** + * End the current frame. + */ + virtual void end() = 0; + + /** + * Get the view matrix + * + * @param view_matrix + */ + virtual void get_view_matrix(nvmath::mat4f* view_matrix) { *view_matrix = nvmath::mat4f(1); } + + /** + * @returns the horizontal aspect ratio + */ + virtual float get_aspect_ratio() = 0; }; } // namespace holoscan::viz diff --git a/modules/holoviz/tests/CMakeLists.txt b/modules/holoviz/tests/CMakeLists.txt index aa16a20d..30c61dae 100644 --- a/modules/holoviz/tests/CMakeLists.txt +++ b/modules/holoviz/tests/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,5 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -add_subdirectory(functional) -add_subdirectory(unit) \ No newline at end of file +find_package(GTest) + +if(GTest_FOUND) + add_subdirectory(functional) + add_subdirectory(unit) +endif() diff --git a/modules/holoviz/tests/functional/CMakeLists.txt b/modules/holoviz/tests/functional/CMakeLists.txt index f109e266..22915d76 100644 --- a/modules/holoviz/tests/functional/CMakeLists.txt +++ b/modules/holoviz/tests/functional/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -set(PROJECT_NAME clara_holoviz_functionaltests) +set(PROJECT_NAME holoviz_functionaltests) # fetch the dependencies include(FetchContent) @@ -28,6 +28,7 @@ FetchContent_Declare( FetchContent_MakeAvailable(stb) find_package(CUDAToolkit REQUIRED) +find_package(Vulkan REQUIRED) add_executable(${PROJECT_NAME}) add_executable(holoscan::viz::functionaltests ALIAS ${PROJECT_NAME}) @@ -49,11 +50,6 @@ target_compile_definitions(${PROJECT_NAME} GTEST_HAS_RTTI=1 GTEST_LANG_CXX11=1 GTEST_HAS_EXCEPTIONS=1 - # the gtest library provided by Holoscan is compiled with _GLIBCXX_USE_CXX11_ABI set to 1 - # if this is not set there are link errors as - # undefined reference to `testing::internal::EqFailure(char const*, char const*, std::string const&, std::string const&, bool) - # or runtime errors of functions in gtest expecting std::string - _GLIBCXX_USE_CXX11_ABI=1 ) target_include_directories(${PROJECT_NAME} @@ -63,12 +59,14 @@ target_include_directories(${PROJECT_NAME} target_link_libraries(${PROJECT_NAME} PRIVATE + glfw + Vulkan::Vulkan X11::X11 holoscan::viz holoscan::viz::imgui GTest::gtest_main CUDA::cuda_driver - nvpro_core # for GLFW + holoscan::logger ) add_test(NAME functional COMMAND ${PROJECT_NAME}) diff --git a/modules/holoviz/tests/functional/geometry_layer_test.cpp b/modules/holoviz/tests/functional/geometry_layer_test.cpp index cf1ab652..dbcb1237 100644 --- a/modules/holoviz/tests/functional/geometry_layer_test.cpp +++ b/modules/holoviz/tests/functional/geometry_layer_test.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,256 +17,430 @@ #include -#include "headless_fixture.hpp" +#include #include +#include "headless_fixture.hpp" namespace viz = holoscan::viz; +namespace holoscan::viz { + // define the '<<' operator to get a nice parameter string -std::ostream &operator<<(std::ostream &os, const viz::PrimitiveTopology &topology) { -#define CASE(VALUE) \ - case VALUE: \ - os << std::string(#VALUE); \ - break; - - switch (topology) { - CASE(viz::PrimitiveTopology::POINT_LIST); - CASE(viz::PrimitiveTopology::LINE_LIST); - CASE(viz::PrimitiveTopology::LINE_STRIP); - CASE(viz::PrimitiveTopology::TRIANGLE_LIST); - CASE(viz::PrimitiveTopology::CROSS_LIST); - CASE(viz::PrimitiveTopology::RECTANGLE_LIST); - CASE(viz::PrimitiveTopology::OVAL_LIST); +std::ostream& operator<<(std::ostream& os, const PrimitiveTopology& topology) { +#define CASE(VALUE) \ + case VALUE: \ + os << std::string(#VALUE); \ + break; + + switch (topology) { + CASE(viz::PrimitiveTopology::POINT_LIST); + CASE(viz::PrimitiveTopology::LINE_LIST); + CASE(viz::PrimitiveTopology::LINE_STRIP); + CASE(viz::PrimitiveTopology::TRIANGLE_LIST); + CASE(viz::PrimitiveTopology::CROSS_LIST); + CASE(viz::PrimitiveTopology::RECTANGLE_LIST); + CASE(viz::PrimitiveTopology::OVAL_LIST); default: - os.setstate(std::ios_base::failbit); - } - return os; + os.setstate(std::ios_base::failbit); + } + return os; #undef CASE } -// Fixture that creates initializes Holoviz -class GeometryLayer - : public TestHeadless - , public testing::WithParamInterface { -}; +} // namespace holoscan::viz -TEST_P(GeometryLayer, Primitive) { - const viz::PrimitiveTopology topology = GetParam(); +// Fixture that initializes Holoviz +class PrimitiveTopology : public TestHeadless, + public testing::WithParamInterface {}; - uint32_t crc; - uint32_t primitive_count; - std::vector data; - switch (topology) { +TEST_P(PrimitiveTopology, Primitive) { + const viz::PrimitiveTopology topology = GetParam(); + + uint32_t crc; + uint32_t primitive_count; + std::vector data; + switch (topology) { case viz::PrimitiveTopology::POINT_LIST: - primitive_count = 1; - data.push_back(0.5f); - data.push_back(0.5f); - crc = 0xE81FD1BB; - break; + primitive_count = 1; + data.push_back(0.5f); + data.push_back(0.5f); + crc = 0xE81FD1BB; + break; case viz::PrimitiveTopology::LINE_LIST: - primitive_count = 2; - data.push_back(0.1f); - data.push_back(0.1f); - data.push_back(0.9f); - data.push_back(0.9f); - - data.push_back(0.7f); - data.push_back(0.3f); - data.push_back(0.2f); - data.push_back(0.4f); - crc = 0xF7E63B21; - break; + primitive_count = 2; + data.push_back(0.1f); + data.push_back(0.1f); + data.push_back(0.9f); + data.push_back(0.9f); + + data.push_back(0.7f); + data.push_back(0.3f); + data.push_back(0.2f); + data.push_back(0.4f); + crc = 0xF7E63B21; + break; case viz::PrimitiveTopology::LINE_STRIP: - primitive_count = 2; - data.push_back(0.1f); - data.push_back(0.1f); - data.push_back(0.7f); - data.push_back(0.9f); - - data.push_back(0.3f); - data.push_back(0.2f); - crc = 0x392E35D8; - break; + primitive_count = 2; + data.push_back(0.1f); + data.push_back(0.1f); + data.push_back(0.7f); + data.push_back(0.9f); + + data.push_back(0.3f); + data.push_back(0.2f); + crc = 0x392E35D8; + break; case viz::PrimitiveTopology::TRIANGLE_LIST: - primitive_count = 2; - data.push_back(0.1f); - data.push_back(0.1f); - data.push_back(0.5f); - data.push_back(0.9f); - data.push_back(0.9f); - data.push_back(0.1f); - - data.push_back(0.05f); - data.push_back(0.7f); - data.push_back(0.15f); - data.push_back(0.8f); - data.push_back(0.25f); - data.push_back(0.6f); - crc = 0xB29BAA37; - break; + primitive_count = 2; + data.push_back(0.1f); + data.push_back(0.1f); + data.push_back(0.5f); + data.push_back(0.9f); + data.push_back(0.9f); + data.push_back(0.1f); + + data.push_back(0.05f); + data.push_back(0.7f); + data.push_back(0.15f); + data.push_back(0.8f); + data.push_back(0.25f); + data.push_back(0.6f); + crc = 0xB29BAA37; + break; case viz::PrimitiveTopology::CROSS_LIST: - primitive_count = 2; - data.push_back(0.5f); - data.push_back(0.5f); - data.push_back(0.1f); - - data.push_back(0.1f); - data.push_back(0.3f); - data.push_back(0.01f); - crc = 0x16056A95; - break; + primitive_count = 2; + data.push_back(0.5f); + data.push_back(0.5f); + data.push_back(0.1f); + + data.push_back(0.1f); + data.push_back(0.3f); + data.push_back(0.01f); + crc = 0xa32f4dcb; + break; case viz::PrimitiveTopology::RECTANGLE_LIST: - primitive_count = 2; - data.push_back(0.1f); - data.push_back(0.1f); - data.push_back(0.9f); - data.push_back(0.9f); - - data.push_back(0.3f); - data.push_back(0.2f); - data.push_back(0.5f); - data.push_back(0.3f); - crc = 0x355A2C00; - break; + primitive_count = 2; + data.push_back(0.1f); + data.push_back(0.1f); + data.push_back(0.9f); + data.push_back(0.9f); + + data.push_back(0.3f); + data.push_back(0.2f); + data.push_back(0.5f); + data.push_back(0.3f); + crc = 0x355A2C00; + break; case viz::PrimitiveTopology::OVAL_LIST: - primitive_count = 2; - data.push_back(0.5f); - data.push_back(0.5f); - data.push_back(0.2f); - data.push_back(0.1f); - - data.push_back(0.6f); - data.push_back(0.4f); - data.push_back(0.05f); - data.push_back(0.07f); - crc = 0xA907614F; - break; + primitive_count = 2; + data.push_back(0.5f); + data.push_back(0.5f); + data.push_back(0.2f); + data.push_back(0.1f); + + data.push_back(0.6f); + data.push_back(0.4f); + data.push_back(0.05f); + data.push_back(0.07f); + crc = 0xA907614F; + break; default: - EXPECT_TRUE(false) << "Unhandled primitive topoplogy"; - } + EXPECT_TRUE(false) << "Unhandled primitive topoplogy"; + } - EXPECT_NO_THROW(viz::Begin()); + EXPECT_NO_THROW(viz::Begin()); - EXPECT_NO_THROW(viz::BeginGeometryLayer()); + EXPECT_NO_THROW(viz::BeginGeometryLayer()); - for (uint32_t i = 0; i < 3; ++i) { - if (i == 1) { - EXPECT_NO_THROW(viz::Color(1.f, 0.5f, 0.25f, 0.75f)); - } else if (i == 2) { - EXPECT_NO_THROW(viz::PointSize(4.f)); - EXPECT_NO_THROW(viz::LineWidth(3.f)); - } + for (uint32_t i = 0; i < 3; ++i) { + if (i == 1) { + EXPECT_NO_THROW(viz::Color(1.f, 0.5f, 0.25f, 0.75f)); + } else if (i == 2) { + EXPECT_NO_THROW(viz::PointSize(4.f)); + EXPECT_NO_THROW(viz::LineWidth(3.f)); + } - EXPECT_NO_THROW(viz::Primitive(topology, primitive_count, data.size(), data.data())); + EXPECT_NO_THROW(viz::Primitive(topology, primitive_count, data.size(), data.data())); - for (auto &&item : data) { - item += 0.1f; - } - } - EXPECT_NO_THROW(viz::EndLayer()); + for (auto&& item : data) { item += 0.1f; } + } + EXPECT_NO_THROW(viz::EndLayer()); - EXPECT_NO_THROW(viz::End()); + EXPECT_NO_THROW(viz::End()); - CompareResultCRC32({crc}); + CompareResultCRC32({crc}); } -INSTANTIATE_TEST_SUITE_P(GeometryLayer, GeometryLayer, - testing::Values(viz::PrimitiveTopology::POINT_LIST, - viz::PrimitiveTopology::LINE_LIST, - viz::PrimitiveTopology::LINE_STRIP, - viz::PrimitiveTopology::TRIANGLE_LIST, - viz::PrimitiveTopology::CROSS_LIST, - viz::PrimitiveTopology::RECTANGLE_LIST, - viz::PrimitiveTopology::OVAL_LIST)); +INSTANTIATE_TEST_SUITE_P( + GeometryLayer, PrimitiveTopology, + testing::Values(viz::PrimitiveTopology::POINT_LIST, viz::PrimitiveTopology::LINE_LIST, + viz::PrimitiveTopology::LINE_STRIP, viz::PrimitiveTopology::TRIANGLE_LIST, + viz::PrimitiveTopology::CROSS_LIST, viz::PrimitiveTopology::RECTANGLE_LIST, + viz::PrimitiveTopology::OVAL_LIST)); + +// Fixture that initializes Holoviz +class GeometryLayer : public TestHeadless {}; TEST_F(GeometryLayer, Text) { - EXPECT_NO_THROW(viz::Begin()); + EXPECT_NO_THROW(viz::Begin()); - EXPECT_NO_THROW(viz::BeginGeometryLayer()); - EXPECT_NO_THROW(viz::Text(0.4f, 0.4f, 0.4f, "Text")); - EXPECT_NO_THROW(viz::Color(0.5f, 0.9f, 0.7f, 0.9f)); - EXPECT_NO_THROW(viz::Text(0.1f, 0.1f, 0.2f, "Colored")); - EXPECT_NO_THROW(viz::EndLayer()); + EXPECT_NO_THROW(viz::BeginGeometryLayer()); + EXPECT_NO_THROW(viz::Text(0.4f, 0.4f, 0.4f, "Text")); + EXPECT_NO_THROW(viz::Color(0.5f, 0.9f, 0.7f, 0.9f)); + EXPECT_NO_THROW(viz::Text(0.1f, 0.1f, 0.2f, "Colored")); + EXPECT_NO_THROW(viz::EndLayer()); - EXPECT_NO_THROW(viz::End()); + EXPECT_NO_THROW(viz::End()); - CompareResultCRC32({0xCA706DD0}); + CompareResultCRC32({0xc68d7716}); } -TEST_F(GeometryLayer, Reuse) { - std::vector data{0.5f, 0.5f}; +TEST_F(GeometryLayer, TextClipped) { + EXPECT_NO_THROW(viz::Begin()); - for (uint32_t i = 0; i < 2; ++i) { - EXPECT_NO_THROW(viz::Begin()); + EXPECT_NO_THROW(viz::BeginGeometryLayer()); + EXPECT_NO_THROW(viz::Text(1.1f, 0.4f, 0.4f, "Text")); - EXPECT_NO_THROW(viz::BeginGeometryLayer()); - EXPECT_NO_THROW(viz::Color(0.1f, 0.2f, 0.3f, 0.4f)); - EXPECT_NO_THROW(viz::LineWidth(2.f)); - EXPECT_NO_THROW(viz::PointSize(3.f)); - EXPECT_NO_THROW(viz::Primitive(viz::PrimitiveTopology::POINT_LIST, 1, data.size(), - data.data())); - EXPECT_NO_THROW(viz::Text(0.4f, 0.4f, 0.1f, "Text")); - EXPECT_NO_THROW(viz::EndLayer()); + EXPECT_NO_THROW(viz::End()); - EXPECT_NO_THROW(viz::End()); - } + CompareResultCRC32({0x8a9c008}); } -TEST_F(GeometryLayer, Errors) { - std::vector data{0.5f, 0.5f}; +class GeometryLayerWithFont : public TestHeadless { + protected: + void SetUp() override { + ASSERT_NO_THROW(viz::SetFont("../modules/holoviz/src/fonts/Roboto-Bold.ttf", 12.f)); - EXPECT_NO_THROW(viz::Begin()); + // call base class + ::TestHeadless::SetUp(); + } - // it's an error to call geometry functions without calling BeginGeometryLayer first - EXPECT_THROW(viz::Color(0.f, 0.f, 0.f, 1.f), std::runtime_error); - EXPECT_THROW(viz::LineWidth(1.0f), std::runtime_error); - EXPECT_THROW(viz::PointSize(1.0f), std::runtime_error); - EXPECT_THROW(viz::Primitive(viz::PrimitiveTopology::POINT_LIST, 1, data.size(), - data.data()), std::runtime_error); - EXPECT_THROW(viz::Text(0.5f, 0.5f, 0.1f, "Text"), std::runtime_error); + void TearDown() override { + // call base class + ::TestHeadless::TearDown(); - // it's an error to call BeginGeometryLayer again without calling EndLayer - EXPECT_NO_THROW(viz::BeginGeometryLayer()); - EXPECT_THROW(viz::BeginGeometryLayer(), std::runtime_error); - EXPECT_NO_THROW(viz::EndLayer()); + ASSERT_NO_THROW(viz::SetFont("", 0.f)); + } +}; - // it's an error to call geometry functions when a different layer is active - EXPECT_NO_THROW(viz::BeginImageLayer()); - EXPECT_THROW(viz::Color(0.f, 0.f, 0.f, 1.f), std::runtime_error); - EXPECT_THROW(viz::LineWidth(1.0f), std::runtime_error); - EXPECT_THROW(viz::PointSize(1.0f), std::runtime_error); - EXPECT_THROW(viz::Primitive(viz::PrimitiveTopology::POINT_LIST, 1, data.size(), - data.data()), std::runtime_error); - EXPECT_THROW(viz::Text(0.5f, 0.5f, 0.1f, "Text"), std::runtime_error); - EXPECT_NO_THROW(viz::EndLayer()); +TEST_F(GeometryLayerWithFont, Text) { + EXPECT_NO_THROW(viz::Begin()); - EXPECT_NO_THROW(viz::BeginGeometryLayer()); + EXPECT_NO_THROW(viz::BeginGeometryLayer()); + EXPECT_NO_THROW(viz::Text(0.1f, 0.1f, 0.7f, "Font")); + EXPECT_NO_THROW(viz::EndLayer()); + + EXPECT_NO_THROW(viz::End()); + + CompareResultCRC32({0xbccffe56}); +} - // Primitive function errors, first call the passing function - EXPECT_NO_THROW(viz::Primitive(viz::PrimitiveTopology::POINT_LIST, 1, - data.size(), data.data())); - // it's an error to call Primitive with a primitive count of zero - EXPECT_THROW(viz::Primitive(viz::PrimitiveTopology::POINT_LIST, 0, data.size(), data.data()), - std::invalid_argument); - // it's an error to call Primitive with a data size of zero - EXPECT_THROW(viz::Primitive(viz::PrimitiveTopology::POINT_LIST, 1, 0, data.data()), - std::invalid_argument); - // it's an error to call Primitive with a null data pointer - EXPECT_THROW(viz::Primitive(viz::PrimitiveTopology::POINT_LIST, 1, data.size(), nullptr), - std::invalid_argument); - // it's an error to call Primitive with a data size which is too small for the primitive count - EXPECT_THROW(viz::Primitive(viz::PrimitiveTopology::POINT_LIST, 1, data.size() - 1, - data.data()), std::runtime_error); - - // Text function errors, first call the passing function - EXPECT_NO_THROW(viz::Text(0.5f, 0.5f, 0.1f, "Text")); - // it's an error to call Text with a size of zero - EXPECT_THROW(viz::Text(0.5f, 0.5f, 0.0f, "Text"), std::invalid_argument); - // it's an error to call Text with null text pointer - EXPECT_THROW(viz::Text(0.5f, 0.5f, 0.1f, nullptr), std::invalid_argument); +// Fixture that initializes Holoviz +class DepthMapRenderMode : public TestHeadless, + public testing::WithParamInterface {}; + +TEST_P(DepthMapRenderMode, DepthMap) { + const viz::DepthMapRenderMode depth_map_render_mode = GetParam(); + const uint32_t map_width = 8; + const uint32_t map_height = 8; + + // allocate device memory + viz::CudaService::ScopedPush cuda_context = viz::CudaService::get().PushContext(); + + viz::UniqueCUdeviceptr depth_ptr; + depth_ptr.reset([this] { + CUdeviceptr device_ptr; + EXPECT_EQ(cuMemAlloc(&device_ptr, map_width * map_height * sizeof(uint8_t)), CUDA_SUCCESS); + return device_ptr; + }()); + std::vector depth_data(map_width * map_height); + for (size_t index = 0; index < depth_data.size(); ++index) { depth_data[index] = index * 4; } + EXPECT_EQ(cuMemcpyHtoD(depth_ptr.get(), depth_data.data(), depth_data.size()), CUDA_SUCCESS); + + viz::UniqueCUdeviceptr color_ptr; + color_ptr.reset([this] { + CUdeviceptr device_ptr; + EXPECT_EQ(cuMemAlloc(&device_ptr, map_width * map_height * sizeof(uint32_t)), CUDA_SUCCESS); + return device_ptr; + }()); + std::vector color_data(map_width * map_height); + for (uint32_t index = 0; index < color_data.size(); ++index) { + color_data[index] = (index << 18) | (index << 9) | index | 0xFF000000; + } + EXPECT_EQ(cuMemcpyHtoD(color_ptr.get(), color_data.data(), color_data.size() * sizeof(uint32_t)), + CUDA_SUCCESS); + + uint32_t crc; + switch (depth_map_render_mode) { + case viz::DepthMapRenderMode::POINTS: + crc = 0x1eb98bfa; + break; + case viz::DepthMapRenderMode::LINES: + crc = 0xbf3be45a; + break; + case viz::DepthMapRenderMode::TRIANGLES: + crc = 0x5ac3bd4b; + break; + } + EXPECT_NO_THROW(viz::Begin()); + + EXPECT_NO_THROW(viz::BeginGeometryLayer()); + EXPECT_NO_THROW(viz::DepthMap(depth_map_render_mode, + map_width, + map_height, + viz::ImageFormat::R8_UNORM, + depth_ptr.get(), + viz::ImageFormat::R8G8B8A8_UNORM, + color_ptr.get())); + EXPECT_NO_THROW(viz::EndLayer()); + + EXPECT_NO_THROW(viz::End()); + + CompareResultCRC32({crc}); +} + +INSTANTIATE_TEST_SUITE_P(GeometryLayer, DepthMapRenderMode, + testing::Values(viz::DepthMapRenderMode::POINTS, + viz::DepthMapRenderMode::LINES, + viz::DepthMapRenderMode::TRIANGLES)); + +TEST_F(GeometryLayer, Reuse) { + std::vector data{0.5f, 0.5f}; + + for (uint32_t i = 0; i < 2; ++i) { + EXPECT_NO_THROW(viz::Begin()); + EXPECT_NO_THROW(viz::BeginGeometryLayer()); + EXPECT_NO_THROW(viz::Color(0.1f, 0.2f, 0.3f, 0.4f)); + EXPECT_NO_THROW(viz::LineWidth(2.f)); + EXPECT_NO_THROW(viz::PointSize(3.f)); + EXPECT_NO_THROW( + viz::Primitive(viz::PrimitiveTopology::POINT_LIST, 1, data.size(), data.data())); + EXPECT_NO_THROW(viz::Text(0.4f, 0.4f, 0.1f, "Text")); EXPECT_NO_THROW(viz::EndLayer()); EXPECT_NO_THROW(viz::End()); + } +} + +TEST_F(GeometryLayer, Errors) { + std::vector data{0.5f, 0.5f}; + + EXPECT_NO_THROW(viz::Begin()); + + // it's an error to call geometry functions without calling BeginGeometryLayer first + EXPECT_THROW(viz::Color(0.f, 0.f, 0.f, 1.f), std::runtime_error); + EXPECT_THROW(viz::LineWidth(1.0f), std::runtime_error); + EXPECT_THROW(viz::PointSize(1.0f), std::runtime_error); + EXPECT_THROW(viz::Primitive(viz::PrimitiveTopology::POINT_LIST, 1, data.size(), data.data()), + std::runtime_error); + EXPECT_THROW(viz::Text(0.5f, 0.5f, 0.1f, "Text"), std::runtime_error); + + // it's an error to call BeginGeometryLayer again without calling EndLayer + EXPECT_NO_THROW(viz::BeginGeometryLayer()); + EXPECT_THROW(viz::BeginGeometryLayer(), std::runtime_error); + EXPECT_NO_THROW(viz::EndLayer()); + + // it's an error to call geometry functions when a different layer is active + EXPECT_NO_THROW(viz::BeginImageLayer()); + EXPECT_THROW(viz::Color(0.f, 0.f, 0.f, 1.f), std::runtime_error); + EXPECT_THROW(viz::LineWidth(1.0f), std::runtime_error); + EXPECT_THROW(viz::PointSize(1.0f), std::runtime_error); + EXPECT_THROW(viz::Primitive(viz::PrimitiveTopology::POINT_LIST, 1, data.size(), data.data()), + std::runtime_error); + EXPECT_THROW(viz::Text(0.5f, 0.5f, 0.1f, "Text"), std::runtime_error); + EXPECT_NO_THROW(viz::EndLayer()); + + EXPECT_NO_THROW(viz::BeginGeometryLayer()); + + // Primitive function errors, first call the passing function + EXPECT_NO_THROW(viz::Primitive(viz::PrimitiveTopology::POINT_LIST, 1, data.size(), data.data())); + // it's an error to call Primitive with a primitive count of zero + EXPECT_THROW(viz::Primitive(viz::PrimitiveTopology::POINT_LIST, 0, data.size(), data.data()), + std::invalid_argument); + // it's an error to call Primitive with a data size of zero + EXPECT_THROW(viz::Primitive(viz::PrimitiveTopology::POINT_LIST, 1, 0, data.data()), + std::invalid_argument); + // it's an error to call Primitive with a null data pointer + EXPECT_THROW(viz::Primitive(viz::PrimitiveTopology::POINT_LIST, 1, data.size(), nullptr), + std::invalid_argument); + // it's an error to call Primitive with a data size which is too small for the primitive count + EXPECT_THROW(viz::Primitive(viz::PrimitiveTopology::POINT_LIST, 1, data.size() - 1, data.data()), + std::runtime_error); + + // Text function errors, first call the passing function + EXPECT_NO_THROW(viz::Text(0.5f, 0.5f, 0.1f, "Text")); + // it's an error to call Text with a size of zero + EXPECT_THROW(viz::Text(0.5f, 0.5f, 0.0f, "Text"), std::invalid_argument); + // it's an error to call Text with null text pointer + EXPECT_THROW(viz::Text(0.5f, 0.5f, 0.1f, nullptr), std::invalid_argument); + + // Depth map function errors, first call the passing function + const uint32_t map_width = 8; + const uint32_t map_height = 8; + + // allocate device memory + viz::CudaService::ScopedPush cuda_context = viz::CudaService::get().PushContext(); + viz::UniqueCUdeviceptr depth_ptr; + depth_ptr.reset([this] { + CUdeviceptr device_ptr; + EXPECT_EQ(cuMemAlloc(&device_ptr, map_width * map_height * sizeof(uint8_t)), CUDA_SUCCESS); + return device_ptr; + }()); + viz::UniqueCUdeviceptr color_ptr; + color_ptr.reset([this] { + CUdeviceptr device_ptr; + EXPECT_EQ(cuMemAlloc(&device_ptr, map_width * map_height * sizeof(uint32_t)), CUDA_SUCCESS); + return device_ptr; + }()); + + // First call the passing function + EXPECT_NO_THROW(viz::DepthMap(viz::DepthMapRenderMode::POINTS, + map_width, + map_height, + viz::ImageFormat::R8_UNORM, + depth_ptr.get(), + viz::ImageFormat::R8G8B8A8_UNORM, + color_ptr.get())); + // it's an error to call DepthMap with a width of zero + EXPECT_THROW(viz::DepthMap(viz::DepthMapRenderMode::POINTS, + 0, + map_height, + viz::ImageFormat::R8_UNORM, + depth_ptr.get(), + viz::ImageFormat::R8G8B8A8_UNORM, + color_ptr.get()), + std::invalid_argument); + // it's an error to call DepthMap with a width of zero + EXPECT_THROW(viz::DepthMap(viz::DepthMapRenderMode::POINTS, + map_width, + 0, + viz::ImageFormat::R8_UNORM, + depth_ptr.get(), + viz::ImageFormat::R8G8B8A8_UNORM, + color_ptr.get()), + std::invalid_argument); + // it's an error to call DepthMap with a depth format other than viz::ImageFormat::R8_UINT + EXPECT_THROW(viz::DepthMap(viz::DepthMapRenderMode::POINTS, + map_width, + map_height, + viz::ImageFormat::R16_UNORM, + depth_ptr.get(), + viz::ImageFormat::R8G8B8A8_UNORM, + color_ptr.get()), + std::invalid_argument); + // it's an error to call DepthMap with no depth map pointer + EXPECT_THROW(viz::DepthMap(viz::DepthMapRenderMode::POINTS, + map_width, + map_height, + viz::ImageFormat::R8_UNORM, + 0, + viz::ImageFormat::R8G8B8A8_UNORM, + color_ptr.get()), + std::invalid_argument); + + EXPECT_NO_THROW(viz::EndLayer()); + + EXPECT_NO_THROW(viz::End()); } diff --git a/modules/holoviz/tests/functional/headless_fixture.cpp b/modules/holoviz/tests/functional/headless_fixture.cpp index a4f9e8a5..9ed42616 100644 --- a/modules/holoviz/tests/functional/headless_fixture.cpp +++ b/modules/holoviz/tests/functional/headless_fixture.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,136 +25,140 @@ #include - namespace viz = holoscan::viz; -template -void Fill(std::vector &data, size_t elements, T or_mask = T(), +template +void Fill(std::vector& data, size_t elements, T or_mask = T(), T and_mask = std::numeric_limits::max()) { - // fill volume with random data and build histogram - for (size_t index = 0; index < elements; ++index) { - if (std::is_same::value) { - reinterpret_cast(data.data())[index] = static_cast(std::rand()) / RAND_MAX; - } else { - reinterpret_cast(data.data())[index] = (static_cast(std::rand()) | - or_mask) & and_mask; - } + // fill volume with random data and build histogram + for (size_t index = 0; index < elements; ++index) { + if (std::is_same::value) { + reinterpret_cast(data.data())[index] = static_cast(std::rand()) / RAND_MAX; + } else { + reinterpret_cast(data.data())[index] = (static_cast(std::rand()) | or_mask) & and_mask; } + } } void TestHeadless::SetUp() { - ASSERT_NO_THROW(viz::Init(width_, height_, "Holoviz test", viz::InitFlags::HEADLESS)); + ASSERT_NO_THROW(viz::Init(width_, height_, "Holoviz test", viz::InitFlags::HEADLESS)); } void TestHeadless::TearDown() { - ASSERT_NO_THROW(viz::Shutdown()); + ASSERT_NO_THROW(viz::Shutdown()); } void TestHeadless::SetupData(viz::ImageFormat format, uint32_t rand_seed) { - std::srand(rand_seed); + std::srand(rand_seed); - uint32_t channels; - uint32_t component_size; - switch (format) { + uint32_t channels; + uint32_t component_size; + switch (format) { case viz::ImageFormat::R8G8B8A8_UNORM: - channels = 4; - component_size = sizeof(uint8_t); - data_.resize(width_ * height_ * channels * component_size); - Fill(data_, width_ * height_, 0xFF000000); - break; + channels = 4; + component_size = sizeof(uint8_t); + data_.resize(width_ * height_ * channels * component_size); + Fill(data_, width_ * height_, 0xFF000000); + break; case viz::ImageFormat::R8_UINT: - channels = 1; - component_size = sizeof(uint32_t); - data_.resize(width_ * height_ * channels * component_size); - Fill(data_, width_ * height_, 0x00000000, lut_size_ - 1); - break; + channels = 1; + component_size = sizeof(uint8_t); + data_.resize(width_ * height_ * channels * component_size); + Fill(data_, width_ * height_, 0x00, lut_size_ - 1); + break; + case viz::ImageFormat::R8_UNORM: + channels = 1; + component_size = sizeof(uint8_t); + data_.resize(width_ * height_ * channels * component_size); + Fill(data_, width_ * height_, 0x00, 0xFF); + break; default: - ASSERT_TRUE(false) << "Unsupported image format " << static_cast(format); - } + ASSERT_TRUE(false) << "Unsupported image format " << static_cast(format); + } } -void TestHeadless::ReadData(std::vector &read_data) { - const size_t data_size = width_ * height_ * sizeof(uint8_t) * 4; +void TestHeadless::ReadData(std::vector& read_data) { + const size_t data_size = width_ * height_ * sizeof(uint8_t) * 4; - const viz::CudaService::ScopedPush cuda_context = viz::CudaService::get().PushContext(); + const viz::CudaService::ScopedPush cuda_context = viz::CudaService::get().PushContext(); - viz::UniqueCUdeviceptr device_ptr; - device_ptr.reset([this](size_t data_size) { - CUdeviceptr device_ptr; - EXPECT_EQ(cuMemAlloc(&device_ptr, data_size), CUDA_SUCCESS); - return device_ptr; - }(data_size)); + viz::UniqueCUdeviceptr device_ptr; + device_ptr.reset([this](size_t data_size) { + CUdeviceptr device_ptr; + EXPECT_EQ(cuMemAlloc(&device_ptr, data_size), CUDA_SUCCESS); + return device_ptr; + }(data_size)); - ASSERT_TRUE(device_ptr); + ASSERT_TRUE(device_ptr); - ASSERT_NO_THROW(viz::ReadFramebuffer(viz::ImageFormat::R8G8B8A8_UNORM, data_size, - device_ptr.get())); + ASSERT_NO_THROW(viz::ReadFramebuffer( + viz::ImageFormat::R8G8B8A8_UNORM, width_, height_, data_size, device_ptr.get())); - read_data.resize(data_size); + read_data.resize(data_size); - ASSERT_EQ(cuMemcpyDtoH(read_data.data(), device_ptr.get(), read_data.size()), CUDA_SUCCESS); + ASSERT_EQ(cuMemcpyDtoH(read_data.data(), device_ptr.get(), read_data.size()), CUDA_SUCCESS); } -static std::string BuildFileName(const std::string &end) { - const testing::TestInfo *test_info = testing::UnitTest::GetInstance()->current_test_info(); +static std::string BuildFileName(const std::string& end) { + const testing::TestInfo* test_info = testing::UnitTest::GetInstance()->current_test_info(); - std::string file_name; - file_name += test_info->test_suite_name(); - file_name += "_"; - file_name += test_info->name(); - file_name += "_fail.png"; + std::string file_name; + file_name += test_info->test_suite_name(); + file_name += "_"; + file_name += test_info->name(); + file_name += "_"; + file_name += end; + file_name += ".png"; - // parameterized tests have '/', replace with '_' - std::replace(file_name.begin(), file_name.end(), '/', '_'); + // parameterized tests have '/', replace with '_' + std::replace(file_name.begin(), file_name.end(), '/', '_'); - return file_name; + return file_name; } bool TestHeadless::CompareResult() { - if (data_.size() != width_ * height_ * sizeof(uint8_t) * 4) { - EXPECT_TRUE(false) << "Can compare R8G8B8A8_UNORM data only"; - return false; - } + if (data_.size() != width_ * height_ * sizeof(uint8_t) * 4) { + EXPECT_TRUE(false) << "Can compare R8G8B8A8_UNORM data only"; + return false; + } - std::vector read_data; - ReadData(read_data); + std::vector read_data; + ReadData(read_data); - for (size_t index = 0; index < data_.size(); ++index) { - if (data_[index] != read_data[index]) { - const std::string ref_file_name = BuildFileName("ref"); - const std::string fail_file_name = BuildFileName("fail"); + for (size_t index = 0; index < data_.size(); ++index) { + if (data_[index] != read_data[index]) { + const std::string ref_file_name = BuildFileName("ref"); + const std::string fail_file_name = BuildFileName("fail"); - stbi_write_png(ref_file_name.c_str(), width_, height_, 4, data_.data(), 0); - stbi_write_png(fail_file_name.c_str(), width_, height_, 4, read_data.data(), 0); + stbi_write_png(ref_file_name.c_str(), width_, height_, 4, data_.data(), 0); + stbi_write_png(fail_file_name.c_str(), width_, height_, 4, read_data.data(), 0); - EXPECT_TRUE(false) << "Data mismatch, wrote images to " << ref_file_name << " and " - << fail_file_name; - return false; - } + EXPECT_TRUE(false) << "Data mismatch, wrote images to " << ref_file_name << " and " + << fail_file_name; + return false; } - return true; + } + return true; } bool TestHeadless::CompareResultCRC32(const std::vector crc32) { - std::vector read_data; - ReadData(read_data); + std::vector read_data; + ReadData(read_data); - const uint32_t read_crc32 = stbiw__crc32(read_data.data(), read_data.size()); + const uint32_t read_crc32 = stbiw__crc32(read_data.data(), read_data.size()); - if (std::find(crc32.begin(), crc32.end(), read_crc32) == crc32.end()) { - const std::string fail_file_name = BuildFileName("fail"); + if (std::find(crc32.begin(), crc32.end(), read_crc32) == crc32.end()) { + const std::string fail_file_name = BuildFileName("fail"); - stbi_write_png(fail_file_name.c_str(), width_, height_, 4, read_data.data(), 0); + stbi_write_png(fail_file_name.c_str(), width_, height_, 4, read_data.data(), 0); - std::ostringstream str; - str << "CRC mismatch, expected {" << std::setw(6) << std::hex; - for (auto &&value : crc32) { - str << "0x" << value << ", "; - } - str << "} but calculated 0x" << read_crc32 << ", wrote failing image to " << fail_file_name; - EXPECT_FALSE(true) << str.str(); - return false; - } + std::ostringstream str; + str << "CRC mismatch, expected {" << std::hex << std::setw(8); + for (auto&& value : crc32) { str << "0x" << value << ", "; } + str << "} but calculated 0x" << read_crc32 << ", wrote failing image to " << fail_file_name; + EXPECT_FALSE(true) << str.str(); + return false; + } - return true; + return true; } diff --git a/modules/holoviz/tests/functional/headless_fixture.hpp b/modules/holoviz/tests/functional/headless_fixture.hpp index 325a4803..8d715f70 100644 --- a/modules/holoviz/tests/functional/headless_fixture.hpp +++ b/modules/holoviz/tests/functional/headless_fixture.hpp @@ -30,65 +30,62 @@ */ class TestHeadless : public ::testing::Test { protected: - /** - * Construct a new TestHeadless object - */ - TestHeadless() = default; + /** + * Construct a new TestHeadless object + */ + TestHeadless() = default; - /** - * Construct a new TestHeadless object with a given window size - * - * @param width window width - * @param height window height - */ - TestHeadless(uint32_t width, uint32_t height) - : width_(width) - , height_(height) { - } + /** + * Construct a new TestHeadless object with a given window size + * + * @param width window width + * @param height window height + */ + TestHeadless(uint32_t width, uint32_t height) : width_(width), height_(height) {} - /// ::testing::Test virtual members - ///@{ - void SetUp() override; - void TearDown() override; - ///@} + /// ::testing::Test virtual members + ///@{ + void SetUp() override; + void TearDown() override; + ///@} - /** - * Setup random pixel data. - * - * @param format pixel format - * @param rand_seed random seed - */ - void SetupData(holoscan::viz::ImageFormat format, uint32_t rand_seed = 1); + /** + * Setup random pixel data. + * + * @param format pixel format + * @param rand_seed random seed + */ + void SetupData(holoscan::viz::ImageFormat format, uint32_t rand_seed = 1); - /** - * Read back data from the Holoviz window. - * - * @param read_data vector to hold the read back data - */ - void ReadData(std::vector &read_data); + /** + * Read back data from the Holoviz window. + * + * @param read_data vector to hold the read back data + */ + void ReadData(std::vector& read_data); - /** - * Read back data and compare with the data generated with SetupData(). - * - * @returns false if read back and generated data do not match - */ - bool CompareResult(); + /** + * Read back data and compare with the data generated with SetupData(). + * + * @returns false if read back and generated data do not match + */ + bool CompareResult(); - /** - * Read back data, generate a CRC32 and compare with the provided CRC32's. - * - * @param crc32 vector of expected CRC32's - * - * @returns false if read back and generated data do not match - */ - bool CompareResultCRC32(const std::vector crc32); + /** + * Read back data, generate a CRC32 and compare with the provided CRC32's. + * + * @param crc32 vector of expected CRC32's + * + * @returns false if read back and generated data do not match + */ + bool CompareResultCRC32(const std::vector crc32); - const uint32_t lut_size_ = 8; + const uint32_t lut_size_ = 8; - const uint32_t width_ = 64; - const uint32_t height_ = 32; + const uint32_t width_ = 64; + const uint32_t height_ = 32; - std::vector data_; + std::vector data_; }; #endif /* HOLOVIZ_TESTS_FUNCTIONAL_HEADLESS_FIXTURE_HPP */ diff --git a/modules/holoviz/tests/functional/im_gui_layer_test.cpp b/modules/holoviz/tests/functional/im_gui_layer_test.cpp index 2cbf7c4e..a68b8e41 100644 --- a/modules/holoviz/tests/functional/im_gui_layer_test.cpp +++ b/modules/holoviz/tests/functional/im_gui_layer_test.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,79 +18,77 @@ #include #include -#include "headless_fixture.hpp" #include #include +#include "headless_fixture.hpp" namespace viz = holoscan::viz; -using UniqueImGuiContext = viz::UniqueValue; +using UniqueImGuiContext = + viz::UniqueValue; -// Fixture that creates initializes Holoviz +// Fixture that initializes Holoviz class ImGuiLayer : public TestHeadless { protected: - ImGuiLayer() - : TestHeadless(256, 256) { - } - - void SetUp() override { - if (!ImGui::GetCurrentContext()) { - im_gui_context_.reset(ImGui::CreateContext()); - ASSERT_TRUE(im_gui_context_); - } - ASSERT_NO_THROW(viz::ImGuiSetCurrentContext(ImGui::GetCurrentContext())); + ImGuiLayer() : TestHeadless(256, 256) {} - // call base class - ::TestHeadless::SetUp(); + void SetUp() override { + if (!ImGui::GetCurrentContext()) { + im_gui_context_.reset(ImGui::CreateContext()); + ASSERT_TRUE(im_gui_context_); } + ASSERT_NO_THROW(viz::ImGuiSetCurrentContext(ImGui::GetCurrentContext())); - void TearDown() override { - // call base class - ::TestHeadless::TearDown(); + // call base class + ::TestHeadless::SetUp(); + } - ASSERT_NO_THROW(viz::ImGuiSetCurrentContext(nullptr)); - } + void TearDown() override { + // call base class + ::TestHeadless::TearDown(); + + ASSERT_NO_THROW(viz::ImGuiSetCurrentContext(nullptr)); + } - UniqueImGuiContext im_gui_context_; + UniqueImGuiContext im_gui_context_; }; TEST_F(ImGuiLayer, Window) { - for (uint32_t i = 0; i < 2; ++i) { - EXPECT_NO_THROW(viz::Begin()); - EXPECT_NO_THROW(viz::BeginImGuiLayer()); + for (uint32_t i = 0; i < 2; ++i) { + EXPECT_NO_THROW(viz::Begin()); + EXPECT_NO_THROW(viz::BeginImGuiLayer()); - ImGui::Begin("Window"); - ImGui::Text("Some Text"); - ImGui::End(); + ImGui::Begin("Window"); + ImGui::Text("Some Text"); + ImGui::End(); - EXPECT_NO_THROW(viz::EndLayer()); - EXPECT_NO_THROW(viz::End()); - } + EXPECT_NO_THROW(viz::EndLayer()); + EXPECT_NO_THROW(viz::End()); + } - CompareResultCRC32({ - 0x7BAF2927, // A5000 - 0x8852CA52, // A6000 - }); + CompareResultCRC32({ + 0xb374795b, // RTX 6000, RTX A5000 + 0x40899a2e // RTX A6000 + }); } TEST_F(ImGuiLayer, Errors) { - std::vector data{0.5f, 0.5f}; + std::vector data{0.5f, 0.5f}; - EXPECT_NO_THROW(viz::Begin()); + EXPECT_NO_THROW(viz::Begin()); - // it's an error to call call BeginImGuiLayer no valid ImGui context is set - EXPECT_NO_THROW(viz::ImGuiSetCurrentContext(nullptr)); - EXPECT_THROW(viz::BeginImGuiLayer(), std::runtime_error); - EXPECT_NO_THROW(viz::ImGuiSetCurrentContext(im_gui_context_.get())); + // it's an error to call call BeginImGuiLayer no valid ImGui context is set + EXPECT_NO_THROW(viz::ImGuiSetCurrentContext(nullptr)); + EXPECT_THROW(viz::BeginImGuiLayer(), std::runtime_error); + EXPECT_NO_THROW(viz::ImGuiSetCurrentContext(im_gui_context_.get())); - // it's an error to call BeginImGuiLayer again without calling EndLayer - EXPECT_NO_THROW(viz::BeginImGuiLayer()); - EXPECT_THROW(viz::BeginImGuiLayer(), std::runtime_error); - EXPECT_NO_THROW(viz::EndLayer()); + // it's an error to call BeginImGuiLayer again without calling EndLayer + EXPECT_NO_THROW(viz::BeginImGuiLayer()); + EXPECT_THROW(viz::BeginImGuiLayer(), std::runtime_error); + EXPECT_NO_THROW(viz::EndLayer()); - // multiple ImGui layers per frame are not supported - EXPECT_THROW(viz::BeginImGuiLayer(), std::runtime_error); + // multiple ImGui layers per frame are not supported + EXPECT_THROW(viz::BeginImGuiLayer(), std::runtime_error); - EXPECT_NO_THROW(viz::End()); + EXPECT_NO_THROW(viz::End()); } diff --git a/modules/holoviz/tests/functional/image_layer_test.cpp b/modules/holoviz/tests/functional/image_layer_test.cpp index dcb923e3..6c4882ad 100644 --- a/modules/holoviz/tests/functional/image_layer_test.cpp +++ b/modules/holoviz/tests/functional/image_layer_test.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,149 +17,170 @@ #include -#include "headless_fixture.hpp" -#include +#include + #include +#include +#include "headless_fixture.hpp" namespace viz = holoscan::viz; -enum class Source { - HOST, - CUDA_DEVICE -}; +enum class Source { HOST, CUDA_DEVICE }; -enum class Reuse { - DISABLE, - ENABLE -}; +enum class Reuse { DISABLE, ENABLE }; -enum class UseLut { - DISABLE, - ENABLE -}; +enum class UseLut { DISABLE, ENABLE, ENABLE_WITH_NORMALIZE }; // define the '<<' operators to get a nice parameter string -#define CASE(VALUE) \ - case VALUE: \ - os << std::string(#VALUE); \ - break; +#define CASE(VALUE) \ + case VALUE: \ + os << std::string(#VALUE); \ + break; // define the '<<' operator to get a nice parameter string -std::ostream &operator<<(std::ostream &os, const Source &source) { - switch (source) { - CASE(Source::HOST) - CASE(Source::CUDA_DEVICE) +std::ostream& operator<<(std::ostream& os, const Source& source) { + switch (source) { + CASE(Source::HOST) + CASE(Source::CUDA_DEVICE) default: - os.setstate(std::ios_base::failbit); - } - return os; + os.setstate(std::ios_base::failbit); + } + return os; } -std::ostream &operator<<(std::ostream &os, const Reuse &reuse) { - switch (reuse) { - CASE(Reuse::DISABLE) - CASE(Reuse::ENABLE) +std::ostream& operator<<(std::ostream& os, const Reuse& reuse) { + switch (reuse) { + CASE(Reuse::DISABLE) + CASE(Reuse::ENABLE) default: - os.setstate(std::ios_base::failbit); - } - return os; + os.setstate(std::ios_base::failbit); + } + return os; } -std::ostream &operator<<(std::ostream &os, const UseLut &use_lut) { - switch (use_lut) { - CASE(UseLut::DISABLE) - CASE(UseLut::ENABLE) +std::ostream& operator<<(std::ostream& os, const UseLut& use_lut) { + switch (use_lut) { + CASE(UseLut::DISABLE) + CASE(UseLut::ENABLE) + CASE(UseLut::ENABLE_WITH_NORMALIZE) default: - os.setstate(std::ios_base::failbit); - } - return os; + os.setstate(std::ios_base::failbit); + } + return os; } #undef CASE -class ImageLayer - : public TestHeadless - , public testing::WithParamInterface> { -}; +class ImageLayer : public TestHeadless, + public testing::WithParamInterface> {}; TEST_P(ImageLayer, Image) { - const Source source = std::get<0>(GetParam()); - const bool reuse = std::get<1>(GetParam()) == Reuse::ENABLE; - const bool use_lut = std::get<2>(GetParam()) == UseLut::ENABLE; + const Source source = std::get<0>(GetParam()); + const bool reuse = std::get<1>(GetParam()) == Reuse::ENABLE; + const UseLut use_lut = std::get<2>(GetParam()); - const viz::ImageFormat kFormat = use_lut ? viz::ImageFormat::R8_UINT : - viz::ImageFormat::R8G8B8A8_UNORM; - const viz::ImageFormat kLutFormat = viz::ImageFormat::R8G8B8A8_UNORM; + const viz::ImageFormat kLutFormat = viz::ImageFormat::R8G8B8A8_UNORM; - SetupData(kFormat); + viz::ImageFormat kFormat = viz::ImageFormat::R8G8B8A8_UNORM; + if (use_lut == UseLut::ENABLE) { + kFormat = viz::ImageFormat::R8_UINT; + } else if (use_lut == UseLut::ENABLE_WITH_NORMALIZE) { + kFormat = viz::ImageFormat::R8_UNORM; + } - std::vector lut; - std::vector data_with_lut; + SetupData(kFormat); - viz::CudaService::ScopedPush cuda_context; - viz::UniqueCUdeviceptr device_ptr; + std::vector lut; + std::vector data_with_lut; - if (source == Source::CUDA_DEVICE) { - cuda_context = viz::CudaService::get().PushContext(); - device_ptr.reset([this] { - CUdeviceptr device_ptr; - EXPECT_EQ(cuMemAlloc(&device_ptr, data_.size()), CUDA_SUCCESS); - return device_ptr; - }()); + viz::CudaService::ScopedPush cuda_context; + viz::UniqueCUdeviceptr device_ptr; - EXPECT_EQ(cuMemcpyHtoD(device_ptr.get(), data_.data(), data_.size()), CUDA_SUCCESS); - } + if (source == Source::CUDA_DEVICE) { + cuda_context = viz::CudaService::get().PushContext(); + device_ptr.reset([this] { + CUdeviceptr device_ptr; + EXPECT_EQ(cuMemAlloc(&device_ptr, data_.size()), CUDA_SUCCESS); + return device_ptr; + }()); - if (use_lut) { - std::srand(1); - lut.resize(lut_size_); - for (uint32_t index = 0; index < lut_size_; ++index) { - lut[index] = static_cast(std::rand()) | 0xFF000000; - } + EXPECT_EQ(cuMemcpyHtoD(device_ptr.get(), data_.data(), data_.size()), CUDA_SUCCESS); + } - // lookup color to produce result - data_with_lut.resize(width_ * height_ * sizeof(uint32_t)); - for (size_t index = 0; index < width_ * height_; ++index) { - const uint8_t value = data_[index]; + if (use_lut != UseLut::DISABLE) { + std::srand(1); + lut.resize(lut_size_); + for (uint32_t index = 0; index < lut_size_; ++index) { + lut[index] = static_cast(std::rand()) | 0xFF000000; + } - reinterpret_cast(data_with_lut.data())[index] = lut[value]; - } + // lookup color to produce result + data_with_lut.resize(width_ * height_ * sizeof(uint32_t)); + if (use_lut == UseLut::ENABLE) { + for (size_t index = 0; index < width_ * height_; ++index) { + const uint8_t src_value = data_[index]; + reinterpret_cast(data_with_lut.data())[index] = lut[src_value]; + } + } else if (use_lut == UseLut::ENABLE_WITH_NORMALIZE) { + for (size_t index = 0; index < width_ * height_; ++index) { + const float offset = (float(data_[index]) / 255.f) * lut_size_; + uint32_t val0 = lut[uint32_t(offset - 0.5f)]; + uint32_t val1 = lut[uint32_t(offset + 0.5f)]; + float dummy; + float frac = std::modf(offset, &dummy); + + float r = float(val1 & 0xFF) + frac * (float(val0 & 0xFF) - float(val1 & 0xFF)); + float g = float((val1 & 0xFF00) >> 8) + + frac * (float((val0 & 0xFF00) >> 8) - float((val1 & 0xFF00) >> 8)); + float b = float((val1 & 0xFF0000) >> 16) + + frac * (float((val0 & 0xFF0000) >> 16) - float((val1 & 0xFF0000) >> 16)); + float a = float((val1 & 0xFF000000) >> 24) + + frac * (float((val0 & 0xFF000000) >> 24) - float((val1 & 0xFF000000) >> 24)); + + reinterpret_cast(data_with_lut.data())[index] = + uint32_t(r + 0.5f) | (uint32_t(g + 0.5f) << 8) | (uint32_t(b + 0.5f) << 16) | + (uint32_t(a + 0.5f) << 24); + } } + } + + for (uint32_t i = 0; i < (reuse ? 2 : 1); ++i) { + EXPECT_NO_THROW(viz::Begin()); - for (uint32_t i = 0; i < (reuse ? 2 : 1); ++i) { - EXPECT_NO_THROW(viz::Begin()); - - EXPECT_NO_THROW(viz::BeginImageLayer()); - - if (use_lut) { - EXPECT_NO_THROW(viz::LUT(lut_size_, kLutFormat, lut.size() * sizeof(uint32_t), - lut.data())); - } - - switch (source) { - case Source::HOST: - EXPECT_NO_THROW(viz::ImageHost(width_, height_, kFormat, - reinterpret_cast(data_.data()))); - break; - case Source::CUDA_DEVICE: - EXPECT_NO_THROW(viz::ImageCudaDevice(width_, height_, kFormat, device_ptr.get())); - break; - default: - EXPECT_TRUE(false) << "Unhandled source type"; - } - EXPECT_NO_THROW(viz::EndLayer()); - - EXPECT_NO_THROW(viz::End()); + EXPECT_NO_THROW(viz::BeginImageLayer()); + + if (use_lut != UseLut::DISABLE) { + EXPECT_NO_THROW(viz::LUT(lut_size_, + kLutFormat, + lut.size() * sizeof(uint32_t), + lut.data(), + use_lut == UseLut::ENABLE_WITH_NORMALIZE)); } - if (use_lut) { - std::swap(data_, data_with_lut); - CompareResult(); - std::swap(data_with_lut, data_); - } else { - CompareResult(); + switch (source) { + case Source::HOST: + EXPECT_NO_THROW( + viz::ImageHost(width_, height_, kFormat, reinterpret_cast(data_.data()))); + break; + case Source::CUDA_DEVICE: + EXPECT_NO_THROW(viz::ImageCudaDevice(width_, height_, kFormat, device_ptr.get())); + break; + default: + EXPECT_TRUE(false) << "Unhandled source type"; } + EXPECT_NO_THROW(viz::EndLayer()); + + EXPECT_NO_THROW(viz::End()); + } + + if (use_lut != UseLut::DISABLE) { + std::swap(data_, data_with_lut); + CompareResult(); + std::swap(data_with_lut, data_); + } else { + CompareResult(); + } } INSTANTIATE_TEST_SUITE_P(ImageLayer, ImageLayer, @@ -168,19 +189,19 @@ INSTANTIATE_TEST_SUITE_P(ImageLayer, ImageLayer, testing::Values(UseLut::DISABLE, UseLut::ENABLE))); TEST_F(ImageLayer, ImageCudaArray) { - constexpr viz::ImageFormat kFormat = viz::ImageFormat::R8G8B8A8_UNORM; + constexpr viz::ImageFormat kFormat = viz::ImageFormat::R8G8B8A8_UNORM; - CUarray array; + CUarray array; - EXPECT_NO_THROW(viz::Begin()); + EXPECT_NO_THROW(viz::Begin()); - EXPECT_NO_THROW(viz::BeginImageLayer()); - // Cuda array support is not yet implemented, the test below should throw. Add the cuda array - // test to the test case above when cuda array support is implemented. - EXPECT_THROW(viz::ImageCudaArray(kFormat, array), std::runtime_error); - EXPECT_NO_THROW(viz::EndLayer()); + EXPECT_NO_THROW(viz::BeginImageLayer()); + // Cuda array support is not yet implemented, the test below should throw. Add the cuda array + // test to the test case above when cuda array support is implemented. + EXPECT_THROW(viz::ImageCudaArray(kFormat, array), std::runtime_error); + EXPECT_NO_THROW(viz::EndLayer()); - EXPECT_NO_THROW(viz::End()); + EXPECT_NO_THROW(viz::End()); } /** @@ -192,104 +213,99 @@ TEST_F(ImageLayer, ImageCudaArray) { * That special case is tested here. */ TEST_F(ImageLayer, SwitchHostDevice) { - const viz::ImageFormat kFormat = viz::ImageFormat::R8G8B8A8_UNORM; - - const viz::CudaService::ScopedPush cuda_context = viz::CudaService::get().PushContext(); - - for (uint32_t i = 0; i < 3; ++i) { - SetupData(kFormat, i); - - viz::UniqueCUdeviceptr device_ptr; - device_ptr.reset([this] { - CUdeviceptr device_ptr; - EXPECT_EQ(cuMemAlloc(&device_ptr, data_.size()), CUDA_SUCCESS); - return device_ptr; - }()); - - EXPECT_EQ(cuMemcpyHtoD(device_ptr.get(), data_.data(), data_.size()), CUDA_SUCCESS); - - EXPECT_NO_THROW(viz::Begin()); - - EXPECT_NO_THROW(viz::BeginImageLayer()); - - if (i & 1) { - EXPECT_NO_THROW(viz::ImageHost(width_, height_, kFormat, - reinterpret_cast(data_.data()))); - } else { - EXPECT_NO_THROW(viz::ImageCudaDevice(width_, height_, kFormat, device_ptr.get())); - } - - EXPECT_NO_THROW(viz::EndLayer()); - - EXPECT_NO_THROW(viz::End()); - - if (!CompareResult()) { - break; - } - } -} + const viz::ImageFormat kFormat = viz::ImageFormat::R8G8B8A8_UNORM; -TEST_F(ImageLayer, Errors) { - constexpr viz::ImageFormat kFormat = viz::ImageFormat::R8G8B8A8_UNORM; - constexpr viz::ImageFormat kLutFormat = viz::ImageFormat::R8G8B8A8_UNORM; - - std::vector host_data(width_ * height_ * 4); - std::vector lut(lut_size_); + const viz::CudaService::ScopedPush cuda_context = viz::CudaService::get().PushContext(); - const viz::CudaService::ScopedPush cuda_context = viz::CudaService::get().PushContext(); + for (uint32_t i = 0; i < 3; ++i) { + SetupData(kFormat, i); viz::UniqueCUdeviceptr device_ptr; device_ptr.reset([this] { - CUdeviceptr device_ptr; - EXPECT_EQ(cuMemAlloc(&device_ptr, width_ * height_ * 4), CUDA_SUCCESS); - return device_ptr; + CUdeviceptr device_ptr; + EXPECT_EQ(cuMemAlloc(&device_ptr, data_.size()), CUDA_SUCCESS); + return device_ptr; }()); - CUarray array; + EXPECT_EQ(cuMemcpyHtoD(device_ptr.get(), data_.data(), data_.size()), CUDA_SUCCESS); EXPECT_NO_THROW(viz::Begin()); - // it's an error to specify both a host and a device image for a layer. EXPECT_NO_THROW(viz::BeginImageLayer()); - EXPECT_NO_THROW(viz::ImageHost(width_, height_, kFormat, - reinterpret_cast(host_data.data()))); - EXPECT_THROW(viz::ImageCudaDevice(width_, height_, kFormat, device_ptr.get()), - std::runtime_error); - EXPECT_NO_THROW(viz::EndLayer()); - EXPECT_NO_THROW(viz::BeginImageLayer()); - EXPECT_NO_THROW(viz::ImageCudaDevice(width_, height_, kFormat, device_ptr.get())); - EXPECT_THROW(viz::ImageHost(width_, height_, kFormat, - reinterpret_cast(host_data.data())), - std::runtime_error); - EXPECT_NO_THROW(viz::EndLayer()); - - // it's an error to call image functions without calling BeginImageLayer first - EXPECT_THROW(viz::ImageCudaDevice(width_, height_, kFormat, device_ptr.get()), - std::runtime_error); - EXPECT_THROW(viz::ImageHost(width_, height_, kFormat, - reinterpret_cast(host_data.data())), - std::runtime_error); - EXPECT_THROW(viz::ImageCudaArray(kFormat, array), std::runtime_error); - EXPECT_THROW(viz::LUT(lut_size_, kLutFormat, lut.size() * sizeof(uint32_t), lut.data()), - std::runtime_error); - - // it's an error to call BeginImageLayer again without calling EndLayer - EXPECT_NO_THROW(viz::BeginImageLayer()); - EXPECT_THROW(viz::BeginImageLayer(), std::runtime_error); - EXPECT_NO_THROW(viz::EndLayer()); + if (i & 1) { + EXPECT_NO_THROW( + viz::ImageHost(width_, height_, kFormat, reinterpret_cast(data_.data()))); + } else { + EXPECT_NO_THROW(viz::ImageCudaDevice(width_, height_, kFormat, device_ptr.get())); + } - // it's an error to call image functions when a different layer is active - EXPECT_NO_THROW(viz::BeginGeometryLayer()); - EXPECT_THROW(viz::ImageCudaDevice(width_, height_, kFormat, device_ptr.get()), - std::runtime_error); - EXPECT_THROW(viz::ImageHost(width_, height_, kFormat, - reinterpret_cast(host_data.data())), - std::runtime_error); - EXPECT_THROW(viz::ImageCudaArray(kFormat, array), std::runtime_error); - EXPECT_THROW(viz::LUT(lut_size_, kLutFormat, lut.size() * sizeof(uint32_t), lut.data()), - std::runtime_error); EXPECT_NO_THROW(viz::EndLayer()); EXPECT_NO_THROW(viz::End()); + + if (!CompareResult()) { break; } + } +} + +TEST_F(ImageLayer, Errors) { + constexpr viz::ImageFormat kFormat = viz::ImageFormat::R8G8B8A8_UNORM; + constexpr viz::ImageFormat kLutFormat = viz::ImageFormat::R8G8B8A8_UNORM; + + std::vector host_data(width_ * height_ * 4); + std::vector lut(lut_size_); + + const viz::CudaService::ScopedPush cuda_context = viz::CudaService::get().PushContext(); + + viz::UniqueCUdeviceptr device_ptr; + device_ptr.reset([this] { + CUdeviceptr device_ptr; + EXPECT_EQ(cuMemAlloc(&device_ptr, width_ * height_ * 4), CUDA_SUCCESS); + return device_ptr; + }()); + + CUarray array; + + EXPECT_NO_THROW(viz::Begin()); + + // it's an error to specify both a host and a device image for a layer. + EXPECT_NO_THROW(viz::BeginImageLayer()); + EXPECT_NO_THROW( + viz::ImageHost(width_, height_, kFormat, reinterpret_cast(host_data.data()))); + EXPECT_THROW(viz::ImageCudaDevice(width_, height_, kFormat, device_ptr.get()), + std::runtime_error); + EXPECT_NO_THROW(viz::EndLayer()); + + EXPECT_NO_THROW(viz::BeginImageLayer()); + EXPECT_NO_THROW(viz::ImageCudaDevice(width_, height_, kFormat, device_ptr.get())); + EXPECT_THROW(viz::ImageHost(width_, height_, kFormat, reinterpret_cast(host_data.data())), + std::runtime_error); + EXPECT_NO_THROW(viz::EndLayer()); + + // it's an error to call image functions without calling BeginImageLayer first + EXPECT_THROW(viz::ImageCudaDevice(width_, height_, kFormat, device_ptr.get()), + std::runtime_error); + EXPECT_THROW(viz::ImageHost(width_, height_, kFormat, reinterpret_cast(host_data.data())), + std::runtime_error); + EXPECT_THROW(viz::ImageCudaArray(kFormat, array), std::runtime_error); + EXPECT_THROW(viz::LUT(lut_size_, kLutFormat, lut.size() * sizeof(uint32_t), lut.data()), + std::runtime_error); + + // it's an error to call BeginImageLayer again without calling EndLayer + EXPECT_NO_THROW(viz::BeginImageLayer()); + EXPECT_THROW(viz::BeginImageLayer(), std::runtime_error); + EXPECT_NO_THROW(viz::EndLayer()); + + // it's an error to call image functions when a different layer is active + EXPECT_NO_THROW(viz::BeginGeometryLayer()); + EXPECT_THROW(viz::ImageCudaDevice(width_, height_, kFormat, device_ptr.get()), + std::runtime_error); + EXPECT_THROW(viz::ImageHost(width_, height_, kFormat, reinterpret_cast(host_data.data())), + std::runtime_error); + EXPECT_THROW(viz::ImageCudaArray(kFormat, array), std::runtime_error); + EXPECT_THROW(viz::LUT(lut_size_, kLutFormat, lut.size() * sizeof(uint32_t), lut.data()), + std::runtime_error); + EXPECT_NO_THROW(viz::EndLayer()); + + EXPECT_NO_THROW(viz::End()); } diff --git a/modules/holoviz/tests/functional/init_test.cpp b/modules/holoviz/tests/functional/init_test.cpp index 066d0f45..c191df06 100644 --- a/modules/holoviz/tests/functional/init_test.cpp +++ b/modules/holoviz/tests/functional/init_test.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,60 +19,108 @@ #define GLFW_INCLUDE_NONE #define GLFW_INCLUDE_VULKAN #include +#include +#include #include namespace viz = holoscan::viz; TEST(Init, GLFWWindow) { - Display *display = XOpenDisplay(NULL); - if (!display) { - GTEST_SKIP() << "X11 server is not running or DISPLAY variable is not set, skipping test."; - } - - EXPECT_EQ(glfwInit(), GLFW_TRUE); - glfwWindowHint(GLFW_CLIENT_API, GLFW_NO_API); - GLFWwindow *const window = glfwCreateWindow(128, 64, "Holoviz test", NULL, NULL); - - EXPECT_NO_THROW(viz::Init(window)); - EXPECT_FALSE(viz::WindowShouldClose()); - EXPECT_FALSE(viz::WindowIsMinimized()); - EXPECT_NO_THROW(viz::Shutdown()); + Display* display = XOpenDisplay(NULL); + if (!display) { + GTEST_SKIP() << "X11 server is not running or DISPLAY variable is not set, skipping test."; + } + + EXPECT_EQ(glfwInit(), GLFW_TRUE); + glfwWindowHint(GLFW_CLIENT_API, GLFW_NO_API); + GLFWwindow* const window = glfwCreateWindow(128, 64, "Holoviz test", NULL, NULL); + + EXPECT_NO_THROW(viz::Init(window)); + EXPECT_FALSE(viz::WindowShouldClose()); + EXPECT_FALSE(viz::WindowIsMinimized()); + EXPECT_NO_THROW(viz::Shutdown()); } TEST(Init, CreateWindow) { - Display *display = XOpenDisplay(NULL); - if (!display) { - GTEST_SKIP() << "X11 server is not running or DISPLAY variable is not set, skipping test."; - } - - EXPECT_NO_THROW(viz::Init(128, 64, "Holoviz test")); - EXPECT_FALSE(viz::WindowShouldClose()); - EXPECT_FALSE(viz::WindowIsMinimized()); - EXPECT_NO_THROW(viz::Shutdown()); + Display* display = XOpenDisplay(NULL); + if (!display) { + GTEST_SKIP() << "X11 server is not running or DISPLAY variable is not set, skipping test."; + } + + EXPECT_NO_THROW(viz::Init(128, 64, "Holoviz test")); + EXPECT_FALSE(viz::WindowShouldClose()); + EXPECT_FALSE(viz::WindowIsMinimized()); + EXPECT_NO_THROW(viz::Shutdown()); } TEST(Init, Fullscreen) { - Display *display = XOpenDisplay(NULL); - if (!display) { - GTEST_SKIP() << "X11 server is not running or DISPLAY variable is not set, skipping test."; - } - - // There is an issue when setting a mode with lower resolution than the current mode, in - // this case the nvidia driver switches to panning mode. This could be avoided when calling - // XRRSetScreenSize() additionally to setting the mode: - // https://www.winehq.org/pipermail/wine-patches/2016-July/152357.html - // For now just use a really big resolution to avoid the issue. - - EXPECT_NO_THROW(viz::Init(4096, 4096, "Holoviz test", viz::InitFlags::FULLSCREEN)); - EXPECT_FALSE(viz::WindowShouldClose()); - EXPECT_FALSE(viz::WindowIsMinimized()); - EXPECT_NO_THROW(viz::Shutdown()); + Display* display = XOpenDisplay(NULL); + if (!display) { + GTEST_SKIP() << "X11 server is not running or DISPLAY variable is not set, skipping test."; + } + + // There is an issue when setting a mode with lower resolution than the current mode, in + // this case the nvidia driver switches to panning mode. This could be avoided when calling + // XRRSetScreenSize() additionally to setting the mode: + // https://www.winehq.org/pipermail/wine-patches/2016-July/152357.html + // For now just use a really big resolution to avoid the issue. + + EXPECT_NO_THROW(viz::Init(4096, 4096, "Holoviz test", viz::InitFlags::FULLSCREEN)); + EXPECT_FALSE(viz::WindowShouldClose()); + EXPECT_FALSE(viz::WindowIsMinimized()); + EXPECT_NO_THROW(viz::Shutdown()); } TEST(Init, Headless) { - EXPECT_NO_THROW(viz::Init(128, 64, "Holoviz test", viz::InitFlags::HEADLESS)); - EXPECT_FALSE(viz::WindowShouldClose()); - EXPECT_FALSE(viz::WindowIsMinimized()); - EXPECT_NO_THROW(viz::Shutdown()); + EXPECT_NO_THROW(viz::Init(128, 64, "Holoviz test", viz::InitFlags::HEADLESS)); + EXPECT_FALSE(viz::WindowShouldClose()); + EXPECT_FALSE(viz::WindowIsMinimized()); + EXPECT_NO_THROW(viz::Shutdown()); +} + +/** + * Check that viz::Init() is returning an error (and not crashing) when the Vulkan loader can't + * find a ICD. + */ +TEST(Init, VulkanLoaderFail) { + // VK_LOADER_DRIVERS_SELECT selects manifest files which matches the provided glob + const char* vk_drivers_files_str = "VK_DRIVER_FILES"; + // VK_ICD_FILENAMES force the loader to use the specific driver JSON files. This is deprecated + // and replaced by VK_LOADER_DRIVERS_SELECT above, but we need to set both variables to support + // both the old loader and the new one. + const char* vk_icd_filenames_str = "VK_ICD_FILENAMES"; + + // get the old value so we can restore it later + char* old_vk_drivers_files_value = getenv(vk_drivers_files_str); + char* old_vk_icd_filenames_value = getenv(vk_icd_filenames_str); + + // set to 'none' so now manifest files ar found + EXPECT_EQ(setenv(vk_drivers_files_str, "none", 1), 0); + EXPECT_EQ(setenv(vk_icd_filenames_str, "none", 1), 0); + + // should throw because no driver is found + EXPECT_THROW(viz::Init(128, 64, "Holoviz test", viz::InitFlags::HEADLESS), std::runtime_error); + + // reset env to old value or remove + if (old_vk_drivers_files_value) { + EXPECT_EQ(setenv(vk_drivers_files_str, old_vk_drivers_files_value, 1), 0); + } else { + EXPECT_EQ(unsetenv(vk_drivers_files_str), 0); + } + if (old_vk_icd_filenames_value) { + EXPECT_EQ(setenv(vk_icd_filenames_str, old_vk_icd_filenames_value, 1), 0); + } else { + EXPECT_EQ(unsetenv(vk_icd_filenames_str), 0); + } + + EXPECT_NO_THROW(viz::Shutdown()); +} + +TEST(Init, Errors) { + // should thrown when specifying an invalid font file + EXPECT_NO_THROW(viz::SetFont("NonExistingFile.ttf", 12.f)); + EXPECT_THROW(viz::Init(128, 64, "Holoviz test", viz::InitFlags::HEADLESS), std::runtime_error); + EXPECT_NO_THROW(viz::Shutdown()); + EXPECT_NO_THROW(viz::SetFont("", 0.f)); } diff --git a/modules/holoviz/tests/functional/layer_test.cpp b/modules/holoviz/tests/functional/layer_test.cpp index e9407b20..b322265f 100644 --- a/modules/holoviz/tests/functional/layer_test.cpp +++ b/modules/holoviz/tests/functional/layer_test.cpp @@ -17,82 +17,79 @@ #include -#include "headless_fixture.hpp" #include +#include "headless_fixture.hpp" namespace viz = holoscan::viz; -class Layer : public TestHeadless { -}; +class Layer : public TestHeadless {}; TEST_F(Layer, Opacity) { - const viz::ImageFormat kFormat = viz::ImageFormat::R8G8B8A8_UNORM; - const float opacity = 0.4f; + const viz::ImageFormat kFormat = viz::ImageFormat::R8G8B8A8_UNORM; + const float opacity = 0.4f; - SetupData(kFormat); + SetupData(kFormat); - std::vector data_with_opacity; + std::vector data_with_opacity; - data_with_opacity.resize(width_ * height_ * sizeof(uint32_t)); - for (size_t index = 0; index < width_ * height_ * 4; ++index) { - data_with_opacity.data()[index] = uint8_t((static_cast(data_[index]) / - 255.f * opacity) * - 255.f + 0.5f); - } + data_with_opacity.resize(width_ * height_ * sizeof(uint32_t)); + for (size_t index = 0; index < width_ * height_ * 4; ++index) { + data_with_opacity.data()[index] = + uint8_t((static_cast(data_[index]) / 255.f * opacity) * 255.f + 0.5f); + } - EXPECT_NO_THROW(viz::Begin()); + EXPECT_NO_THROW(viz::Begin()); - EXPECT_NO_THROW(viz::BeginImageLayer()); + EXPECT_NO_THROW(viz::BeginImageLayer()); - EXPECT_NO_THROW(viz::LayerOpacity(opacity)); + EXPECT_NO_THROW(viz::LayerOpacity(opacity)); - EXPECT_NO_THROW( - viz::ImageHost(width_, height_, viz::ImageFormat::R8G8B8A8_UNORM, - reinterpret_cast(data_.data()))); + EXPECT_NO_THROW(viz::ImageHost( + width_, height_, viz::ImageFormat::R8G8B8A8_UNORM, reinterpret_cast(data_.data()))); - EXPECT_NO_THROW(viz::EndLayer()); + EXPECT_NO_THROW(viz::EndLayer()); - EXPECT_NO_THROW(viz::End()); + EXPECT_NO_THROW(viz::End()); - std::swap(data_, data_with_opacity); - CompareResult(); - std::swap(data_with_opacity, data_); + std::swap(data_, data_with_opacity); + CompareResult(); + std::swap(data_with_opacity, data_); } TEST_F(Layer, Priority) { - const viz::ImageFormat kFormat = viz::ImageFormat::R8G8B8A8_UNORM; + const viz::ImageFormat kFormat = viz::ImageFormat::R8G8B8A8_UNORM; - const uint32_t red = 0xFF0000FF; - const uint32_t green = 0xFF00FF00; + const uint32_t red = 0xFF0000FF; + const uint32_t green = 0xFF00FF00; - for (uint32_t i = 0; i < 2; ++i) { - EXPECT_NO_THROW(viz::Begin()); + for (uint32_t i = 0; i < 2; ++i) { + EXPECT_NO_THROW(viz::Begin()); - EXPECT_NO_THROW(viz::BeginImageLayer()); - EXPECT_NO_THROW(viz::LayerPriority(i)); - EXPECT_NO_THROW(viz::ImageHost(1, 1, viz::ImageFormat::R8G8B8A8_UNORM, - reinterpret_cast(&red))); - EXPECT_NO_THROW(viz::EndLayer()); + EXPECT_NO_THROW(viz::BeginImageLayer()); + EXPECT_NO_THROW(viz::LayerPriority(i)); + EXPECT_NO_THROW(viz::ImageHost( + 1, 1, viz::ImageFormat::R8G8B8A8_UNORM, reinterpret_cast(&red))); + EXPECT_NO_THROW(viz::EndLayer()); - EXPECT_NO_THROW(viz::BeginImageLayer()); - EXPECT_NO_THROW(viz::LayerPriority(1 - i)); - EXPECT_NO_THROW(viz::ImageHost(1, 1, viz::ImageFormat::R8G8B8A8_UNORM, - reinterpret_cast(&green))); - EXPECT_NO_THROW(viz::EndLayer()); + EXPECT_NO_THROW(viz::BeginImageLayer()); + EXPECT_NO_THROW(viz::LayerPriority(1 - i)); + EXPECT_NO_THROW(viz::ImageHost( + 1, 1, viz::ImageFormat::R8G8B8A8_UNORM, reinterpret_cast(&green))); + EXPECT_NO_THROW(viz::EndLayer()); - EXPECT_NO_THROW(viz::End()); + EXPECT_NO_THROW(viz::End()); - std::vector read_data; - ReadData(read_data); - EXPECT_EQ(reinterpret_cast(read_data.data())[0], (i == 0) ? green : red); - } + std::vector read_data; + ReadData(read_data); + EXPECT_EQ(reinterpret_cast(read_data.data())[0], (i == 0) ? green : red); + } } TEST_F(Layer, Errors) { - // it's an error to call EndLayer without an active layer - EXPECT_THROW(viz::EndLayer(), std::runtime_error); + // it's an error to call EndLayer without an active layer + EXPECT_THROW(viz::EndLayer(), std::runtime_error); - // it's an error to call layer functions without an active layer - EXPECT_THROW(viz::LayerOpacity(1.f), std::runtime_error); - EXPECT_THROW(viz::LayerPriority(0), std::runtime_error); + // it's an error to call layer functions without an active layer + EXPECT_THROW(viz::LayerOpacity(1.f), std::runtime_error); + EXPECT_THROW(viz::LayerPriority(0), std::runtime_error); } diff --git a/modules/holoviz/tests/unit/CMakeLists.txt b/modules/holoviz/tests/unit/CMakeLists.txt index 8bae9b0a..127c2ff7 100644 --- a/modules/holoviz/tests/unit/CMakeLists.txt +++ b/modules/holoviz/tests/unit/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -set(PROJECT_NAME clara_holoviz_unittests) +set(PROJECT_NAME holoviz_unittests) add_executable(${PROJECT_NAME}) add_executable(holoscan::viz::unittests ALIAS ${PROJECT_NAME}) @@ -28,10 +28,6 @@ target_compile_definitions(${PROJECT_NAME} GTEST_HAS_RTTI=1 GTEST_LANG_CXX11=1 GTEST_HAS_EXCEPTIONS=1 - # the gtest library provided by Holoscan is compiled with _GLIBCXX_USE_CXX11_ABI set to 1 - # if this is not set there are link errors or runtime errors of functions in gtest expecting - # std::string - _GLIBCXX_USE_CXX11_ABI=1 ) target_link_libraries(${PROJECT_NAME} diff --git a/modules/holoviz/tests/unit/util/unique_value_test.cpp b/modules/holoviz/tests/unit/util/unique_value_test.cpp index f7f8caf9..8d35ac7f 100644 --- a/modules/holoviz/tests/unit/util/unique_value_test.cpp +++ b/modules/holoviz/tests/unit/util/unique_value_test.cpp @@ -24,118 +24,117 @@ using namespace holoscan::viz; namespace { bool g_has_been_destroyed; -using ValueType = uint32_t; +using ValueType = uint32_t; ValueType test_value = 0xDEADBEEF; void destroy(ValueType value) { - EXPECT_EQ(test_value, value); - g_has_been_destroyed = true; + EXPECT_EQ(test_value, value); + g_has_been_destroyed = true; } } // anonymous namespace TEST(UniqueValue, All) { - g_has_been_destroyed = false; + g_has_been_destroyed = false; - { - UniqueValue unique_value; + { + UniqueValue unique_value; - // default state - EXPECT_EQ(ValueType(), unique_value.get()); - EXPECT_FALSE(static_cast(unique_value)); - EXPECT_EQ(ValueType(), unique_value.release()); - } + // default state + EXPECT_EQ(ValueType(), unique_value.get()); + EXPECT_FALSE(static_cast(unique_value)); + EXPECT_EQ(ValueType(), unique_value.release()); + } - EXPECT_FALSE(g_has_been_destroyed) << "destroy should not be called on default (zero) value"; + EXPECT_FALSE(g_has_been_destroyed) << "destroy should not be called on default (zero) value"; - // reset and release - { - UniqueValue unique_value; + // reset and release + { + UniqueValue unique_value; - // set to guard to the test value - EXPECT_NO_THROW(unique_value.reset(test_value)); + // set to guard to the test value + EXPECT_NO_THROW(unique_value.reset(test_value)); - EXPECT_EQ(test_value, unique_value.get()) << "did not return the test value"; - EXPECT_TRUE(static_cast(unique_value)) << - "expect boolean operator to return true if value is set"; + EXPECT_EQ(test_value, unique_value.get()) << "did not return the test value"; + EXPECT_TRUE(static_cast(unique_value)) + << "expect boolean operator to return true if value is set"; - // release the test value - EXPECT_EQ(test_value, unique_value.release()) << "failed to release"; - EXPECT_FALSE(static_cast(unique_value)) << - "expect boolean operator to return false if value is not set"; - EXPECT_EQ(ValueType(), unique_value.get()) << - "did not return default value if value is not set"; - } + // release the test value + EXPECT_EQ(test_value, unique_value.release()) << "failed to release"; + EXPECT_FALSE(static_cast(unique_value)) + << "expect boolean operator to return false if value is not set"; + EXPECT_EQ(ValueType(), unique_value.get()) + << "did not return default value if value is not set"; + } - EXPECT_FALSE(g_has_been_destroyed) << "destroy should not be called after release"; + EXPECT_FALSE(g_has_been_destroyed) << "destroy should not be called after release"; + // set with constructor + { + UniqueValue unique_value(test_value); + + EXPECT_EQ(test_value, unique_value.get()) << "did not return the test value"; + + // release the test value + EXPECT_EQ(test_value, unique_value.release()) << "failed to release"; + } + + EXPECT_FALSE(g_has_been_destroyed) << "action should not be called after release"; + + // call action of test value + { + UniqueValue unique_value; + + // set to guard to the test value + EXPECT_NO_THROW(unique_value.reset(test_value)); + } + + EXPECT_TRUE(g_has_been_destroyed) << "action should be set after UniqueValue had been reset"; + g_has_been_destroyed = false; + + // call action of test value + { // set with constructor - { - UniqueValue unique_value(test_value); - - EXPECT_EQ(test_value, unique_value.get()) << "did not return the test value"; - - // release the test value - EXPECT_EQ(test_value, unique_value.release()) << "failed to release"; - } - - EXPECT_FALSE(g_has_been_destroyed) << "action should not be called after release"; - - // call action of test value - { - UniqueValue unique_value; - - // set to guard to the test value - EXPECT_NO_THROW(unique_value.reset(test_value)); - } - - EXPECT_TRUE(g_has_been_destroyed) << "action should be set after UniqueValue had been reset"; - g_has_been_destroyed = false; - - // call action of test value - { - // set with constructor - UniqueValue unique_value(test_value); - } - - EXPECT_TRUE(g_has_been_destroyed) << - "action should be set after UniqueValue had been destroyed"; - g_has_been_destroyed = false; - - // move constructor - { - // construct - UniqueValue unique_value(test_value); - // move construct - UniqueValue other_unique_value = - std::move(unique_value); - EXPECT_FALSE(g_has_been_destroyed) << - "action should not be set after UniqueValue had been move constructed"; - - EXPECT_EQ(ValueType(), unique_value.get()) << "old unique_value is released"; - EXPECT_EQ(test_value, other_unique_value.get()) - << "new unique_value has the test_value after move construction"; - } - - EXPECT_TRUE(g_has_been_destroyed) << "action should be set after UniqueValue had been reset"; - g_has_been_destroyed = false; - - // move assignment - { - // construct - UniqueValue unique_value(test_value); - // move construct - UniqueValue other_unique_value; - EXPECT_EQ(ValueType(), other_unique_value.get()) << "other_unique_value is empty"; - other_unique_value = std::move(unique_value); - EXPECT_FALSE(g_has_been_destroyed) << - "action should not be set after UniqueValue had been move assigned"; - - EXPECT_EQ(ValueType(), unique_value.get()) << "old unique_value is released"; - EXPECT_EQ(test_value, other_unique_value.get()) << - "new unique_value has the test_value after move assignment"; - } - - EXPECT_TRUE(g_has_been_destroyed) << "action should be set after UniqueValue had been reset"; - g_has_been_destroyed = false; + UniqueValue unique_value(test_value); + } + + EXPECT_TRUE(g_has_been_destroyed) << "action should be set after UniqueValue had been destroyed"; + g_has_been_destroyed = false; + + // move constructor + { + // construct + UniqueValue unique_value(test_value); + // move construct + UniqueValue other_unique_value = + std::move(unique_value); + EXPECT_FALSE(g_has_been_destroyed) + << "action should not be set after UniqueValue had been move constructed"; + + EXPECT_EQ(ValueType(), unique_value.get()) << "old unique_value is released"; + EXPECT_EQ(test_value, other_unique_value.get()) + << "new unique_value has the test_value after move construction"; + } + + EXPECT_TRUE(g_has_been_destroyed) << "action should be set after UniqueValue had been reset"; + g_has_been_destroyed = false; + + // move assignment + { + // construct + UniqueValue unique_value(test_value); + // move construct + UniqueValue other_unique_value; + EXPECT_EQ(ValueType(), other_unique_value.get()) << "other_unique_value is empty"; + other_unique_value = std::move(unique_value); + EXPECT_FALSE(g_has_been_destroyed) + << "action should not be set after UniqueValue had been move assigned"; + + EXPECT_EQ(ValueType(), unique_value.get()) << "old unique_value is released"; + EXPECT_EQ(test_value, other_unique_value.get()) + << "new unique_value has the test_value after move assignment"; + } + + EXPECT_TRUE(g_has_been_destroyed) << "action should be set after UniqueValue had been reset"; + g_has_been_destroyed = false; } diff --git a/modules/holoviz/thirdparty/nvpro_core/CMakeLists.txt b/modules/holoviz/thirdparty/nvpro_core/CMakeLists.txt index 187dc10d..cb6068ff 100644 --- a/modules/holoviz/thirdparty/nvpro_core/CMakeLists.txt +++ b/modules/holoviz/thirdparty/nvpro_core/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,44 +24,87 @@ FetchContent_Declare( GIT_PROGRESS TRUE SOURCE_DIR nvpro_core PATCH_COMMAND git apply "${CMAKE_CURRENT_SOURCE_DIR}/nvpro_core.patch" || true -) + ) FetchContent_GetProperties(nvpro_core) if(NOT nvpro_core_POPULATED) FetchContent_Populate(nvpro_core) - # setup nvpro_core - set(BASE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) - include(${BASE_DIRECTORY}/nvpro_core/cmake/setup.cmake) + # Create the nvpro_core project + # We do this manually instead of using the nvpro_core CMakeFile.txt since nvpro_core is + # - not using the target_??? commands, this leads to exported private dependencies + # - setting multiple global CMAKE_??? variables which overwrite our project variables + set(nvpro_core_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/nvpro_core) - _add_project_definitions(${CMAKE_PROJECT_NAME}) + find_package(Vulkan REQUIRED) - # additions from packages needed for this app - # add refs in LIBRARIES_OPTIMIZED - # add refs in LIBRARIES_DEBUG - # add files in PACKAGE_SOURCE_FILES - _add_package_VulkanSDK() - _add_package_IMGUI() + add_library(nvpro_core STATIC) - # process the rest of some cmake code that needs to be done *after* the packages add - _add_nvpro_core_lib() + target_sources(nvpro_core + PRIVATE + ${nvpro_core_SOURCE_DIR}/nvh/cameramanipulator.cpp + ${nvpro_core_SOURCE_DIR}/nvh/nvprint.cpp + ${nvpro_core_SOURCE_DIR}/nvp/perproject_globals.cpp + ) - # nvpro_core is using add_definitions() and include_directories(), convert these to target properties - get_directory_property(NVPRO_CORE_INCLUDE_DIRECTORIES INCLUDE_DIRECTORIES) - target_include_directories(nvpro_core - PUBLIC - ${NVPRO_CORE_INCLUDE_DIRECTORIES} + set(nvvk_SOURCE_DIR ${nvpro_core_SOURCE_DIR}/nvvk) + + target_sources(nvpro_core + PRIVATE + ${nvvk_SOURCE_DIR}/buffersuballocator_vk.cpp + ${nvvk_SOURCE_DIR}/commands_vk.cpp + ${nvvk_SOURCE_DIR}/context_vk.cpp + ${nvvk_SOURCE_DIR}/debug_util_vk.cpp + ${nvvk_SOURCE_DIR}/descriptorsets_vk.cpp + ${nvvk_SOURCE_DIR}/error_vk.cpp + ${nvvk_SOURCE_DIR}/extensions_vk.cpp + ${nvvk_SOURCE_DIR}/images_vk.cpp + ${nvvk_SOURCE_DIR}/memallocator_dedicated_vk.cpp + ${nvvk_SOURCE_DIR}/memallocator_vk.cpp + ${nvvk_SOURCE_DIR}/memorymanagement_vk.cpp + ${nvvk_SOURCE_DIR}/nsight_aftermath_vk.cpp + ${nvvk_SOURCE_DIR}/pipeline_vk.cpp + ${nvvk_SOURCE_DIR}/resourceallocator_vk.cpp + ${nvvk_SOURCE_DIR}/samplers_vk.cpp + ${nvvk_SOURCE_DIR}/shadermodulemanager_vk.cpp + ${nvvk_SOURCE_DIR}/stagingmemorymanager_vk.cpp + ${nvvk_SOURCE_DIR}/swapchain_vk.cpp + ) + + set(imgui_SOURCE_DIR ${nvpro_core_SOURCE_DIR}/third_party/imgui) + + target_sources(nvpro_core + PRIVATE + ${imgui_SOURCE_DIR}/backends/imgui_impl_glfw.cpp + ${imgui_SOURCE_DIR}/backends/imgui_impl_vulkan.cpp ) - get_directory_property(NVPRO_CORE_COMPILE_DEFINITIONS COMPILE_DEFINITIONS) - # nvpro_core expects GLFW 3.4 which is not yet release. 3.4 added GLFW_CONTEXT_DEBUG - # as an alias to GLFW_OPENGL_DEBUG_CONTEXT, we do this manually. + target_compile_definitions(nvpro_core - PRIVATE -DGLFW_CONTEXT_DEBUG=GLFW_OPENGL_DEBUG_CONTEXT - PUBLIC ${NVPRO_CORE_COMPILE_DEFINITIONS} - ) + PRIVATE + PROJECT_NAME="Holoviz" + # nvpro_core expects GLFW 3.4 which is not yet released. 3.4 added GLFW_CONTEXT_DEBUG + # as an alias to GLFW_OPENGL_DEBUG_CONTEXT, we do this manually. + -DGLFW_CONTEXT_DEBUG=GLFW_OPENGL_DEBUG_CONTEXT + NVP_SUPPORTS_VULKANSDK + ) + + target_include_directories(nvpro_core + PUBLIC + ${imgui_SOURCE_DIR} + ${nvpro_core_SOURCE_DIR} + ${nvpro_core_SOURCE_DIR}/nvp + ) - target_sources(nvpro_core PRIVATE ${COMMON_SOURCE_FILES}) + target_link_libraries(nvpro_core + PRIVATE + Vulkan::Vulkan + glfw + ) + + set_target_properties(nvpro_core + PROPERTIES POSITION_INDEPENDENT_CODE ON + ) # export the nvpro_core cmake directory to provide access to e.g. utilities.cmake for compile_glsl_directory() set(nvpro_core_CMAKE_DIR "${nvpro_core_SOURCE_DIR}/cmake" CACHE INTERNAL "nvpro_core cmake dir" FORCE) @@ -72,39 +115,39 @@ if(NOT nvpro_core_POPULATED) CACHE INTERNAL "nvpro_core external imgui dir" FORCE) # export the ImGUI installation of nvpro_core as a static library - set(PROJECT_NAME clara_holoviz_imgui) - add_library(${PROJECT_NAME} STATIC) - add_library(holoscan::viz::imgui ALIAS ${PROJECT_NAME}) + add_library(holoviz_imgui STATIC) + add_library(holoscan::viz::imgui ALIAS holoviz_imgui) - target_sources(${PROJECT_NAME} + target_sources(holoviz_imgui PRIVATE ${nvpro_core_EXT_IMGUI_DIR}/imgui.cpp ${nvpro_core_EXT_IMGUI_DIR}/imgui_draw.cpp ${nvpro_core_EXT_IMGUI_DIR}/imgui_tables.cpp ${nvpro_core_EXT_IMGUI_DIR}/imgui_widgets.cpp + ${nvpro_core_EXT_IMGUI_DIR}/misc/cpp/imgui_stdlib.cpp ) - target_include_directories(${PROJECT_NAME} + target_include_directories(holoviz_imgui PUBLIC $ $/holoviz/imgui ) - install(TARGETS ${PROJECT_NAME} EXPORT ${PROJECT_NAME}Config - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - COMPONENT "holoscan-dependencies" + set_target_properties(holoviz_imgui + PROPERTIES POSITION_INDEPENDENT_CODE ON ) - # TODO: Commenting the install since it's included in the SDK - # install(EXPORT ${PROJECT_NAME}Config - # DESTINATION cmake - # ) + install(TARGETS holoviz_imgui EXPORT holoviz_imguiConfig + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT "holoscan-dependencies" + ) install( FILES ${nvpro_core_EXT_IMGUI_DIR}/imconfig.h ${nvpro_core_EXT_IMGUI_DIR}/imgui_internal.h ${nvpro_core_EXT_IMGUI_DIR}/imgui.h + ${nvpro_core_EXT_IMGUI_DIR}/misc/cpp/imgui_stdlib.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/holoviz/imgui COMPONENT "holoscan-dependencies" ) diff --git a/patches/README.md b/patches/README.md new file mode 100644 index 00000000..3e3a32e9 --- /dev/null +++ b/patches/README.md @@ -0,0 +1,15 @@ +This folder contains patches that are either applied during the build process of Holoscan SDK +or have been used to build artifacts used by the SDK + +# gxf_remove_complex_primitives_support.patch +This patch is applied at build time of the Holoscan SDK and removes complex primitive support from +GXF since this feature does not compile with CUDA 11.6 and C++17. + +# v4l2-plugin.patch +This patch is released under GNU LGPL License v2.1 + +This patch applies to the NVIDIA tegra v4l2 repository +https://nv-tegra.nvidia.com/tegra/v4l2-src/v4l2_libs.git (branch l4t/l4t-r35.2.1) + +This patch brings modification to v4l2-plugin to load plugins in the local directory +(where the library is loaded) instead of hard-coded path. diff --git a/patches/v4l2-plugin.patch b/patches/v4l2-plugin.patch new file mode 100644 index 00000000..edcbac07 --- /dev/null +++ b/patches/v4l2-plugin.patch @@ -0,0 +1,88 @@ +# This patch is released under GNU LGPL License v2.1 +--- a/libv4l2/v4l2-plugin.c ++++ b/libv4l2/v4l2-plugin.c +@@ -25,6 +25,9 @@ + #include "libv4l2.h" + #include "libv4l2-priv.h" + #include "libv4l-plugin.h" ++#include ++#include ++#include + + /* libv4l plugin support: + it is provided by functions v4l2_plugin_[open,close,etc]. +@@ -45,12 +48,29 @@ + and if it was not then v4l2_* functions proceed with their usual behavior. + */ + +-#define PLUGINS_PATTERN LIBV4L2_PLUGIN_DIR "/*.so" ++#define PLUGINS_PATTERN "*v4l2_plugin.so" ++ ++#include + + void v4l2_plugin_init(int fd, void **plugin_lib_ret, void **plugin_priv_ret, + const struct libv4l_dev_ops **dev_ops_ret) + { +- char *error; ++ // Get the current directory where the shared lib is loaded ++ char* dlidname = NULL; ++ char* dli_fname = NULL; ++ Dl_info info; ++ if (dladdr(v4l2_plugin_init,&info) != 0) { ++ dli_fname = strdup((char*)info.dli_fname); ++ dlidname = dirname(dli_fname); ++ } ++ ++ if(dlidname == NULL){ ++ printf("v4l2_plugin_init : cannot find directory dlidname\n"); ++ free(dli_fname); ++ return; ++ } ++ ++ char *error; + int glob_ret; + uint32_t i; + void *plugin_library = NULL; +@@ -61,7 +81,10 @@ void v4l2_plugin_init(int fd, void **plugin_lib_ret, void **plugin_priv_ret, + *plugin_lib_ret = NULL; + *plugin_priv_ret = NULL; + +- glob_ret = glob(PLUGINS_PATTERN, 0, NULL, &globbuf); ++ char globpattern[2048]; ++ sprintf(globpattern,"%s/%s",dlidname,PLUGINS_PATTERN); ++ glob_ret = glob(globpattern, 0, NULL, &globbuf); ++ free(dli_fname); + + if (glob_ret == GLOB_NOSPACE) + return; +@@ -70,18 +93,28 @@ void v4l2_plugin_init(int fd, void **plugin_lib_ret, void **plugin_priv_ret, + goto leave; + + for (i = 0; i < globbuf.gl_pathc; i++) { ++ + V4L2_LOG("PLUGIN: dlopen(%s);\n", globbuf.gl_pathv[i]); + + plugin_library = dlopen(globbuf.gl_pathv[i], RTLD_LAZY); +- if (!plugin_library) ++ ++ error = dlerror(); ++ if (error != NULL) { ++ V4L2_LOG("PLUGIN: dlopen failed: %s\n", error); ++ continue; ++ } ++ ++ if (!plugin_library) { ++ V4L2_LOG("PLUGIN: dlopen failed"); + continue; ++ } + + dlerror(); /* Clear any existing error */ + libv4l2_plugin = (struct libv4l_dev_ops *) + dlsym(plugin_library, "libv4l2_plugin"); + error = dlerror(); + if (error != NULL) { +- V4L2_LOG_ERR("PLUGIN: dlsym failed: %s\n", error); ++ V4L2_LOG("PLUGIN: dlsym failed: %s\n", error); + dlclose(plugin_library); + continue; + } diff --git a/pyproject.toml b/pyproject.toml index 4038e071..a6e7ccd6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,14 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. - [tool.isort] profile = "black" line_length = 100 known_first_party = "holoscan" skip_glob = ["build*/*", "install*/*", ".cache/*"] -# make sure we don't change the import order in top-level Python __init__.py -skip = ["./python/pybind11/__init__.py"] [tool.black] line-length = 100 @@ -39,3 +36,26 @@ extend-exclude = ''' [tool.codespell] skip = "_deps,build*,.cache,html,_build,_static,generated,latex,install*,.git,xml,vale" ignore-words-list = "bu,dne,unexpect" + +[tool.ruff] +line-length = 103 # 103 to avoid E501 warnings from the copyright header +exclude = [ + "_build", + "_deps", + "_static", + "build*", + ".cache", + "html", + "generated", + "latex", + "install*", + ".git", + "xml" +] + +[tool.ruff.per-file-ignores] +# commented ignore ported from flake8 as E203 is not checked by ruff +# "scripts/convert_video_to_gxf_entities.py" = ["E203"] + +[tool.ruff.isort] +known-first-party = ["holoscan"] diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 3e3e163f..33b347b0 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -20,146 +20,7 @@ file(MAKE_DIRECTORY ${CMAKE_PYBIND11_MODULE_OUT_DIR}) # custom target to encapsulate all add_custom_target(holoscan-python ALL) -# This function takes in a test name and test source and handles setting all of the associated -# properties and linking to build the test -function(holoscan_pybind11_module CMAKE_PYBIND11_MODULE_NAME) - - pybind11_add_module(holoscan_${CMAKE_PYBIND11_MODULE_NAME} - MODULE - ${ARGN} - ) - add_dependencies(holoscan-python holoscan_${CMAKE_PYBIND11_MODULE_NAME}) - - target_link_libraries(holoscan_${CMAKE_PYBIND11_MODULE_NAME} - PRIVATE - ${HOLOSCAN_PACKAGE_NAME} - ) - - # Sets the rpath of the module - file(RELATIVE_PATH install_lib_relative_path - ${CMAKE_CURRENT_LIST_DIR}/pybind11/${CMAKE_PYBIND11_MODULE_NAME}/__init__.py - ${CMAKE_SOURCE_DIR}/${HOLOSCAN_INSTALL_LIB_DIR} ) - list(APPEND _rpath - "\$ORIGIN/${install_lib_relative_path}" # in our install tree (same layout as src) - "\$ORIGIN/../lib" # in our python wheel" - ) - list(JOIN _rpath ":" _rpath) - set_property(TARGET holoscan_${CMAKE_PYBIND11_MODULE_NAME} - APPEND PROPERTY BUILD_RPATH ${_rpath} - ) - unset(_rpath) - - # make submodule folder - file(MAKE_DIRECTORY ${CMAKE_PYBIND11_MODULE_OUT_DIR}/${CMAKE_PYBIND11_MODULE_NAME}) - - # custom target to ensure the module's __init__.py file is copied - set(CMAKE_SUBMODULE_INIT_FILE - ${CMAKE_CURRENT_LIST_DIR}/pybind11/${CMAKE_PYBIND11_MODULE_NAME}/__init__.py - ) - set(CMAKE_SUBMODULE_OUT_DIR ${CMAKE_PYBIND11_MODULE_OUT_DIR}/${CMAKE_PYBIND11_MODULE_NAME}) - add_custom_target(holoscan-${CMAKE_PYBIND11_MODULE_NAME}-init - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_SUBMODULE_INIT_FILE}" "${CMAKE_SUBMODULE_OUT_DIR}/" - DEPENDS "${CMAKE_SUBMODULE_INIT_FILE}" - ) - add_dependencies(holoscan_${CMAKE_PYBIND11_MODULE_NAME} - holoscan-${CMAKE_PYBIND11_MODULE_NAME}-init - ) - - # Note: OUTPUT_NAME filename (_holoscan_core) must match the module name in the PYBIND11_MODULE macro - set_target_properties(holoscan_${CMAKE_PYBIND11_MODULE_NAME} PROPERTIES - LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR} - OUTPUT_NAME holoscan/${CMAKE_PYBIND11_MODULE_NAME}/_${CMAKE_PYBIND11_MODULE_NAME} - ) - -endfunction() - -# custom target for top-level __init__.py file is copied -set(CMAKE_PYBIND11_PRIMARY_INIT_FILE ${CMAKE_CURRENT_LIST_DIR}/pybind11/__init__.py) -add_custom_target(holoscan-python-pyinit - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_PYBIND11_PRIMARY_INIT_FILE}" "${CMAKE_PYBIND11_MODULE_OUT_DIR}/" - DEPENDS "${CMAKE_PYBIND11_PRIMARY_INIT_FILE}" -) -add_dependencies(holoscan-python holoscan-python-pyinit) - -# custom target for top-level config.py file -set(CMAKE_PYBIND11_CONFIG_FILE ${CMAKE_CURRENT_LIST_DIR}/pybind11/config.py) -add_custom_target(holoscan-python-config - COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_PYBIND11_CONFIG_FILE}" "${CMAKE_PYBIND11_MODULE_OUT_DIR}/" - DEPENDS "${CMAKE_PYBIND11_CONFIG_FILE}" -) -add_dependencies(holoscan-python holoscan-python-config) - -###################### -# Add pybind11 modules -###################### - -holoscan_pybind11_module( - core - pybind11/core/core.cpp - pybind11/core/core.hpp - pybind11/core/core_pydoc.hpp - pybind11/core/dl_converter.cpp - pybind11/core/dl_converter.hpp - pybind11/core/trampolines.cpp - pybind11/core/trampolines.hpp - pybind11/kwarg_handling.cpp - pybind11/kwarg_handling.hpp - pybind11/macros.hpp -) - -holoscan_pybind11_module( - gxf - pybind11/core/dl_converter.cpp - pybind11/core/dl_converter.hpp - pybind11/core/trampolines.cpp - pybind11/core/trampolines.hpp - pybind11/macros.hpp - pybind11/gxf/gxf.cpp - pybind11/gxf/gxf.hpp - pybind11/gxf/gxf_pydoc.hpp -) - -holoscan_pybind11_module( - logger - pybind11/macros.hpp - pybind11/logger/logger.cpp - pybind11/logger/logger_pydoc.hpp -) - -holoscan_pybind11_module( - conditions - pybind11/macros.hpp - pybind11/conditions/conditions.cpp - pybind11/conditions/conditions_pydoc.hpp -) - -holoscan_pybind11_module( - resources - pybind11/macros.hpp - pybind11/resources/resources.cpp - pybind11/resources/resources_pydoc.hpp -) - -holoscan_pybind11_module( - graphs - pybind11/macros.hpp - pybind11/graphs/graphs.cpp - pybind11/graphs/graphs_pydoc.hpp -) - -holoscan_pybind11_module( - operators - pybind11/macros.hpp - pybind11/operators/operators.cpp - pybind11/operators/operators_pydoc.hpp -) - -holoscan_pybind11_module( - executors - pybind11/macros.hpp - pybind11/executors/executors.cpp - pybind11/executors/executors_pydoc.hpp -) +add_subdirectory(src) # For convenience, we currently copy the tests folder to build/python/lib as well @@ -171,15 +32,6 @@ add_custom_target(holoscan-python-tests ) add_dependencies(holoscan-python holoscan-python-tests) -install(DIRECTORY - "${CMAKE_CURRENT_BINARY_DIR}/lib" - DESTINATION python - FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE - DIRECTORY_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE - COMPONENT "holoscan-python_libs" - PATTERN "__pycache__" EXCLUDE -) - if(HOLOSCAN_BUILD_TESTS) add_test(NAME python-api-tests COMMAND ${PYTHON_EXECUTABLE} -m pytest ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}/tests diff --git a/python/src/CMakeLists.txt b/python/src/CMakeLists.txt new file mode 100644 index 00000000..46ab158e --- /dev/null +++ b/python/src/CMakeLists.txt @@ -0,0 +1,181 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Utility function to generate pybind11 modules +function(holoscan_pybind11_module CMAKE_PYBIND11_MODULE_NAME) + set(module_name holoscan_${CMAKE_PYBIND11_MODULE_NAME}_python) + pybind11_add_module(${module_name} MODULE ${ARGN}) + + add_dependencies(holoscan-python ${module_name}) + + target_link_libraries(${module_name} + PRIVATE holoscan::core + ) + + # Sets the rpath of the module + file(RELATIVE_PATH install_lib_relative_path + ${CMAKE_CURRENT_LIST_DIR}/${CMAKE_PYBIND11_MODULE_NAME}/__init__.py + ${CMAKE_SOURCE_DIR}/${HOLOSCAN_INSTALL_LIB_DIR} ) + list(APPEND _rpath + "\$ORIGIN/${install_lib_relative_path}" # in our install tree (same layout as src) + "\$ORIGIN/../lib" # in our python wheel" + ) + list(JOIN _rpath ":" _rpath) + set_property(TARGET ${module_name} + APPEND PROPERTY BUILD_RPATH ${_rpath} + ) + unset(_rpath) + + # make submodule folder + file(MAKE_DIRECTORY ${CMAKE_PYBIND11_MODULE_OUT_DIR}/${CMAKE_PYBIND11_MODULE_NAME}) + + # custom target to ensure the module's __init__.py file is copied + set(CMAKE_SUBMODULE_INIT_FILE + ${CMAKE_CURRENT_LIST_DIR}/${CMAKE_PYBIND11_MODULE_NAME}/__init__.py + ) + set(CMAKE_SUBMODULE_OUT_DIR ${CMAKE_PYBIND11_MODULE_OUT_DIR}/${CMAKE_PYBIND11_MODULE_NAME}) + add_custom_target(holoscan-${CMAKE_PYBIND11_MODULE_NAME}-init + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_SUBMODULE_INIT_FILE}" "${CMAKE_SUBMODULE_OUT_DIR}/" + DEPENDS "${CMAKE_SUBMODULE_INIT_FILE}" + ) + add_dependencies(${module_name} + holoscan-${CMAKE_PYBIND11_MODULE_NAME}-init + ) + + # Note: OUTPUT_NAME filename (_holoscan_core) must match the module name in the PYBIND11_MODULE macro + set_target_properties(${module_name} PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/../${CMAKE_INSTALL_LIBDIR} + OUTPUT_NAME holoscan/${CMAKE_PYBIND11_MODULE_NAME}/_${CMAKE_PYBIND11_MODULE_NAME} + ) + +endfunction() + +# custom target for top-level __init__.py file is copied +set(CMAKE_PYBIND11_PRIMARY_INIT_FILE ${CMAKE_CURRENT_LIST_DIR}/__init__.py) +add_custom_target(holoscan-python-pyinit + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_PYBIND11_PRIMARY_INIT_FILE}" "${CMAKE_PYBIND11_MODULE_OUT_DIR}/" + DEPENDS "${CMAKE_PYBIND11_PRIMARY_INIT_FILE}" +) +add_dependencies(holoscan-python holoscan-python-pyinit) + +# custom target for top-level config.py file +set(CMAKE_PYBIND11_CONFIG_FILE ${CMAKE_CURRENT_LIST_DIR}/config.py) +add_custom_target(holoscan-python-config + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_PYBIND11_CONFIG_FILE}" "${CMAKE_PYBIND11_MODULE_OUT_DIR}/" + DEPENDS "${CMAKE_PYBIND11_CONFIG_FILE}" +) +add_dependencies(holoscan-python holoscan-python-config) + +###################### +# Add pybind11 modules +###################### + +holoscan_pybind11_module( + core + core/core.cpp + core/core.hpp + core/core_pydoc.hpp + core/dl_converter.cpp + core/dl_converter.hpp + core/trampolines.cpp + core/trampolines.hpp + kwarg_handling.cpp + kwarg_handling.hpp + macros.hpp +) + +holoscan_pybind11_module( + gxf + core/dl_converter.cpp + core/dl_converter.hpp + core/trampolines.cpp + core/trampolines.hpp + macros.hpp + gxf/gxf.cpp + gxf/gxf.hpp + gxf/gxf_pydoc.hpp +) + +holoscan_pybind11_module( + logger + macros.hpp + logger/logger.cpp + logger/logger_pydoc.hpp +) + +holoscan_pybind11_module( + conditions + macros.hpp + conditions/conditions.cpp + conditions/conditions_pydoc.hpp +) + +holoscan_pybind11_module( + resources + macros.hpp + resources/resources.cpp + resources/resources_pydoc.hpp +) + +holoscan_pybind11_module( + graphs + macros.hpp + graphs/graphs.cpp + graphs/graphs_pydoc.hpp +) + +holoscan_pybind11_module( + operators + macros.hpp + operators/operators.cpp + operators/operators_pydoc.hpp +) +target_link_libraries(holoscan_operators_python + PUBLIC + holoscan::ops::aja + holoscan::ops::bayer_demosaic + holoscan::ops::format_converter + holoscan::ops::holoviz + holoscan::ops::multiai_inference + holoscan::ops::multiai_postprocessor + holoscan::ops::segmentation_postprocessor + holoscan::ops::tensor_rt + holoscan::ops::video_stream_recorder + holoscan::ops::video_stream_replayer +) + +# custom target to copy operators/_native.py file +set(CMAKE_PYBIND11_NATIVE_OPS_FILE ${CMAKE_CURRENT_LIST_DIR}/operators/_native.py) +add_custom_target(holoscan-python-native-ops + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_PYBIND11_NATIVE_OPS_FILE}" "${CMAKE_PYBIND11_MODULE_OUT_DIR}/operators/_native.py" + DEPENDS "${CMAKE_PYBIND11_NATIVE_OPS_FILE}" +) +add_dependencies(holoscan-python holoscan-python-native-ops) + +holoscan_pybind11_module( + executors + macros.hpp + executors/executors.cpp + executors/executors_pydoc.hpp +) + +install(DIRECTORY + "${CMAKE_CURRENT_BINARY_DIR}/../lib" + DESTINATION python + FILE_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE + DIRECTORY_PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE + COMPONENT "holoscan-python_libs" + PATTERN "__pycache__" EXCLUDE +) diff --git a/python/pybind11/__init__.py b/python/src/__init__.py similarity index 80% rename from python/pybind11/__init__.py rename to python/src/__init__.py index 6f869539..899fb54e 100644 --- a/python/pybind11/__init__.py +++ b/python/src/__init__.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,11 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -# fmt: off +# isort: off # must keep import of core and gxf before other modules that rely on them from . import core, gxf -from . import conditions, config, executors, graphs, logger, operators, resources -# fmt: on +from . import conditions, config, executors, graphs, logger, operators, resources + +# isort: on as_tensor = core.Tensor.as_tensor diff --git a/python/pybind11/conditions/__init__.py b/python/src/conditions/__init__.py similarity index 100% rename from python/pybind11/conditions/__init__.py rename to python/src/conditions/__init__.py diff --git a/python/pybind11/conditions/conditions.cpp b/python/src/conditions/conditions.cpp similarity index 86% rename from python/pybind11/conditions/conditions.cpp rename to python/src/conditions/conditions.cpp index ab04140a..36966547 100644 --- a/python/pybind11/conditions/conditions.cpp +++ b/python/src/conditions/conditions.cpp @@ -158,7 +158,25 @@ PYBIND11_MODULE(_conditions, m) { .def("check_tick_enabled", &BooleanCondition::check_tick_enabled, doc::BooleanCondition::doc_check_tick_enabled) - .def("setup", &BooleanCondition::setup, "spec"_a, doc::BooleanCondition::doc_setup); + .def("setup", &BooleanCondition::setup, "spec"_a, doc::BooleanCondition::doc_setup) + .def( + "__repr__", + // had to remove const here to access check_tick_enabled() + [](BooleanCondition& cond) { + try { + std::string enabled = fmt::format("{}", cond.check_tick_enabled()); + // Python True, False are uppercase + enabled[0] = toupper(enabled[0]); + return fmt::format( + "BooleanCondition(self, enable_tick={}, name={})", enabled, cond.name()); + } catch (const std::runtime_error& e) { + // fallback for when initialize() has not yet been called + return fmt::format( + "BooleanCondition(self, enable_tick=, name={})", cond.name()); + } + }, + py::call_guard(), + R"doc(Return repr(self).)doc"); py::class_>( m, "CountCondition", doc::CountCondition::doc_CountCondition) @@ -173,7 +191,23 @@ PYBIND11_MODULE(_conditions, m) { .def_property("count", py::overload_cast<>(&CountCondition::count), py::overload_cast(&CountCondition::count), - doc::CountCondition::doc_count); + doc::CountCondition::doc_count) + .def( + "__repr__", + // had to remove const here to access count() property getter + [](CountCondition& cond) { + try { + auto cnt = cond.count(); + return fmt::format( + "CountCondition(self, count={}, name={})", cnt, cond.name()); + } catch (const std::runtime_error& e) { + // fallback for when initialize() has not yet been called + return fmt::format( + "CountCondition(self, count=, name={})", cond.name()); + } + }, + py::call_guard(), + R"doc(Return repr(self).)doc"); py::class_([](ParameterWrapper& param_wrap, Arg& arg) { std::any& any_param = param_wrap.value(); - std::any& any_arg = arg.value(); - // Note that the type of any_param is Parameter*, not Parameter. auto& param = *std::any_cast*>(any_param); - // `PyOperatorSpec.py_param` will have stored the actual Operator class from Python - // into a Parameter param, so `param.get()` here is `PyOperatorSpec.py_op`. - // `param.key()` is the name of the attribute on the Python class. - // We can then set that Class attribute's value to the value contained in Arg. - // Here we use the `arg_to_py_object` utility to convert from `Arg` to `py::object`. - py::setattr(param.get(), param.key().c_str(), arg_to_py_object(arg)); - }); - - auto& gxf_param_adaptor = ::holoscan::gxf::GXFParameterAdaptor::get_instance(); - gxf_param_adaptor.add_param_handler([](gxf_context_t context, - gxf_uid_t uid, - const char* key, - const ArgType& arg_type, - const std::any& any_value) { - (void)context; - (void)uid; - (void)arg_type; - try { - auto& param = *std::any_cast*>(any_value); - + // If arg has no name and value, that indicates that we want to set the default value for + // the native operator if it is not specified. + if (arg.name().empty() && !arg.has_value()) { + const char* key = param.key().c_str(); // If the attribute is not None, we do not need to set it. // Otherwise, we set it to the default value (if it exists). if (!py::hasattr(param.get(), key) || (py::getattr(param.get(), key).is(py::none()) && param.has_default_value())) { py::setattr(param.get(), key, param.default_value()); } - - // If the parameter (any_value) is from the native operator - // (which means 'uid == -1'), return here without setting GXF parameter. - if (uid == -1) { return GXF_SUCCESS; } - } catch (const std::bad_any_cast& e) { - HOLOSCAN_LOG_ERROR("Bad any cast exception: {}", e.what()); + return; } - return GXF_FAILURE; + // `PyOperatorSpec.py_param` will have stored the actual Operator class from Python + // into a Parameter param, so `param.get()` here is `PyOperatorSpec.py_op`. + // `param.key()` is the name of the attribute on the Python class. + // We can then set that Class attribute's value to the value contained in Arg. + // Here we use the `arg_to_py_object` utility to convert from `Arg` to `py::object`. + py::setattr(param.get(), param.key().c_str(), arg_to_py_object(arg)); }); } @@ -198,14 +180,10 @@ PYBIND11_MODULE(_core, m) { } return py::str(); }) + .def_property_readonly("description", &Arg::description, doc::Arg::doc_description) .def( "__repr__", - [](const Arg& arg) { - return fmt::format("", - arg.name(), - element_type_namemap.at(arg.arg_type().element_type()), - container_type_namemap.at(arg.arg_type().container_type())); - }, + [](const Arg& arg) { return arg.description(); }, py::call_guard(), R"doc(Return repr(self).)doc"); @@ -220,23 +198,10 @@ PYBIND11_MODULE(_core, m) { py::overload_cast(&ArgList::add), "arg"_a, doc::ArgList::doc_add_ArgList) + .def_property_readonly("description", &ArgList::description, doc::ArgList::doc_description) .def( "__repr__", - [](const ArgList& args) { - auto msg_buf = fmt::memory_buffer(); - fmt::format_to(std::back_inserter(msg_buf), "[\n"); - for (Arg arg : args) { - fmt::format_to(std::back_inserter(msg_buf), - " ,\n", - arg.name(), - element_type_namemap.at(arg.arg_type().element_type()), - container_type_namemap.at(arg.arg_type().container_type())); - } - fmt::format_to(std::back_inserter(msg_buf), "]"); - - std::string repr = fmt::format("{:.{}}", msg_buf.data(), msg_buf.size()); - return repr; - }, + [](const ArgList& list) { return list.description(); }, py::call_guard(), R"doc(Return repr(self).)doc"); @@ -249,13 +214,11 @@ PYBIND11_MODULE(_core, m) { .def_property_readonly("element_type", &ArgType::element_type, doc::ArgType::doc_element_type) .def_property_readonly( "container_type", &ArgType::container_type, doc::ArgType::doc_container_type) + .def_property_readonly("dimension", &ArgType::dimension, doc::ArgType::doc_dimension) + .def_property_readonly("to_string", &ArgType::to_string, doc::ArgType::doc_to_string) .def( "__repr__", - [](const ArgType& t) { - return fmt::format("", - element_type_namemap.at(t.element_type()), - container_type_namemap.at(t.container_type())); - }, + [](const ArgType& t) { return t.to_string(); }, py::call_guard(), R"doc(Return repr(self).)doc"); @@ -276,7 +239,14 @@ PYBIND11_MODULE(_core, m) { .def_property_readonly("args", &Component::args, doc::Component::doc_args) .def("initialize", &Component::initialize, - doc::Component::doc_initialize); // note: virtual function + doc::Component::doc_initialize) // note: virtual function + .def_property_readonly( + "description", &Component::description, doc::Component::doc_description) + .def( + "__repr__", + [](const Component& c) { return c.description(); }, + py::call_guard(), + R"doc(Return repr(self).)doc"); py::enum_(m, "ConditionType", doc::ConditionType::doc_ConditionType) .value("NONE", ConditionType::kNone) @@ -297,14 +267,20 @@ PYBIND11_MODULE(_core, m) { py::overload_cast<>(&Condition::fragment), py::overload_cast(&Condition::fragment), doc::Condition::doc_fragment) - // .def_property( - // "spec", - // py::overload_cast<>(&Condition::spec), - // py::overload_cast< std::unique_ptr >(&Condition::spec)); + .def_property("spec", + &Condition::spec_shared, + py::overload_cast&>(&Condition::spec)) .def("setup", &Condition::setup, doc::Condition::doc_setup) // note: virtual .def("initialize", &Condition::initialize, - doc::Condition::doc_initialize); // note: virtual function + doc::Condition::doc_initialize) // note: virtual function + .def_property_readonly( + "description", &Condition::description, doc::Condition::doc_description) + .def( + "__repr__", + [](const Condition& c) { return c.description(); }, + py::call_guard(), + R"doc(Return repr(self).)doc"); py::class_>( m, "Resource", doc::Resource::doc_Resource) @@ -317,14 +293,19 @@ PYBIND11_MODULE(_core, m) { py::overload_cast<>(&Resource::fragment), py::overload_cast(&Resource::fragment), doc::Resource::doc_fragment) - // .def_property( - // "spec", - // py::overload_cast<>(&Resource::spec), - // py::overload_cast< std::unique_ptr >(&Resource::spec)); + .def_property("spec", + &Resource::spec_shared, + py::overload_cast&>(&Resource::spec)) .def("setup", &Resource::setup, doc::Resource::doc_setup) // note: virtual .def("initialize", &Resource::initialize, - doc::Resource::doc_initialize); // note: virtual function + doc::Resource::doc_initialize) // note: virtual function + .def_property_readonly("description", &Resource::description, doc::Resource::doc_description) + .def( + "__repr__", + [](const Resource& c) { return c.description(); }, + py::call_guard(), + R"doc(Return repr(self).)doc"); py::class_> input_context( m, "InputContext", doc::InputContext::doc_InputContext); @@ -353,8 +334,17 @@ PYBIND11_MODULE(_core, m) { py::class_>( m, "ComponentSpec", R"doc(Component specification class.)doc") .def(py::init(), "fragment"_a, doc::ComponentSpec::doc_ComponentSpec) - .def_property_readonly("fragment", &ComponentSpec::fragment, doc::ComponentSpec::doc_fragment) - .def_property_readonly("params", &ComponentSpec::params, doc::ComponentSpec::doc_params); + .def_property_readonly("fragment", + py::overload_cast<>(&ComponentSpec::fragment), + doc::ComponentSpec::doc_fragment) + .def_property_readonly("params", &ComponentSpec::params, doc::ComponentSpec::doc_params) + .def_property_readonly( + "description", &ComponentSpec::description, doc::ComponentSpec::doc_description) + .def( + "__repr__", + [](const ComponentSpec& spec) { return spec.description(); }, + py::call_guard(), + R"doc(Return repr(self).)doc"); py::class_ iospec(m, "IOSpec", R"doc(I/O specification class.)doc"); iospec @@ -395,18 +385,29 @@ PYBIND11_MODULE(_core, m) { .def(py::init(), "fragment"_a, doc::OperatorSpec::doc_OperatorSpec) .def("input", py::overload_cast<>(&OperatorSpec::input), - doc::OperatorSpec::doc_input) + doc::OperatorSpec::doc_input, + py::return_value_policy::reference_internal) .def("input", py::overload_cast(&OperatorSpec::input), "name"_a, - doc::OperatorSpec::doc_input_name) + doc::OperatorSpec::doc_input_name, + py::return_value_policy::reference_internal) .def("output", py::overload_cast<>(&OperatorSpec::output), - doc::OperatorSpec::doc_output) + doc::OperatorSpec::doc_output, + py::return_value_policy::reference_internal) .def("output", py::overload_cast(&OperatorSpec::output), "name"_a, - doc::OperatorSpec::doc_output_name); + doc::OperatorSpec::doc_output_name, + py::return_value_policy::reference_internal) + .def_property_readonly( + "description", &OperatorSpec::description, doc::OperatorSpec::doc_description) + .def( + "__repr__", + [](const OperatorSpec& spec) { return spec.description(); }, + py::call_guard(), + R"doc(Return repr(self).)doc"); // Note: In the case of OperatorSpec, InputContext, OutputContext, ExecutionContext, // there are a separate, independent wrappers for PyOperatorSpec, PyInputContext, @@ -442,10 +443,10 @@ PYBIND11_MODULE(_core, m) { // note: added py::dynamic_attr() to allow dynamically adding attributes in a Python subclass py::class_> operator_class( - m, "Operator", py::dynamic_attr(), doc::Operator::doc_Operator); + m, "Operator", py::dynamic_attr(), doc::Operator::doc_Operator_args_kwargs); operator_class - .def(py::init(), + .def(py::init(), doc::Operator::doc_Operator_args_kwargs) .def_property("name", py::overload_cast<>(&Operator::name, py::const_), @@ -477,7 +478,13 @@ PYBIND11_MODULE(_core, m) { .def("compute", &Operator::compute, // note: virtual function doc::Operator::doc_compute, - py::call_guard()); // note: should release GIL + py::call_guard()) // note: should release GIL + .def_property_readonly("description", &Operator::description, doc::Operator::doc_description) + .def( + "__repr__", + [](const Operator& op) { return op.description(); }, + py::call_guard(), + R"doc(Return repr(self).)doc"); py::enum_(operator_class, "OperatorType") .value("NATIVE", Operator::OperatorType::kNative) @@ -495,7 +502,8 @@ PYBIND11_MODULE(_core, m) { m, "Executor", R"doc(Executor class.)doc") .def(py::init(), "fragment"_a, doc::Executor::doc_Executor) .def("run", &Executor::run, doc::Executor::doc_run) // note: virtual function - .def_property_readonly("fragment", &Executor::fragment, doc::Executor::doc_fragment) + .def_property_readonly( + "fragment", py::overload_cast<>(&Executor::fragment), doc::Executor::doc_fragment) .def_property("context", py::overload_cast<>(&Executor::context), py::overload_cast(&Executor::context), diff --git a/python/pybind11/core/core.hpp b/python/src/core/core.hpp similarity index 93% rename from python/pybind11/core/core.hpp rename to python/src/core/core.hpp index c187df7b..be453c23 100644 --- a/python/pybind11/core/core.hpp +++ b/python/src/core/core.hpp @@ -98,6 +98,14 @@ class PyTensor : public Tensor { static std::shared_ptr from_dlpack(const py::object& obj); }; +template +std::vector get_names_from_map(ObjT& map_obj) { + std::vector names; + names.reserve(map_obj.size()); + for (auto& i : map_obj) { names.push_back(i.first); } + return names; +} + } // namespace holoscan #endif /* PYBIND11_CORE_CORE_HPP */ diff --git a/python/pybind11/core/core_pydoc.hpp b/python/src/core/core_pydoc.hpp similarity index 74% rename from python/pybind11/core/core_pydoc.hpp rename to python/src/core/core_pydoc.hpp index 550b2799..6366585f 100644 --- a/python/pybind11/core/core_pydoc.hpp +++ b/python/src/core/core_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -27,14 +27,14 @@ namespace holoscan::doc { namespace ArgElementType { PYDOC(ArgElementType, R"doc( -Enum class for an Arg's element type. +Enum class for an `Arg`'s element type. )doc") } // namespace ArgElementType namespace ArgContainerType { PYDOC(ArgContainerType, R"doc( -Enum class for an Arg's container type. +Enum class for an `Arg`'s container type. )doc") } // namespace ArgContainerType @@ -64,13 +64,17 @@ ArgType info corresponding to the argument. Returns ------- -arg_type : ``holoscan.core.ArgType`` +arg_type : holoscan.core.ArgType )doc") PYDOC(has_value, R"doc( Boolean flag indicating whether a value has been assigned to the argument. )doc") +PYDOC(description, R"doc( +YAML formatted string describing the argument. +)doc") + } // namespace Arg namespace ArgList { @@ -93,7 +97,7 @@ The number of arguments in the list. )doc") PYDOC(args, R"doc( -The underlying list of Arg objects. +The underlying list of `Arg` objects. )doc") PYDOC(clear, R"doc( @@ -108,6 +112,10 @@ PYDOC(add_ArgList, R"doc( Add a list of arguments to the list. )doc") +PYDOC(description, R"doc( +YAML formatted string describing the list. +)doc") + // note: docs for overloadded add method are defined in core.cpp } // namespace ArgList @@ -124,10 +132,10 @@ Class containing argument type info. Parameters ---------- -element_type : ``holoscan.core.ArgElementType`` +element_type : holoscan.core.ArgElementType Element type of the argument. -container_type : ``holoscan.core.ArgContainerType`` +container_type : holoscan.core.ArgContainerType Container type of the argument. )doc") @@ -140,8 +148,15 @@ PYDOC(container_type, R"doc( The container type of the argument. )doc") -// note: docs for overloadded add method are defined in core.cpp +PYDOC(dimension, R"doc( +The dimension of the argument container. +)doc") + +PYDOC(to_string, R"doc( +String describing the argument type. +)doc") +// note: docs for overloaded add method are defined in core.cpp } // namespace ArgType namespace Component { @@ -164,7 +179,7 @@ The fragment containing the component. Returns ------- -name : ``holoscan.core.Fragment`` +name : holoscan.core.Fragment )doc") PYDOC(id, R"doc( @@ -173,7 +188,7 @@ The identifier of the component. The identifier is initially set to -1, and will become a valid value when the component is initialized. -With the default executor (``holoscan.gxf.GXFExecutor``), the identifier is set to the GXF +With the default executor (`holoscan.gxf.GXFExecutor`), the identifier is set to the GXF component ID. Returns @@ -194,13 +209,17 @@ The list of arguments associated with the component. Returns ------- -arglist : ``holoscan.core.ArgList`` +arglist : holoscan.core.ArgList )doc") PYDOC(initialize, R"doc( Initialize the component. )doc") +PYDOC(description, R"doc( +YAML formatted string describing the component. +)doc") + } // namespace Component namespace ConditionType { @@ -224,17 +243,17 @@ Class representing a condition. Can be initialized with any number of Python positional and keyword arguments. -If a ``'name'`` keyword argument is provided, it must be a ``str`` and will be +If a `name` keyword argument is provided, it must be a `str` and will be used to set the name of the Operator. -If a ``'fragment'`` keyword argument is provided, it must be of type -``holoscan.core.Fragment`` (or -``holoscan.core.Application``). A single ``Fragment`` object can also be +If a `fragment` keyword argument is provided, it must be of type +`holoscan.core.Fragment` (or +`holoscan.core.Application`). A single `Fragment` object can also be provided positionally instead. -Any other arguments will be cast from a Python argument type to a C++ ``Arg`` +Any other arguments will be cast from a Python argument type to a C++ `Arg` and stored in ``self.args``. (For details on how the casting is done, see the -``py_object_to_arg`` utility). +`py_object_to_arg` utility). Parameters ---------- @@ -246,9 +265,9 @@ Parameters Raises ------ RuntimeError - If ``'name'`` kwarg is provided, but is not of str type. - If multiple arguments of type ``Fragment`` are provided. - If any other arguments cannot be converted to Arg type via ``py_object_to_arg``. + If `name` kwarg is provided, but is not of `str` type. + If multiple arguments of type `Fragment` are provided. + If any other arguments cannot be converted to `Arg` type via `py_object_to_arg`. )doc") PYDOC(name, R"doc( @@ -264,7 +283,11 @@ Fragment that the condition belongs to. Returns ------- -name : ``holoscan.core.Fragment`` +name : holoscan.core.Fragment +)doc") + +PYDOC(spec, R"doc( +The condition's ComponentSpec. )doc") PYDOC(setup, R"doc( @@ -275,6 +298,10 @@ PYDOC(initialize, R"doc( initialization method for the condition. )doc") +PYDOC(description, R"doc( +YAML formatted string describing the condition. +)doc") + } // namespace Condition namespace Resource { @@ -289,17 +316,17 @@ Class representing a resource. Can be initialized with any number of Python positional and keyword arguments. -If a ``'name'`` keyword argument is provided, it must be a ``str`` and will be +If a `name` keyword argument is provided, it must be a `str` and will be used to set the name of the Operator. -If a ``'fragment'`` keyword argument is provided, it must be of type -``holoscan.core.Fragment`` (or -``holoscan.core.Application``). A single ``Fragment`` object can also be +If a `fragment` keyword argument is provided, it must be of type +`holoscan.core.Fragment` (or +`holoscan.core.Application`). A single `Fragment` object can also be provided positionally instead. -Any other arguments will be cast from a Python argument type to a C++ ``Arg`` +Any other arguments will be cast from a Python argument type to a C++ `Arg` and stored in ``self.args``. (For details on how the casting is done, see the -``py_object_to_arg`` utility). +`py_object_to_arg` utility). Parameters ---------- @@ -311,9 +338,9 @@ Parameters Raises ------ RuntimeError - If ``'name'`` kwarg is provided, but is not of str type. - If multiple arguments of type ``Fragment`` are provided. - If any other arguments cannot be converted to Arg type via ``py_object_to_arg``. + If `name` kwarg is provided, but is not of `str` type. + If multiple arguments of type `Fragment` are provided. + If any other arguments cannot be converted to `Arg` type via `py_object_to_arg`. )doc") PYDOC(name, R"doc( @@ -329,7 +356,11 @@ Fragment that the resource belongs to. Returns ------- -name : ``holoscan.core.Fragment`` +name : holoscan.core.Fragment +)doc") + +PYDOC(spec, R"doc( +The resources's ComponentSpec. )doc") PYDOC(setup, R"doc( @@ -340,6 +371,10 @@ PYDOC(initialize, R"doc( initialization method for the resource. )doc") +PYDOC(description, R"doc( +YAML formatted string describing the resource. +)doc") + } // namespace Resource namespace InputContext { @@ -375,8 +410,8 @@ Class representing a message. A message is a data structure that is used to pass data between operators. It wraps a ``std::any`` object and provides a type-safe interface to access the data. -This class is used by the ``holoscan::gxf::GXFWrapper`` to support the Holoscan native operator. -The ``holoscan::gxf::GXFWrapper`` will hold the object of this class and delegate the message to the +This class is used by the `holoscan::gxf::GXFWrapper` to support the Holoscan native operator. +The `holoscan::gxf::GXFWrapper` will hold the object of this class and delegate the message to the Holoscan native operator. )doc") @@ -390,7 +425,7 @@ Component specification class. Parameters ---------- -fragment : ``holoscan.core.Fragment`` +fragment : holoscan.core.Fragment The fragment that the component belongs to. )doc") @@ -399,13 +434,17 @@ The fragment that the component belongs to. Returns ------- -name : ``holoscan.core.Fragment`` +name : holoscan.core.Fragment )doc") PYDOC(params, R"doc( The parameters associated with the component. )doc") +PYDOC(description, R"doc( +YAML formatted string describing the component spec. +)doc") + } // namespace ComponentSpec namespace IOSpec { @@ -416,11 +455,11 @@ I/O specification class. Parameters ---------- -op_spec : ``holoscan.core.OperatorSpec`` +op_spec : holoscan.core.OperatorSpec Operator specification class of the associated operator. name : str The name of the IOSpec object. -io_type : ``holoscan.core.IOSpec.IOType`` +io_type : holoscan.core.IOSpec.IOType Enum indicating whether this is an input or output specification. )doc") @@ -437,7 +476,7 @@ The type (input or output) of the I/O specification class. Returns ------- -io_type : ``holoscan.core.IOSpec.IOType`` +io_type : holoscan.core.IOSpec.IOType )doc") PYDOC(resource, R"doc( @@ -445,7 +484,7 @@ Resource class associated with this I/O specification. Returns ------- -resource : ``holoscan.core.Resource`` +resource : holoscan.core.Resource )doc") PYDOC(conditions, R"doc( @@ -453,7 +492,7 @@ List of Condition objects associated with this I/O specification. Returns ------- -condition : list of ``holoscan.core.Condition`` +condition : list of holoscan.core.Condition )doc") PYDOC(condition, R"doc( @@ -461,23 +500,23 @@ Add a condition to this input/output. The following ConditionTypes are supported: -- ConditionType.NONE -- ConditionType.MESSAGE_AVAILABLE -- ConditionType.DOWNSTREAM_MESSAGE_AFFORDABLE -- ConditionType.COUNT -- ConditionType.BOOLEAN +- `ConditionType.NONE` +- `ConditionType.MESSAGE_AVAILABLE` +- `ConditionType.DOWNSTREAM_MESSAGE_AFFORDABLE` +- `ConditionType.COUNT` +- `ConditionType.BOOLEAN` Parameters ---------- -kind : ``holoscan.core.ConditionType`` +kind : holoscan.core.ConditionType The type of the condition. **kwargs - Python keyword arguments that will be cast to an ArgList associated + Python keyword arguments that will be cast to an `ArgList` associated with the condition. Returns ------- -obj : ``holoscan.core.IOSpec`` +obj : holoscan.core.IOSpec The self object. )doc") @@ -498,7 +537,7 @@ Operator specification class. Parameters ---------- -fragment : ``holoscan.core.Fragment`` +fragment : holoscan.core.Fragment The fragment that this operator belongs to. )doc") @@ -528,36 +567,35 @@ name : str The name of the output port. )doc") +PYDOC(description, R"doc( +YAML formatted string describing the operator spec. +)doc") + } // namespace OperatorSpec namespace Operator { -PYDOC(Operator, R"doc( -Operator class.)doc") - // Constructor PYDOC(Operator_args_kwargs, R"doc( Operator class. Can be initialized with any number of Python positional and keyword arguments. -If a ``'name'`` keyword argument is provided, it must be a ``str`` and will be +If a `name` keyword argument is provided, it must be a `str` and will be used to set the name of the Operator. -If a ``fragment`` keyword argument is provided, it must be of type -``holoscan.core.Fragment`` (or -``holoscan.core.Application``). A single ``Fragment`` object can also be -provided positionally instead. - -``Condition`` classes will be added to ``self.conditions``, ``Resource`` +`Condition` classes will be added to ``self.conditions``, `Resource` classes will be added to ``self.resources``, and any other arguments will be -cast from a Python argument type to a C++ ``Arg`` and stored in ``self.args``. -(For details on how the casting is done, see the ``py_object_to_arg`` +cast from a Python argument type to a C++ `Arg` and stored in ``self.args``. +(For details on how the casting is done, see the `py_object_to_arg` utility). When a Condition or Resource is provided via a kwarg, it's name will be automatically be updated to the name of the kwarg. Parameters ---------- +fragment : holoscan.core.Fragment + The `holoscan.core.Fragment` (or `holoscan.core.Application`) to which this Operator will + belong. \*args Positional arguments. \*\*kwargs @@ -566,9 +604,9 @@ Parameters Raises ------ RuntimeError - If ``'name'`` kwarg is provided, but is not of str type. - If multiple arguments of type ``Fragment`` are provided. - If any other arguments cannot be converted to Arg type via ``py_object_to_arg``. + If `name` kwarg is provided, but is not of `str` type. + If multiple arguments of type `Fragment` are provided. + If any other arguments cannot be converted to `Arg` type via `py_object_to_arg`. )doc") PYDOC(name, R"doc( @@ -584,7 +622,7 @@ The fragment that the operator belongs to. Returns ------- -name : ``holoscan.core.Fragment`` +name : holoscan.core.Fragment )doc") PYDOC(conditions, R"doc( @@ -619,10 +657,14 @@ executed by the operator. PYDOC(operator_type, R"doc( The operator type. -``holoscan.core.Operator.OperatorType`` enum representing the type of +`holoscan.core.Operator.OperatorType` enum representing the type of the operator. The two types currently implemented are native and GXF. )doc") +PYDOC(description, R"doc( +YAML formatted string describing the operator. +)doc") + } // namespace Operator namespace Config { @@ -665,7 +707,7 @@ Executor class. Parameters ---------- -fragment : ``holoscan.core.Fragment`` +fragment : holoscan.core.Fragment The fragment that the executor is associated with. )doc") @@ -674,7 +716,7 @@ The fragment that the executor belongs to. Returns ------- -name : ``holoscan.core.Fragment`` +name : holoscan.core.Fragment )doc") PYDOC(run, R"doc( @@ -711,7 +753,7 @@ The application associated with the fragment. Returns ------- -app : ``holoscan.core.Application`` +app : holoscan.core.Application )doc") PYDOC(config_kwargs, R"doc( @@ -730,7 +772,7 @@ Get the configuration associated with the fragment. Returns ------- -config : ``holoscan.core.Config`` +config : holoscan.core.Config )doc") PYDOC(graph, R"doc( @@ -752,7 +794,7 @@ key : str Returns ------- -args : ``holoscan.core.ArgList`` +args : holoscan.core.ArgList An argument list associated with the key. )doc") @@ -777,7 +819,7 @@ Add an operator to the fragment. Parameters ---------- -op : ``holoscan.core.Operator`` +op : holoscan.core.Operator The operator to add. )doc") @@ -786,9 +828,9 @@ Connect two operators associated with the fragment. Parameters ---------- -upstream_op : ``holoscan.core.Operator`` +upstream_op : holoscan.core.Operator Source operator. -downstream_op : ``holoscan.core.Operator`` +downstream_op : holoscan.core.Operator Destination operator. port_pairs : Sequence of 2-tuples Sequence of ports to connect. The first element of each 2-tuple is a port @@ -799,7 +841,7 @@ port_pairs : Sequence of 2-tuples PYDOC(compose, R"doc( The compose method of the Fragment. -This method should be called after ``config``, but before ``run`` in order to +This method should be called after `config`, but before `run` in order to compose the computation graph. )doc") @@ -807,7 +849,7 @@ PYDOC(run, R"doc( The run method of the Fragment. This method runs the computation. It must have first been initialized via -``config`` and ``compose``. +`config` and `compose`. )doc") } // namespace Fragment @@ -832,7 +874,7 @@ Add an operator to the application. Parameters ---------- -op : ``holoscan.core.Operator`` +op : holoscan.core.Operator The operator to add. )doc") @@ -841,9 +883,9 @@ Connect two operators associated with the fragment. Parameters ---------- -upstream_op : ``holoscan.core.Operator`` +upstream_op : holoscan.core.Operator Source operator. -downstream_op : ``holoscan.core.Operator`` +downstream_op : holoscan.core.Operator Destination operator. port_pairs : Sequence of (str, str) tuples Sequence of ports to connect. The first element of each 2-tuple is a port @@ -854,7 +896,7 @@ port_pairs : Sequence of (str, str) tuples PYDOC(compose, R"doc( The compose method of the application. -This method should be called after ``config``, but before ``run`` in order to +This method should be called after `config`, but before `run` in order to compose the computation graph. )doc") @@ -862,7 +904,7 @@ PYDOC(run, R"doc( The run method of the application. This method runs the computation. It must have first been initialized via -``config`` and ``compose``. +`config` and `compose`. )doc") } // namespace Application @@ -875,14 +917,14 @@ DLDevice class. )doc") PYDOC(device_type, R"doc( -The device type (DLDeviceType). +The device type (`DLDeviceType`). The following device types are supported: -- DLDeviceType.DLCPU: system memory (kDLCPU) -- DLDeviceType.DLCUDA: CUDA GPU memory (kDLCUDA) -- DLDeviceType.DLCUDAHost: CUDA pinned memory (kDLCUDAHost) -- DLDeviceType.DLCUDAManaged: CUDA managed memory (kDLCUDAManaged) +- `DLDeviceType.DLCPU`: system memory (kDLCPU) +- `DLDeviceType.DLCUDA`: CUDA GPU memory (kDLCUDA) +- `DLDeviceType.DLCUDAHost`: CUDA pinned memory (kDLCUDAHost) +- `DLDeviceType.DLCUDAManaged`: CUDA managed memory (kDLCUDAManaged) )doc") PYDOC(device_id, R"doc( @@ -954,7 +996,7 @@ The size of the tensor's data in bytes. namespace Core { PYDOC(py_object_to_arg, R"doc( -Utility that converts a single python argument to a corresponding Arg type. +Utility that converts a single python argument to a corresponding `Arg` type. Parameters ---------- @@ -963,16 +1005,16 @@ value : Any Returns ------- -obj : ``holoscan.core.Arg`` - Arg class corresponding to the provided value. For example a Python float - will become an Arg containing a C++ double while a list of Python ints - would become an Arg corresponding to a ``std::vector``. +obj : holoscan.core.Arg + `Arg` class corresponding to the provided value. For example a Python float + will become an `Arg` containing a C++ double while a list of Python ints + would become an `Arg` corresponding to a ``std::vector``. name : str, optional A name to assign to the argument. )doc") PYDOC(kwargs_to_arglist, R"doc( -Utility that converts a set of python keyword arguments to an ArgList. +Utility that converts a set of python keyword arguments to an `ArgList`. Parameters ---------- @@ -981,18 +1023,18 @@ Parameters Returns ------- -arglist : ``holoscan.core.ArgList`` - ArgList class corresponding to the provided keyword values. The argument +arglist : holoscan.core.ArgList + `ArgList` class corresponding to the provided keyword values. The argument names will match the keyword names. Values will be converted as for - ``py_object_to_arg``. + `py_object_to_arg`. )doc") PYDOC(arg_to_py_object, R"doc( -Utility that converts an Arg to a corresponding Python object. +Utility that converts an `Arg` to a corresponding Python object. Parameters ---------- -arg : ``holoscan.core.Arg`` +arg : holoscan.core.Arg The argument to convert. Returns @@ -1004,18 +1046,18 @@ obj : Any )doc") PYDOC(arglist_to_kwargs, R"doc( -Utility that converts an ArgList to a Python kwargs dictionary. +Utility that converts an `ArgList` to a Python kwargs dictionary. Parameters ---------- -arglist : ``holoscan.core.ArgList`` +arglist : holoscan.core.ArgList The argument list to convert. Returns ------- kwargs : dict Python dictionary with keys matching the names of the arguments in - ArgList. The values will be converted as for ``arg_to_py_object``. + `ArgList`. The values will be converted as for `arg_to_py_object`. )doc") } // namespace Core diff --git a/python/pybind11/core/dl_converter.cpp b/python/src/core/dl_converter.cpp similarity index 100% rename from python/pybind11/core/dl_converter.cpp rename to python/src/core/dl_converter.cpp diff --git a/python/pybind11/core/dl_converter.hpp b/python/src/core/dl_converter.hpp similarity index 100% rename from python/pybind11/core/dl_converter.hpp rename to python/src/core/dl_converter.hpp diff --git a/python/pybind11/core/trampolines.cpp b/python/src/core/trampolines.cpp similarity index 84% rename from python/pybind11/core/trampolines.cpp rename to python/src/core/trampolines.cpp index 4bed20fc..5cf3b0a0 100644 --- a/python/pybind11/core/trampolines.cpp +++ b/python/src/core/trampolines.cpp @@ -60,10 +60,16 @@ py::object PyInputContext::py_receive(const std::string& name) { // Check element type (querying the first element using the name '{name}:0') auto& element = any_result[0]; - if (element.type() == typeid(holoscan::gxf::Entity)) { + auto& element_type = element.type(); + + if (element_type == typeid(holoscan::gxf::Entity)) { std::vector result; try { for (auto& any_item : any_result) { + if (element_type == typeid(nullptr_t)) { + result.emplace_back(); + continue; + } result.push_back(std::any_cast(any_item)); } } catch (const std::bad_any_cast& e) { @@ -75,10 +81,14 @@ py::object PyInputContext::py_receive(const std::string& name) { } py::tuple result_tuple = vector2pytuple(result); return result_tuple; - } else if (element.type() == typeid(std::shared_ptr)) { + } else if (element_type == typeid(std::shared_ptr)) { std::vector> result; try { for (auto& any_item : any_result) { + if (element_type == typeid(nullptr_t)) { + result.emplace_back(nullptr); + continue; + } result.push_back(std::any_cast>(any_item)); } } catch (const std::bad_any_cast& e) { @@ -91,22 +101,25 @@ py::object PyInputContext::py_receive(const std::string& name) { py::tuple result_tuple = vector2pytuple(result); return result_tuple; } else { - HOLOSCAN_LOG_ERROR("Messages are not available"); return py::none(); } } else { auto result = receive(name.c_str()); - if (result.type() == typeid(holoscan::gxf::Entity)) { + auto& result_type = result.type(); + + if (result_type == typeid(holoscan::gxf::Entity)) { auto in_entity = std::any_cast(result); // Create a shared Entity (increase ref count) holoscan::PyEntity entity_wrapper(in_entity); return py::cast(entity_wrapper); - } else if (result.type() == typeid(std::shared_ptr)) { + } else if (result_type == typeid(std::shared_ptr)) { auto in_message = std::any_cast>(result); return in_message->obj(); + } else if (result_type == typeid(nullptr_t)) { + return py::none(); } else { - HOLOSCAN_LOG_ERROR("Message is not available"); + HOLOSCAN_LOG_ERROR("Invalid message type: {}", result_type.name()); return py::none(); } } diff --git a/python/pybind11/core/trampolines.hpp b/python/src/core/trampolines.hpp similarity index 93% rename from python/pybind11/core/trampolines.hpp rename to python/src/core/trampolines.hpp index 30a9bab3..85fcd86b 100644 --- a/python/pybind11/core/trampolines.hpp +++ b/python/src/core/trampolines.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -344,11 +344,13 @@ class PyOperator : public Operator { // Define a kwargs-based constructor that can create an ArgList // for passing on to the variadic-template based constructor. - PyOperator(py::object op, const py::args& args, const py::kwargs& kwargs) : Operator() { + PyOperator(py::object op, Fragment* fragment, const py::args& args, const py::kwargs& kwargs) + : Operator() { using std::string_literals::operator""s; HOLOSCAN_LOG_TRACE("PyOperator::PyOperator()"); py_op_ = op; + fragment_ = fragment; int n_fragments = 0; for (auto& item : args) { @@ -358,9 +360,7 @@ class PyOperator : public Operator { } else if (py::isinstance(arg_value)) { this->add_arg(arg_value.cast>()); } else if (py::isinstance(arg_value)) { - if (n_fragments > 0) { throw std::runtime_error("multiple Fragment objects provided"); } - fragment_ = arg_value.cast(); - n_fragments += 1; + throw std::runtime_error("multiple Fragment objects provided"); } else if (py::isinstance(arg_value)) { this->add_arg(arg_value.cast()); } else if (py::isinstance(arg_value)) { @@ -380,11 +380,8 @@ class PyOperator : public Operator { } } else if (kwarg_name == "fragment"s) { if (py::isinstance(kwarg_value)) { - if (n_fragments > 0) { - throw std::runtime_error( - "Cannot add kwarg fragment, when a Fragment was also provided positionally"); - } - fragment_ = kwarg_value.cast(); + throw std::runtime_error( + "Cannot add kwarg fragment. Fragment can only be provided positionally"); } else { throw std::runtime_error("fragment kwarg must be a Fragment"); } @@ -439,7 +436,21 @@ class PyOperator : public Operator { // Get the compute method of the Python Operator class and call it py::gil_scoped_acquire scope_guard; py::object py_compute = py::getattr(py_op_, "compute"); - py_compute.operator()(py::cast(py_op_input), py::cast(py_op_output), py::cast(py_context)); + try { + py_compute.operator()(py::cast(py_op_input), py::cast(py_op_output), py::cast(py_context)); + } catch (const py::error_already_set& e) { + // Print the Python error to stderr + auto stderr = py::module::import("sys").attr("stderr"); + + py::print(fmt::format("Exception occurred for operator: '{}'", name_), + py::arg("file") = stderr); + py::module::import("traceback") + .attr("print_exception")(e.type(), e.value(), e.trace(), py::none(), stderr); + + // Note:: We don't want to throw an exception here, because it will cause the Python + // interpreter to exit. Instead, we'll just log the error and continue. + // throw std::runtime_error(fmt::format("Python error in compute method: {}", e.what())); + } } } diff --git a/python/pybind11/executors/__init__.py b/python/src/executors/__init__.py similarity index 100% rename from python/pybind11/executors/__init__.py rename to python/src/executors/__init__.py diff --git a/python/pybind11/executors/executors.cpp b/python/src/executors/executors.cpp similarity index 92% rename from python/pybind11/executors/executors.cpp rename to python/src/executors/executors.cpp index 1b404356..8472df46 100644 --- a/python/pybind11/executors/executors.cpp +++ b/python/src/executors/executors.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -63,6 +63,7 @@ PYBIND11_MODULE(_executors, m) { "gxf_context"_a, "eid"_a, "io_spec"_a, + "bind_port"_a = false, doc::GXFExecutor::doc_create_input_port); m.def("create_output_port", @@ -71,6 +72,7 @@ PYBIND11_MODULE(_executors, m) { "gxf_context"_a, "eid"_a, "io_spec"_a, + "bind_port"_a = false, doc::GXFExecutor::doc_create_output_port); } // PYBIND11_MODULE } // namespace holoscan diff --git a/python/pybind11/executors/executors_pydoc.hpp b/python/src/executors/executors_pydoc.hpp similarity index 84% rename from python/pybind11/executors/executors_pydoc.hpp rename to python/src/executors/executors_pydoc.hpp index 5cc5a54b..909ff575 100644 --- a/python/pybind11/executors/executors_pydoc.hpp +++ b/python/src/executors/executors_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -54,7 +54,7 @@ types. Parameters ---------- -fragment : Fragment +fragment : holoscan.core.Fragment The fragment that this operator belongs to. gxf_context : PyCapsule object The operator's associated GXF context. @@ -62,6 +62,9 @@ eid : int The GXF entity ID. io_spec : IOSpec Input port specification. +bind_port : bool + If ``True``, bind the port to the existing GXF Receiver component. + Otherwise, create a new GXF Receiver component. )doc") PYDOC(create_output_port, R"doc( @@ -77,7 +80,7 @@ condition types. Parameters ---------- -fragment : Fragment +fragment : holoscan.core.Fragment The fragment that this operator belongs to. gxf_context : PyCapsule object The operator's associated GXF context. @@ -85,6 +88,9 @@ eid : int The GXF entity ID. io_spec : IOSpec Output port specification. +bind_port : bool + If ``True``, bind the port to the existing GXF Transmitter component. + Otherwise, create a new GXF Transmitter component. )doc") } // namespace GXFExecutor diff --git a/python/pybind11/graphs/__init__.py b/python/src/graphs/__init__.py similarity index 100% rename from python/pybind11/graphs/__init__.py rename to python/src/graphs/__init__.py diff --git a/python/pybind11/graphs/graphs.cpp b/python/src/graphs/graphs.cpp similarity index 100% rename from python/pybind11/graphs/graphs.cpp rename to python/src/graphs/graphs.cpp diff --git a/python/pybind11/graphs/graphs_pydoc.hpp b/python/src/graphs/graphs_pydoc.hpp similarity index 90% rename from python/pybind11/graphs/graphs_pydoc.hpp rename to python/src/graphs/graphs_pydoc.hpp index 144aa233..806648d8 100644 --- a/python/pybind11/graphs/graphs_pydoc.hpp +++ b/python/src/graphs/graphs_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -45,7 +45,7 @@ Add a node to the graph. Parameters ---------- -op : Operator +op : holoscan.core.Operator The node to add. )doc") @@ -68,9 +68,9 @@ Get a port mapping dictionary between two operators in the graph. Parameters ---------- -op_u : Operator +op_u : holoscan.core.Operator The source operator. -op_v : Operator +op_v : holoscan.core.Operator The destination operator. Returns @@ -85,7 +85,7 @@ Check if an operator is a root operator. Parameters ---------- -op : Operator +op : holoscan.core.Operator A node in the graph. Returns @@ -99,7 +99,7 @@ Check if an operator is a leaf operator. Parameters ---------- -op : Operator +op : holoscan.core.Operator A node in the graph. Returns @@ -131,7 +131,7 @@ Get the operators immediately downstream of a given operator. Parameters ---------- -op : Operator +op : holoscan.core.Operator A node in the graph. Returns diff --git a/python/pybind11/gxf/__init__.py b/python/src/gxf/__init__.py similarity index 84% rename from python/pybind11/gxf/__init__.py rename to python/src/gxf/__init__.py index fc71f700..812acb52 100644 --- a/python/pybind11/gxf/__init__.py +++ b/python/src/gxf/__init__.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,6 @@ holoscan.gxf.GXFComponent holoscan.gxf.GXFCondition holoscan.gxf.GXFExecutionContext - holoscan.gxf.GXFExtensionRegistrar holoscan.gxf.GXFInputContext holoscan.gxf.GXFOperator holoscan.gxf.GXFOutputContext @@ -30,24 +29,22 @@ from ._gxf import ( - PyEntity as Entity, GXFComponent, GXFCondition, GXFExecutionContext, - GXFExtensionRegistrar, GXFInputContext, GXFOperator, GXFOutputContext, GXFResource, - load_extensions, ) +from ._gxf import PyEntity as Entity +from ._gxf import load_extensions __all__ = [ "Entity", "GXFComponent", "GXFCondition", "GXFExecutionContext", - "GXFExtensionRegistrar", "GXFInputContext", "GXFOperator", "GXFOutputContext", diff --git a/python/pybind11/gxf/gxf.cpp b/python/src/gxf/gxf.cpp similarity index 68% rename from python/pybind11/gxf/gxf.cpp rename to python/src/gxf/gxf.cpp index 3c1afb10..c500186f 100644 --- a/python/pybind11/gxf/gxf.cpp +++ b/python/src/gxf/gxf.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-3023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -31,6 +31,7 @@ #include "holoscan/core/gxf/gxf_component.hpp" #include "holoscan/core/gxf/gxf_condition.hpp" #include "holoscan/core/gxf/gxf_execution_context.hpp" +#include "holoscan/core/gxf/gxf_extension_manager.hpp" #include "holoscan/core/gxf/gxf_extension_registrar.hpp" #include "holoscan/core/gxf/gxf_io_context.hpp" #include "holoscan/core/gxf/gxf_operator.hpp" @@ -91,39 +92,26 @@ PYBIND11_MODULE(_gxf, m) { m.attr("__version__") = "dev"; #endif - // Note: here we copy load_extensions from GXF's Python bindings for convenience + // TODO: This method can be removed once Executor::extension_manager(), + // ExtensionManager, GXFExtensionManager are exposed to Python. m.def( "load_extensions", [](uint64_t context, const std::vector& extension_filenames, - const std::vector& manifest_filenames, - const std::string& base_directory) { - std::vector extension_filenames_cstr(extension_filenames.size()); - for (size_t i = 0; i < extension_filenames.size(); i++) { - extension_filenames_cstr[i] = extension_filenames[i].c_str(); + const std::vector& manifest_filenames) { + gxf::GXFExtensionManager extension_manager(reinterpret_cast(context)); + for (const auto& extension_filename : extension_filenames) { + extension_manager.load_extension(extension_filename); } - - std::vector manifest_filenames_cstr(manifest_filenames.size()); - for (size_t i = 0; i < manifest_filenames.size(); i++) { - manifest_filenames_cstr[i] = manifest_filenames[i].c_str(); + for (const auto& manifest_filename : manifest_filenames) { + auto node = YAML::LoadFile(manifest_filename); + extension_manager.load_extensions_from_yaml(node); } - - const GxfLoadExtensionsInfo info{ - extension_filenames_cstr.empty() ? nullptr : extension_filenames_cstr.data(), - static_cast(extension_filenames_cstr.size()), - manifest_filenames_cstr.empty() ? nullptr : manifest_filenames_cstr.data(), - static_cast(manifest_filenames_cstr.size()), - base_directory.c_str()}; - - const gxf_result_t result = - GxfLoadExtensions(reinterpret_cast(context), &info); - if (result != GXF_SUCCESS) { throw pybind11::value_error(GxfResultStr(result)); } }, "Loads GXF extension libraries", "context"_a, "extension_filenames"_a = std::vector{}, - "manifest_filenames"_a = std::vector{}, - "base_directory"_a = ""); + "manifest_filenames"_a = std::vector{}); py::class_>(m, "Entity", doc::Entity::doc_Entity) .def(py::init<>(), doc::Entity::doc_Entity); @@ -131,6 +119,7 @@ PYBIND11_MODULE(_gxf, m) { py::class_>( m, "PyEntity", doc::Entity::doc_Entity) .def(py::init(&PyEntity::py_create), doc::Entity::doc_Entity) + .def("__bool__", &PyEntity::operator bool) .def("get", &PyEntity::py_get, "name"_a = "", doc::Entity::doc_get) .def("add", &PyEntity::py_add, "obj"_a, "name"_a = ""); @@ -184,8 +173,75 @@ PYBIND11_MODULE(_gxf, m) { .def_property("gxf_cid", py::overload_cast<>(&ops::GXFOperator::gxf_cid, py::const_), py::overload_cast(&ops::GXFOperator::gxf_cid), - doc::GXFOperator::doc_gxf_cid); - // Note: have not wrapped register_converter as it is currently not needed from Python. + doc::GXFOperator::doc_gxf_cid) + .def( + "__repr__", + // had to remove const here to access conditions(), etc. + [](ops::GXFOperator& op) { + auto msg_buf = fmt::memory_buffer(); + // print names of all input ports + fmt::format_to(std::back_inserter(msg_buf), "GXFOperator:\n\tname = {}", op.name()); + fmt::format_to(std::back_inserter(msg_buf), "\n\tconditions = ("); + for (auto cond : op.conditions()) { + fmt::format_to(std::back_inserter(msg_buf), "{}, ", cond.first); + } + fmt::format_to(std::back_inserter(msg_buf), ")"); + + // print names of all output ports + fmt::format_to(std::back_inserter(msg_buf), "\n\tresources = ("); + for (auto res : op.resources()) { + fmt::format_to(std::back_inserter(msg_buf), "{}, ", res.first); + } + fmt::format_to(std::back_inserter(msg_buf), ")"); + + // print names of all args + fmt::format_to(std::back_inserter(msg_buf), "\n\targs = ("); + for (auto arg : op.args()) { + fmt::format_to(std::back_inserter(msg_buf), "{}, ", arg.name()); + } + fmt::format_to(std::back_inserter(msg_buf), ")"); + + // Print attribute of the OperatorSpec corresponding to this operator + + // fmt::format_to(std::back_inserter(msg_buf), "\n\tspec:\n\t\tinputs = ("); + // for (auto name : get_names_from_map(op.spec()->inputs())) { + // fmt::format_to(std::back_inserter(msg_buf), "{}, ", name); + // } + // fmt::format_to(std::back_inserter(msg_buf), ")"); + + // // print names of all output ports + // fmt::format_to(std::back_inserter(msg_buf), "\n\t\toutputs = ("); + // for (auto name : get_names_from_map(op.spec()->outputs())) { + // fmt::format_to(std::back_inserter(msg_buf), "{}, ", name); + // } + // fmt::format_to(std::back_inserter(msg_buf), ")"); + + // // print names of all conditions + // fmt::format_to(std::back_inserter(msg_buf), "\n\t\tconditions = ("); + // for (auto name : get_names_from_map(op.spec()->conditions())) { + // fmt::format_to(std::back_inserter(msg_buf), "{}, ", name); + // } + // fmt::format_to(std::back_inserter(msg_buf), ")"); + + // // print names of all resources + // fmt::format_to(std::back_inserter(msg_buf), "\n\t\tresources = ("); + // for (auto name : get_names_from_map(op.spec()->resources())) { + // fmt::format_to(std::back_inserter(msg_buf), "{}, ", name); + // } + // fmt::format_to(std::back_inserter(msg_buf), ")"); + + // // print names of all parameters + // fmt::format_to(std::back_inserter(msg_buf), "\n\t\tparams = ("); + // for (auto& p : op.spec()->params()) { + // fmt::format_to(std::back_inserter(msg_buf), "{}, ", p.first); + // } + // fmt::format_to(std::back_inserter(msg_buf), ")"); + + std::string repr = fmt::format("{:.{}}", msg_buf.data(), msg_buf.size()); + return repr; + }, + py::call_guard(), + R"doc(Return repr(self).)doc"); py::class_>( m, "GXFInputContext", R"doc(GXF input context.)doc") @@ -215,59 +271,6 @@ PYBIND11_MODULE(_gxf, m) { "op"_a, doc::GXFExecutionContext::doc_GXFExecutionContext); - py::class_ gxf_extension_registrar( - m, "GXFExtensionRegistrar", doc::GXFExtensionRegistrar::doc_GXFExtensionRegistrar); - - py::class_(m, "gxf_tid_t"); - - gxf_extension_registrar - .def(py::init(), - "context"_a, - "extension_name"_a, - "extension_description"_a = "", - "tid"_a = default_tid, - doc::GXFExtensionRegistrar::doc_GXFExtensionRegistrar) - .def("create_random_tid", - &gxf::GXFExtensionRegistrar::create_random_tid, - doc::GXFExtensionRegistrar::doc_create_random_tid) - .def("is_allocated", - &gxf::GXFExtensionRegistrar::is_allocated, - "tid"_a, - "kind"_a, - doc::GXFExtensionRegistrar::doc_is_allocated) - .def("allocate_tid", - &gxf::GXFExtensionRegistrar::allocate_tid, - "kind"_a, - doc::GXFExtensionRegistrar::doc_allocate_tid) - .def("register_extension", - &gxf::GXFExtensionRegistrar::register_extension, - doc::GXFExtensionRegistrar::doc_register_extension) - .def("reset", - &gxf::GXFExtensionRegistrar::reset, - "context"_a, - "extension_name"_a, - "extension_description"_a = "", - "tid"_a = default_tid, - doc::GXFExtensionRegistrar::doc_reset) - // templated add_component methods - .def("add_component", - py::overload_cast( - &gxf::GXFExtensionRegistrar::add_component), - "description"_a = "", - "tid"_a = default_tid, - doc::GXFExtensionRegistrar::doc_add_component) - // templated add_type methods - .def( - "add_type", - py::overload_cast(&gxf::GXFExtensionRegistrar::add_type), - "description"_a = "", - "tid"_a = default_tid, - doc::GXFExtensionRegistrar::doc_add_type); - - py::enum_(gxf_extension_registrar, "TypeKind") - .value("Extension", gxf::GXFExtensionRegistrar::TypeKind::kExtension) - .value("Component", gxf::GXFExtensionRegistrar::TypeKind::kComponent); - py::class_(m, "GXFWrapper", doc::GXFWrapper::doc_GXFWrapper) .def(py::init<>(), doc::GXFWrapper::doc_GXFWrapper) .def("initialize", &gxf::GXFWrapper::initialize, doc::GXFWrapper::doc_initialize) diff --git a/python/pybind11/gxf/gxf.hpp b/python/src/gxf/gxf.hpp similarity index 95% rename from python/pybind11/gxf/gxf.hpp rename to python/src/gxf/gxf.hpp index bde9841d..079be6f6 100644 --- a/python/pybind11/gxf/gxf.hpp +++ b/python/src/gxf/gxf.hpp @@ -39,6 +39,8 @@ class PyEntity : public gxf::Entity { static PyEntity py_create(const PyExecutionContext& ctx); + using gxf::Entity::operator bool; // inherit operator bool + py::object py_get(const char* name = nullptr) const; py::object py_add(const py::object& value, const char* name = nullptr); }; diff --git a/python/pybind11/gxf/gxf_pydoc.hpp b/python/src/gxf/gxf_pydoc.hpp similarity index 90% rename from python/pybind11/gxf/gxf_pydoc.hpp rename to python/src/gxf/gxf_pydoc.hpp index bff45a83..9c633393 100644 --- a/python/pybind11/gxf/gxf_pydoc.hpp +++ b/python/src/gxf/gxf_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -231,35 +231,6 @@ op : holoscan.gxf.GXFOperator } // namespace GXFExecutionContext -namespace GXFExtensionRegistrar { - -// Constructor -PYDOC(GXFExtensionRegistrar, R"doc( -)doc") - -PYDOC(is_allocated, R"doc( -)doc") - -PYDOC(create_random_tid, R"doc( -)doc") - -PYDOC(allocate_tid, R"doc( -)doc") - -PYDOC(register_extension, R"doc( -)doc") - -PYDOC(reset, R"doc( -)doc") - -PYDOC(add_component, R"doc( -)doc") - -PYDOC(add_type, R"doc( -)doc") - -} // namespace GXFExtensionRegistrar - namespace GXFWrapper { // Constructor diff --git a/python/pybind11/kwarg_handling.cpp b/python/src/kwarg_handling.cpp similarity index 100% rename from python/pybind11/kwarg_handling.cpp rename to python/src/kwarg_handling.cpp diff --git a/python/pybind11/kwarg_handling.hpp b/python/src/kwarg_handling.hpp similarity index 100% rename from python/pybind11/kwarg_handling.hpp rename to python/src/kwarg_handling.hpp diff --git a/python/pybind11/logger/__init__.py b/python/src/logger/__init__.py similarity index 100% rename from python/pybind11/logger/__init__.py rename to python/src/logger/__init__.py diff --git a/python/pybind11/logger/logger.cpp b/python/src/logger/logger.cpp similarity index 94% rename from python/pybind11/logger/logger.cpp rename to python/src/logger/logger.cpp index 575563aa..89f95b94 100644 --- a/python/pybind11/logger/logger.cpp +++ b/python/src/logger/logger.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,8 +19,8 @@ #include +#include "holoscan/logger/logger.hpp" #include "logger_pydoc.hpp" -#include "holoscan/core/logger.hpp" #define STRINGIFY(x) #x #define MACRO_STRINGIFY(x) STRINGIFY(x) diff --git a/python/pybind11/logger/logger_pydoc.hpp b/python/src/logger/logger_pydoc.hpp similarity index 92% rename from python/pybind11/logger/logger_pydoc.hpp rename to python/src/logger/logger_pydoc.hpp index 1a9807ae..b3b13fb8 100644 --- a/python/pybind11/logger/logger_pydoc.hpp +++ b/python/src/logger/logger_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -51,7 +51,7 @@ Set the global logging level. Parameters ---------- -level : ``holoscan.logger.LogLevel`` +level : holoscan.logger.LogLevel The logging level to set )doc") @@ -110,7 +110,7 @@ Determine the minimum log level that will trigger an automatic flush. Returns ------- -level : ``holoscan.logger.LogLevel`` +level : holoscan.logger.LogLevel The level at which the flush occurs. )doc") @@ -119,7 +119,7 @@ Sets the minimum log level that will trigger an automatic flush. Parameters ---------- -level : ``holoscan.logger.LogLevel`` +level : holoscan.logger.LogLevel The level at which the logger should automatically flush. )doc") diff --git a/python/pybind11/macros.hpp b/python/src/macros.hpp similarity index 100% rename from python/pybind11/macros.hpp rename to python/src/macros.hpp diff --git a/python/pybind11/operators/__init__.py b/python/src/operators/__init__.py similarity index 71% rename from python/pybind11/operators/__init__.py rename to python/src/operators/__init__.py index 074c8dd3..1cf4c408 100644 --- a/python/pybind11/operators/__init__.py +++ b/python/src/operators/__init__.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,72 +20,50 @@ holoscan.operators.BayerDemosaicOp holoscan.operators.FormatConverterOp holoscan.operators.HolovizOp - holoscan.operators.LSTMTensorRTInferenceOp holoscan.operators.MultiAIInferenceOp holoscan.operators.MultiAIPostprocessorOp holoscan.operators.NTV2Channel + holoscan.operators.PingRxOp + holoscan.operators.PingTxOp holoscan.operators.SegmentationPostprocessorOp holoscan.operators.TensorRTInferenceOp - holoscan.operators.ToolTrackingPostprocessorOp holoscan.operators.VideoStreamRecorderOp holoscan.operators.VideoStreamReplayerOp - holoscan.operators.VisualizerICardioOp """ -import os from collections.abc import MutableMapping, Sequence -from os.path import join -from ..config import holoscan_gxf_extensions_path from ..core import IOSpec from ..resources import UnboundedAllocator +from ._native import PingRxOp, PingTxOp from ._operators import AJASourceOp, BayerDemosaicOp, FormatConverterOp from ._operators import HolovizOp as _HolovizOp -from ._operators import LSTMTensorRTInferenceOp as _LSTMTensorRTInferenceOp from ._operators import ( MultiAIInferenceOp, MultiAIPostprocessorOp, NTV2Channel, SegmentationPostprocessorOp, TensorRTInferenceOp, - ToolTrackingPostprocessorOp, VideoStreamRecorderOp, VideoStreamReplayerOp, - VisualizerICardioOp, ) -# emergent source operator is not always available, depending on build options -try: - from ._operators import EmergentSourceOp - - have_emergent_op = True -except ImportError: - have_emergent_op = False - - __all__ = [ "AJASourceOp", "BayerDemosaicOp", "FormatConverterOp", "HolovizOp", - "LSTMTensorRTInferenceOp", "MultiAIInferenceOp", "MultiAIPostprocessorOp", "NTV2Channel", + "PingRxOp", + "PingTxOp", "SegmentationPostprocessorOp", "TensorRTInferenceOp", - "ToolTrackingPostprocessorOp", "VideoStreamRecorderOp", "VideoStreamReplayerOp", - "VisualizerICardioOp", ] -if have_emergent_op: - __all__ += ["EmergentSourceOp"] - -shader_path = join(holoscan_gxf_extensions_path, "visualizer_tool_tracking", "glsl") -font_path = join(holoscan_gxf_extensions_path, "visualizer_tool_tracking", "fonts") -del holoscan_gxf_extensions_path VIZ_TOOL_DEFAULT_COLORS = [ [0.12, 0.47, 0.71], @@ -102,68 +80,6 @@ [1.00, 1.00, 0.60], ] - -class LSTMTensorRTInferenceOp(_LSTMTensorRTInferenceOp): - def __init__( - self, - fragment, - pool, - cuda_stream_pool, - model_file_path, - engine_cache_dir, - input_tensor_names, - input_state_tensor_names, - input_binding_names, - output_tensor_names, - output_state_tensor_names, - output_binding_names, - max_workspace_size, - force_engine_update=False, - verbose=True, - enable_fp16_=True, - name="lstm_inferer", - ): - if not os.path.exists(model_file_path): - raise ValueError(f"Could not locate model file: ({model_file_path=})") - if not os.path.exists(engine_cache_dir): - raise ValueError(f"Could not engine cache directory: ({engine_cache_dir=})") - - if len(input_binding_names) != len(input_tensor_names): - raise ValueError("lengths of `input_binding_names` and `input_tensor_names` must match") - if len(output_binding_names) != len(output_tensor_names): - raise ValueError( - "lengths of `output_binding_names` and `output_tensor_names` must match" - ) - - if pool is None: - cuda_stream_pool = UnboundedAllocator(fragment=self, name="pool") - - super().__init__( - fragment=fragment, - pool=pool, - cuda_stream_pool=cuda_stream_pool, - model_file_path=model_file_path, - engine_cache_dir=engine_cache_dir, - input_tensor_names=input_tensor_names, - input_state_tensor_names=input_state_tensor_names, - input_binding_names=input_binding_names, - output_tensor_names=output_tensor_names, - output_state_tensor_names=output_state_tensor_names, - output_binding_names=output_binding_names, - max_workspace_size=max_workspace_size, - force_engine_update=force_engine_update, - verbose=verbose, - enable_fp16_=enable_fp16_, - name=name, - ) - - -# copy docstrings defined in operators_pydoc.hpp -LSTMTensorRTInferenceOp.__doc__ = _LSTMTensorRTInferenceOp.__doc__ -LSTMTensorRTInferenceOp.__init__.__doc__ = _LSTMTensorRTInferenceOp.__init__.__doc__ # noqa -# TODO: remove from operators_pydoc.hpp and just define docstrings in this file? - - _holoviz_str_to_input_type = { "unknown": _HolovizOp.InputType.UNKNOWN, "color": _HolovizOp.InputType.COLOR, @@ -176,6 +92,14 @@ def __init__( "rectangles": _HolovizOp.InputType.RECTANGLES, "ovals": _HolovizOp.InputType.OVALS, "text": _HolovizOp.InputType.TEXT, + "depth_map": _HolovizOp.InputType.DEPTH_MAP, + "depth_map_color": _HolovizOp.InputType.DEPTH_MAP_COLOR, +} + +_holoviz_str_to_depth_map_render_mode = { + "points": _HolovizOp.DepthMapRenderMode.POINTS, + "lines": _HolovizOp.DepthMapRenderMode.LINES, + "triangles": _HolovizOp.DepthMapRenderMode.TRIANGLES, } @@ -197,9 +121,10 @@ def __init__( headless=False, enable_render_buffer_input=False, enable_render_buffer_output=False, + font_path="", + cuda_stream_pool=None, name="holoviz_op", ): - if allocator is None: allocator = UnboundedAllocator(fragment) @@ -245,6 +170,7 @@ def __init__( "line_width", "point_size", "text", + "depth_map_render_mode", } unrecognized_keys = set(tensor.keys()) - valid_keys if unrecognized_keys: @@ -287,6 +213,26 @@ def __init__( else: text = [] ispec._text = tensor.get("text", text) + if "depth_map_render_mode" in tensor: + depth_map_render_mode = tensor["depth_map_render_mode"] + if isinstance(depth_map_render_mode, str): + depth_map_render_mode.lower() + if depth_map_render_mode not in _holoviz_str_to_depth_map_render_mode: + raise ValueError( + f"unrecognized depth_map_render_mode name: {depth_map_render_mode}" + ) + depth_map_render_mode = _holoviz_str_to_depth_map_render_mode[ + depth_map_render_mode + ] + elif not isinstance(input_type, _HolovizOp.DepthMapRenderMode): + raise ValueError( + "value corresponding to key 'depth_map_render_mode' must be either a " + "HolovizOp.DepthMapRenderMode object or one of the following " + f"strings: {tuple(_holoviz_str_to_depth_map_render_mode.keys())}" + ) + else: + depth_map_render_mode = _HolovizOp.DepthMapRenderMode.POINTS + ispec._depth_map_render_mode = depth_map_render_mode tensor_input_specs.append(ispec) super().__init__( @@ -305,6 +251,8 @@ def __init__( headless=headless, enable_render_buffer_input=enable_render_buffer_input, enable_render_buffer_output=enable_render_buffer_output, + font_path=font_path, + cuda_stream_pool=cuda_stream_pool, name=name, ) diff --git a/python/src/operators/_native.py b/python/src/operators/_native.py new file mode 100644 index 00000000..9e9e00cf --- /dev/null +++ b/python/src/operators/_native.py @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from holoscan.core import Operator, OperatorSpec + + +class PingTxOp(Operator): + """Simple transmitter operator. + + This operator has a single output port: + output: "out" + + On each tick, it transmits an integer to the "out" port. + """ + + def __init__(self, fragment, *args, **kwargs): + self.index = 1 + # Need to call the base class constructor last + super().__init__(fragment, *args, **kwargs) + + def setup(self, spec: OperatorSpec): + spec.output("out") + + def compute(self, op_input, op_output, context): + op_output.emit(self.index, "out") + self.index += 1 + + +class PingRxOp(Operator): + """Simple receiver operator. + + This operator has a single input port: + input: "in" + + This is an example of a native operator with one input port. + On each tick, it receives an integer from the "in" port. + """ + + def __init__(self, fragment, *args, **kwargs): + # Need to call the base class constructor last + super().__init__(fragment, *args, **kwargs) + + def setup(self, spec: OperatorSpec): + spec.input("in") + + def compute(self, op_input, op_output, context): + value = op_input.receive("in") + print(f"Rx message value: {value}") diff --git a/python/pybind11/operators/operators.cpp b/python/src/operators/operators.cpp similarity index 72% rename from python/pybind11/operators/operators.cpp rename to python/src/operators/operators.cpp index d6d55df1..f29e9a95 100644 --- a/python/pybind11/operators/operators.cpp +++ b/python/src/operators/operators.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -36,7 +36,19 @@ #include "holoscan/core/resources/gxf/block_memory_pool.hpp" #include "holoscan/core/resources/gxf/cuda_stream_pool.hpp" #include "holoscan/core/resources/gxf/unbounded_allocator.hpp" -#include "holoscan/std_ops.hpp" + +#include "holoscan/operators/aja_source/aja_source.hpp" +#include "holoscan/operators/bayer_demosaic/bayer_demosaic.hpp" +#include "holoscan/operators/format_converter/format_converter.hpp" +#include "holoscan/operators/holoviz/holoviz.hpp" +#include "holoscan/operators/multiai_inference/multiai_inference.hpp" +#include "holoscan/operators/multiai_postprocessor/multiai_postprocessor.hpp" +#include "holoscan/operators/ping_rx/ping_rx.hpp" +#include "holoscan/operators/ping_tx/ping_tx.hpp" +#include "holoscan/operators/segmentation_postprocessor/segmentation_postprocessor.hpp" +#include "holoscan/operators/tensor_rt/tensor_rt_inference.hpp" +#include "holoscan/operators/video_stream_recorder/video_stream_recorder.hpp" +#include "holoscan/operators/video_stream_replayer/video_stream_replayer.hpp" using std::string_literals::operator""s; using pybind11::literals::operator""_a; @@ -84,6 +96,8 @@ class PyFormatConverterOp : public FormatConverterOp { float scale_max = 1.f, uint8_t alpha_value = static_cast(255), int32_t resize_height = 0, int32_t resize_width = 0, int32_t resize_mode = 0, const std::vector out_channel_order = std::vector{}, + std::shared_ptr cuda_stream_pool = + std::shared_ptr(), const std::string& name = "format_converter") : FormatConverterOp(ArgList{Arg{"in_tensor_name", in_tensor_name}, Arg{"in_dtype", in_dtype}, @@ -96,7 +110,8 @@ class PyFormatConverterOp : public FormatConverterOp { Arg{"resize_height", resize_height}, Arg{"resize_mode", resize_mode}, Arg{"out_channel_order", out_channel_order}, - Arg{"pool", pool}}) { + Arg{"pool", pool}, + Arg{"cuda_stream_pool", cuda_stream_pool}}) { name_ = name; fragment_ = fragment; spec_ = std::make_shared(fragment); @@ -181,55 +196,6 @@ class PyVideoStreamRecorderOp : public VideoStreamRecorderOp { } }; -class PyLSTMTensorRTInferenceOp : public LSTMTensorRTInferenceOp { - public: - /* Inherit the constructors */ - using LSTMTensorRTInferenceOp::LSTMTensorRTInferenceOp; - - // Define a constructor that fully initializes the object. - PyLSTMTensorRTInferenceOp( - Fragment* fragment, const std::vector& input_tensor_names, - const std::vector& output_tensor_names, - const std::vector& input_binding_names, - const std::vector& output_binding_names, const std::string& model_file_path, - const std::string& engine_cache_dir, - // int64_t dla_core, - std::shared_ptr pool, - std::shared_ptr cuda_stream_pool, - // std::shared_ptr clock, - const std::string& plugins_lib_namespace = "", - const std::vector& input_state_tensor_names = std::vector{}, - const std::vector& output_state_tensor_names = std::vector{}, - bool force_engine_update = false, bool enable_fp16_ = false, bool verbose = false, - bool relaxed_dimension_check = true, int64_t max_workspace_size = 67108864l, - int32_t max_batch_size = 1, const std::string& name = "lstm_tensor_rt_inference") - : LSTMTensorRTInferenceOp(ArgList{Arg{"input_tensor_names", input_tensor_names}, - Arg{"output_tensor_names", output_tensor_names}, - Arg{"input_binding_names", input_binding_names}, - Arg{"output_binding_names", output_binding_names}, - Arg{"model_file_path", model_file_path}, - Arg{"engine_cache_dir", engine_cache_dir}, - // Arg{"dla_core", dla_core}, - Arg{"pool", pool}, - Arg{"cuda_stream_pool", cuda_stream_pool}, - // Arg{"clock", clock}, - Arg{"plugins_lib_namespace", plugins_lib_namespace}, - Arg{"input_state_tensor_names", input_state_tensor_names}, - Arg{"output_state_tensor_names", output_state_tensor_names}, - Arg{"force_engine_update", force_engine_update}, - Arg{"enable_fp16_", enable_fp16_}, - Arg{"verbose", verbose}, - Arg{"relaxed_dimension_check", relaxed_dimension_check}, - Arg{"max_workspace_size", max_workspace_size}, - Arg{"max_batch_size", max_batch_size}}) { - name_ = name; - fragment_ = fragment; - spec_ = std::make_shared(fragment); - setup(*spec_.get()); - initialize(); - } -}; - class PyTensorRTInferenceOp : public TensorRTInferenceOp { public: /* Inherit the constructors */ @@ -288,9 +254,12 @@ class PyHolovizOp : public HolovizOp { const std::vector& tensors = std::vector(), const std::vector>& color_lut = std::vector>(), const std::string& window_title = "Holoviz", const std::string& display_name = "DP-0", - uint32_t width = 1920, uint32_t height = 1080, uint32_t framerate = 60, + uint32_t width = 1920, uint32_t height = 1080, float framerate = 60.f, bool use_exclusive_display = false, bool fullscreen = false, bool headless = false, bool enable_render_buffer_input = false, bool enable_render_buffer_output = false, + const std::string& font_path = "", + std::shared_ptr cuda_stream_pool = + std::shared_ptr(), const std::string& name = "holoviz_op") : HolovizOp(ArgList{Arg{"allocator", allocator}, Arg{"color_lut", color_lut}, @@ -303,7 +272,9 @@ class PyHolovizOp : public HolovizOp { Arg{"fullscreen", fullscreen}, Arg{"headless", headless}, Arg{"enable_render_buffer_input", enable_render_buffer_input}, - Arg{"enable_render_buffer_output", enable_render_buffer_output}}) { + Arg{"enable_render_buffer_output", enable_render_buffer_output}, + Arg{"font_path", font_path}, + Arg{"cuda_stream_pool", cuda_stream_pool}}) { // only append tensors argument if it is not empty // avoids [holoscan] [error] [gxf_operator.hpp:126] Unable to handle parameter 'tensors' if (tensors.size() > 0) { this->add_arg(Arg{"tensors", tensors}); } @@ -327,34 +298,14 @@ class PySegmentationPostprocessorOp : public SegmentationPostprocessorOp { const std::string& in_tensor_name = "", const std::string& network_output_type = "softmax"s, const std::string& data_format = "hwc"s, + std::shared_ptr cuda_stream_pool = + std::shared_ptr(), const std::string& name = "segmentation_postprocessor"s) : SegmentationPostprocessorOp(ArgList{Arg{"in_tensor_name", in_tensor_name}, Arg{"network_output_type", network_output_type}, Arg{"data_format", data_format}, - Arg{"allocator", allocator}}) { - name_ = name; - fragment_ = fragment; - spec_ = std::make_shared(fragment); - setup(*spec_.get()); - initialize(); - } -}; - -class PyToolTrackingPostprocessorOp : public ToolTrackingPostprocessorOp { - public: - /* Inherit the constructors */ - using ToolTrackingPostprocessorOp::ToolTrackingPostprocessorOp; - - // Define a constructor that fully initializes the object. - PyToolTrackingPostprocessorOp( - Fragment* fragment, std::shared_ptr device_allocator, - std::shared_ptr host_allocator, float min_prob = 0.5f, - std::vector> overlay_img_colors = VIZ_TOOL_DEFAULT_COLORS, - const std::string& name = "tool_tracking_postprocessor") - : ToolTrackingPostprocessorOp(ArgList{Arg{"device_allocator", device_allocator}, - Arg{"host_allocator", host_allocator}, - Arg{"min_prob", min_prob}, - Arg{"overlay_img_colors", overlay_img_colors}}) { + Arg{"allocator", allocator}, + Arg{"cuda_stream_pool", cuda_stream_pool}}) { name_ = name; fragment_ = fragment; spec_ = std::make_shared(fragment); @@ -461,15 +412,14 @@ class PyMultiAIPostprocessorOp : public MultiAIPostprocessorOp { const std::vector& in_tensor_names, // = {std::string("")}, const std::vector& out_tensor_names, // = {std::string("")}, bool input_on_cuda = false, bool output_on_cuda = false, bool transmit_on_cuda = false, - // TODO(grelee): handle receivers similarly to HolovizOp? (default: {}) - // TODO(grelee): handle transmitter similarly to HolovizOp? - const std::string& name = "multi_ai_postprocessor") + bool disable_transmitter = false, const std::string& name = "multi_ai_postprocessor") : MultiAIPostprocessorOp(ArgList{Arg{"allocator", allocator}, Arg{"in_tensor_names", in_tensor_names}, Arg{"out_tensor_names", out_tensor_names}, Arg{"input_on_cuda", input_on_cuda}, Arg{"output_on_cuda", output_on_cuda}, - Arg{"transmit_on_cuda", transmit_on_cuda}}) { + Arg{"transmit_on_cuda", transmit_on_cuda}, + Arg{"disable_transmitter", disable_transmitter}}) { name_ = name; fragment_ = fragment; @@ -489,31 +439,6 @@ class PyMultiAIPostprocessorOp : public MultiAIPostprocessorOp { } }; -class PyVisualizerICardioOp : public VisualizerICardioOp { - public: - /* Inherit the constructors */ - using VisualizerICardioOp::VisualizerICardioOp; - - // Define a constructor that fully initializes the object. - PyVisualizerICardioOp(Fragment* fragment, std::shared_ptr<::holoscan::Allocator> allocator, - const std::vector& in_tensor_names = {std::string("")}, - const std::vector& out_tensor_names = {std::string("")}, - bool input_on_cuda = false, - // TODO(grelee): handle receivers similarly to HolovizOp? (default: {}) - // TODO(grelee): handle transmitter similarly to HolovizOp? - const std::string& name = "visualizer_icardio") - : VisualizerICardioOp(ArgList{Arg{"allocator", allocator}, - Arg{"in_tensor_names", in_tensor_names}, - Arg{"out_tensor_names", out_tensor_names}, - Arg{"input_on_cuda", input_on_cuda}}) { - name_ = name; - fragment_ = fragment; - spec_ = std::make_shared(fragment); - setup(*spec_.get()); - initialize(); - } -}; - class PyBayerDemosaicOp : public BayerDemosaicOp { public: /* Inherit the constructors */ @@ -541,32 +466,6 @@ class PyBayerDemosaicOp : public BayerDemosaicOp { } }; -#if HOLOSCAN_BUILD_EMERGENT == 1 - -class PyEmergentSourceOp : public EmergentSourceOp { - public: - /* Inherit the constructors */ - using EmergentSourceOp::EmergentSourceOp; - - // Define a constructor that fully initializes the object. - PyEmergentSourceOp(Fragment* fragment, - // defaults here should match constexpr values in EmergentSourceOp::Setup - uint32_t width = 4200, uint32_t height = 2160, uint32_t framerate = 240, - bool rdma = false, const std::string& name = "emergent_source") - : EmergentSourceOp(ArgList{Arg{"width", width}, - Arg{"height", height}, - Arg{"framerate", framerate}, - Arg{"rdma", rdma}}) { - name_ = name; - fragment_ = fragment; - spec_ = std::make_shared(fragment); - setup(*spec_.get()); - initialize(); - } -}; - -#endif // HOLOSCAN_BUILD_EMERGENT == 1 - PYBIND11_MODULE(_operators, m) { m.doc() = R"pbdoc( Holoscan SDK Python Bindings @@ -588,10 +487,7 @@ PYBIND11_MODULE(_operators, m) { * operators inheriting from GXFOperator * *****************************************/ - py::class_>( + py::class_>( m, "FormatConverterOp", doc::FormatConverterOp::doc_FormatConverterOp) .def(py::init, @@ -606,6 +502,7 @@ PYBIND11_MODULE(_operators, m) { int32_t, int32_t, const std::vector, + std::shared_ptr, const std::string&>(), "fragment"_a, "pool"_a, @@ -620,11 +517,9 @@ PYBIND11_MODULE(_operators, m) { "resize_width"_a = 0, "resize_mode"_a = 0, "out_channel_order"_a = std::vector{}, + "cuda_stream_pool"_a = std::shared_ptr(), "name"_a = "format_converter"s, doc::FormatConverterOp::doc_FormatConverterOp_python) - .def_property_readonly("gxf_typename", - &FormatConverterOp::gxf_typename, - doc::FormatConverterOp::doc_gxf_typename) .def("initialize", &FormatConverterOp::initialize, doc::FormatConverterOp::doc_initialize) .def("setup", &FormatConverterOp::setup, "spec"_a, doc::FormatConverterOp::doc_setup); @@ -640,7 +535,7 @@ PYBIND11_MODULE(_operators, m) { .value("NTV2_MAX_NUM_CHANNELS", NTV2Channel::NTV2_MAX_NUM_CHANNELS) .value("NTV2_CHANNEL_INVALID", NTV2Channel::NTV2_CHANNEL_INVALID); - py::class_>( + py::class_>( m, "AJASourceOp", doc::AJASourceOp::doc_AJASourceOp) .def(py::init>( - m, "LSTMTensorRTInferenceOp", doc::LSTMTensorRTInferenceOp::doc_LSTMTensorRTInferenceOp) - .def(py::init&, - const std::vector&, - const std::vector&, - const std::vector&, - const std::string&, - const std::string&, - // int64_t, // dla_core - std::shared_ptr, - std::shared_ptr, - // std::shared_ptr, // clock - const std::string&, - const std::vector&, - const std::vector&, - bool, - bool, - bool, - bool, - int64_t, - int32_t, - const std::string&>(), - "fragment"_a, - "input_tensor_names"_a, - "output_tensor_names"_a, - "input_binding_names"_a, - "output_binding_names"_a, - "model_file_path"_a, - "engine_cache_dir"_a, - // "dla_core"_a, - "pool"_a, - "cuda_stream_pool"_a, - // "clock"_a, - "plugins_lib_namespace"_a = "", - "input_state_tensor_names"_a = std::vector{}, - "output_state_tensor_names"_a = std::vector{}, - "force_engine_update"_a = false, - "enable_fp16_"_a = false, - "verbose"_a = false, - "relaxed_dimension_check"_a = true, - "max_workspace_size"_a = 67108864l, - "max_batch_size"_a = 1, - "name"_a = "lstm_tensor_rt_inference"s, - doc::LSTMTensorRTInferenceOp::doc_LSTMTensorRTInferenceOp_python) - .def_property_readonly("gxf_typename", - &LSTMTensorRTInferenceOp::gxf_typename, - doc::LSTMTensorRTInferenceOp::doc_gxf_typename) - .def("initialize", - &LSTMTensorRTInferenceOp::initialize, - doc::LSTMTensorRTInferenceOp::doc_initialize) - .def("setup", - &LSTMTensorRTInferenceOp::setup, - "spec"_a, - doc::LSTMTensorRTInferenceOp::doc_setup); - py::class_>( m, "VideoStreamRecorderOp", doc::VideoStreamRecorderOp::doc_VideoStreamRecorderOp) .def(py::init(), @@ -791,9 +625,6 @@ PYBIND11_MODULE(_operators, m) { "flush_on_tick"_a = false, "name"_a = "recorder"s, doc::VideoStreamRecorderOp::doc_VideoStreamRecorderOp_python) - .def_property_readonly("gxf_typename", - &VideoStreamRecorderOp::gxf_typename, - doc::VideoStreamRecorderOp::doc_gxf_typename) .def("initialize", &VideoStreamRecorderOp::initialize, doc::VideoStreamRecorderOp::doc_initialize) @@ -801,7 +632,7 @@ PYBIND11_MODULE(_operators, m) { py::class_>( m, "VideoStreamReplayerOp", doc::VideoStreamReplayerOp::doc_VideoStreamReplayerOp) .def(py::init> holoviz_op( + py::class_> holoviz_op( m, "HolovizOp", doc::HolovizOp::doc_HolovizOp); holoviz_op .def(py::init, const std::string&>(), "fragment"_a, "allocator"_a, @@ -867,10 +697,10 @@ PYBIND11_MODULE(_operators, m) { "headless"_a = false, "enable_render_buffer_input"_a = false, "enable_render_buffer_output"_a = false, + "font_path"_a = "", + "cuda_stream_pool"_a = std::shared_ptr(), "name"_a = "holoviz_op"s, doc::HolovizOp::doc_HolovizOp_python) - .def_property_readonly( - "gxf_typename", &HolovizOp::gxf_typename, doc::HolovizOp::doc_gxf_typename) .def("initialize", &HolovizOp::initialize, doc::HolovizOp::doc_initialize) .def("setup", &HolovizOp::setup, "spec"_a, doc::HolovizOp::doc_setup); @@ -885,7 +715,14 @@ PYBIND11_MODULE(_operators, m) { .value("CROSSES", HolovizOp::InputType::CROSSES) .value("RECTANGLES", HolovizOp::InputType::RECTANGLES) .value("OVALS", HolovizOp::InputType::OVALS) - .value("TEXT", HolovizOp::InputType::TEXT); + .value("TEXT", HolovizOp::InputType::TEXT) + .value("DEPTH_MAP", HolovizOp::InputType::DEPTH_MAP) + .value("DEPTH_MAP_COLOR", HolovizOp::InputType::DEPTH_MAP_COLOR); + + py::enum_(holoviz_op, "DepthMapRenderMode") + .value("POINTS", HolovizOp::DepthMapRenderMode::POINTS) + .value("LINES", HolovizOp::DepthMapRenderMode::LINES) + .value("TRIANGLES", HolovizOp::DepthMapRenderMode::TRIANGLES); py::class_(holoviz_op, "InputSpec") .def(py::init()) @@ -895,11 +732,12 @@ PYBIND11_MODULE(_operators, m) { .def_readwrite("_priority", &HolovizOp::InputSpec::priority_) .def_readwrite("_line_width", &HolovizOp::InputSpec::line_width_) .def_readwrite("_point_size", &HolovizOp::InputSpec::point_size_) - .def_readwrite("_text", &HolovizOp::InputSpec::text_); + .def_readwrite("_text", &HolovizOp::InputSpec::text_) + .def_readwrite("_depth_map_render_mode", &HolovizOp::InputSpec::depth_map_render_mode_); py::class_>( m, "SegmentationPostprocessorOp", @@ -909,53 +747,24 @@ PYBIND11_MODULE(_operators, m) { const std::string&, const std::string&, const std::string&, + std::shared_ptr, const std::string&>(), "fragment"_a, "allocator"_a, "in_tensor_name"_a = ""s, "network_output_type"_a = "softmax"s, "data_format"_a = "hwc"s, + "cuda_stream_pool"_a = std::shared_ptr(), "name"_a = "segmentation_postprocessor"s, doc::SegmentationPostprocessorOp::doc_SegmentationPostprocessorOp_python) - .def_property_readonly("gxf_typename", - &SegmentationPostprocessorOp::gxf_typename, - doc::SegmentationPostprocessorOp::doc_gxf_typename) .def("setup", &SegmentationPostprocessorOp::setup, "spec"_a, doc::SegmentationPostprocessorOp::doc_setup); - py::class_>( - m, - "ToolTrackingPostprocessorOp", - doc::ToolTrackingPostprocessorOp::doc_ToolTrackingPostprocessorOp) - .def(py::init, - std::shared_ptr, - float, - std::vector>, - const std::string&>(), - "fragment"_a, - "device_allocator"_a, - "host_allocator"_a, - "min_prob"_a = 0.5f, - "overlay_img_colors"_a = VIZ_TOOL_DEFAULT_COLORS, - "name"_a = "tool_tracking_postprocessor"s, - doc::ToolTrackingPostprocessorOp::doc_ToolTrackingPostprocessorOp_python) - .def_property_readonly("gxf_typename", - &ToolTrackingPostprocessorOp::gxf_typename, - doc::ToolTrackingPostprocessorOp::doc_gxf_typename) - .def("setup", - &ToolTrackingPostprocessorOp::setup, - "spec"_a, - doc::ToolTrackingPostprocessorOp::doc_setup); - py::class_> multiai_infererence_op( m, "MultiAIInferenceOp", doc::MultiAIInferenceOp::doc_MultiAIInferenceOp); @@ -994,9 +803,6 @@ PYBIND11_MODULE(_operators, m) { "is_engine_path"_a = false, "name"_a = "multi_ai_inference"s, doc::MultiAIInferenceOp::doc_MultiAIInferenceOp_python) - .def_property_readonly("gxf_typename", - &MultiAIInferenceOp::gxf_typename, - doc::MultiAIInferenceOp::doc_gxf_typename) .def("initialize", &MultiAIInferenceOp::initialize, doc::MultiAIInferenceOp::doc_initialize) .def("setup", &MultiAIInferenceOp::setup, "spec"_a, doc::MultiAIInferenceOp::doc_setup); @@ -1012,7 +818,7 @@ PYBIND11_MODULE(_operators, m) { py::class_> multiai_postprocessor_op( m, "MultiAIPostprocessorOp", doc::MultiAIPostprocessorOp::doc_MultiAIPostprocessorOp); @@ -1027,21 +833,20 @@ PYBIND11_MODULE(_operators, m) { bool, bool, bool, + bool, const std::string&>(), "fragment"_a, "allocator"_a, - "process_operations"_a, - "processed_map"_a, - "in_tensor_names"_a = std::vector{{std::string("")}}, - "out_tensor_names"_a = std::vector{{std::string("")}}, + "process_operations"_a = py::dict(), + "processed_map"_a = py::dict(), + "in_tensor_names"_a = std::vector{}, + "out_tensor_names"_a = std::vector{}, "input_on_cuda"_a = false, "output_on_cuda"_a = false, "transmit_on_cuda"_a = false, + "disable_transmitter"_a = false, "name"_a = "multi_ai_postprocessor"s, doc::MultiAIPostprocessorOp::doc_MultiAIPostprocessorOp_python) - .def_property_readonly("gxf_typename", - &MultiAIPostprocessorOp::gxf_typename, - doc::MultiAIPostprocessorOp::doc_gxf_typename) .def("initialize", &MultiAIPostprocessorOp::initialize, doc::MultiAIPostprocessorOp::doc_initialize) @@ -1060,30 +865,6 @@ PYBIND11_MODULE(_operators, m) { .def("insert", &MultiAIPostprocessorOp::DataVecMap::get_map) .def("get_map", &MultiAIPostprocessorOp::DataVecMap::get_map); - py::class_>( - m, "VisualizerICardioOp", doc::VisualizerICardioOp::doc_VisualizerICardioOp) - .def(py::init, - const std::vector&, - const std::vector&, - bool, - const std::string&>(), - "fragment"_a, - "allocator"_a, - "in_tensor_names"_a, // = {std::string("")}, - "out_tensor_names"_a, // = {std::string("")}, - "input_on_cuda"_a = false, - "name"_a = "visualizer_icardio"s, - doc::VisualizerICardioOp::doc_VisualizerICardioOp_python) - .def_property_readonly("gxf_typename", - &VisualizerICardioOp::gxf_typename, - doc::VisualizerICardioOp::doc_gxf_typename) - .def("initialize", &VisualizerICardioOp::initialize, doc::VisualizerICardioOp::doc_initialize) - .def("setup", &VisualizerICardioOp::setup, "spec"_a, doc::VisualizerICardioOp::doc_setup); - py::class_>( m, "BayerDemosaicOp", doc::BayerDemosaicOp::doc_BayerDemosaicOp) .def(py::init<>(), doc::BayerDemosaicOp::doc_BayerDemosaicOp) @@ -1112,24 +893,5 @@ PYBIND11_MODULE(_operators, m) { "gxf_typename", &BayerDemosaicOp::gxf_typename, doc::BayerDemosaicOp::doc_gxf_typename) .def("initialize", &BayerDemosaicOp::initialize, doc::BayerDemosaicOp::doc_initialize) .def("setup", &BayerDemosaicOp::setup, "spec"_a, doc::BayerDemosaicOp::doc_setup); - -#if HOLOSCAN_BUILD_EMERGENT == 1 - - py::class_>( - m, "EmergentSourceOp", doc::EmergentSourceOp::doc_EmergentSourceOp) - .def(py::init(), - "fragment"_a, - // defaults values here should match constexpr values in C++ EmergentSourceOp::Setup - "width"_a = 4200, - "height"_a = 2160, - "framerate"_a = 240, - "rdma"_a = false, - "name"_a = "emergent_source"s, - doc::EmergentSourceOp::doc_EmergentSourceOp_python) - .def_property_readonly( - "gxf_typename", &EmergentSourceOp::gxf_typename, doc::EmergentSourceOp::doc_gxf_typename) - .def("initialize", &EmergentSourceOp::initialize, doc::EmergentSourceOp::doc_initialize) - .def("setup", &EmergentSourceOp::setup, "spec"_a, doc::EmergentSourceOp::doc_setup); -#endif // HOLOSCAN_BUILD_EMERGENT == 1 } // PYBIND11_MODULE NOLINT } // namespace holoscan::ops diff --git a/python/pybind11/operators/operators_pydoc.hpp b/python/src/operators/operators_pydoc.hpp similarity index 62% rename from python/pybind11/operators/operators_pydoc.hpp rename to python/src/operators/operators_pydoc.hpp index 68ad56e5..89519475 100644 --- a/python/pybind11/operators/operators_pydoc.hpp +++ b/python/src/operators/operators_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -36,9 +36,9 @@ Format conversion operator. Parameters ---------- -fragment : Fragment +fragment : holoscan.core.Fragment The fragment that the operator belongs to. -pool : ``holoscan.resources.Allocator`` +pool : holoscan.resources.Allocator Memory pool allocator used by the operator. out_dtype : str Destination data type (e.g. "RGB888" or "RGBA8888"). @@ -63,19 +63,12 @@ resize_mode : int, optional Resize mode enum value corresponding to NPP's nppiInterpolationMode (default=NPPI_INTER_CUBIC). channel_order : sequence of int Sequence of integers describing how channel values are permuted. +cuda_stream_pool : holoscan.resources.CudaStreamPool, optional + CudaStreamPool instance to allocate CUDA streams. name : str, optional The name of the operator. )doc") -PYDOC(gxf_typename, R"doc( -The GXF type name of the resource. - -Returns -------- -str - The GXF type name of the resource -)doc") - PYDOC(initialize, R"doc( Initialize the operator. @@ -88,7 +81,7 @@ Define the operator specification. Parameters ---------- -spec : ``holoscan.core.OperatorSpec`` +spec : holoscan.core.OperatorSpec The operator specification. )doc") @@ -106,11 +99,11 @@ Operator to get a video stream from an AJA capture card. Parameters ---------- -fragment : Fragment +fragment : holoscan.core.Fragment The fragment that the operator belongs to. device : str, optional The device to target (e.g. "0" for device 0) -channel : ``holoscan.operators.NTV2Channel`` or int, optional +channel : holoscan.operators.NTV2Channel or int, optional The camera NTV2Channel to use for output. width : int, optional Width of the video stream. @@ -122,7 +115,7 @@ rdma : bool, optional Boolean indicating whether RDMA is enabled. enable_overlay : bool, optional Boolean indicating whether a separate overlay channel is enabled. -overlay_channel : ``holoscan.operators.NTV2Channel`` or int, optional +overlay_channel : holoscan.operators.NTV2Channel or int, optional The camera NTV2Channel to use for overlay output. overlay_rdma : bool, optional Boolean indicating whether RDMA is enabled for the overlay. @@ -130,21 +123,12 @@ name : str, optional The name of the operator. )doc") -PYDOC(gxf_typename, R"doc( -The GXF type name of the resource. - -Returns -------- -str - The GXF type name of the resource -)doc") - PYDOC(setup, R"doc( Define the operator specification. Parameters ---------- -spec : ``holoscan.core.OperatorSpec`` +spec : holoscan.core.OperatorSpec The operator specification. )doc") @@ -157,89 +141,6 @@ and uses a light-weight initialization. } // namespace AJASourceOp -namespace LSTMTensorRTInferenceOp { - -PYDOC(LSTMTensorRTInferenceOp, R"doc( -Operator class to perform inference using an LSTM model. -)doc") - -// PyLSTMTensorRTInferenceOp Constructor -PYDOC(LSTMTensorRTInferenceOp_python, R"doc( -Operator class to perform inference using an LSTM model. - -Parameters ----------- -fragment : Fragment - The fragment that the operator belongs to. -input_tensor_names : sequence of str - Names of input tensors in the order to be fed into the model. -output_tensor_names : sequence of str - Names of output tensors in the order to be retrieved from the model. -input_binding_names : sequence of str - Names of input bindings as in the model in the same order of - what is provided in `input_tensor_names`. -output_binding_names : sequence of str - Names of output bindings as in the model in the same order of - what is provided in `output_tensor_names`. -model_file_path : str - Path to the ONNX model to be loaded. -engine_cache_dir : str - Path to a folder containing cached engine files to be serialized and loaded from. -pool : ``holoscan.resources.Allocator`` - Allocator instance for output tensors. -cuda_stream_pool : ``holoscan.resources.CudaStreamPool`` - CudaStreamPool instance to allocate CUDA streams. -plugins_lib_namespace : str - Namespace used to register all the plugins in this library. -input_state_tensor_names : sequence of str, optional - Names of input state tensors that are used internally by TensorRT. -output_state_tensor_names : sequence of str, optional - Names of output state tensors that are used internally by TensorRT. -force_engine_update : bool, optional - Always update engine regardless of whether there is an existing engine file. - Warning: this may take minutes to complete, so is False by default. -enable_fp16 : bool, optional - Enable inference with FP16 and FP32 fallback. -verbose : bool, optional - Enable verbose logging to the console. -relaxed_dimension_check : bool, optional - Ignore dimensions of 1 for input tensor dimension check. -max_workspace_size : int, optional - Size of working space in bytes. -max_batch_size : int, optional - Maximum possible batch size in case the first dimension is dynamic and used - as batch size. -name : str, optional - The name of the operator. -)doc") - -PYDOC(gxf_typename, R"doc( -The GXF type name of the resource. - -Returns -------- -str - The GXF type name of the resource -)doc") - -PYDOC(initialize, R"doc( -Initialize the operator. - -This method is called only once when the operator is created for the first time, -and uses a light-weight initialization. -)doc") - -PYDOC(setup, R"doc( -Define the operator specification. - -Parameters ----------- -spec : ``holoscan.core.OperatorSpec`` - The operator specification. -)doc") - -} // namespace LSTMTensorRTInferenceOp - namespace HolovizOp { PYDOC(HolovizOp, R"doc( @@ -256,16 +157,17 @@ This is a Vulkan-based visualizer. Parameters ---------- -fragment : Fragment +fragment : holoscan.core.Fragment The fragment that the operator belongs to. -allocator : ``holoscan.core.Allocator``, optional +allocator : holoscan.core.Allocator, optional Allocator used to allocate render buffer output. If None, will default to `holoscan.core.UnboundedAllocator`. receivers : sequence of holoscan.core.IOSpec, optional List of input receivers. -tensors : sequence of holoscan.core.InputSpec, optional - List of input tensors. 'name' is required, 'type' is optional (unknown, color, color_lut, - points, lines, line_strip, triangles, crosses, rectangles, ovals, text). +tensors : sequence of dict, optional + List of input tensors. Each tensor is defined by a dictionary where the 'name' key must + correspond to a tensor sent to the operator's input. See the notes section below for further + details on how the tensor dictionary is defined. color_lut : list of list of float, optional Color lookup table for tensors of type 'color_lut'. Should be shape `(n_colors, 4)`. window_title : str, optional @@ -276,8 +178,8 @@ width : int, optional Window width or display resolution width if in exclusive or fullscreen mode. height : int, optional Window height or display resolution width if in exclusive or fullscreen mode. -framerate : int, optional - Display framerate if in exclusive mode. +framerate : float, optional + Display framerate in Hz if in exclusive mode. use_exclusive_display : bool, optional Enable exclusive display. fullscreen : bool, optional @@ -286,22 +188,49 @@ headless : bool, optional Enable headless mode. No window is opened, the render buffer is output to port `render_buffer_output`. enable_render_buffer_input : bool, optional - If True, an additional input port, named `render_buffer_input` is added to the + If ``True``, an additional input port, named 'render_buffer_input' is added to the operator. enable_render_buffer_output : bool, optional - If True, an additional output port, named `render_buffer_output` is added to the + If ``True``, an additional output port, named 'render_buffer_output' is added to the operator. +font_path : str, optional + File path for the font used for rendering text. +cuda_stream_pool : holoscan.resources.CudaStreamPool, optional + CudaStreamPool instance to allocate CUDA streams. name : str, optional The name of the operator. -)doc") -PYDOC(gxf_typename, R"doc( -The GXF type name of the resource. +Notes +----- +The `tensors` argument is used to specify the tensors to display. Each tensor is defined using a +dictionary, that must, at minimum include a 'name' key that corresponds to a tensor found on the +operator's input. A 'type' key should also be provided to indicate the type of entry to display. +The 'type' key will be one of {"color", "color_lut", "crosses", "lines", "line_strip", "ovals", +"points", "rectangles", "text", "triangles", "depth_map", "depth_map_color", "unknown"}. The +default type is "unknown" which will attempt to guess the corresponding type based on the tensor +dimensions. Concrete examples are given below. -Returns -------- -str - The GXF type name of the resource +To show a single 2D RGB or RGBA image, use a list containing a single tensor of type 'color'. + +.. code-block:: python + + tensors = [dict(name="video", type="color", opacity=1.0, priority=0)] + +Here, the optional key `opacity` is used to scale the opacity of the tensor. The `priority` key +is used to specify the render priority for layers. Layers with a higher priority will be rendered +on top of those with a lower priority. + +If we also had a "boxes" tensor representing rectangular bounding boxes, we could display them +on top of the image like this. + +.. code-block:: python + + tensors = [ + dict(name="video", type="color", priority=0), + dict(name="boxes", type="rectangles", color=[1.0, 0.0, 0.0], line_width=2, priority=1), + ] + +where the `color` and `line_width` keys specify the color and line width of the bounding box. )doc") PYDOC(initialize, R"doc( @@ -316,7 +245,7 @@ Define the operator specification. Parameters ---------- -spec : ``holoscan.core.OperatorSpec`` +spec : holoscan.core.OperatorSpec The operator specification. )doc") @@ -334,9 +263,9 @@ Operator carrying out post-processing operations used in the ultrasound demo app Parameters ---------- -fragment : Fragment +fragment : holoscan.core.Fragment The fragment that the operator belongs to. -allocator : ``holoscan.resources.Allocator`` +allocator : holoscan.resources.Allocator Memory allocator to use for the output. in_tensor_name : str, optional Name of the input tensor. @@ -344,20 +273,13 @@ network_output_type : str, optional Network output type (e.g. 'softmax'). data_format : str, optional Data format of network output. +cuda_stream_pool : holoscan.resources.CudaStreamPool, optional + CudaStreamPool instance to allocate CUDA streams. name : str, optional The name of the operator. )doc") -PYDOC(gxf_typename, R"doc( -The GXF type name of the resource. - -Returns -------- -str - The GXF type name of the resource -)doc") - PYDOC(initialize, R"doc( Initialize the operator. @@ -370,7 +292,7 @@ Define the operator specification. Parameters ---------- -spec : ``holoscan.core.OperatorSpec`` +spec : holoscan.core.OperatorSpec The operator specification. )doc") @@ -388,27 +310,18 @@ Operator class to record the video stream to a file. Parameters ---------- -fragment : Fragment +fragment : holoscan.core.Fragment The fragment that the operator belongs to. directory : str Directory path for storing files. basename : str User specified file name without extension. flush_on_tick : bool, optional - Flushes output buffer on every tick when True. + Flushes output buffer on every tick when ``True``. name : str, optional The name of the operator. )doc") -PYDOC(gxf_typename, R"doc( -The GXF type name of the resource. - -Returns -------- -str - The GXF type name of the resource -)doc") - PYDOC(initialize, R"doc( Initialize the operator. @@ -421,7 +334,7 @@ Define the operator specification. Parameters ---------- -spec : ``holoscan.core.OperatorSpec`` +spec : holoscan.core.OperatorSpec The operator specification. )doc") @@ -439,7 +352,7 @@ Operator class to replay a video stream from a file. Parameters ---------- -fragment : Fragment +fragment : holoscan.core.Fragment The fragment that the operator belongs to. directory : str Directory path for reading files from. @@ -465,15 +378,6 @@ name : str, optional The name of the operator. )doc") -PYDOC(gxf_typename, R"doc( -The GXF type name of the resource. - -Returns -------- -str - The GXF type name of the resource -)doc") - PYDOC(initialize, R"doc( Initialize the operator. @@ -486,7 +390,7 @@ Define the operator specification. Parameters ---------- -spec : ``holoscan.core.OperatorSpec`` +spec : holoscan.core.OperatorSpec The operator specification. )doc") @@ -571,83 +475,30 @@ spec : ``holoscan.core.OperatorSpec`` } // namespace TensorRTInferenceOp -namespace ToolTrackingPostprocessorOp { - -PYDOC(ToolTrackingPostprocessorOp, R"doc( -Operator performing post-processing for the endoscopy tool tracking demo. -)doc") - -// PyToolTrackingPostprocessorOp Constructor -PYDOC(ToolTrackingPostprocessorOp_python, R"doc( -Operator performing post-processing for the endoscopy tool tracking demo. - -Parameters ----------- -fragment : Fragment - The fragment that the operator belongs to. -device_allocator : ``holoscan.resources.Allocator`` - Output allocator used on the device side. -host_allocator : ``holoscan.resources.Allocator`` - Output allocator used on the host side. -min_prob : float, optional - Minimum probability (in range [0, 1]). -overlay_img_colors : sequence of sequence of float, optional - Color of the image overlays, a list of RGB values with components between 0 and 1. -name : str, optional - The name of the operator. -)doc") - -PYDOC(gxf_typename, R"doc( -The GXF type name of the resource. - -Returns -------- -str - The GXF type name of the resource -)doc") - -PYDOC(initialize, R"doc( -Initialize the operator. - -This method is called only once when the operator is created for the first time, -and uses a light-weight initialization. -)doc") - -PYDOC(setup, R"doc( -Define the operator specification. - -Parameters ----------- -spec : ``holoscan.core.OperatorSpec`` - The operator specification. -)doc") - -} // namespace ToolTrackingPostprocessorOp - namespace MultiAIInferenceOp { PYDOC(MultiAIInferenceOp, R"doc( Multi-AI inference operator. )doc") -// PyHolovizOp Constructor +// PyMultiAIInferenceOp_python Constructor PYDOC(MultiAIInferenceOp_python, R"doc( Multi-AI inference operator. Parameters ---------- -fragment : Fragment +fragment : holoscan.core.Fragment The fragment that the operator belongs to. backend : {"trt", "onnxrt"} Backend to use for inference. Set "trt" for TensorRT and "onnxrt" for the ONNX runtime. -allocator : ``holoscan.resources.Allocator`` +allocator : holoscan.resources.Allocator Memory allocator to use for the output. -inference_map : ``holoscan.operators.MultiAIInferenceOp.DataMap`` +inference_map : holoscan.operators.MultiAIInferenceOp.DataVecMap Tensor to model map. -model_path_map : ``holoscan.operators.MultiAIInferenceOp.DataMap`` +model_path_map : holoscan.operators.MultiAIInferenceOp.DataMap Path to the ONNX model to be loaded. -pre_processor_map : ``holoscan.operators.MultiAIInferenceOp::DataVecMap`` +pre_processor_map : holoscan.operators.MultiAIInferenceOp::DataVecMap Pre processed data to model map. in_tensor_names : sequence of str, optional Input tensors. @@ -671,15 +522,6 @@ name : str, optional The name of the operator. )doc") -PYDOC(gxf_typename, R"doc( -The GXF type name of the resource. - -Returns -------- -str - The GXF type name of the resource -)doc") - PYDOC(initialize, R"doc( Initialize the operator. @@ -692,7 +534,7 @@ Define the operator specification. Parameters ---------- -spec : ``holoscan.core.OperatorSpec`` +spec : holoscan.core.OperatorSpec The operator specification. )doc") @@ -704,21 +546,21 @@ PYDOC(MultiAIPostprocessorOp, R"doc( Multi-AI post-processing operator. )doc") -// PyHolovizOp Constructor +// PyMultiAIPostprocessorOp Constructor PYDOC(MultiAIPostprocessorOp_python, R"doc( Multi-AI post-processing operator. Parameters ---------- -fragment : Fragment +fragment : holoscan.core.Fragment The fragment that the operator belongs to. -allocator : ``holoscan.resources.Allocator`` +allocator : holoscan.resources.Allocator Memory allocator to use for the output. -post_processor_map : ``holoscan.operators.MultiAIPostprocessorOp.DataVecMap`` +post_processor_map : holoscan.operators.MultiAIPostprocessorOp.DataVecMap All post processor settings for each model. -process_operations : ``holoscan.operators.MultiAIPostprocessorOp.DataVecMap`` +process_operations : holoscan.operators.MultiAIPostprocessorOp.DataVecMap Operations in sequence on tensors. -processed_map : ``holoscan.operators.MultiAIPostprocessorOp::DataMap`` +processed_map : holoscan.operators.MultiAIPostprocessorOp::DataVecMap Input-output tensor mapping. in_tensor_names : sequence of str, optional Names of input tensors in the order to be fed into the operator. @@ -730,19 +572,12 @@ output_on_cuda : bool, optional Whether the output buffer is on the GPU. transmit_on_cuda : bool, optional Whether to transmit the message on the GPU. +disable_transmitter : bool, optional + If ``True``, disable the transmitter output port of the operator. name : str, optional The name of the operator. )doc") -PYDOC(gxf_typename, R"doc( -The GXF type name of the resource. - -Returns -------- -str - The GXF type name of the resource -)doc") - PYDOC(initialize, R"doc( Initialize the operator. @@ -755,65 +590,12 @@ Define the operator specification. Parameters ---------- -spec : ``holoscan.core.OperatorSpec`` +spec : holoscan.core.OperatorSpec The operator specification. )doc") } // namespace MultiAIPostprocessorOp -namespace VisualizerICardioOp { - -PYDOC(VisualizerICardioOp, R"doc( -iCardio Multi-AI demo application visualization operator. -)doc") - -// PyHolovizOp Constructor -PYDOC(VisualizerICardioOp_python, R"doc( -iCardio Multi-AI demo application visualization operator. - -Parameters ----------- -fragment : Fragment - The fragment that the operator belongs to. -allocator : ``holoscan.resources.Allocator`` - Memory allocator to use for the output. -in_tensor_names : sequence of str, optional - Names of input tensors in the order to be fed into the operator. -out_tensor_names : sequence of str, optional - Names of output tensors in the order to be fed into the operator. -input_on_cuda : bool, optional - Boolean indicating whether the input tensors are on the GPU. -name : str, optional - The name of the operator. -)doc") - -PYDOC(gxf_typename, R"doc( -The GXF type name of the resource. - -Returns -------- -str - The GXF type name of the resource -)doc") - -PYDOC(initialize, R"doc( -Initialize the operator. - -This method is called only once when the operator is created for the first time, -and uses a light-weight initialization. -)doc") - -PYDOC(setup, R"doc( -Define the operator specification. - -Parameters ----------- -spec : ``holoscan.core.OperatorSpec`` - The operator specification. -)doc") - -} // namespace VisualizerICardioOp - namespace BayerDemosaicOp { // Constructor @@ -827,11 +609,11 @@ Format conversion operator. Parameters ---------- -fragment : Fragment +fragment : holoscan.core.Fragment The fragment that the operator belongs to. -pool : ``holoscan.resources.Allocator`` +pool : holoscan.resources.Allocator Memory pool allocator used by the operator. -cuda_stream_pool : ``holoscan.resources.CudaStreamPool`` +cuda_stream_pool : holoscan.resources.CudaStreamPool CUDA Stream pool to create CUDA streams in_tensor_name : str, optional The name of the input tensor. @@ -846,7 +628,7 @@ bayer_grid_pos : int, optional generate_alpha : bool, optional Generate alpha channel. alpha_value : int, optional - Alpha value to be generated if `generate_alpha` is set to `True`. + Alpha value to be generated if `generate_alpha` is set to ``True``. name : str, optional The name of the operator. )doc") @@ -872,66 +654,12 @@ Define the operator specification. Parameters ---------- -spec : ``holoscan.core.OperatorSpec`` +spec : holoscan.core.OperatorSpec The operator specification. )doc") } // namespace BayerDemosaicOp -namespace EmergentSourceOp { - -// Constructor -PYDOC(EmergentSourceOp, R"doc( -Operator to get a video stream from an Emergent Vision Technologies camera. -)doc") - -// PyEmergentSourceOp Constructor -PYDOC(EmergentSourceOp_python, R"doc( -Operator to get a video stream from an Emergent Vision Technologies camera. - -Parameters ----------- -fragment : Fragment - The fragment that the operator belongs to. -width : int, optional - Width of the video stream. -height : int, optional - Height of the video stream. -framerate : int, optional - Frame rate of the video stream. -rdma : bool, optional - Boolean indicating whether RDMA is enabled. -name : str, optional - The name of the operator. -)doc") - -PYDOC(gxf_typename, R"doc( -The GXF type name of the resource. - -Returns -------- -str - The GXF type name of the resource -)doc") - -PYDOC(setup, R"doc( -Define the operator specification. - -Parameters ----------- -spec : ``holoscan.core.OperatorSpec`` - The operator specification. -)doc") - -PYDOC(initialize, R"doc( -Initialize the operator. - -This method is called only once when the operator is created for the first time, -and uses a light-weight initialization. -)doc") - -} // namespace EmergentSourceOp - } // namespace holoscan::doc #endif // PYHOLOSCAN_OPERATORS_PYDOC_HPP diff --git a/python/pybind11/resources/__init__.py b/python/src/resources/__init__.py similarity index 100% rename from python/pybind11/resources/__init__.py rename to python/src/resources/__init__.py diff --git a/python/pybind11/resources/resources.cpp b/python/src/resources/resources.cpp similarity index 100% rename from python/pybind11/resources/resources.cpp rename to python/src/resources/resources.cpp diff --git a/python/pybind11/resources/resources_pydoc.hpp b/python/src/resources/resources_pydoc.hpp similarity index 93% rename from python/pybind11/resources/resources_pydoc.hpp rename to python/src/resources/resources_pydoc.hpp index f97245c6..5c156669 100644 --- a/python/pybind11/resources/resources_pydoc.hpp +++ b/python/src/resources/resources_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -91,7 +91,7 @@ Provides a maximum number of equally sized blocks of memory. Parameters ---------- -fragment : Fragment +fragment : holoscan.core.Fragment The fragment to assign the resource to. storage_type : int or holoscan.resources.MemoryStorageType The storage type (0=Host, 1=Device, 2=System). @@ -117,7 +117,7 @@ Define the component specification. Parameters ---------- -spec : ComponentSpec +spec : holoscan.core.ComponentSpec Component specification associated with the resource. )doc") @@ -135,7 +135,7 @@ CUDA stream pool. Parameters ---------- -fragment : Fragment +fragment : holoscan.core.Fragment The fragment to assign the resource to. dev_id : int CUDA device ID. @@ -165,7 +165,7 @@ Define the component specification. Parameters ---------- -spec : ComponentSpec +spec : holoscan.core.ComponentSpec Component specification associated with the resource. )doc") @@ -187,7 +187,7 @@ New messages are first pushed to a back stage. Parameters ---------- -fragment : Fragment +fragment : holoscan.core.Fragment The fragment to assign the resource to. capacity : int, optional The capacity of the receiver. @@ -211,7 +211,7 @@ Define the component specification. Parameters ---------- -spec : ComponentSpec +spec : holoscan.core.ComponentSpec Component specification associated with the resource. )doc") @@ -233,7 +233,7 @@ Messages are pushed to a back stage after they are published. Parameters ---------- -fragment : Fragment +fragment : holoscan.core.Fragment The fragment to assign the resource to. capacity : int, optional The capacity of the transmitter. @@ -257,7 +257,7 @@ Define the component specification. Parameters ---------- -spec : ComponentSpec +spec : holoscan.core.ComponentSpec Component specification associated with the resource. )doc") @@ -292,7 +292,7 @@ Serializer for GXF Timestamp and Tensor components. Parameters ---------- -fragment : Fragment +fragment : holoscan.core.Fragment The fragment to assign the resource to. name : str, optional The name of the serializer. @@ -312,7 +312,7 @@ Define the component specification. Parameters ---------- -spec : ComponentSpec +spec : holoscan.core.ComponentSpec Component specification associated with the resource. )doc") @@ -358,7 +358,7 @@ This allocator uses dynamic memory allocation without an upper bound. Parameters ---------- -fragment : Fragment +fragment : holoscan.core.Fragment The fragment to assign the resource to. name : str, optional The name of the serializer. @@ -378,7 +378,7 @@ Define the component specification. Parameters ---------- -spec : ComponentSpec +spec : holoscan.core.ComponentSpec Component specification associated with the resource. )doc") @@ -396,7 +396,7 @@ Serializer for video streams. Parameters ---------- -fragment : Fragment +fragment : holoscan.core.Fragment The fragment to assign the resource to. name : str, optional The name of the serializer. @@ -416,7 +416,7 @@ Define the component specification. Parameters ---------- -spec : ComponentSpec +spec : holoscan.core.ComponentSpec Component specification associated with the resource. )doc") diff --git a/python/tests/app_config_ping.yaml b/python/tests/app_config_ping.yaml index f873482a..7f8323bd 100644 --- a/python/tests/app_config_ping.yaml +++ b/python/tests/app_config_ping.yaml @@ -1,5 +1,5 @@ %YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,6 +14,5 @@ # See the License for the specific language governing permissions and # limitations under the License. --- - mx: multiplier: 3 diff --git a/python/tests/operator_parameters.yaml b/python/tests/operator_parameters.yaml index 0b49e113..6b76d955 100644 --- a/python/tests/operator_parameters.yaml +++ b/python/tests/operator_parameters.yaml @@ -1,5 +1,5 @@ %YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,7 +29,6 @@ aja: overlay_rdma: true replayer: - directory: "../data/endoscopy/video" basename: "surgical_video" frame_rate: 0 # as specified in timestamps repeat: true # default: false @@ -59,54 +58,6 @@ format_converter_aja: resize_width: 854 resize_height: 480 -lstm_inference: - input_tensor_names: - - source_video - - cellstate_in - - hiddenstate_in - input_state_tensor_names: - - cellstate_in - - hiddenstate_in - input_binding_names: - - data_ph:0 # (shape=[1, 480, 854, 3], dtype=float32) <==> source_video - - cellstate_ph:0 # (shape=[1, 60, 107, 7], dtype=float32) == internal state - - hiddenstate_ph:0 # (shape=[1, 60, 107, 7], dtype=float32) == internal state - output_tensor_names: - - cellstate_out - - hiddenstate_out - - probs - - scaled_coords - - binary_masks - output_state_tensor_names: - - cellstate_out - - hiddenstate_out - output_binding_names: - - Model/net_states:0 # (shape=[ 1, 60, 107, 7], dtype=float32) - - Model/net_hidden:0 # (shape=[ 1, 60, 107, 7], dtype=float32) - - probs:0 # (shape=[1, 7], dtype=float32) - - Localize/scaled_coords:0 # (shape=[1, 7, 2], dtype=float32) - - Localize_1/binary_masks:0 # (shape=[1, 7, 60, 107], dtype=float32) - force_engine_update: false - verbose: true - max_workspace_size: 2147483648 - enable_fp16_: true - - -segmentation_inference: # TensorRtInference - input_tensor_names: - - source_video - input_binding_names: - - INPUT__0 - output_tensor_names: - - inference_output_tensor - output_binding_names: - - OUTPUT__0 - force_engine_update: false - verbose: true - max_workspace_size: 2147483648 - enable_fp16_: false - - visualizer_format_converter_replayer: in_dtype: "rgb888" out_dtype: "rgba8888" @@ -153,8 +104,6 @@ visualizer: in_channels: 4 in_bytes_per_pixel: 1 -tool_tracking_postprocessor: - holoviz: width: 854 height: 480 @@ -190,16 +139,30 @@ holoviz: - Irrigator - Spec.Bag +segmentation_inference: # TensorRtInference + input_tensor_names: + - source_video + input_binding_names: + - INPUT__0 + output_tensor_names: + - inference_output_tensor + output_binding_names: + - OUTPUT__0 + force_engine_update: false + verbose: true + max_workspace_size: 2147483648 + enable_fp16_: false + multiai_inference: backend: "trt" pre_processor_map: - "icardio_plax_chamber": ["plax_cham_pre_proc"] - "icardio_aortic_stenosis": ["aortic_pre_proc"] - "icardio_bmode_perspective": ["bmode_pre_proc"] + "plax_chamber": ["plax_cham_pre_proc"] + "aortic_stenosis": ["aortic_pre_proc"] + "bmode_perspective": ["bmode_pre_proc"] inference_map: - "icardio_plax_chamber": "plax_cham_infer" - "icardio_aortic_stenosis": "aortic_infer" - "icardio_bmode_perspective": "bmode_infer" + "plax_chamber": "plax_cham_infer" + "aortic_stenosis": "aortic_infer" + "bmode_perspective": "bmode_infer" in_tensor_names: ["plax_cham_pre_proc", "aortic_pre_proc", "bmode_pre_proc"] out_tensor_names: ["plax_cham_infer", "aortic_infer", "bmode_infer"] parallel_inference: true @@ -221,18 +184,7 @@ multiai_postprocessor: input_on_cuda: false output_on_cuda: false transmit_on_cuda: false - -visualizer_icardio: - in_tensor_names: ["plax_chamber_processed"] - out_tensor_names: ["keypoints", "keyarea_1", "keyarea_2", - "keyarea_3", "keyarea_4", "keyarea_5", "lines"] - input_on_cuda: false - -emergent: - width: 4200 - height: 2160 - framerate: 240 - rdma: false + disable_transmitter: false demosaic: generate_alpha: false diff --git a/python/tests/system/test_application_exception_in_compute.py b/python/tests/system/test_application_exception_in_compute.py new file mode 100644 index 00000000..9b924acc --- /dev/null +++ b/python/tests/system/test_application_exception_in_compute.py @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from holoscan.conditions import CountCondition +from holoscan.core import Application, Operator +from holoscan.logger import load_env_log_level + + +class BrokenOp(Operator): + def compute(self, op_input, op_output, context): + # intentionally cause a ZeroDivisionError to test exception_handling + 1 / 0 + + +class BadOperatorApp(Application): + def compose(self): + mx = BrokenOp(self, CountCondition(self, 1), name="mx") + self.add_operator(mx) + + +def test_exception_handling(capfd): + load_env_log_level() + app = BadOperatorApp() + app.run() + + # assert that the exception was logged + captured = capfd.readouterr() + assert "ZeroDivisionError: division by zero" in captured.err + assert captured.err.count("Traceback") == 1 diff --git a/python/tests/test_application.py b/python/tests/system/test_application_minimal.py similarity index 57% rename from python/tests/test_application.py rename to python/tests/system/test_application_minimal.py index 4b5149af..980fd868 100644 --- a/python/tests/test_application.py +++ b/python/tests/system/test_application_minimal.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,13 +13,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -from test_operators_native import MinimalOp, PingMiddleOp, PingRxOp, PingTxOp - from holoscan.conditions import CountCondition -from holoscan.core import Application +from holoscan.core import Application, Operator, OperatorSpec from holoscan.logger import load_env_log_level +class MinimalOp(Operator): + def __init__(self, *args, **kwargs): + self.count = 1 + self.param_value = None + # Need to call the base class constructor last + super().__init__(*args, **kwargs) + + def setup(self, spec: OperatorSpec): + spec.param("param_value", 500) + + def compute(self, op_input, op_output, context): + self.count += 1 + + class MinimalApp(Application): def compose(self): mx = MinimalOp(self, CountCondition(self, 10), name="mx") @@ -31,19 +43,3 @@ def test_minimal_app(ping_config_file): app = MinimalApp() app.config(ping_config_file) app.run() - - -class MyPingApp(Application): - def compose(self): - tx = PingTxOp(self, CountCondition(self, 10), name="tx") - mx = PingMiddleOp(self, self.from_config("mx"), name="mx") - rx = PingRxOp(self, name="rx") - self.add_flow(tx, mx, {("out1", "in1"), ("out2", "in2")}) - self.add_flow(mx, rx, {("out1", "receivers"), ("out2", "receivers")}) - - -def test_my_ping_app(ping_config_file): - load_env_log_level() - app = MyPingApp() - app.config(ping_config_file) - app.run() diff --git a/python/tests/system/test_application_ping.py b/python/tests/system/test_application_ping.py new file mode 100644 index 00000000..8c1bb97c --- /dev/null +++ b/python/tests/system/test_application_ping.py @@ -0,0 +1,116 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from holoscan.conditions import CountCondition +from holoscan.core import Application, Operator, OperatorSpec +from holoscan.logger import load_env_log_level + + +class ValueData: + def __init__(self, value): + self.data = value + + def __repr__(self): + return f"ValueData({self.data})" + + def __eq__(self, other): + return self.data == other.data + + def __hash__(self): + return hash(self.data) + + +class PingTxOp(Operator): + def __init__(self, *args, **kwargs): + self.index = 0 + # Need to call the base class constructor last + super().__init__(*args, **kwargs) + + def setup(self, spec: OperatorSpec): + spec.output("out1") + spec.output("out2") + + def compute(self, op_input, op_output, context): + value1 = ValueData(self.index) + self.index += 1 + op_output.emit(value1, "out1") + + value2 = ValueData(self.index) + self.index += 1 + op_output.emit(value2, "out2") + + +class PingMiddleOp(Operator): + def __init__(self, *args, **kwargs): + # If `self.multiplier` is set here (e.g., `self.multiplier = 4`), then + # the default value by `param()` in `setup()` will be ignored. + # (you can just call `spec.param("multiplier")` in `setup()` to use the + # default value) + # + # self.multiplier = 4 + self.count = 1 + + # Need to call the base class constructor last + super().__init__(*args, **kwargs) + + def setup(self, spec: OperatorSpec): + spec.input("in1") + spec.input("in2") + spec.output("out1") + spec.output("out2") + spec.param("multiplier", 2) + + def compute(self, op_input, op_output, context): + value1 = op_input.receive("in1") + value2 = op_input.receive("in2") + self.count += 1 + + # Multiply the values by the multiplier parameter + value1.data *= self.multiplier + value2.data *= self.multiplier + + op_output.emit(value1, "out1") + op_output.emit(value2, "out2") + + +class PingRxOp(Operator): + def __init__(self, *args, **kwargs): + self.count = 1 + # Need to call the base class constructor last + super().__init__(*args, **kwargs) + + def setup(self, spec: OperatorSpec): + spec.param("receivers", kind="receivers") + + def compute(self, op_input, op_output, context): + values = op_input.receive("receivers") + assert values is not None + self.count += 1 + + +class MyPingApp(Application): + def compose(self): + tx = PingTxOp(self, CountCondition(self, 10), name="tx") + mx = PingMiddleOp(self, self.from_config("mx"), name="mx") + rx = PingRxOp(self, name="rx") + self.add_flow(tx, mx, {("out1", "in1"), ("out2", "in2")}) + self.add_flow(mx, rx, {("out1", "receivers"), ("out2", "receivers")}) + + +def test_my_ping_app(ping_config_file): + load_env_log_level() + app = MyPingApp() + app.config(ping_config_file) + app.run() diff --git a/python/tests/test_operators_gxf.py b/python/tests/test_operators_gxf.py deleted file mode 100644 index c915b1fc..00000000 --- a/python/tests/test_operators_gxf.py +++ /dev/null @@ -1,475 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os - -import pytest - -from holoscan.core import Operator, _Operator -from holoscan.gxf import GXFOperator -from holoscan.operators import ( - AJASourceOp, - BayerDemosaicOp, - FormatConverterOp, - HolovizOp, - LSTMTensorRTInferenceOp, - MultiAIInferenceOp, - MultiAIPostprocessorOp, - NTV2Channel, - SegmentationPostprocessorOp, - TensorRTInferenceOp, - ToolTrackingPostprocessorOp, - VideoStreamRecorderOp, - VideoStreamReplayerOp, - VisualizerICardioOp, - _holoviz_str_to_input_type, -) -from holoscan.resources import ( - BlockMemoryPool, - CudaStreamPool, - MemoryStorageType, - UnboundedAllocator, -) - -try: - from holoscan.operators import EmergentSourceOp - - have_emergent_op = True -except ImportError: - have_emergent_op = False - -sample_data_path = os.environ.get("HOLOSCAN_SAMPLE_DATA_PATH", "../data") - - -class TestAJASourceOp: - def test_kwarg_based_initialization(self, app, config_file, capfd): - app.config(config_file) - op = AJASourceOp( - fragment=app, - name="source", - channel=NTV2Channel.NTV2_CHANNEL1, - **app.kwargs("aja"), - ) - assert isinstance(op, GXFOperator) - assert isinstance(op, _Operator) - assert op.id != -1 - assert op.operator_type == Operator.OperatorType.GXF - assert op.gxf_typename == "nvidia::holoscan::AJASource" - - # assert no warnings or errors logged - captured = capfd.readouterr() - # assert "error" not in captured.err - # TODO: resolve error logged by GXF: "Unable to handle parameter 'channel'" - assert "warning" not in captured.err - - -class TestFormatConverterOp: - def test_kwarg_based_initialization(self, app, config_file, capfd): - app.config(config_file) - op = FormatConverterOp( - fragment=app, - name="recorder_format_converter", - pool=BlockMemoryPool( - name="pool", - fragment=app, - storage_type=MemoryStorageType.DEVICE, - block_size=16 * 1024**2, - num_blocks=4, - ), - **app.kwargs("recorder_format_converter"), - ) - assert isinstance(op, GXFOperator) - assert isinstance(op, _Operator) - len(op.args) == 12 - assert op.id != -1 - assert op.operator_type == Operator.OperatorType.GXF - assert op.gxf_typename == "nvidia::holoscan::formatconverter::FormatConverter" - - # assert no warnings or errors logged - captured = capfd.readouterr() - assert "error" not in captured.err - assert "warning" not in captured.err - - -class TestLSTMTensorRTInferenceOp: - def test_kwarg_based_initialization(self, app, config_file, capfd): - app.config(config_file) - lstm_cuda_stream_pool = CudaStreamPool( - fragment=app, - name="cuda_stream", - dev_id=0, - stream_flags=0, - stream_priority=0, - reserved_size=1, - max_size=5, - ) - op = LSTMTensorRTInferenceOp( - fragment=app, - name="lstm_inferer", - pool=UnboundedAllocator(fragment=app, name="pool"), - cuda_stream_pool=lstm_cuda_stream_pool, - model_file_path=os.path.join( - sample_data_path, "ultrasound", "model", "us_unet_256x256_nhwc.onnx" - ), - engine_cache_dir=os.path.join( - sample_data_path, "ultrasound", "model", "us_unet_256x256_nhwc_engines" - ), - **app.kwargs("lstm_inference"), - ) - assert isinstance(op, GXFOperator) - assert isinstance(op, _Operator) - assert len(op.args) == 17 - assert op.id != -1 - assert op.operator_type == Operator.OperatorType.GXF - assert op.gxf_typename == "nvidia::holoscan::custom_lstm_inference::TensorRtInference" - - # assert no warnings or errors logged - # captured = capfd.readouterr() - # assert "error" not in captured.err - # assert "warning" not in captured.err - # Note: currently warning and error are present due to non-specified - # optional parameters - # error GXF_FAILURE setting GXF parameter 'clock' - # error GXF_FAILURE setting GXF parameter 'dla_core' - - -class TestTensorRTInferenceOp: - def test_kwarg_based_initialization(self, app, config_file, capfd): - app.config(config_file) - lstm_cuda_stream_pool = CudaStreamPool( - fragment=app, - name="cuda_stream", - dev_id=0, - stream_flags=0, - stream_priority=0, - reserved_size=1, - max_size=5, - ) - op = TensorRTInferenceOp( - fragment=app, - name="inferer", - pool=UnboundedAllocator(fragment=app, name="pool"), - cuda_stream_pool=lstm_cuda_stream_pool, - model_file_path=os.path.join( - sample_data_path, "ultrasound", "model", "us_unet_256x256_nhwc.onnx" - ), - engine_cache_dir=os.path.join( - sample_data_path, "ultrasound", "model", "us_unet_256x256_nhwc_engines" - ), - **app.kwargs("segmentation_inference"), - ) - assert isinstance(op, GXFOperator) - assert isinstance(op, _Operator) - assert op.id != -1 - assert op.operator_type == Operator.OperatorType.GXF - assert op.gxf_typename == "nvidia::gxf::TensorRtInference" - - # assert no warnings or errors logged - # captured = capfd.readouterr() - # assert "error" not in captured.err - # assert "warning" not in captured.err - # Note: currently warning and error are present due to non-specified - # optional parameters - # error GXF_FAILURE setting GXF parameter 'clock' - # error GXF_FAILURE setting GXF parameter 'dla_core' - - -class TestVideoStreamRecorderOp: - def test_kwarg_based_initialization(self, app, config_file, capfd): - app.config(config_file) - op = VideoStreamRecorderOp(name="recorder", fragment=app, **app.kwargs("recorder")) - assert isinstance(op, GXFOperator) - assert isinstance(op, _Operator) - assert op.id != -1 - assert op.operator_type == Operator.OperatorType.GXF - assert op.gxf_typename == "nvidia::gxf::EntityRecorder" - - # assert no warnings or errors logged - captured = capfd.readouterr() - assert "error" not in captured.err - assert "warning" not in captured.err - - -class TestVideoStreamReplayerOp: - def test_kwarg_based_initialization(self, app, config_file, capfd): - app.config(config_file) - op = VideoStreamReplayerOp(name="replayer", fragment=app, **app.kwargs("replayer")) - assert isinstance(op, GXFOperator) - assert isinstance(op, _Operator) - assert op.id != -1 - assert op.operator_type == Operator.OperatorType.GXF - assert op.gxf_typename == "nvidia::holoscan::stream_playback::VideoStreamReplayer" - - # assert no warnings or errors logged - captured = capfd.readouterr() - assert "error" not in captured.err - assert "warning" not in captured.err - - -class TestSegmentationPostprocessorOp: - def test_kwarg_based_initialization(self, app, capfd): - op = SegmentationPostprocessorOp( - fragment=app, - allocator=UnboundedAllocator(fragment=app, name="allocator"), - name="segmentation_postprocessor", - ) - assert isinstance(op, GXFOperator) - assert isinstance(op, _Operator) - assert op.id != -1 - assert op.operator_type == Operator.OperatorType.GXF - assert op.gxf_typename == "nvidia::holoscan::segmentation_postprocessor::Postprocessor" - - # assert no warnings or errors logged - captured = capfd.readouterr() - assert "error" not in captured.err - assert "warning" not in captured.err - - -class TestToolTrackingPostprocessorOp: - def test_kwarg_based_initialization(self, app, capfd): - op = ToolTrackingPostprocessorOp( - fragment=app, - name="tool_tracking_postprocessor", - device_allocator=BlockMemoryPool( - fragment=app, - name="device_alloc", - storage_type=MemoryStorageType.DEVICE, - block_size=16 * 1024**2, - num_blocks=4, - ), - host_allocator=UnboundedAllocator(fragment=app, name="host_alloc"), - ) - assert isinstance(op, GXFOperator) - assert isinstance(op, _Operator) - assert op.id != -1 - assert op.operator_type == Operator.OperatorType.GXF - assert op.gxf_typename == "nvidia::holoscan::tool_tracking_postprocessor::Postprocessor" - - # assert no warnings or errors logged - captured = capfd.readouterr() - assert "error" not in captured.err - assert "warning" not in captured.err - - -# TODO: -# For holoviz, need to implement std::vector "tensors" argument - - -@pytest.mark.parametrize( - "type_str", - [ - "unknown", - "color", - "color_lut", - "points", - "lines", - "line_strip", - "triangles", - "crosses", - "rectangles", - "ovals", - "text", - ], -) -def test_holoviz_input_types(type_str): - assert isinstance(_holoviz_str_to_input_type[type_str], HolovizOp.InputType) - - -class TestHolovizOp: - def test_kwarg_based_initialization(self, app, config_file, capfd): - app.config(config_file) - op = HolovizOp(app, name="visualizer", **app.kwargs("holoviz")) - assert isinstance(op, GXFOperator) - assert isinstance(op, _Operator) - - # assert no warnings or errors logged - captured = capfd.readouterr() - assert "error" not in captured.err - assert "warning" not in captured.err - assert op.id != -1 - assert op.operator_type == Operator.OperatorType.GXF - assert op.gxf_typename == "nvidia::holoscan::Holoviz" - - @pytest.mark.parametrize( - "tensor", - [ - # color not length 4 - dict( - name="scaled_coords", - type="crosses", - line_width=4, - color=[1.0, 0.0], - ), - # color cannot be a str - dict( - name="scaled_coords", - type="crosses", - line_width=4, - color="red", - ), - # color values out of range - dict( - name="scaled_coords", - type="crosses", - line_width=4, - color=[255, 255, 255, 255], - ), - # unrecognized type - dict( - name="scaled_coords", - type="invalid", - line_width=4, - ), - # type not specified - dict( - name="scaled_coords", - line_width=4, - ), - # name not specified - dict( - type="crosses", - line_width=4, - ), - # unrecognized key specified - dict( - name="scaled_coords", - type="crosses", - line_width=4, - color=[1.0, 1.0, 1.0, 0.0], - invalid_key=None, - ), - ], - ) - def test_invalid_tensors(self, tensor, app): - - with pytest.raises(ValueError): - HolovizOp( - name="visualizer", - fragment=app, - tensors=[tensor], - ) - - -class TestMultiAIInferenceOp: - def test_kwarg_based_initialization(self, app, config_file, capfd): - app.config(config_file) - model_path = os.path.join(sample_data_path, "multiai_ultrasound", "models") - - model_path_map = { - "icardio_plax_chamber": os.path.join(model_path, "plax_chamber.onnx"), - "icardio_aortic_stenosis": os.path.join(model_path, "aortic_stenosis.onnx"), - "icardio_bmode_perspective": os.path.join(model_path, "bmode_perspective.onnx"), - } - - op = MultiAIInferenceOp( - app, - name="multiai_inference", - allocator=UnboundedAllocator(app, name="pool"), - model_path_map=model_path_map, - **app.kwargs("multiai_inference"), - ) - assert isinstance(op, GXFOperator) - assert isinstance(op, _Operator) - assert op.id != -1 - assert op.gxf_typename == "nvidia::holoscan::multiai::MultiAIInference" - - # assert no warnings or errors logged - captured = capfd.readouterr() - assert "error" not in captured.err - assert "warning" not in captured.err - - -class TestMultiAIPostprocessorOp: - def test_kwarg_based_initialization(self, app, config_file, capfd): - app.config(config_file) - op = MultiAIPostprocessorOp( - app, - name="multiai_postprocessor", - allocator=UnboundedAllocator(app, name="pool"), - **app.kwargs("multiai_postprocessor"), - ) - assert isinstance(op, GXFOperator) - assert isinstance(op, _Operator) - assert op.gxf_typename == "nvidia::holoscan::multiai::MultiAIPostprocessor" - - -class TestVisualizerICardioOp: - def test_kwarg_based_initialization(self, app, config_file, capfd): - app.config(config_file) - op = VisualizerICardioOp( - app, - name="visualizer_icardio", - allocator=UnboundedAllocator(app, name="pool"), - **app.kwargs("visualizer_icardio"), - ) - assert isinstance(op, GXFOperator) - assert isinstance(op, _Operator) - assert op.gxf_typename == "nvidia::holoscan::multiai::VisualizerICardio" - - -class TestBayerDemosaicOp: - def test_kwarg_based_initialization(self, app, config_file, capfd): - app.config(config_file) - demosaic_stream_pool = CudaStreamPool( - app, - name="cuda_stream", - dev_id=0, - stream_flags=0, - stream_priority=0, - reserved_size=1, - max_size=5, - ) - op = BayerDemosaicOp( - app, - name="demosaic", - pool=BlockMemoryPool( - fragment=app, - name="device_alloc", - storage_type=MemoryStorageType.DEVICE, - block_size=16 * 1024**2, - num_blocks=4, - ), - cuda_stream_pool=demosaic_stream_pool, - **app.kwargs("demosaic"), - ) - assert isinstance(op, GXFOperator) - assert isinstance(op, _Operator) - assert op.id != -1 - assert op.gxf_typename == "nvidia::holoscan::BayerDemosaic" - - # assert no warnings or errors logged - captured = capfd.readouterr() - assert "error" not in captured.err - assert "warning" not in captured.err - - -@pytest.mark.skipif(not have_emergent_op, reason="requires EmergentSourceOp") -class TestEmergentSourceOp: - def test_kwarg_based_initialization(self, app, config_file, capfd): - app.config(config_file) - op = EmergentSourceOp( - app, - name="emergent", - **app.kwargs("emergent"), - ) - assert isinstance(op, GXFOperator) - assert isinstance(op, _Operator) - assert op.id != -1 - assert op.gxf_typename == "nvidia::holoscan::EmergentSource" - - # assert no warnings or errors logged - captured = capfd.readouterr() - assert "error" not in captured.err - assert "warning" not in captured.err diff --git a/python/tests/test_operators_native.py b/python/tests/test_operators_native.py deleted file mode 100644 index 21105f62..00000000 --- a/python/tests/test_operators_native.py +++ /dev/null @@ -1,394 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest - -from holoscan.conditions import CountCondition -from holoscan.core import Arg, Operator, OperatorSpec, Tensor -from holoscan.gxf import GXFExtensionRegistrar -from holoscan.resources import UnboundedAllocator - -try: - import numpy as np - - unsigned_dtypes = [np.uint8, np.uint16, np.uint32, np.uint64] - signed_dtypes = [np.int8, np.int16, np.int32, np.int64] - float_dtypes = [np.float16, np.float32, np.float64] - complex_dtypes = [np.complex64, np.complex128] - -except ImportError: - unsigned_dtypes = signed_dtypes = float_dtypes = [] - - -class TestOperator: - def test_default_init(self, fragment, capfd): - op = Operator() - assert op.name == "" - assert op.fragment is None - assert op.id == -1 - assert op.operator_type == Operator.OperatorType.NATIVE - capfd.readouterr() - - def test_operator_type(self): - assert hasattr(Operator, "OperatorType") - assert hasattr(Operator.OperatorType, "NATIVE") - assert hasattr(Operator.OperatorType, "GXF") - - @pytest.mark.parametrize("with_fragment", [False, True, "as_kwarg"]) - @pytest.mark.parametrize("with_name", [False, True]) - @pytest.mark.parametrize("with_condition", [False, True, "as_kwarg"]) - @pytest.mark.parametrize("with_resource", [False, True, "as_kwarg"]) - def test_init(self, app, with_fragment, with_name, with_condition, with_resource, capfd): - kwargs = dict(a=5, b=(13.7, 15.2), c="abcd") - args = () - if with_name: - kwargs["name"] = "my op" - if with_fragment == "as_kwarg": - kwargs["fragment"] = app - elif with_fragment: - args = (app,) - else: - args = () - if with_condition: - if with_condition == "as_kwarg": - kwargs["count"] = CountCondition(app, count=15) - else: - args += (CountCondition(app, count=15, name="count"),) - if with_resource: - if with_condition == "as_kwarg": - kwargs["pool"] = UnboundedAllocator(app) - else: - args += (UnboundedAllocator(app, name="pool"),) - op = Operator(*args, **kwargs) - - # check operator name - expected_name = "my op" if with_name else "" - assert op.name == expected_name - - # check operator fragment - if with_fragment: - assert op.fragment is app - else: - assert op.fragment is None - - # check all args that were not of Condition or Resource type - assert len(op.args) == 3 - assert [arg.name for arg in op.args] == ["a", "b", "c"] - - # check conditions - if with_condition: - assert len(op.conditions) == 1 - assert "count" in op.conditions - assert isinstance(op.conditions["count"], CountCondition) - else: - assert len(op.conditions) == 0 - - # check resources - if with_resource: - assert len(op.resources) == 1 - assert "pool" in op.resources - assert isinstance(op.resources["pool"], UnboundedAllocator) - else: - assert len(op.resources) == 0 - capfd.readouterr() - - def test_error_on_multiple_fragments(self, app, capfd): - with pytest.raises(RuntimeError): - Operator(app, app) - with pytest.raises(RuntimeError): - Operator(app, fragment=app) - capfd.readouterr() - - def test_name(self, capfd): - op = Operator() - op.name = "op1" - assert op.name == "op1" - - op = Operator(name="op3") - assert op.name == "op3" - capfd.readouterr() - - def test_fragment(self, fragment, capfd): - op = Operator() - assert op.fragment is None - op.fragment = fragment - assert op.fragment is fragment - - def test_add_arg(self, capfd): - op = Operator() - op.add_arg(Arg("a1")) - capfd.readouterr() - - def test_initialize(self, app, config_file, capfd): - spec = OperatorSpec(app) - - op = Operator() - # Operator.__init__ will have added op.spec for us - assert isinstance(op.spec, OperatorSpec) - - app.config(config_file) - - # initialize context - context = app.executor.context - assert context is not None - - # follow order of operations in Fragment::make_operator - op.name = "my operator" - - # initialize() will segfault op.fragment is not assigned? - op.fragment = app - - op.setup(spec) - op.spec = spec - assert isinstance(op.spec, OperatorSpec) - - # Have to first register the GXFWrapper type before we can call - # op.initialize() on an operator of type kNative. - registrar = GXFExtensionRegistrar(context, "my_registrar") - # add_component is currently templated only for - # - registrar.add_component("GXF wrapper to support Holoscan SDK native operators") - # add_type is currently templated only for - registrar.add_type("Holoscan message type") - # will be True if extension was registered successfully - assert registrar.register_extension() - - op.initialize() - op.id != -1 - op.operator_type == Operator.OperatorType.NATIVE - capfd.readouterr() - - def test_operator_setup_and_assignment(self, fragment, capfd): - spec = OperatorSpec(fragment) - op = Operator(fragment) - op.setup(spec) - op.spec = spec - capfd.readouterr() - - def test_dynamic_attribute_allowed(self, capfd): - obj = Operator() - obj.custom_attribute = 5 - capfd.readouterr() - - -class TestTensor: - def _check_dlpack_attributes(self, t): - assert hasattr(t, "__dlpack__") - type(t.__dlpack__()).__name__ == "PyCapsule" - - assert hasattr(t, "__dlpack_device__") - dev = t.__dlpack_device__() - assert isinstance(dev, tuple) and len(dev) == 2 - - def _check_array_interface_attribute(self, t, arr, cuda=False): - if cuda: - assert hasattr(t, "__cuda_array_interface__") - interface = t.__cuda_array_interface__ - reference_interface = arr.__cuda_array_interface__ - else: - assert hasattr(t, "__array_interface__") - interface = t.__array_interface__ - reference_interface = arr.__array_interface__ - - assert interface["version"] == 3 - - assert interface["typestr"] == arr.dtype.str - assert interface["shape"] == arr.shape - assert len(interface["data"]) == 2 - if cuda: - assert interface["data"][0] == arr.data.mem.ptr - # no writeable flag present on CuPy arrays - else: - assert interface["data"][0] == arr.ctypes.data - assert interface["data"][1] == (not arr.flags.writeable) - if interface["strides"] is None: - assert arr.flags.c_contiguous - else: - assert interface["strides"] == arr.strides - assert interface["descr"] == [("", arr.dtype.str)] - - if reference_interface["version"] == interface["version"]: - interface["shape"] == reference_interface["shape"] - interface["typestr"] == reference_interface["typestr"] - interface["descr"] == reference_interface["descr"] - interface["data"] == reference_interface["data"] - if reference_interface["strides"] is not None: - interface["strides"] == reference_interface["strides"] - - def _check_tensor_property_values(self, t, arr, cuda=False): - assert t.size == arr.size - assert t.nbytes == arr.nbytes - assert t.ndim == arr.ndim - assert t.itemsize == arr.dtype.itemsize - - assert t.shape == arr.shape - assert t.strides == arr.strides - - type(t.data).__name__ == "PyCapsule" - - @pytest.mark.parametrize( - "dtype", unsigned_dtypes + signed_dtypes + float_dtypes + complex_dtypes - ) - @pytest.mark.parametrize("order", ["F", "C"]) - def test_numpy_as_tensor(self, dtype, order): - np = pytest.importorskip("numpy") - a = np.zeros((4, 8, 12), dtype=dtype, order=order) - t = Tensor.as_tensor(a) - assert isinstance(t, Tensor) - - self._check_dlpack_attributes(t) - self._check_array_interface_attribute(t, a, cuda=False) - self._check_tensor_property_values(t, a) - - @pytest.mark.parametrize( - "dtype", unsigned_dtypes + signed_dtypes + float_dtypes + complex_dtypes - ) - @pytest.mark.parametrize("order", ["F", "C"]) - def test_cupy_as_tensor(self, dtype, order): - cp = pytest.importorskip("cupy") - a = cp.zeros((4, 8, 12), dtype=dtype, order=order) - t = Tensor.as_tensor(a) - assert isinstance(t, Tensor) - - self._check_dlpack_attributes(t) - self._check_array_interface_attribute(t, a, cuda=True) - self._check_tensor_property_values(t, a) - - def test_tensor_properties_are_readonly(self): - np = pytest.importorskip("numpy") - a = np.zeros((4, 8, 12), dtype=np.uint8) - t = Tensor.as_tensor(a) - with pytest.raises(AttributeError): - t.size = 8 - with pytest.raises(AttributeError): - t.nbytes = 8 - with pytest.raises(AttributeError): - t.ndim = 2 - with pytest.raises(AttributeError): - t.itemsize = 3 - with pytest.raises(AttributeError): - t.shape = (t.size,) - with pytest.raises(AttributeError): - t.strides = (8,) - with pytest.raises(AttributeError): - t.data = 0 - - @pytest.mark.parametrize( - "dtype", unsigned_dtypes + signed_dtypes + float_dtypes + complex_dtypes - ) - @pytest.mark.parametrize("order", ["F", "C"]) - @pytest.mark.parametrize("module", ["cupy", "numpy"]) - def test_tensor_round_trip(self, dtype, order, module): - xp = pytest.importorskip(module) - a = xp.zeros((4, 8, 12), dtype=dtype, order=order) - t = Tensor.as_tensor(a) - b = xp.asarray(t) - xp.testing.assert_array_equal(a, b) - - -class MinimalOp(Operator): - def __init__(self, *args, **kwargs): - self.count = 1 - self.param_value = None - # Need to call the base class constructor last - super().__init__(*args, **kwargs) - - def setup(self, spec: OperatorSpec): - spec.param("param_value", 500) - - def compute(self, op_input, op_output, context): - self.count += 1 - - -class ValueData: - def __init__(self, value): - self.data = value - - def __repr__(self): - return f"ValueData({self.data})" - - def __eq__(self, other): - return self.data == other.data - - def __hash__(self): - return hash(self.data) - - -class PingTxOp(Operator): - def __init__(self, *args, **kwargs): - self.index = 0 - # Need to call the base class constructor last - super().__init__(*args, **kwargs) - - def setup(self, spec: OperatorSpec): - spec.output("out1") - spec.output("out2") - - def compute(self, op_input, op_output, context): - value1 = ValueData(self.index) - self.index += 1 - op_output.emit(value1, "out1") - - value2 = ValueData(self.index) - self.index += 1 - op_output.emit(value2, "out2") - - -class PingMiddleOp(Operator): - def __init__(self, *args, **kwargs): - # If `self.multiplier` is set here (e.g., `self.multiplier = 4`), then - # the default value by `param()` in `setup()` will be ignored. - # (you can just call `spec.param("multiplier")` in `setup()` to use the - # default value) - # - # self.multiplier = 4 - self.count = 1 - - # Need to call the base class constructor last - super().__init__(*args, **kwargs) - - def setup(self, spec: OperatorSpec): - spec.input("in1") - spec.input("in2") - spec.output("out1") - spec.output("out2") - spec.param("multiplier", 2) - - def compute(self, op_input, op_output, context): - value1 = op_input.receive("in1") - value2 = op_input.receive("in2") - self.count += 1 - - # Multiply the values by the multiplier parameter - value1.data *= self.multiplier - value2.data *= self.multiplier - - op_output.emit(value1, "out1") - op_output.emit(value2, "out2") - - -class PingRxOp(Operator): - def __init__(self, *args, **kwargs): - self.count = 1 - # Need to call the base class constructor last - super().__init__(*args, **kwargs) - - def setup(self, spec: OperatorSpec): - spec.param("receivers", kind="receivers") - - def compute(self, op_input, op_output, context): - values = op_input.receive("receivers") - assert values is not None - self.count += 1 diff --git a/python/tests/test_conditions.py b/python/tests/unit/test_conditions.py similarity index 59% rename from python/tests/test_conditions.py rename to python/tests/unit/test_conditions.py index 21a505df..f03c97e2 100644 --- a/python/tests/test_conditions.py +++ b/python/tests/unit/test_conditions.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,8 +19,8 @@ DownstreamMessageAffordableCondition, MessageAvailableCondition, ) -from holoscan.core import Condition -from holoscan.gxf import GXFCondition +from holoscan.core import Application, Condition, ConditionType, Operator +from holoscan.gxf import Entity, GXFCondition class TestBooleanCondition: @@ -135,3 +135,85 @@ def test_default_initialization(self, app): def test_positional_initialization(self, app): MessageAvailableCondition(app, 1, 4, "available") + + +#################################################################################################### +# Test Ping app with no conditions on Rx operator +#################################################################################################### + + +class PingTxOpNoCondition(Operator): + def __init__(self, *args, **kwargs): + self.index = 0 + # Need to call the base class constructor last + super().__init__(*args, **kwargs) + + def setup(self, spec): + spec.output("out1") + spec.output("out2") + + def compute(self, op_input, op_output, context): + self.index += 1 + if self.index == 1: + print(f"#TX{self.index}") + elif self.index == 2: + print(f"#T1O{self.index}") + op_output.emit(self.index, "out1") + elif self.index == 3: + print(f"#T2O{self.index}") + entity = Entity(context) + op_output.emit(entity, "out2") + elif self.index == 4: + print(f"#TO{self.index}") + op_output.emit(self.index, "out1") + entity = Entity(context) + op_output.emit(entity, "out2") + else: + print(f"#TX{self.index}") + + +class PingRxOpNoInputCondition(Operator): + def __init__(self, *args, **kwargs): + self.index = 0 + # Need to call the base class constructor last + super().__init__(*args, **kwargs) + + def setup(self, spec): + # No input condition + spec.input("in1").condition(ConditionType.NONE) + spec.input("in2").condition(ConditionType.NONE) + + def compute(self, op_input, op_output, context): + self.index += 1 + value1 = op_input.receive("in1") + value2 = op_input.receive("in2") + + if value1 and not value2: + print(f"#R1O{self.index}") + elif not value1 and value2: + print(f"#R2O{self.index}") + elif value1 and value2: + print(f"#RO{self.index}") + else: + print(f"#RX{self.index}") + + +class PingRxOpNoInputConditionApp(Application): + def compose(self): + tx = PingTxOpNoCondition(self, CountCondition(self, 5), name="tx") + rx = PingRxOpNoInputCondition(self, CountCondition(self, 5), name="rx") + self.add_flow(tx, rx, {("out1", "in1"), ("out2", "in2")}) + + +def test_ping_no_input_condition(capfd): + app = PingRxOpNoInputConditionApp() + app.run() + + captured = capfd.readouterr() + + sequence = (line[1:] if line.startswith("#") else "" for line in captured.out.splitlines()) + assert "".join(sequence) == "TX1RX1T1O2R1O2T2O3R2O3TO4RO4TX5RX5" + + error_msg = captured.err.lower() + assert "error" not in error_msg + assert "warning" not in error_msg diff --git a/python/tests/test_config.py b/python/tests/unit/test_config.py similarity index 88% rename from python/tests/test_config.py rename to python/tests/unit/test_config.py index 6a4b54b5..2b617783 100644 --- a/python/tests/test_config.py +++ b/python/tests/unit/test_config.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/python/tests/test_core.py b/python/tests/unit/test_core.py similarity index 92% rename from python/tests/test_core.py rename to python/tests/unit/test_core.py index 08cb1512..b022f497 100644 --- a/python/tests/test_core.py +++ b/python/tests/unit/test_core.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -35,13 +35,12 @@ Operator, OperatorSpec, OutputContext, - py_object_to_arg, Resource, _Fragment, + py_object_to_arg, ) from holoscan.executors import GXFExecutor from holoscan.graphs import FlowGraph -from holoscan.gxf import GXFExtensionRegistrar class OpTx(Operator): @@ -65,31 +64,7 @@ def setup(self, spec: OperatorSpec): spec.input("in2") -def register_gxf_wrapper(_fragment): - # Register the GXFWrapper type - registrar = GXFExtensionRegistrar(_fragment.executor.context, "my_registrar") - # add_component is add_component - registrar.add_component("GXF wrapper to support Holoscan SDK native operators") - # add_type is add_type - registrar.add_type("Holoscan message type") - code = registrar.register_extension() - # code is True if the registration succeeded - assert code - return registrar - - def get_tx_and_rx_ops(_fragment): - # Explicit register_gxf_wrapper call to avoid logged errors such as: - # ERROR gxf/std/type_registry.cpp@48: Unknown type: holoscan::gxf::GXFWrapper - # Usually we don't explicitly register this from the Python API because - # the underlying C++ `Fragment::run()` or `Application::run()` method calls - # it for us when running an application. In this file, for - # `TestApplication`, `TestFragment` classes, we want to test the - # `add_flow` and `add_operator` methods outside of the context of their - # usual use within the `run()` method. - - register_gxf_wrapper(_fragment) - op_tx = OpTx(_fragment, name="op_tx") op_rx = OpRx(_fragment, name="op_rx") @@ -109,7 +84,7 @@ def test_init(self): def test_repr(self): t = ArgType(ArgElementType.FLOAT64, ArgContainerType.ARRAY) - assert repr(t) == "" + assert repr(t) == "std::array" def test_dynamic_attribute_not_allowed(self): obj = ArgType() @@ -119,7 +94,6 @@ def test_dynamic_attribute_not_allowed(self): class TestArg: def test_name(self): - a = Arg("arg_1") assert a.name == "arg_1" @@ -127,7 +101,6 @@ def test_name(self): Arg() def test_arg_type(self): - a = Arg("arg_1") isinstance(a.arg_type, ArgType) isinstance(a.arg_type.container_type, ArgContainerType) @@ -146,7 +119,7 @@ def test_has_value(self): def test_repr(self): a = Arg("my_arg") s = a.__repr__() - assert s == "" + assert s == "name: my_arg\ntype: CustomType" def test_dynamic_attribute_not_allowed(self): obj = Arg("a") @@ -529,11 +502,7 @@ def test_from_config_missing_key(self, fragment, config_file, capfd): assert msg in captured.err def test_uninitialized_config(self, fragment, config_file, capfd): - fragment.config() - msg = "Config object was not created. Call config(config_file, prefix) first." - captured = capfd.readouterr() - assert "warning" in captured.err - assert msg in captured.err + assert fragment.config().config_file == "" def test_add_operator(self, fragment, config_file): fragment.config(config_file) @@ -613,7 +582,7 @@ def test_from_config_missing_key(self, app, config_file, capfd): assert msg in captured.err def test_uninitialized_config(self, app, config_file, capfd): - app.config() + assert app.config().config_file == "" def test_add_operator(self, app, config_file): app.config(config_file) diff --git a/python/tests/test_executors.py b/python/tests/unit/test_executors.py similarity index 77% rename from python/tests/test_executors.py rename to python/tests/unit/test_executors.py index 880d468e..2aeb3151 100644 --- a/python/tests/test_executors.py +++ b/python/tests/unit/test_executors.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,7 +17,8 @@ from holoscan.core import Executor, IOSpec from holoscan.executors import GXFExecutor, create_input_port, create_output_port -from holoscan.operators import HolovizOp +from holoscan.operators import BayerDemosaicOp +from holoscan.resources import CudaStreamPool, UnboundedAllocator class TestGXFExecutor: @@ -41,7 +42,20 @@ def test_dynamic_attribute_not_allowed(self, app): @pytest.mark.parametrize("port_type", ("input", "output")) def test_create_port_methods(port_type, app): - op = HolovizOp(app) + # op here could be any operator that is a GXFOperator + op = BayerDemosaicOp( + app, + pool=UnboundedAllocator(app), + cuda_stream_pool=CudaStreamPool( + app, + name="cuda_stream", + dev_id=0, + stream_flags=0, + stream_priority=0, + reserved_size=1, + max_size=5, + ), + ) if port_type == "input": io_spec = IOSpec(op.spec, "my_" + port_type, IOSpec.IOType.INPUT) creation_func = create_input_port diff --git a/python/tests/test_graphs.py b/python/tests/unit/test_graphs.py similarity index 95% rename from python/tests/test_graphs.py rename to python/tests/unit/test_graphs.py index 21781345..f2468682 100644 --- a/python/tests/test_graphs.py +++ b/python/tests/unit/test_graphs.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/python/tests/test_gxf.py b/python/tests/unit/test_gxf.py similarity index 75% rename from python/tests/test_gxf.py rename to python/tests/unit/test_gxf.py index 31c4cbfb..b3005981 100644 --- a/python/tests/test_gxf.py +++ b/python/tests/unit/test_gxf.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,13 +21,14 @@ GXFComponent, GXFCondition, GXFExecutionContext, - GXFExtensionRegistrar, GXFInputContext, GXFOperator, GXFOutputContext, GXFResource, ) -from holoscan.operators import VideoStreamReplayerOp + +# need any operator based on GXFOperator for testing here +# from holoscan.operators import BayerDemosaicOp class TestEntity: @@ -37,7 +38,6 @@ def test_not_constructable(self): class TestGXFComponent: - GXFClass = GXFComponent def test_init(self): @@ -81,7 +81,6 @@ def test_dynamic_attribute_not_allowed(self): class TestGXFCondition(TestGXFComponent): - GXFClass = GXFCondition def test_type(self): @@ -97,7 +96,6 @@ def test_dynamic_attribute_not_allowed(self): class TestGXFResource(TestGXFComponent): - GXFClass = GXFResource def test_type(self): @@ -116,8 +114,7 @@ class TestGXFInputContext: def test_init(self, app, config_file): app.config(config_file) context = app.executor.context - op = VideoStreamReplayerOp(name="replayer", fragment=app, **app.kwargs("replayer")) - assert isinstance(op, GXFOperator) + op = GXFOperator() input_context = GXFInputContext(context, op) assert isinstance(input_context, GXFInputContext) @@ -126,8 +123,7 @@ class TestGXFOutputContext: def test_init(self, app, config_file): app.config(config_file) context = app.executor.context - op = VideoStreamReplayerOp(name="replayer", fragment=app, **app.kwargs("replayer")) - assert isinstance(op, GXFOperator) + op = GXFOperator() output_context = GXFOutputContext(context, op) assert isinstance(output_context, GXFOutputContext) @@ -136,32 +132,12 @@ class TestGXFExecutionContext: def test_init(self, app, config_file): app.config(config_file) context = app.executor.context - op = VideoStreamReplayerOp(name="replayer", fragment=app, **app.kwargs("replayer")) - assert isinstance(op, GXFOperator) + op = GXFOperator() output_context = GXFExecutionContext(context, op) assert isinstance(output_context, GXFExecutionContext) -class TestGXFExtensionRegistrar: - def test_register_GXFWrapper(self, app): - context = app.executor.context - registrar = GXFExtensionRegistrar(context, "my_registrar") - # add_component is currently templated only for - # - registrar.add_component("GXF wrapper to support Holoscan SDK native operators") - # add_type is currently templated only for - registrar.add_type("Holoscan message type") - # will be True if extension was registered successfully - assert registrar.register_extension() - - def test_typekind_enum(self): - assert hasattr(GXFExtensionRegistrar, "TypeKind") - assert hasattr(GXFExtensionRegistrar.TypeKind, "Extension") - assert hasattr(GXFExtensionRegistrar.TypeKind, "Component") - - class TestGXFOperator: - GXFClass = GXFOperator def test_init(self): diff --git a/python/tests/test_kwargs.py b/python/tests/unit/test_kwargs.py similarity index 93% rename from python/tests/test_kwargs.py rename to python/tests/unit/test_kwargs.py index cda6f169..e03b1753 100644 --- a/python/tests/test_kwargs.py +++ b/python/tests/unit/test_kwargs.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,6 +19,7 @@ import numpy as np import pytest +from holoscan.conditions import BooleanCondition from holoscan.core import ( Arg, ArgContainerType, @@ -28,7 +29,6 @@ kwargs_to_arglist, ) from holoscan.core._core import arg_to_py_object, py_object_to_arg -from holoscan.conditions import BooleanCondition from holoscan.operators import HolovizOp from holoscan.resources import UnboundedAllocator @@ -295,7 +295,7 @@ def test_resource_to_arg(app, as_vector): assert arg.arg_type.element_type == ArgElementType.RESOURCE -@pytest.mark.skipif(not hasattr(np, 'float128'), reason="float128 dtype not available") +@pytest.mark.skipif(not hasattr(np, "float128"), reason="float128 dtype not available") def test_unsupported_numpy_dtype_raises(): """Test that unknown dtypes raise an error.""" with pytest.raises(RuntimeError): @@ -321,7 +321,7 @@ def test_unknown_scalar_numeric_type(): # nested sequences of depth > 2 aren't supported [[[1, 2]]], # arbitrary Python objectds aren't supported - dict(a=6) + dict(a=6), ], ) def test_py_object_to_arg_error(value): @@ -409,27 +409,39 @@ def test_arglist_from_kwargs(): # also test __repr__ method of ArgList here rstr = repr(arglist) - repr_strings = rstr.split("\n") - assert repr_strings[0] == "[" - assert ( - repr_strings[1] == " ," - ) # noqa - assert ( - repr_strings[2] == " ," - ) # noqa assert ( - repr_strings[3] == " ," - ) # noqa - assert ( - repr_strings[4] == " ," - ) # noqa - assert ( - repr_strings[5] == " ," - ) # noqa - assert ( - repr_strings[6] == " ," - ) # noqa - assert repr_strings[7] == "]" + rstr + == """name: arglist +args: + - name: alpha + type: double + value: 5 + - name: beta + type: float + value: 3 + - name: offsets + type: std::vector + value: + - 1 + - 0 + - 3 + - name: names + type: std::vector + value: + - abc + - def + - name: verbose + type: bool + value: false + - name: flags + type: std::vector + value: + - false + - true + - true + - false + - true""" + ) def test_arg_to_py_object_unsupported(fragment): diff --git a/python/tests/test_logger.py b/python/tests/unit/test_logger.py similarity index 95% rename from python/tests/test_logger.py rename to python/tests/unit/test_logger.py index ace51171..a1b50000 100644 --- a/python/tests/test_logger.py +++ b/python/tests/unit/test_logger.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/python/tests/unit/test_operators_gxf.py b/python/tests/unit/test_operators_gxf.py new file mode 100644 index 00000000..474a2811 --- /dev/null +++ b/python/tests/unit/test_operators_gxf.py @@ -0,0 +1,104 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from holoscan.core import _Operator +from holoscan.gxf import GXFOperator +from holoscan.operators import BayerDemosaicOp, TensorRTInferenceOp +from holoscan.resources import ( + BlockMemoryPool, + CudaStreamPool, + MemoryStorageType, + UnboundedAllocator, +) + +sample_data_path = os.environ.get("HOLOSCAN_SAMPLE_DATA_PATH", "../data") + + +class TestBayerDemosaicOp: + def test_kwarg_based_initialization(self, app, config_file, capfd): + app.config(config_file) + demosaic_stream_pool = CudaStreamPool( + app, + name="cuda_stream", + dev_id=0, + stream_flags=0, + stream_priority=0, + reserved_size=1, + max_size=5, + ) + op = BayerDemosaicOp( + app, + name="demosaic", + pool=BlockMemoryPool( + fragment=app, + name="device_alloc", + storage_type=MemoryStorageType.DEVICE, + block_size=16 * 1024**2, + num_blocks=4, + ), + cuda_stream_pool=demosaic_stream_pool, + **app.kwargs("demosaic"), + ) + assert isinstance(op, GXFOperator) + assert isinstance(op, _Operator) + assert op.id != -1 + assert op.gxf_typename == "nvidia::holoscan::BayerDemosaic" + + # assert no warnings or errors logged + captured = capfd.readouterr() + assert "error" not in captured.err + assert "warning" not in captured.err + + +class TestTensorRTInferenceOp: + def test_kwarg_based_initialization(self, app, config_file, capfd): + app.config(config_file) + lstm_cuda_stream_pool = CudaStreamPool( + fragment=app, + name="cuda_stream", + dev_id=0, + stream_flags=0, + stream_priority=0, + reserved_size=1, + max_size=5, + ) + op = TensorRTInferenceOp( + fragment=app, + name="inferer", + pool=UnboundedAllocator(fragment=app, name="pool"), + cuda_stream_pool=lstm_cuda_stream_pool, + model_file_path=os.path.join( + sample_data_path, "ultrasound", "model", "us_unet_256x256_nhwc.onnx" + ), + engine_cache_dir=os.path.join( + sample_data_path, "ultrasound", "model", "us_unet_256x256_nhwc_engines" + ), + **app.kwargs("segmentation_inference"), + ) + assert isinstance(op, GXFOperator) + assert isinstance(op, _Operator) + assert op.id != -1 + assert op.gxf_typename == "nvidia::gxf::TensorRtInference" + + # assert no warnings or errors logged + # captured = capfd.readouterr() + # assert "error" not in captured.err + # assert "warning" not in captured.err + # Note: currently warning and error are present due to non-specified + # optional parameters + # error GXF_FAILURE setting GXF parameter 'clock' + # error GXF_FAILURE setting GXF parameter 'dla_core' diff --git a/python/tests/unit/test_operators_native.py b/python/tests/unit/test_operators_native.py new file mode 100644 index 00000000..962ac2d3 --- /dev/null +++ b/python/tests/unit/test_operators_native.py @@ -0,0 +1,678 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import pytest + +from holoscan.conditions import CountCondition +from holoscan.core import Application, Arg, Operator, OperatorSpec, Tensor, _Operator +from holoscan.gxf import Entity +from holoscan.operators import ( + AJASourceOp, + FormatConverterOp, + HolovizOp, + MultiAIInferenceOp, + MultiAIPostprocessorOp, + NTV2Channel, + SegmentationPostprocessorOp, + VideoStreamRecorderOp, + VideoStreamReplayerOp, + _holoviz_str_to_depth_map_render_mode, + _holoviz_str_to_input_type, +) +from holoscan.resources import BlockMemoryPool, MemoryStorageType, UnboundedAllocator + +try: + import numpy as np + + unsigned_dtypes = [np.uint8, np.uint16, np.uint32, np.uint64] + signed_dtypes = [np.int8, np.int16, np.int32, np.int64] + float_dtypes = [np.float16, np.float32, np.float64] + complex_dtypes = [np.complex64, np.complex128] + +except ImportError: + unsigned_dtypes = signed_dtypes = float_dtypes = [] + +sample_data_path = os.environ.get("HOLOSCAN_SAMPLE_DATA_PATH", "../data") + + +class TestOperator: + def test_default_init(self): + with pytest.raises(TypeError): + Operator() + + def test_invalid_init(self): + with pytest.raises(ValueError): + Operator(5) + + def test_invalid_init2(self, fragment): + # C++ PyOperator class will throw std::runtime_error if >1 fragment provided + with pytest.raises(RuntimeError): + Operator(fragment, fragment) + + def test_basic_init(self, fragment, capfd): + op = Operator(fragment) + assert op.name == "" + assert op.fragment is fragment + assert op.id != -1 + assert op.operator_type == Operator.OperatorType.NATIVE + capfd.readouterr() + + def test_basic_kwarg_init(self, fragment, capfd): + op = Operator(fragment=fragment) + assert op.name == "" + assert op.fragment is fragment + assert op.id != -1 + assert op.operator_type == Operator.OperatorType.NATIVE + capfd.readouterr() + + def test_operator_type(self): + assert hasattr(Operator, "OperatorType") + assert hasattr(Operator.OperatorType, "NATIVE") + assert hasattr(Operator.OperatorType, "GXF") + + @pytest.mark.parametrize("with_name", [False, True]) + @pytest.mark.parametrize("with_condition", [False, True, "as_kwarg"]) + @pytest.mark.parametrize("with_resource", [False, True, "as_kwarg"]) + def test_init(self, app, with_name, with_condition, with_resource, capfd): + kwargs = dict(a=5, b=(13.7, 15.2), c="abcd") + args = () + if with_name: + kwargs["name"] = "my op" + args = (app,) + if with_condition: + if with_condition == "as_kwarg": + kwargs["count"] = CountCondition(app, count=15) + else: + args += (CountCondition(app, count=15, name="count"),) + if with_resource: + if with_condition == "as_kwarg": + kwargs["pool"] = UnboundedAllocator(app) + else: + args += (UnboundedAllocator(app, name="pool"),) + op = Operator(*args, **kwargs) + + # check operator name + expected_name = "my op" if with_name else "" + assert op.name == expected_name + assert op.fragment is app + + # check all args that were not of Condition or Resource type + assert len(op.args) == 3 + assert [arg.name for arg in op.args] == ["a", "b", "c"] + + # check conditions + if with_condition: + assert len(op.conditions) == 1 + assert "count" in op.conditions + assert isinstance(op.conditions["count"], CountCondition) + else: + assert len(op.conditions) == 0 + + # check resources + if with_resource: + assert len(op.resources) == 1 + assert "pool" in op.resources + assert isinstance(op.resources["pool"], UnboundedAllocator) + else: + assert len(op.resources) == 0 + capfd.readouterr() + + def test_error_on_multiple_fragments(self, app, capfd): + with pytest.raises(RuntimeError): + Operator(app, app) + with pytest.raises(TypeError): + Operator(app, fragment=app) + capfd.readouterr() + + def test_name(self, fragment, capfd): + op = Operator(fragment) + op.name = "op1" + assert op.name == "op1" + + op = Operator(fragment, name="op3") + assert op.name == "op3" + capfd.readouterr() + + def test_add_arg(self, fragment, capfd): + op = Operator(fragment) + op.add_arg(Arg("a1")) + capfd.readouterr() + + def test_initialize(self, app, config_file, capfd): + spec = OperatorSpec(app) + + op = Operator(app) + # Operator.__init__ will have added op.spec for us + assert isinstance(op.spec, OperatorSpec) + + app.config(config_file) + + # initialize context + context = app.executor.context + assert context is not None + + # follow order of operations in Fragment::make_operator + op.name = "my operator" + + # initialize() will segfault op.fragment is not assigned? + op.fragment = app + + op.setup(spec) + op.spec = spec + assert isinstance(op.spec, OperatorSpec) + + op.initialize() + op.id != -1 + op.operator_type == Operator.OperatorType.NATIVE + capfd.readouterr() + + def test_operator_setup_and_assignment(self, fragment, capfd): + spec = OperatorSpec(fragment) + op = Operator(fragment) + op.setup(spec) + op.spec = spec + capfd.readouterr() + + def test_dynamic_attribute_allowed(self, fragment, capfd): + obj = Operator(fragment) + obj.custom_attribute = 5 + capfd.readouterr() + + +class TestTensor: + def _check_dlpack_attributes(self, t): + assert hasattr(t, "__dlpack__") + type(t.__dlpack__()).__name__ == "PyCapsule" + + assert hasattr(t, "__dlpack_device__") + dev = t.__dlpack_device__() + assert isinstance(dev, tuple) and len(dev) == 2 + + def _check_array_interface_attribute(self, t, arr, cuda=False): + if cuda: + assert hasattr(t, "__cuda_array_interface__") + interface = t.__cuda_array_interface__ + reference_interface = arr.__cuda_array_interface__ + else: + assert hasattr(t, "__array_interface__") + interface = t.__array_interface__ + reference_interface = arr.__array_interface__ + + assert interface["version"] == 3 + + assert interface["typestr"] == arr.dtype.str + assert interface["shape"] == arr.shape + assert len(interface["data"]) == 2 + if cuda: + assert interface["data"][0] == arr.data.mem.ptr + # no writeable flag present on CuPy arrays + else: + assert interface["data"][0] == arr.ctypes.data + assert interface["data"][1] == (not arr.flags.writeable) + if interface["strides"] is None: + assert arr.flags.c_contiguous + else: + assert interface["strides"] == arr.strides + assert interface["descr"] == [("", arr.dtype.str)] + + if reference_interface["version"] == interface["version"]: + interface["shape"] == reference_interface["shape"] + interface["typestr"] == reference_interface["typestr"] + interface["descr"] == reference_interface["descr"] + interface["data"] == reference_interface["data"] + if reference_interface["strides"] is not None: + interface["strides"] == reference_interface["strides"] + + def _check_tensor_property_values(self, t, arr, cuda=False): + assert t.size == arr.size + assert t.nbytes == arr.nbytes + assert t.ndim == arr.ndim + assert t.itemsize == arr.dtype.itemsize + + assert t.shape == arr.shape + assert t.strides == arr.strides + + type(t.data).__name__ == "PyCapsule" + + @pytest.mark.parametrize( + "dtype", unsigned_dtypes + signed_dtypes + float_dtypes + complex_dtypes + ) + @pytest.mark.parametrize("order", ["F", "C"]) + def test_numpy_as_tensor(self, dtype, order): + np = pytest.importorskip("numpy") + a = np.zeros((4, 8, 12), dtype=dtype, order=order) + t = Tensor.as_tensor(a) + assert isinstance(t, Tensor) + + self._check_dlpack_attributes(t) + self._check_array_interface_attribute(t, a, cuda=False) + self._check_tensor_property_values(t, a) + + @pytest.mark.parametrize( + "dtype", unsigned_dtypes + signed_dtypes + float_dtypes + complex_dtypes + ) + @pytest.mark.parametrize("order", ["F", "C"]) + def test_cupy_as_tensor(self, dtype, order): + cp = pytest.importorskip("cupy") + a = cp.zeros((4, 8, 12), dtype=dtype, order=order) + t = Tensor.as_tensor(a) + assert isinstance(t, Tensor) + + self._check_dlpack_attributes(t) + self._check_array_interface_attribute(t, a, cuda=True) + self._check_tensor_property_values(t, a) + + def test_tensor_properties_are_readonly(self): + np = pytest.importorskip("numpy") + a = np.zeros((4, 8, 12), dtype=np.uint8) + t = Tensor.as_tensor(a) + with pytest.raises(AttributeError): + t.size = 8 + with pytest.raises(AttributeError): + t.nbytes = 8 + with pytest.raises(AttributeError): + t.ndim = 2 + with pytest.raises(AttributeError): + t.itemsize = 3 + with pytest.raises(AttributeError): + t.shape = (t.size,) + with pytest.raises(AttributeError): + t.strides = (8,) + with pytest.raises(AttributeError): + t.data = 0 + + @pytest.mark.parametrize( + "dtype", unsigned_dtypes + signed_dtypes + float_dtypes + complex_dtypes + ) + @pytest.mark.parametrize("order", ["F", "C"]) + @pytest.mark.parametrize("module", ["cupy", "numpy"]) + def test_tensor_round_trip(self, dtype, order, module): + xp = pytest.importorskip(module) + a = xp.zeros((4, 8, 12), dtype=dtype, order=order) + t = Tensor.as_tensor(a) + b = xp.asarray(t) + xp.testing.assert_array_equal(a, b) + + +# Test cases for specific bundled native operators + + +class TestAJASourceOp: + def test_kwarg_based_initialization(self, app, config_file, capfd): + app.config(config_file) + op = AJASourceOp( + fragment=app, + name="source", + channel=NTV2Channel.NTV2_CHANNEL1, + **app.kwargs("aja"), + ) + assert isinstance(op, _Operator) + assert op.operator_type == Operator.OperatorType.NATIVE + + # assert no warnings or errors logged + captured = capfd.readouterr() + + # Initializing outside the context of app.run() will result in the + # following error being logged because the GXFWrapper will not have + # been created for the operator: + # [error] [gxf_executor.cpp:452] Unable to get GXFWrapper for Operator 'segmentation_postprocessor # noqa: E501 + assert captured.err.count("[error]") <= 1 + assert "warning" not in captured.err + + +class TestFormatConverterOp: + def test_kwarg_based_initialization(self, app, config_file, capfd): + app.config(config_file) + op = FormatConverterOp( + fragment=app, + name="recorder_format_converter", + pool=BlockMemoryPool( + name="pool", + fragment=app, + storage_type=MemoryStorageType.DEVICE, + block_size=16 * 1024**2, + num_blocks=4, + ), + **app.kwargs("recorder_format_converter"), + ) + assert isinstance(op, _Operator) + len(op.args) == 12 + assert op.operator_type == Operator.OperatorType.NATIVE + + # assert no warnings or errors logged + captured = capfd.readouterr() + + # Initializing outside the context of app.run() will result in the + # following error being logged because the GXFWrapper will not have + # been created for the operator: + # [error] [gxf_executor.cpp:452] Unable to get GXFWrapper for Operator 'recorder_format_converter' # noqa: E501 + assert captured.err.count("[error]") <= 1 + assert "warning" not in captured.err + + +class TestMultiAIInferenceOp: + def test_kwarg_based_initialization(self, app, config_file, capfd): + app.config(config_file) + model_path = os.path.join(sample_data_path, "multiai_ultrasound", "models") + + model_path_map = { + "plax_chamber": os.path.join(model_path, "plax_chamber.onnx"), + "aortic_stenosis": os.path.join(model_path, "aortic_stenosis.onnx"), + "bmode_perspective": os.path.join(model_path, "bmode_perspective.onnx"), + } + + op = MultiAIInferenceOp( + app, + name="multiai_inference", + allocator=UnboundedAllocator(app, name="pool"), + model_path_map=model_path_map, + **app.kwargs("multiai_inference"), + ) + assert isinstance(op, _Operator) + assert op.id != -1 + assert op.operator_type == Operator.OperatorType.NATIVE + + # assert no warnings or errors logged + captured = capfd.readouterr() + + # Initializing outside the context of app.run() will result in the + # following error being logged because the GXFWrapper will not have + # been created for the operator: + # [error] [gxf_executor.cpp:452] Unable to get GXFWrapper for Operator 'multi_ai_inference' # noqa: E501 + assert captured.err.count("[error]") <= 1 + assert "warning" not in captured.err + + +class TestMultiAIPostprocessorOp: + def test_kwarg_based_initialization(self, app, config_file, capfd): + app.config(config_file) + op = MultiAIPostprocessorOp( + app, + name="multiai_postprocessor", + allocator=UnboundedAllocator(app, name="pool"), + **app.kwargs("multiai_postprocessor"), + ) + assert isinstance(op, _Operator) + assert op.id != -1 + assert op.operator_type == Operator.OperatorType.NATIVE + + # assert no warnings or errors logged + captured = capfd.readouterr() + + # Initializing outside the context of app.run() will result in the + # following error being logged because the GXFWrapper will not have + # been created for the operator: + # [error] [gxf_executor.cpp:452] Unable to get GXFWrapper for Operator 'multi_ai_inference' # noqa: E501 + assert captured.err.count("[error]") <= 1 + assert "warning" not in captured.err + + +class TestSegmentationPostprocessorOp: + def test_kwarg_based_initialization(self, app, capfd): + op = SegmentationPostprocessorOp( + fragment=app, + allocator=UnboundedAllocator(fragment=app, name="allocator"), + name="segmentation_postprocessor", + ) + assert isinstance(op, _Operator) + assert op.id != -1 + assert op.operator_type == Operator.OperatorType.NATIVE + + # assert no warnings or errors logged + captured = capfd.readouterr() + + # Initializing outside the context of app.run() will result in the + # following error being logged because the GXFWrapper will not have + # been created for the operator: + # [error] [gxf_executor.cpp:452] Unable to get GXFWrapper for Operator 'segmentation_postprocessor' # noqa: E501 + assert captured.err.count("[error]") <= 1 + assert "warning" not in captured.err + + +class TestVideoStreamRecorderOp: + def test_kwarg_based_initialization(self, app, config_file, capfd): + app.config(config_file) + op = VideoStreamRecorderOp(name="recorder", fragment=app, **app.kwargs("recorder")) + assert isinstance(op, _Operator) + assert op.id != -1 + assert op.operator_type == Operator.OperatorType.NATIVE + + # assert no warnings or errors logged + captured = capfd.readouterr() + # Initializing outside the context of app.run() will result in the + # following error being logged because the GXFWrapper will not have + # been created for the operator: + # [error] [gxf_executor.cpp:452] Unable to get GXFWrapper for Operator 'recorder' # noqa: E501 + assert captured.err.count("[error]") <= 1 + assert "warning" not in captured.err + + +class TestVideoStreamReplayerOp: + def test_kwarg_based_initialization(self, app, config_file, capfd): + app.config(config_file) + data_path = os.environ.get("HOLOSCAN_SAMPLE_DATA_PATH", "../data") + op = VideoStreamReplayerOp( + name="replayer", + fragment=app, + directory=os.path.join(data_path, "endoscopy", "video"), + **app.kwargs("replayer"), + ) + assert isinstance(op, _Operator) + assert op.id != -1 + assert op.operator_type == Operator.OperatorType.NATIVE + + # assert no warnings or errors logged + captured = capfd.readouterr() + # Initializing outside the context of app.run() will result in the + # following error being logged because the GXFWrapper will not have + # been created for the operator: + # [error] [gxf_executor.cpp:452] Unable to get GXFWrapper for Operator 'replayer' # noqa: E501 + assert captured.err.count("[error]") <= 1 + assert "warning" not in captured.err + + +@pytest.mark.parametrize( + "type_str", + [ + "unknown", + "color", + "color_lut", + "points", + "lines", + "line_strip", + "triangles", + "crosses", + "rectangles", + "ovals", + "text", + "depth_map", + "depth_map_color", + ], +) +def test_holoviz_input_types(type_str): + assert isinstance(_holoviz_str_to_input_type[type_str], HolovizOp.InputType) + + +@pytest.mark.parametrize( + "depth_type_str", + [ + "points", + "lines", + "triangles", + ], +) +def test_holoviz_depth_types(depth_type_str): + assert isinstance( + _holoviz_str_to_depth_map_render_mode[depth_type_str], HolovizOp.DepthMapRenderMode + ) + + +class TestHolovizOp: + def test_kwarg_based_initialization(self, app, config_file, capfd): + app.config(config_file) + op = HolovizOp(app, name="visualizer", **app.kwargs("holoviz")) + assert isinstance(op, _Operator) + assert op.id != -1 + assert op.operator_type == Operator.OperatorType.NATIVE + + # assert no warnings or errors logged + captured = capfd.readouterr() + # Initializing outside the context of app.run() will result in the + # following error being logged because the GXFWrapper will not have + # been created for the operator: + # [error] [gxf_executor.cpp:452] Unable to get GXFWrapper for Operator 'visualizer # noqa: E501 + assert captured.err.count("[error]") <= 1 + assert "warning" not in captured.err + + @pytest.mark.parametrize( + "tensor", + [ + # color not length 4 + dict( + name="scaled_coords", + type="crosses", + line_width=4, + color=[1.0, 0.0], + ), + # color cannot be a str + dict( + name="scaled_coords", + type="crosses", + line_width=4, + color="red", + ), + # color values out of range + dict( + name="scaled_coords", + type="crosses", + line_width=4, + color=[255, 255, 255, 255], + ), + # unrecognized type + dict( + name="scaled_coords", + type="invalid", + line_width=4, + ), + # type not specified + dict( + name="scaled_coords", + line_width=4, + ), + # name not specified + dict( + type="crosses", + line_width=4, + ), + # unrecognized key specified + dict( + name="scaled_coords", + type="crosses", + line_width=4, + color=[1.0, 1.0, 1.0, 0.0], + invalid_key=None, + ), + ], + ) + def test_invalid_tensors(self, tensor, app): + with pytest.raises(ValueError): + HolovizOp( + name="visualizer", + fragment=app, + tensors=[tensor], + ) + + +class HolovizDepthMapSourceOp(Operator): + def __init__(self, fragment, width, height, *args, **kwargs): + self.width = width + self.height = height + super().__init__(fragment, *args, **kwargs) + + def setup(self, spec: OperatorSpec): + spec.output("out") + + def compute(self, op_input, op_output, context): + import cupy as cp + + depth_map = np.empty((self.height, self.width, 1), dtype=np.uint8) + index = 0 + for y in range(self.height): + for x in range(self.width): + depth_map[y][x][0] = index + index *= 4 + out_message = Entity(context) + out_message.add(Tensor.as_tensor(cp.asarray(depth_map)), "depth_map") + op_output.emit(out_message, "out") + + +class HolovizDepthMapSinkOp(Operator): + def __init__(self, fragment, *args, **kwargs): + super().__init__(fragment, *args, **kwargs) + + def setup(self, spec: OperatorSpec): + spec.input("in") + + def compute(self, op_input, op_output, context): + op_input.receive("in") + # Holoviz outputs a video buffer, but there is no support for video buffers in Python yet + # message = op_input.receive("in") + # image = message.get("render_buffer_output") + + +class MyHolovizDepthMapApp(Application): + def compose(self): + depth_map_width = 8 + depth_map_height = 4 + + render_width = 32 + render_height = 32 + + source = HolovizDepthMapSourceOp( + self, depth_map_width, depth_map_height, CountCondition(self, 1), name="source" + ) + + alloc = UnboundedAllocator( + fragment=self, + name="allocator", + ) + holoviz = HolovizOp( + self, + name="holoviz", + width=render_width, + height=render_height, + headless=True, + enable_render_buffer_output=True, + allocator=alloc, + tensors=[ + dict(name="depth_map", type="depth_map"), + ], + ) + + sink = HolovizDepthMapSinkOp(self, name="sink") + + self.add_flow(source, holoviz, {("", "receivers")}) + self.add_flow(holoviz, sink, {("render_buffer_output", "")}) + + +def test_holoviz_depth_map_app(capfd): + pytest.importorskip("cupy") + app = MyHolovizDepthMapApp() + app.run() + + # assert no errors logged + captured = capfd.readouterr() + assert captured.err.count("[error]") == 0 diff --git a/python/tests/test_resources.py b/python/tests/unit/test_resources.py similarity index 98% rename from python/tests/test_resources.py rename to python/tests/unit/test_resources.py index 6f7f2a5f..813dc6db 100644 --- a/python/tests/test_resources.py +++ b/python/tests/unit/test_resources.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/run b/run index 2bb4a2e9..e5a6bd45 100755 --- a/run +++ b/run @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Error out if a command fails +set -e + #=============================================================================== # Default values for environment variables. #=============================================================================== @@ -26,12 +29,7 @@ init_globals() { export RUN_SCRIPT_FILE="$(readlink -f "${BASH_SOURCE[0]}")" fi - export TOP=$(git rev-parse --show-toplevel 2> /dev/null || dirname "${RUN_SCRIPT_FILE}") - - # If this repository is dev repository, consider public folder as TOP folder - if [ -d ${TOP}/public ]; then - export TOP=${TOP}/public - fi + export TOP=$(dirname "${RUN_SCRIPT_FILE}") HOLOSCAN_PY_EXE=${HOLOSCAN_PY_EXE:-"python3"} export HOLOSCAN_PY_EXE @@ -286,9 +284,9 @@ run_command() { local cmd="$*" if [ "${DO_DRY_RUN}" != "true" ]; then - c_echo B "$(date -u '+%Y-%m-%d %H:%M:%S') " W "\$ " G "${cmd}" + c_echo_err B "$(date -u '+%Y-%m-%d %H:%M:%S') " W "\$ " G "${cmd}" else - c_echo B "$(date -u '+%Y-%m-%d %H:%M:%S') " C "[dryrun] " W "\$ " G "${cmd}" + c_echo_err B "$(date -u '+%Y-%m-%d %H:%M:%S') " C "[dryrun] " W "\$ " G "${cmd}" fi [ "$(echo -n "$@")" = "" ] && return 1 # return 1 if there is no command available @@ -321,7 +319,7 @@ Environments: ' } install_gxf() { - local gxf_tag=${GXF_TAG:-2.5.0-8fbe43cb} + local gxf_tag=${GXF_TAG:-2.5.0-6b2e34ec} local ngc_cli_org=${NGC_CLI_ORG:-nvidia} local ngc_cli_team=${NGC_CLI_TEAM:-clara-holoscan} local GXF_MODIFIED=false @@ -440,26 +438,38 @@ get_buildtype_str() { echo -n "${build_type_str}" } -get_build_platform_option() { - local build_opt="${1:-}" +get_platform_str() { + local platform="${1:-}" + local platform_str + + case "${platform}" in + amd64|x86_64|x86|linux/amd64) + platform_str="linux/amd64" + ;; + arm64|aarch64|arm|linux/arm64) + platform_str="linux/arm64" + ;; + esac + + echo -n "${platform_str}" +} + +get_cuda_archs() { + local cuda_archs="${1:-}" - case "${build_opt}" in - amd64|x86_64|x86) - build_opt="--platform linux/amd64" + case "${cuda_archs}" in + native|NATIVE) + cuda_archs_str="NATIVE" ;; - arm64|aarch64|arm) - build_opt="--platform linux/arm64" + all|ALL) + cuda_archs_str="ALL" ;; *) - if [ -n "${HOLOSCAN_BUILD_PLATFORM}" ]; then - build_opt="--platform ${HOLOSCAN_BUILD_PLATFORM}" - else - build_opt="" - fi + cuda_archs_str="${1:-}" ;; esac - echo -n "${build_opt}" + echo -n "${cuda_archs_str}" } clear_cache_desc() { c_echo 'Clear cache folders (including build/install folders) @@ -508,45 +518,107 @@ setup() { } build_image_desc() { c_echo 'Build dev image + --platform [linux/amd64 | linux/arm64] : Specify the platform (for cross-compilation) + Default: current host platform + Associated environment variable: HOLOSCAN_BUILD_PLATFORM ' } build_image() { - local platform=${1:-} - local build_opts=$(get_build_platform_option "${platform}") - c_echo W "Using docker build option: '${build_opts}'" + # Parse env variables first or set default values + local platform="${HOLOSCAN_BUILD_PLATFORM:-$(get_platform_str $(uname -p))}" + + # Parse CLI arguments next + ARGS=("$@") + local i + local arg + for i in "${!ARGS[@]}"; do + arg="${ARGS[i]}" + if [ "$arg" = "--platform" ]; then + platform=$(get_platform_str "${ARGS[i+1]}") + fi + done run_command export DOCKER_BUILDKIT=1 - run_command docker build ${build_opts} \ + run_command docker build \ --build-arg BUILDKIT_INLINE_CACHE=1 \ + --platform ${platform} \ --network=host \ -t holoscan-sdk-dev \ ${TOP} } -build_desc() { c_echo 'Build (args: one of [debug, release, rel-debug]) - -This command will build the project. +build_desc() { c_echo 'Build the project -Export CMAKE_BUILD_PATH (default: "build") to change the build path. - e.g., - - export CMAKE_BUILD_PATH=build-arm64 +This command will build the project with optional configuration arguments. Arguments: - $1 - CMAKE_BUILD_TYPE (one of ["debug", "release", "rel-debug"]) + --platform [linux/amd64 | linux/arm64] : Specify the platform (for cross-compilation) + Default: current host platform + Associated environment variable: HOLOSCAN_BUILD_PLATFORM + --cudaarchs [native | all | ] + Default: native + Associated environment variable: CMAKE_CUDA_ARCHITECTURES + --type [debug | release | rel-debug] : Specify the type of build + Default: release + Associated environment variable: CMAKE_BUILD_TYPE + --buildpath : Change the build path. + Default: build + Associated environment variable: CMAKE_BUILD_PATH + --installprefix : Specify the install directory + Default: install + Associated environment variable: CMAKE_INSTALL_PREFIX + --reconfigure: Force reconfiguration of the CMake project. By default CMake is only run if + a CMakeCache.txt does not exist in the build directory or if CMake detects a reconfigure + is needed. + Default: false ' } build() { - local buildtype=$(get_buildtype_str $1) + # Parse env variables first or set default values + local platform="${HOLOSCAN_BUILD_PLATFORM:-$(get_platform_str $(uname -p))}" + local cuda_archs="${CMAKE_CUDA_ARCHITECTURES:-NATIVE}" + local build_type="${CMAKE_BUILD_TYPE:-release}" local build_path="${CMAKE_BUILD_PATH:-build}" local install_prefix="${CMAKE_INSTALL_PREFIX:-install}" + local reconfigure=false + + # Parse CLI arguments next + ARGS=("$@") + local i + local arg + for i in "${!ARGS[@]}"; do + arg="${ARGS[i]}" + if [ "$arg" = "--platform" ]; then + platform=$(get_platform_str "${ARGS[i+1]}") + fi + if [ "$arg" = "--cudaarchs" ]; then + cuda_archs=$(get_cuda_archs "${ARGS[i+1]}") + reconfigure=true + fi + if [ "$arg" = "--type" ]; then + build_type=$(get_buildtype_str "${ARGS[i+1]}") + reconfigure=true + fi + if [ "$arg" = "--buildpath" ]; then + build_path="${ARGS[i+1]}" + fi + if [ "$arg" = "--installprefix" ]; then + install_prefix="${ARGS[i+1]}" + fi + if [ "$arg" = "--reconfigure" ]; then + reconfigure=true + fi + done setup - build_image + build_image --platform ${platform} install_gxf # DOCKER PARAMETERS # + # -it + # Making the container interactive allows cancelling the build + # # --rm # Deletes the container after the command runs # @@ -568,22 +640,30 @@ build() { # -S . -B ${build_path} # Generic configuration # - # -D CMAKE_BUILD_TYPE=${buildtype} + # -D CMAKE_BUILD_TYPE=${build_type} # Define the build type (release, debug...). Can be passed as env to docker also. # - run_command ${HOLOSCAN_DOCKER_EXE} run --rm --net host \ + # -D CMAKE_CUDA_ARCHITECTURES=${cuda_archs} + # Define the cuda architectures to build for (NATIVE, ALL, custom). If custom or ALL, the + # last arch will be used for real and virtual architectures (PTX, forward compatible) while + # the previous archs will be real only. + # + run_command ${HOLOSCAN_DOCKER_EXE} run --rm --net host -it \ -u $(id -u):$(id -g) \ -v ${TOP}:/workspace/holoscan-sdk \ -w /workspace/holoscan-sdk \ -e CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) \ holoscan-sdk-dev \ bash -c " - cmake -S . -B ${build_path} -G Ninja \ - -D CMAKE_BUILD_TYPE=${buildtype} \ + if [ ! -f '${build_path}/build.ninja' ] || ${reconfigure} ; then \ + cmake -S . -B ${build_path} -G Ninja \ + -D CMAKE_CUDA_ARCHITECTURES=\"${cuda_archs}\" \ + -D CMAKE_BUILD_TYPE=${build_type}; \ + fi \ && cmake --build ${build_path} -j \ && cmake --install ${build_path} --prefix ${install_prefix} --component holoscan-core \ && cmake --install ${build_path} --prefix ${install_prefix} --component holoscan-gxf_extensions \ - && cmake --install ${build_path} --prefix ${install_prefix} --component holoscan-apps \ + && cmake --install ${build_path} --prefix ${install_prefix} --component holoscan-examples \ && cmake --install ${build_path} --prefix ${install_prefix} --component holoscan-gxf_libs \ && cmake --install ${build_path} --prefix ${install_prefix} --component holoscan-gxf_bins \ && cmake --install ${build_path} --prefix ${install_prefix} --component holoscan-modules \ @@ -596,44 +676,86 @@ build() { # Section: Test #=============================================================================== -lint_desc() { c_echo 'Lint code +lint_desc() { c_echo 'Lint the repository + +Python linting: black, isort, ruff +C++ linting: cpplint +Spelling: codespell + +Arguments: + $@ - directories to lint (default: .) ' } + lint() { - c_echo W "Linting code..." - pushd ${TOP} > /dev/null - run_command ${HOLOSCAN_PY_EXE} scripts/include_checker.py apps gxf_extensions tests - popd > /dev/null -} + local DIR_TO_RUN=${@:-"."} -cpplint_desc() { c_echo 'CPPLint code -' -} + # We use $(command) || exit_code=1 to run all linting tools, and exit + # with failure after all commands were executed if any of them failed + local exit_code=0 -cpplint() { - c_echo W "CPPLinting code..." pushd ${TOP} > /dev/null - local DIR_TO_RUN="apps examples gxf_extensions modules python/pybind11 tests" - if [ -n "$1" ]; then # directories was specified - run_command ${HOLOSCAN_PY_EXE} -m cpplint --recursive "$@" - else - run_command ${HOLOSCAN_PY_EXE} -m cpplint --recursive ${DIR_TO_RUN} - fi + + c_echo W "Linting Python" + run_command ruff $DIR_TO_RUN || exit_code=1 + run_command ${HOLOSCAN_PY_EXE} -m isort -c $DIR_TO_RUN || exit_code=1 + run_command ${HOLOSCAN_PY_EXE} -m black --check $DIR_TO_RUN || exit_code=1 + + c_echo W "Linting C++" + # We use `grep -v` to hide verbose output that drowns actual errors + # Since we care about the success/failure of cpplitn and not of grep, we: + # 1. use `set -o pipefail` to fail if `cpplint` fails + # 2. use `grep -v ... || true` to ignore whether grep hid any output + run_command set -o pipefail; ${HOLOSCAN_PY_EXE} -m cpplint \ + --exclude .cache \ + --exclude build \ + --exclude install \ + --exclude build-\* \ + --exclude install-\* \ + --recursive $DIR_TO_RUN \ + | { grep -v "Ignoring\|Done processing" || true; } || exit_code=1 + + + c_echo W "Code spelling" + run_command codespell $DIR_TO_RUN || exit_code=1 + popd > /dev/null + + exit $exit_code } coverage_desc() { c_echo 'Execute code coverage + --type [debug | release | rel-debug] : Specify the type of build + Default: release + Associated environment variable: CMAKE_BUILD_TYPE + --buildpath : Change the build path. + Default: build + Associated environment variable: CMAKE_BUILD_PATH ' } - coverage() { - local buildtype=$(get_buildtype_str $1) + # Parse env variables first or set default values + local build_type="${CMAKE_BUILD_TYPE:-release}" local build_path="${CMAKE_BUILD_PATH:-build}" + + # Parse CLI arguments next + ARGS=("$@") + local i + local arg + for i in "${!ARGS[@]}"; do + arg="${ARGS[i]}" + if [ "$arg" = "--type" ]; then + build_type=$(get_buildtype_str "${ARGS[i+1]}") + fi + if [ "$arg" = "--buildpath" ]; then + build_path="${ARGS[i+1]}" + fi + done local working_dir=${1:-${build_path}} # Find the nvidia_icd.json file which could reside at different paths # Needed due to https://github.com/NVIDIA/nvidia-container-toolkit/issues/16 - nvidia_icd_json=$(find /usr/share /etc -path '*/vulkan/icd.d/nvidia_icd.json' -type f 2>/dev/null | grep .) || (echo "nvidia_icd.json not found" >&2 && false) + nvidia_icd_json=$(find /usr/share /etc -path '*/vulkan/icd.d/nvidia_icd.json' -type f -print -quit 2>/dev/null | grep .) || (echo "nvidia_icd.json not found" >&2 && false) run_command ${HOLOSCAN_DOCKER_EXE} run --rm --net host \ -u $(id -u):$(id -g) \ @@ -649,12 +771,12 @@ coverage() { -e HOLOSCAN_TESTS_DATA_PATH=/workspace/holoscan-sdk/tests/data \ -e HOLOSCAN_LOG_LEVEL=INFO \ -e PYTHONPATH=/workspace/holoscan-sdk/${working_dir}/python/lib \ - -e CMAKE_BUILD_TYPE=${buildtype} \ -e CMAKE_BUILD_PARALLEL_LEVEL=$(nproc) \ holoscan-sdk-dev \ bash -c " cmake -S . -B ${build_path} -G Ninja \ - -D HOLOSCAN_BUILD_COVERAGE=ON \ + -D HOLOSCAN_BUILD_COVERAGE=ON \ + -D CMAKE_BUILD_TYPE=${build_type} \ && cmake --build ${build_path} \ && cmake --build ${build_path} --target coverage " @@ -719,7 +841,7 @@ launch() { # Find the nvidia_icd.json file which could reside at different paths # Needed due to https://github.com/NVIDIA/nvidia-container-toolkit/issues/16 - nvidia_icd_json=$(find /usr/share /etc -path '*/vulkan/icd.d/nvidia_icd.json' -type f 2>/dev/null | grep .) || (echo "nvidia_icd.json not found" >&2 && false) + nvidia_icd_json=$(find /usr/share /etc -path '*/vulkan/icd.d/nvidia_icd.json' -type f -print -quit 2>/dev/null | grep .) || (echo "nvidia_icd.json not found" >&2 && false) # DOCKER PARAMETERS # @@ -790,6 +912,94 @@ launch() { holoscan-sdk-dev "$@" } +vscode_desc() { c_echo 'Launch VSCode in DevContainer + +Launch a VSCode instance in a Docker container with the development environment. +' +} +vscode() { + if ! command -v code > /dev/null; then + fatal R "Please install " W "VSCode" R " to use VSCode DevContainer. Follow the instructions in https://code.visualstudio.com/Download" + fi + + local workspace_path=$(git rev-parse --show-toplevel 2> /dev/null || dirname $(realpath -s $0)) + local workspace_path_hex + + if [ "${workspace_path}" != "${TOP}" ]; then + # Set the environment variable 'HOLOSCAN_PUBLIC_FOLDER' to the resolved path relative to the + # workspace path if the workspace path is different than the TOP folder so that + # the DevContainer can build files under the Holoscan source root folder while mounting the + # workspace folder. + run_command export HOLOSCAN_PUBLIC_FOLDER=$(realpath --relative-to=${workspace_path} ${TOP}) + fi + + # Allow connecting from docker. This is not needed for WSL2 (`SI:localuser:wslg` is added by default) + run_command xhost +local:docker + + # Install the VSCode Remote Development extension. + # Checking if the extension is already installed is slow, so we always install it so that + # the installation can be skipped if the extension is already installed. + run_command code --force --install-extension ms-vscode-remote.vscode-remote-extensionpack + + # For now, there is no easy way to launch DevContainer directly from the command line. + # ('devcontainer' CLI is only available after manually executing 'Remote-Containers: Install devcontainer CLI' command) + # Here, we launch VSCode with .devcontainer/devcontainer.json with '--folder-uri' option to open the current folder in the container. + # See https://github.com/microsoft/vscode-remote-release/issues/2133 + workspace_path_hex=$(echo -n ${workspace_path} | xxd -p) + workspace_path_hex="${workspace_path_hex//[[:space:]]/}" + c_echo B "Local workspace path: " W "${workspace_path} (${workspace_path_hex})" + run_command code --folder-uri "vscode-remote://dev-container+${workspace_path_hex}/workspace/holoscan-sdk" +} + +vscode_remote_desc() { c_echo 'Attach to the existing VSCode DevContainer + +This command is useful when you want to attach to the existing VSCode DevContainer from the remote machine. + +1) Launch VSCode DevContainer on the local machine with "./run vscode" command. +2) In the remote machine, launch VSCode Remote-SSH extension to connect to the local machine. +3) Launch this command on the remote machine to attach to the existing VSCode DevContainer in the local machine. +' +} +vscode_remote() { + if ! command -v code > /dev/null; then + fatal R "Please install " W "VSCode" R " to use VSCode DevContainer. Follow the instructions in https://code.visualstudio.com/Download" + fi + + local workspace_path=$(git rev-parse --show-toplevel 2> /dev/null || dirname $(realpath -s $0)) + + local container_name=$(run_command docker ps --filter "label=devcontainer.local_folder=${workspace_path}" --format "{{.Names}}" | head -1) + if [ -z "${container_name}" ]; then + fatal R "No DevContainer (with the label 'devcontainer.local_folder=${workspace_path}') is found in the local machine. Please launch VSCode DevContainer with " W "./run vscode" R " command first." + fi + c_echo W "Container name: " y "${container_name}" + + # The following command is based on the information of the following issue: + # https://github.com/microsoft/vscode-remote-release/issues/2133#issuecomment-618328138 + python - > /tmp/holoscan_remote_uri.txt << EOF +import json +from urllib.parse import urlunparse +from subprocess import check_output +hostname = check_output("hostname" , shell=True).decode('utf-8').strip() +remote_description = json.dumps({"containerName": f"/${container_name}", "settings":{"host": f"ssh://{hostname}"}}) +remote_uri = urlunparse(('vscode-remote', f'attached-container+{remote_description.encode("utf8").hex()}', '/workspace/holoscan-sdk', '', '', '')) +print(remote_uri) +EOF + local remote_uri=$(cat /tmp/holoscan_remote_uri.txt) + c_echo W "Remote URI: " y "${remote_uri}" + + # VSCode's attached-container mode does not support forwarding SSH keys to the container well. + # For the reason, we need to copy SSH keys to the container so that the container can access + # the local machine's SSH keys. + c_echo W "Copying SSH keys to the container..." + for i in identity id_rsa id_ed25519 id_ed25519_sk id_dsa id_cdsa id_ecdsa_sk; do + if [ -e ~/.ssh/${i} ]; then + run_command docker cp ~/.ssh/${i} ${container_name}:/home/holoscan-sdk/.ssh/ + fi + done; + + run_command code --folder-uri ${remote_uri} +} + #=============================================================================== parse_args() { diff --git a/scripts/README.md b/scripts/README.md index b9ba15a2..6bcd964e 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -4,7 +4,6 @@ This folder includes the following scripts: - [`convert_video_to_gxf_entities.py`](#convert_video_to_gxf_entitiespy) - [`generate_extension_uuids.py`](#generate_extension_uuidspy) - [`graph_surgeon.py`](#graph_surgeonpy) -- [`include_checker.py`](#include_checkerpy) ## convert_video_to_gxf_entities.py @@ -41,13 +40,3 @@ Note that this script modifies the names of the output by appending `_old` to th ```bash python3 scripts/graph_surgeon.py input_model.onnx output_model.onnx ``` - -## include_checker.py - -If double-quotes are used, this script checks whether there is a file located at that location relative to the file being checked. If the file does not exist, it will suggest changing to an `#include` using angle brackets. Similarly, if the include uses angle brackets, but the file is present locally it will suggest changing to a double-quote style import. - -### Usage - -```bash -python3 scripts/include_checker.py apps examples include modules src test python/pybind11 -``` diff --git a/scripts/include_checker.py b/scripts/include_checker.py deleted file mode 100644 index fbb6ff2b..00000000 --- a/scripts/include_checker.py +++ /dev/null @@ -1,137 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import print_function - -import argparse -import os -import re -import sys - -IncludeRegex = re.compile(r"\s*#include\s*(\S+)") -RemoveComments = re.compile(r"//.*") - -exclusion_regex_list = [ - re.compile(r".*_deps.*"), - re.compile(r".*third_?party.*"), - re.compile(r".*\.cache.*"), - re.compile(r".*public/install*"), - re.compile(r".*public/build*"), -] - - -def parse_args(): - argparser = argparse.ArgumentParser("Checks for a consistent '#include' syntax") - argparser.add_argument( - "--regex", - type=str, - default=r"[.](cu|cuh|h|hpp|hxx|cpp)$", - help="Regex string to filter in sources", - ) - argparser.add_argument("dirs", type=str, nargs="*", help="List of dirs where to find sources") - argparser.add_argument( - "--include", - dest="include", - action="append", - required=False, - default=["include"], - help=("Specify include paths (default: ['include']). " "Can be specified multiple times."), - ) - argparser.add_argument( - "-v", - "--verbose", - dest="verbose", - action="store_true", - help="verbose listing of files checked", - ) - argparser.add_argument( - "--allow", - dest="allow", - action="append", - required=False, - default=["common", "gxf", "gxf_extensions", "holoscan", "holoviz"], - help=( - "Allow prefixes specified (default: ['gxf', 'holoscan', 'holoviz']). " - "Can be specified multiple times." - ), - ) - args = argparser.parse_args() - args.regex_compiled = re.compile(args.regex) - return args - - -def list_all_source_file(file_regex, srcdirs): - all_files = [] - for srcdir in srcdirs: - for root, dirs, files in os.walk(srcdir): - for f in files: - if not any(re.search(exr, root) for exr in exclusion_regex_list) and re.search( - file_regex, f - ): - src = os.path.join(root, f) - all_files.append(src) - return all_files - - -def check_includes_in(src, include_folder_list, allow_prefix_set): - errs = [] - curr_dir = os.path.dirname(src) - prefixes = [curr_dir] - prefixes.extend(include_folder_list) - for line_number, line in enumerate(open(src)): - line = RemoveComments.sub("", line) - match = IncludeRegex.search(line) - if match is None: - continue - val = match.group(1) - inc_file = val[1:-1] # strip out " or < - - include_exists = any(map(os.path.exists, (os.path.join(p, inc_file) for p in prefixes))) - - line_num = line_number + 1 - if val[0] == '"' and not include_exists: - if inc_file.split("/")[0] not in allow_prefix_set: - errs.append("Line:%d use #include <...>" % line_num) - elif val[0] == "<" and include_exists: - if inc_file.split("/")[0] not in allow_prefix_set: - errs.append('Line:%d use #include "..."' % line_num) - return errs - - -def main(): - args = parse_args() - all_files = list_all_source_file(args.regex_compiled, args.dirs) - include_folder_list = args.include - allow_prefix_set = set(args.allow) - all_errs = {} - for f in all_files: - if args.verbose: - print(f"checking: {f}") - errs = check_includes_in(f, include_folder_list, allow_prefix_set) - if len(errs) > 0: - all_errs[f] = errs - if len(all_errs) == 0: - print("include-check PASSED") - else: - print("include-check FAILED! See below for errors...") - for f, errs in all_errs.items(): - print("File: %s" % f) - for e in errs: - print(" %s" % e) - sys.exit(-1) - - -if __name__ == "__main__": - main() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt new file mode 100644 index 00000000..25790d3b --- /dev/null +++ b/src/CMakeLists.txt @@ -0,0 +1,152 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ############################################################################## +# # Common properties +# ############################################################################## +# TODO (grelee): I removed -Wold-style-cast due to it raising errors for ajantv2 code in .cache. After restructuring we can add it back in +add_compile_options($<$:-Werror$-Wall$-Wextra>) + +function(add_holoscan_library target) + add_library(${target} SHARED ${ARGN}) + add_library(${HOLOSCAN_PACKAGE_NAME}::${target} ALIAS ${target}) + set_target_properties(${target} PROPERTIES + OUTPUT_NAME ${HOLOSCAN_PACKAGE_NAME}_${target} + SOVERSION ${PROJECT_VERSION_MAJOR} + VERSION ${PROJECT_VERSION} + ) + target_compile_features(${target} + PUBLIC cxx_std_17 + ) + target_include_directories(${target} + PUBLIC + $ + $ + PRIVATE + $ + ) +endfunction() + +function(add_holoscan_operator operator) + set(target op_${operator}) + add_library(${target} SHARED ${ARGN}) + add_library(${HOLOSCAN_PACKAGE_NAME}::ops::${operator} ALIAS ${target}) + set_target_properties(${target} PROPERTIES + OUTPUT_NAME ${HOLOSCAN_PACKAGE_NAME}_${target} + EXPORT_NAME ops::${operator} + SOVERSION ${PROJECT_VERSION_MAJOR} + VERSION ${PROJECT_VERSION} + ) + target_link_libraries(${target} + PUBLIC ${HOLOSCAN_PACKAGE_NAME}::core + ) +endfunction() + +# ############################################################################## +# # Add library holoscan::logger +# ############################################################################## +add_holoscan_library(logger logger/logger.cpp) +target_link_libraries(logger + PUBLIC + fmt::fmt-header-only + + PRIVATE + spdlog::spdlog_header_only +) + +if(CMAKE_BUILD_TYPE STREQUAL "Release") + target_compile_definitions(logger + PRIVATE HOLOSCAN_LOG_ACTIVE_LEVEL=1 + ) +endif() + +# ############################################################################## +# # Add library: holoscan::core +# ############################################################################## +add_holoscan_library(core + core/application.cpp + core/arg.cpp + core/argument_setter.cpp + core/component.cpp + core/component_spec.cpp + core/condition.cpp + core/conditions/gxf/boolean.cpp + core/conditions/gxf/count.cpp + core/conditions/gxf/downstream_affordable.cpp + core/conditions/gxf/message_available.cpp + core/config.cpp + core/domain/tensor.cpp + core/executors/gxf/gxf_executor.cpp + core/executors/gxf/gxf_parameter_adaptor.cpp + core/fragment.cpp + core/graphs/flow_graph.cpp + core/gxf/entity.cpp + core/gxf/gxf_condition.cpp + core/gxf/gxf_execution_context.cpp + core/gxf/gxf_extension_manager.cpp + core/gxf/gxf_io_context.cpp + core/gxf/gxf_operator.cpp + core/gxf/gxf_resource.cpp + core/gxf/gxf_tensor.cpp + core/gxf/gxf_wrapper.cpp + core/io_spec.cpp + core/operator.cpp + core/operator_spec.cpp + core/resource.cpp + core/resources/gxf/allocator.cpp + core/resources/gxf/block_memory_pool.cpp + core/resources/gxf/cuda_stream_pool.cpp + core/resources/gxf/double_buffer_receiver.cpp + core/resources/gxf/double_buffer_transmitter.cpp + core/resources/gxf/receiver.cpp + core/resources/gxf/std_component_serializer.cpp + core/resources/gxf/transmitter.cpp + core/resources/gxf/video_stream_serializer.cpp +) + +target_link_libraries(core + PUBLIC + holoscan::logger + CUDA::cudart + fmt::fmt-header-only + GXF::core + GXF::cuda + GXF::std + yaml-cpp +) + +target_include_directories(core + PUBLIC + $ + $ + $ +) + +# ############################################################################## +# # Add library: holoscan::infer_utils +# ############################################################################## +add_holoscan_library(infer_utils + utils/holoinfer_utils.cpp +) +target_link_libraries(infer_utils + PUBLIC + holoscan::core + holoscan::infer +) + +# ############################################################################## +# # Add libraries: holoscan::ops::... +# ############################################################################## +add_subdirectory(operators) diff --git a/src/core/arg.cpp b/src/core/arg.cpp index d212ebeb..b3f9338a 100644 --- a/src/core/arg.cpp +++ b/src/core/arg.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,4 +19,120 @@ namespace holoscan { +std::string ArgType::to_string() const { + std::string el_type_str = element_type_name_map_.at(element_type_); + if (container_type_ == ArgContainerType::kNative) { return el_type_str; } + std::string nested_str = + container_type_ == ArgContainerType::kVector ? "std::vector<{}>" : "std::array<{},N>"; + std::string container_str = nested_str; + for (int32_t i = 1; i < dimension_; ++i) { container_str = fmt::format(nested_str, nested_str); } + return fmt::format(container_str, el_type_str); +} + +template +inline static YAML::Node scalar_as_node(const std::any& val) { + return YAML::Node(std::any_cast(val)); +} + +template +inline static YAML::Node vector_as_node(const std::any& val) { + try { + return YAML::Node(std::any_cast>(val)); + } catch (const std::bad_cast& e) { // 2d: std::vector> + try { + return YAML::Node(std::any_cast>>(val)); + } catch (const std::bad_cast& e) { + return YAML::Node(YAML::NodeType::Undefined); + } + } +} + +template +inline static YAML::Node any_as_node(const std::any& val, ArgContainerType type) { + switch (type) { + case ArgContainerType::kNative: + return scalar_as_node(val); + case ArgContainerType::kVector: + return vector_as_node(val); + case ArgContainerType::kArray: + // Don't know size of arrays, abort + return YAML::Node(YAML::NodeType::Undefined); + } + return YAML::Node(YAML::NodeType::Undefined); +} + +inline static YAML::Node any_as_node(const std::any& val, ArgType type) { + ArgContainerType container_t = type.container_type(); + switch (type.element_type()) { + case ArgElementType::kBoolean: + return any_as_node(val, container_t); + case ArgElementType::kInt8: + return any_as_node(val, container_t); + case ArgElementType::kUnsigned8: + return any_as_node(val, container_t); + case ArgElementType::kInt16: + return any_as_node(val, container_t); + case ArgElementType::kUnsigned16: + return any_as_node(val, container_t); + case ArgElementType::kInt32: + return any_as_node(val, container_t); + case ArgElementType::kUnsigned32: + return any_as_node(val, container_t); + case ArgElementType::kInt64: + return any_as_node(val, container_t); + case ArgElementType::kUnsigned64: + return any_as_node(val, container_t); + case ArgElementType::kFloat32: + return any_as_node(val, container_t); + case ArgElementType::kFloat64: + return any_as_node(val, container_t); + case ArgElementType::kString: + return any_as_node(val, container_t); + case ArgElementType::kYAMLNode: + return any_as_node(val, container_t); + default: // kCustom, kHandle, kIOSpec, kCondition, kResource + return YAML::Node(YAML::NodeType::Undefined); + } + return YAML::Node(YAML::NodeType::Undefined); +} + +YAML::Node Arg::to_yaml_node() const { + YAML::Node node; + node["name"] = name_; + node["type"] = arg_type_.to_string(); + node["value"] = any_as_node(value_, arg_type_); + return node; +} + +std::string Arg::description() const { + YAML::Emitter emitter; + emitter << to_yaml_node(); + return emitter.c_str(); +} + +/** + * @brief Get a YAML representation of the argument list. + * + * @return YAML node including the name, and arguments of the argument list. + */ +YAML::Node ArgList::to_yaml_node() const { + YAML::Node node; + node["name"] = name_; + node["args"] = YAML::Node(YAML::NodeType::Sequence); + for (const Arg& arg : args_) { node["args"].push_back(arg.to_yaml_node()); } + return node; +} + +/** + * @brief Get a description of the argument list. + * + * @see to_yaml_node() + * @return YAML string. + */ +std::string ArgList::description() const { + YAML::Emitter emitter; + emitter << to_yaml_node(); + return emitter.c_str(); +} + } // namespace holoscan diff --git a/src/core/component.cpp b/src/core/component.cpp new file mode 100644 index 00000000..135ac8bd --- /dev/null +++ b/src/core/component.cpp @@ -0,0 +1,44 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "holoscan/core/component.hpp" +#include "holoscan/core/fragment.hpp" + +namespace holoscan { + +YAML::Node Component::to_yaml_node() const { + YAML::Node node; + node["id"] = id_; + node["name"] = name_; + if (fragment_) { + node["fragment"] = fragment_->name(); + } else { + node["fragment"] = YAML::Null; + } + node["args"] = YAML::Node(YAML::NodeType::Sequence); + for (const Arg& arg : args_) { + node["args"].push_back(arg.to_yaml_node()); + } + return node; +} + +std::string Component::description() const { + YAML::Emitter emitter; + emitter << to_yaml_node(); + return emitter.c_str(); +} + +} // namespace holoscan diff --git a/src/core/component_spec.cpp b/src/core/component_spec.cpp new file mode 100644 index 00000000..b8221bdc --- /dev/null +++ b/src/core/component_spec.cpp @@ -0,0 +1,48 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "holoscan/core/component_spec.hpp" +#include "holoscan/core/fragment.hpp" + +namespace holoscan { + +YAML::Node ComponentSpec::to_yaml_node() const { + YAML::Node node; + if (fragment_) { + node["fragment"] = fragment_->name(); + } else { + node["fragment"] = YAML::Null; + } + node["params"] = YAML::Node(YAML::NodeType::Sequence); + for (auto& [name, wrapper] : params_) { + const std::string& type = wrapper.arg_type().to_string(); + auto param = static_cast*>(wrapper.storage_ptr()); + YAML::Node param_node; + param_node["name"] = name; + param_node["type"] = type; + param_node["description"] = param->description(); + node["params"].push_back(param_node); + } + return node; +} + +std::string ComponentSpec::description() const { + YAML::Emitter emitter; + emitter << to_yaml_node(); + return emitter.c_str(); +} + +} // namespace holoscan diff --git a/src/core/condition.cpp b/src/core/condition.cpp index bc7a0c01..9d47d5b7 100644 --- a/src/core/condition.cpp +++ b/src/core/condition.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,4 +21,14 @@ namespace holoscan { +YAML::Node Condition::to_yaml_node() const { + YAML::Node node = Component::to_yaml_node(); + if (spec_) { + node["spec"] = spec_->to_yaml_node(); + } else { + node["spec"] = YAML::Null; + } + return node; +} + } // namespace holoscan diff --git a/src/core/conditions/gxf/boolean.cpp b/src/core/conditions/gxf/boolean.cpp index c6ea4929..b1b09513 100644 --- a/src/core/conditions/gxf/boolean.cpp +++ b/src/core/conditions/gxf/boolean.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,6 +23,12 @@ namespace holoscan { +BooleanCondition::BooleanCondition(const std::string& name, + nvidia::gxf::BooleanSchedulingTerm* term) + : GXFCondition(name, term) { + enable_tick_ = term->checkTickEnabled(); +} + void BooleanCondition::setup(ComponentSpec& spec) { spec.param(enable_tick_, "enable_tick", diff --git a/src/core/conditions/gxf/message_available.cpp b/src/core/conditions/gxf/message_available.cpp index f37e1d19..331f9fc8 100644 --- a/src/core/conditions/gxf/message_available.cpp +++ b/src/core/conditions/gxf/message_available.cpp @@ -33,6 +33,7 @@ void MessageAvailableCondition::setup(ComponentSpec& spec) { "The scheduling term permits execution if the given receiver has at least the " "given number of messages available.", 1UL); + // TODO(gbae): This is an optional parameter. Should support optional parameters. spec.param( front_stage_max_size_, "front_stage_max_size", diff --git a/src/core/executors/gxf/gxf_executor.cpp b/src/core/executors/gxf/gxf_executor.cpp index 3c90af98..9da71a3a 100644 --- a/src/core/executors/gxf/gxf_executor.cpp +++ b/src/core/executors/gxf/gxf_executor.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -48,43 +48,60 @@ namespace holoscan::gxf { +static const std::vector kDefaultGXFExtensions{ + "libgxf_std.so", + "libgxf_cuda.so", + "libgxf_multimedia.so", + "libgxf_serialization.so", +}; + +static const std::vector kDefaultHoloscanGXFExtensions{ + "libgxf_bayer_demosaic.so", + "libgxf_stream_playback.so", // keep for use of VideoStreamSerializer + "libgxf_tensor_rt.so", +}; + /// Global context for signal() to interrupt with Ctrl+C gxf_context_t s_signal_context; -GXFExecutor::GXFExecutor(holoscan::Fragment* app) : Executor(app) { - gxf_result_t code; - GXF_LOG_INFO("Creating context"); - code = GxfContextCreate(&context_); - if (code != GXF_SUCCESS) { - GXF_LOG_ERROR("GxfContextCreate Error: %s", GxfResultStr(code)); - return; +GXFExecutor::GXFExecutor(holoscan::Fragment* fragment, bool create_gxf_context) + : Executor(fragment) { + if (create_gxf_context) { + gxf_result_t code; + GXF_LOG_INFO("Creating context"); + code = GxfContextCreate(&context_); + if (code != GXF_SUCCESS) { + GXF_LOG_ERROR("GxfContextCreate Error: %s", GxfResultStr(code)); + return; + } + own_gxf_context_ = true; + gxf_extension_manager_ = std::make_shared(context_); + // Register extensions for holoscan (GXFWrapper codelet) + register_extensions(); } } GXFExecutor::~GXFExecutor() { - gxf_result_t code; - GXF_LOG_INFO("Destroying context"); - code = GxfContextDestroy(context_); - if (code != GXF_SUCCESS) { - GXF_LOG_ERROR("GxfContextDestroy Error: %s", GxfResultStr(code)); - return; + // Deinitialize GXF context only if `own_gxf_context_` is true + if (own_gxf_context_) { + gxf_result_t code; + GXF_LOG_INFO("Destroying context"); + code = GxfContextDestroy(context_); + if (code != GXF_SUCCESS) { + GXF_LOG_ERROR("GxfContextDestroy Error: %s", GxfResultStr(code)); + return; + } } } void GXFExecutor::run(Graph& graph) { auto context = context_; - // Load extensions - std::string config_file = fragment_->config().config_file(); - if (config_file != "") { - const char* manifest_filename = config_file.c_str(); - - const GxfLoadExtensionsInfo load_ext_info{nullptr, 0, &manifest_filename, 1, nullptr}; - HOLOSCAN_LOG_INFO("Loading extensions..."); - GXF_ASSERT_SUCCESS(GxfLoadExtensions(context, &load_ext_info)); + HOLOSCAN_LOG_INFO("Loading extensions from configs..."); + // Load extensions from config file if exists. + for (const auto& yaml_node : fragment_->config().yaml_nodes()) { + gxf_extension_manager_->load_extensions_from_yaml(yaml_node); } - // Register extensions for holoscan (GXFWrapper codelet) - register_extensions(); // Compose the graph fragment_->compose(); @@ -105,7 +122,6 @@ void GXFExecutor::run(Graph& graph) { code = GxfComponentTypeId(context, "nvidia::gxf::GreedyScheduler", &sched_tid); gxf_uid_t sched_cid; code = GxfComponentAdd(context, eid, sched_tid, nullptr, &sched_cid); - code = GxfParameterSetHandle(context, sched_cid, "clock", clock_cid); // Add connections @@ -167,7 +183,6 @@ void GXFExecutor::run(Graph& graph) { next_op_spec->inputs()[target_port]->resource()); gxf_uid_t source_cid = source_gxf_resource->gxf_cid(); gxf_uid_t target_cid = target_gxf_resource->gxf_cid(); - if (connections.find(source_cid) == connections.end()) { connections[source_cid] = std::set(); } @@ -194,8 +209,11 @@ void GXFExecutor::run(Graph& graph) { // Insert GXF's Broadcast component if source port is connected to multiple targets if (target_cids.size() > 1) { + const char* source_cname = ""; + code = GxfComponentName(context, source_cid, &source_cname); gxf_uid_t broadcast_eid; - const GxfEntityCreateInfo broadcast_entity_create_info = {"", + auto broadcast_entity_name = fmt::format("_broadcast_{}_{}", op_name, source_cname); + const GxfEntityCreateInfo broadcast_entity_create_info = {broadcast_entity_name.c_str(), GXF_ENTITY_CREATE_PROGRAM_BIT}; code = GxfCreateEntity(context, &broadcast_entity_create_info, &broadcast_eid); @@ -280,12 +298,66 @@ void GXFExecutor::run(Graph& graph) { GXF_ASSERT_SUCCESS(GxfGraphDeactivate(context)); } +void GXFExecutor::context(void* context) { + context_ = context; + gxf_extension_manager_ = std::make_shared(context_); +} + +std::shared_ptr GXFExecutor::extension_manager() { + return gxf_extension_manager_; +} + void GXFExecutor::create_input_port(Fragment* fragment, gxf_context_t gxf_context, gxf_uid_t eid, - IOSpec* io_spec) { + IOSpec* io_spec, bool bind_port) { + const char* rx_name = io_spec->name().c_str(); // input port name + + // If this executor is used by OperatorWrapper (bind_port == true) to wrap Native Operator, + // then we need to call `io_spec->resource(...)` to set the existing GXF Receiver for this input. + if (bind_port) { + const char* entity_name = ""; + GxfComponentName(gxf_context, eid, &entity_name); + + gxf_tid_t receiver_find_tid{}; + GxfComponentTypeId(gxf_context, "nvidia::gxf::Receiver", &receiver_find_tid); + + gxf_uid_t receiver_cid = 0; + GxfComponentFind(gxf_context, eid, receiver_find_tid, rx_name, nullptr, &receiver_cid); + + gxf_tid_t receiver_tid{}; + GxfComponentType(gxf_context, receiver_cid, &receiver_tid); + + gxf_tid_t double_buffer_receiver_tid{}; + GxfComponentTypeId( + gxf_context, "nvidia::gxf::DoubleBufferReceiver", &double_buffer_receiver_tid); + + if (receiver_tid == double_buffer_receiver_tid) { + nvidia::gxf::DoubleBufferReceiver* double_buffer_receiver_ptr = nullptr; + GxfComponentPointer(gxf_context, + receiver_cid, + receiver_tid, + reinterpret_cast(&double_buffer_receiver_ptr)); + + if (double_buffer_receiver_ptr) { + auto receiver = + std::make_shared(rx_name, double_buffer_receiver_ptr); + // Set the existing DoubleBufferReceiver for this input + io_spec->resource(receiver); + } else { + HOLOSCAN_LOG_ERROR( + "Unable to get DoubleBufferReceiver pointer for the handle: '{}' in '{}' entity", + rx_name, + entity_name); + } + } else { + HOLOSCAN_LOG_ERROR("Unsupported GXF receiver type for the handle: '{}' in '{}' entity", + rx_name, + entity_name); + } + return; + } + gxf_result_t code; // Create Receiver component for this input - const char* rx_name = io_spec->name().c_str(); - auto rx_resource = std::make_shared(); rx_resource->name(rx_name); rx_resource->fragment(fragment); @@ -343,11 +415,57 @@ void GXFExecutor::create_input_port(Fragment* fragment, gxf_context_t gxf_contex } void GXFExecutor::create_output_port(Fragment* fragment, gxf_context_t gxf_context, gxf_uid_t eid, - IOSpec* io_spec) { - gxf_result_t code; - // Create Transmitter component for this output + IOSpec* io_spec, bool bind_port) { const char* tx_name = io_spec->name().c_str(); + // If this executor is used by OperatorWrapper (bind_port == true) to wrap Native Operator, + // then we need to call `io_spec->resource(...)` to set the existing GXF Transmitter for this + // output. + if (bind_port) { + const char* entity_name = ""; + GxfComponentName(gxf_context, eid, &entity_name); + + gxf_tid_t transmitter_find_tid{}; + GxfComponentTypeId(gxf_context, "nvidia::gxf::Transmitter", &transmitter_find_tid); + + gxf_uid_t transmitter_cid = 0; + GxfComponentFind(gxf_context, eid, transmitter_find_tid, tx_name, nullptr, &transmitter_cid); + + gxf_tid_t transmitter_tid{}; + GxfComponentType(gxf_context, transmitter_cid, &transmitter_tid); + + gxf_tid_t double_buffer_transmitter_tid{}; + GxfComponentTypeId( + gxf_context, "nvidia::gxf::DoubleBufferTransmitter", &double_buffer_transmitter_tid); + + if (transmitter_tid == double_buffer_transmitter_tid) { + nvidia::gxf::DoubleBufferTransmitter* double_buffer_transmitter_ptr = nullptr; + GxfComponentPointer(gxf_context, + transmitter_cid, + transmitter_tid, + reinterpret_cast(&double_buffer_transmitter_ptr)); + + if (double_buffer_transmitter_ptr) { + auto transmitter = std::make_shared( + tx_name, double_buffer_transmitter_ptr); + // Set the existing DoubleBufferTransmitter for this output + io_spec->resource(transmitter); + } else { + HOLOSCAN_LOG_ERROR( + "Unable to get DoubleBufferTransmitter pointer for the handle: '{}' in '{}' entity", + tx_name, + entity_name); + } + } else { + HOLOSCAN_LOG_ERROR("Unsupported GXF transmitter type for the handle: '{}' in '{}' entity", + tx_name, + entity_name); + } + return; + } + + gxf_result_t code; + // Create Transmitter component for this output auto tx_resource = std::make_shared(); tx_resource->name(tx_name); tx_resource->fragment(fragment); @@ -425,49 +543,58 @@ bool GXFExecutor::initialize_operator(Operator* op) { auto& spec = *(op->spec()); - // Create Entity for the operator - gxf_uid_t eid; + gxf_uid_t eid = 0; gxf_result_t code; - const GxfEntityCreateInfo entity_create_info = {op->name().c_str(), - GXF_ENTITY_CREATE_PROGRAM_BIT}; - code = GxfCreateEntity(context_, &entity_create_info, &eid); - // Create Codelet component - gxf_tid_t codelet_tid; - gxf_uid_t codelet_cid; - code = GxfComponentTypeId(context_, codelet_typename, &codelet_tid); - code = GxfComponentAdd(context_, eid, codelet_tid, op->name().c_str(), &codelet_cid); - - // Set GXF Codelet ID as the ID of the operator - op->id(codelet_cid); + // Create Entity for the operator if `op_eid_` is 0 + if (op_eid_ == 0) { + const GxfEntityCreateInfo entity_create_info = {op->name().c_str(), + GXF_ENTITY_CREATE_PROGRAM_BIT}; + code = GxfCreateEntity(context_, &entity_create_info, &eid); + } else { + eid = op_eid_; + } - // Set the operator to the GXFWrapper if it is a native operator - if (is_native_operator) { - holoscan::gxf::GXFWrapper* gxf_wrapper = nullptr; - code = GxfComponentPointer( - context_, codelet_cid, codelet_tid, reinterpret_cast(&gxf_wrapper)); - if (gxf_wrapper) { - gxf_wrapper->set_operator(op); + gxf_uid_t codelet_cid; + // Create Codelet component if `op_cid_` is 0 + if (op_cid_ == 0) { + gxf_tid_t codelet_tid; + code = GxfComponentTypeId(context_, codelet_typename, &codelet_tid); + code = GxfComponentAdd(context_, eid, codelet_tid, op->name().c_str(), &codelet_cid); + + // Set the operator to the GXFWrapper if it is a native operator + if (is_native_operator) { + holoscan::gxf::GXFWrapper* gxf_wrapper = nullptr; + code = GxfComponentPointer( + context_, codelet_cid, codelet_tid, reinterpret_cast(&gxf_wrapper)); + if (gxf_wrapper) { + gxf_wrapper->set_operator(op); + } else { + HOLOSCAN_LOG_ERROR("Unable to get GXFWrapper for Operator '{}'", op->name()); + } } else { - HOLOSCAN_LOG_ERROR("Unable to get GXFWrapper for Operator '{}'", op->name()); + // Set the entity id + gxf_op->gxf_eid(eid); + // Set the codelet component id + gxf_op->gxf_cid(codelet_cid); } } else { - // Set the entity id - gxf_op->gxf_eid(eid); - // Set the codelet component id - gxf_op->gxf_cid(codelet_cid); + codelet_cid = op_cid_; } + // Set GXF Codelet ID as the ID of the operator + op->id(codelet_cid); + // Create Components for input const auto& inputs = spec.inputs(); for (const auto& [name, io_spec] : inputs) { - gxf::GXFExecutor::create_input_port(fragment(), context_, eid, io_spec.get()); + gxf::GXFExecutor::create_input_port(fragment(), context_, eid, io_spec.get(), op_eid_ != 0); } // Create Components for output const auto& outputs = spec.outputs(); for (const auto& [name, io_spec] : outputs) { - gxf::GXFExecutor::create_output_port(fragment(), context_, eid, io_spec.get()); + gxf::GXFExecutor::create_output_port(fragment(), context_, eid, io_spec.get(), op_eid_ != 0); } // Create Components for condition @@ -531,8 +658,9 @@ bool GXFExecutor::initialize_operator(Operator* op) { } else { // Set only default parameter values for (auto& [key, param_wrap] : params) { - // Set default value (by setting 'uid' to -1 for set_param() method) if not set. - code = ::holoscan::gxf::GXFParameterAdaptor::set_param(context_, -1, key.c_str(), param_wrap); + // If no value is specified, the default value will be used by setting an empty argument. + Arg empty_arg(""); + ArgumentSetter::set_param(param_wrap, empty_arg); } } (void)code; @@ -551,6 +679,10 @@ bool GXFExecutor::add_receivers(const std::shared_ptr& op, const std::string& new_input_label = fmt::format("{}:{}", receivers_name, iospec_vector.size()); HOLOSCAN_LOG_TRACE("Creating new input port with label '{}'", new_input_label); auto& input_port = downstream_op_spec->input(new_input_label); + // TODO: Currently, there is no convenient API to set the condition of the receivers (input ports) + // from the setup() method of the operator. We need to add a new API to set the condition + // of the receivers (input ports) from the setup() method of the operator. + // Add the new input port to the vector. iospec_vector.push_back(&input_port); @@ -592,21 +724,39 @@ bool GXFExecutor::add_receivers(const std::shared_ptr& op, } void GXFExecutor::register_extensions() { - GXFExtensionRegistrar extension_factory( - context_, - "HoloscanSdkInternalExtension", - "A runtime hidden extension used by Holoscan SDK to provide the native operators"); + // Register the default GXF extensions + for (auto& gxf_extension_file_name : kDefaultGXFExtensions) { + gxf_extension_manager_->load_extension(gxf_extension_file_name); + } + + // Register the default Holoscan GXF extensions + for (auto& gxf_extension_file_name : kDefaultHoloscanGXFExtensions) { + gxf_extension_manager_->load_extension(gxf_extension_file_name); + } - extension_factory.add_component( - "GXF wrapper to support Holoscan SDK native operators"); - extension_factory.add_type("Holoscan message type"); + // Register the GXF extension that provides the native operators + gxf_tid_t gxf_wrapper_tid{0xd4e7c16bcae741f8, 0xa5eb93731de9ccf6}; - extension_factory.add_component("Holoscan's GXF Tensor type"); - extension_factory.add_type("Holoscan's Tensor type"); + if (!gxf_extension_manager_->is_extension_loaded(gxf_wrapper_tid)) { + GXFExtensionRegistrar extension_factory( + context_, + "HoloscanSdkInternalExtension", + "A runtime hidden extension used by Holoscan SDK to provide the native operators", + gxf_wrapper_tid); - if (!extension_factory.register_extension()) { - HOLOSCAN_LOG_ERROR("Failed to register Holoscan SDK internal extension"); - return; + extension_factory.add_component( + "GXF wrapper to support Holoscan SDK native operators"); + extension_factory.add_type("Holoscan message type", + {0x61510ca06aa9493b, 0x8a777d0bf87476b7}); + + extension_factory.add_component( + "Holoscan's GXF Tensor type", {0xa02945eaf20e418c, 0x8e6992b68672ce40}); + extension_factory.add_type("Holoscan's Tensor type", + {0xa5eb0ed57d7f4aa2, 0xb5865ccca0ef955c}); + + if (!extension_factory.register_extension()) { + HOLOSCAN_LOG_ERROR("Failed to register Holoscan SDK internal extension"); + } } } diff --git a/src/core/fragment.cpp b/src/core/fragment.cpp index 6c80a455..7c0d788e 100644 --- a/src/core/fragment.cpp +++ b/src/core/fragment.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -59,10 +59,7 @@ void Fragment::config(const std::string& config_file, const std::string& prefix) } Config& Fragment::config() { - if (!config_) { - HOLOSCAN_LOG_WARN("Config object was not created. Call config(config_file, prefix) first."); - config_ = make_config(); - } + if (!config_) { config_ = make_config(); } return *config_; } diff --git a/src/core/gxf/entity.cpp b/src/core/gxf/entity.cpp index d18d4f8b..6ef27edf 100644 --- a/src/core/gxf/entity.cpp +++ b/src/core/gxf/entity.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -41,4 +41,32 @@ Entity Entity::New(ExecutionContext* context) { } } +nvidia::gxf::Handle get_videobuffer(Entity entity, const char* name) { + // We should use nullptr as a default name because In GXF, 'nullptr' should be used with + // GxfComponentFind() if we want to get the first component of the given type. + + // We first try to get holoscan::gxf::GXFTensor from GXF Entity. + gxf_tid_t tid; + auto tid_result = GxfComponentTypeId( + entity.context(), nvidia::TypenameAsString(), &tid); + if (tid_result != GXF_SUCCESS) { + throw std::runtime_error(fmt::format("Unable to get component type id: {}", tid_result)); + } + + gxf_uid_t cid; + auto cid_result = GxfComponentFind(entity.context(), entity.eid(), tid, name, nullptr, &cid); + if (cid_result != GXF_SUCCESS) { + std::string msg = fmt::format( + "Unable to find nvidia::gxf::VideoBuffer component from the name '{}' (error code: {})", + name == nullptr ? "" : name, + cid_result); + throw std::runtime_error(msg); + } + + // Create a handle to the GXF VideoBuffer object. + auto result = nvidia::gxf::Handle::Create(entity.context(), cid); + if (!result) { throw std::runtime_error("Failed to create Handle to nvidia::gxf::VideoBuffer"); } + return result.value(); +} + } // namespace holoscan::gxf diff --git a/src/core/gxf/gxf_condition.cpp b/src/core/gxf/gxf_condition.cpp index c406e857..a3f6a07b 100644 --- a/src/core/gxf/gxf_condition.cpp +++ b/src/core/gxf/gxf_condition.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,6 +26,17 @@ namespace holoscan::gxf { +GXFCondition::GXFCondition(const std::string& name, nvidia::gxf::SchedulingTerm* term) { + id_ = term->cid(); + name_ = name; + gxf_context_ = term->context(); + gxf_eid_ = term->eid(); + gxf_cid_ = term->cid(); + GxfComponentType(gxf_context_, gxf_cid_, &gxf_tid_); + gxf_cname_ = name; + gxf_cptr_ = term; +} + void GXFCondition::initialize() { Condition::initialize(); gxf_context_ = fragment()->executor().context(); diff --git a/src/core/gxf/gxf_extension_manager.cpp b/src/core/gxf/gxf_extension_manager.cpp new file mode 100644 index 00000000..e5a69294 --- /dev/null +++ b/src/core/gxf/gxf_extension_manager.cpp @@ -0,0 +1,255 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "holoscan/core/gxf/gxf_extension_manager.hpp" + +#include +#include + +#include +#include +#include +#include + +namespace holoscan::gxf { + +// Prefix list to search for the extension +static const std::vector kExtensionSearchPrefixes{"", "gxf_extensions"}; + +GXFExtensionManager::GXFExtensionManager(gxf_context_t context) : ExtensionManager(context) { + refresh(); +} + +GXFExtensionManager::~GXFExtensionManager() { + for (auto& handle : extension_handles_) { dlclose(handle); } +} + +void GXFExtensionManager::refresh() { + if (context_ == nullptr) { return; } + + // Configure request data + runtime_info_ = {nullptr, kGXFExtensionsMaxSize, extension_tid_list_}; + + gxf_result_t result = GxfRuntimeInfo(context_, &runtime_info_); + if (result != GXF_SUCCESS) { + HOLOSCAN_LOG_ERROR("Unable to get runtime info: {}", GxfResultStr(result)); + return; + } + auto& num_extensions = runtime_info_.num_extensions; + auto& extensions = runtime_info_.extensions; + + // Clear the extension handles + extension_tids_.clear(); + + // Add the extension tids to the set + for (uint64_t i = 0; i < num_extensions; ++i) { extension_tids_.insert(extensions[i]); } +} + +bool GXFExtensionManager::load_extension(const std::string& file_name, bool no_error_message, + const std::string& search_path_envs) { + // Skip if file name is empty + if (file_name.empty() || file_name == "null") { + HOLOSCAN_LOG_DEBUG("Extension filename is empty. Skipping extension loading."); + return true; // return true to avoid breaking the pipeline + } + + HOLOSCAN_LOG_DEBUG("Loading extension from '{}'", file_name); + + // We set RTLD_NODELETE to avoid unloading the extension when the library handle is closed. + // This is because it can cause a crash when the extension is used by another library or it + // uses thread_local variables. + void* handle = dlopen(file_name.c_str(), RTLD_LAZY | RTLD_NODELETE); + if (handle == nullptr) { + // Try to load the extension from the environment variable (search_path_env) indicating the + // folder where the extension is located. + if (!search_path_envs.empty()) { + bool found_extension = false; + const auto search_path_env_list = tokenize(search_path_envs, ","); + std::filesystem::path file_path = std::filesystem::path(file_name); + for (const auto& search_path_env : search_path_env_list) { + auto env_value = std::getenv(search_path_env.c_str()); + if (env_value != nullptr) { + const auto search_paths_str = std::string(env_value); + HOLOSCAN_LOG_DEBUG( + "Extension search path found in the env({}): {}", search_path_env, search_paths_str); + // Tokenize the search path and try to load the extension from each + // path in the search path. We stop when the extension is loaded successfully or when + // we run out of paths. + const auto search_paths = tokenize(search_paths_str, ":"); + + std::vector base_names; + // Try to load the extension from the full path of the file (if it is relative) + if (file_path.is_relative() && + (file_path.filename().lexically_normal() != file_path.lexically_normal())) { + base_names.push_back(file_path); + } + // Try to load the extension from the base name of the file + base_names.push_back(file_path.filename()); + + for (const auto& search_path : search_paths) { + for (const auto& prefix : kExtensionSearchPrefixes) { + const std::filesystem::path prefix_path{prefix}; + for (const auto& base_name : base_names) { + const std::filesystem::path candidate_parent_path = search_path / prefix_path; + const std::filesystem::path candidate_path = candidate_parent_path / base_name; + if (std::filesystem::exists(candidate_path)) { + HOLOSCAN_LOG_DEBUG("Trying extension {} found in search path {}", + base_name.c_str(), + candidate_parent_path.c_str()); + handle = dlopen(candidate_path.c_str(), RTLD_LAZY | RTLD_NODELETE); + if (handle != nullptr) { + HOLOSCAN_LOG_DEBUG("Loaded extension {} from search path '{}'", + base_name.c_str(), + candidate_parent_path.c_str()); + found_extension = true; + break; + } + } + } + } + if (found_extension) { break; } + } + } + if (found_extension) { break; } + } + } + if (handle == nullptr) { + if (!no_error_message) { + HOLOSCAN_LOG_WARN("Unable to load extension from '{}' (error: {})", file_name, dlerror()); + } + return false; + } + } else { + HOLOSCAN_LOG_DEBUG("Loaded extension {}", file_name); + } + void* func_ptr = dlsym(handle, kGxfExtensionFactoryName); + if (func_ptr == nullptr) { + if (!no_error_message) { + HOLOSCAN_LOG_ERROR( + "Unable to find extension factory in '{}' (error: {})", file_name, dlerror()); + } + dlclose(handle); + return false; + } + + // Get the factory function + const auto factory_func = reinterpret_cast(func_ptr); + + // Get the extension from the factory + void* result = nullptr; + gxf_result_t code = factory_func(&result); + if (code != GXF_SUCCESS) { + if (!no_error_message) { + HOLOSCAN_LOG_ERROR( + "Failed to create extension from '{}' (error: {})", file_name, GxfResultStr(code)); + } + dlclose(handle); + return false; + } + nvidia::gxf::Extension* extension = static_cast(result); + + // Load the extension from the pointer + const bool is_loaded = load_extension(extension, handle); + if (!is_loaded) { + HOLOSCAN_LOG_ERROR("Unable to load extension from '{}'", file_name); + dlclose(handle); + return false; + } + return true; +} + +bool GXFExtensionManager::load_extensions_from_yaml(const YAML::Node& node, bool no_error_message, + const std::string& search_path_envs, + const std::string& key) { + try { + for (const auto& entry : node[key.c_str()]) { + auto file_name = entry.as(); + auto result = load_extension(file_name, no_error_message, search_path_envs); + if (!result) { return false; } + } + } catch (std::exception& e) { + HOLOSCAN_LOG_ERROR("Error loading extension from yaml: {}", e.what()); + return false; + } + return true; +} + +bool GXFExtensionManager::load_extension(nvidia::gxf::Extension* extension, void* handle) { + if (extension == nullptr) { + HOLOSCAN_LOG_DEBUG("Extension pointer is null. Skipping extension loading."); + return true; // return true to avoid breaking the pipeline + } + + // Check if the extension is already loaded + gxf_extension_info_t info; + info.num_components = 0; + auto info_result = extension->getInfo(&info); + if (!info_result != GXF_SUCCESS) { + HOLOSCAN_LOG_ERROR("Unable to get extension info"); + return false; + } + + // Check and ignore if the extension is already loaded + if (extension_tids_.find(info.id) != extension_tids_.end()) { + HOLOSCAN_LOG_DEBUG("Extension '{}' is already loaded. Skipping extension loading.", info.name); + return true; // return true to avoid breaking the pipeline + } + + extension_tids_.insert(info.id); + + // Load the extension + gxf_result_t result = GxfLoadExtensionFromPointer(context_, extension); + if (result != GXF_SUCCESS) { + HOLOSCAN_LOG_ERROR("Unable to load extension: {}", GxfResultStr(result)); + return false; + } + + // Add the extension handle to the set of handles to be closed when the manager is destroyed + if (handle != nullptr) { extension_handles_.insert(handle); } + + return true; +} + +bool GXFExtensionManager::is_extension_loaded(gxf_tid_t tid) { + return extension_tids_.find(tid) != extension_tids_.end(); +} + +std::vector GXFExtensionManager::tokenize(const std::string& str, + const std::string& delimiters) { + std::vector search_paths; + if (str.size() == 0) { return search_paths; } + + // Find the number of possible paths based on str and delimiters + const auto possible_path_count = + std::count_if(str.begin(), + str.end(), + [&delimiters](char value) { + return delimiters.find_first_of(value) != std::string::npos; + }) + + 1; + search_paths.reserve(possible_path_count); + + size_t start = 0; + size_t end = 0; + while ((start = str.find_first_not_of(delimiters, end)) != std::string::npos) { + end = str.find_first_of(delimiters, start); + search_paths.push_back(str.substr(start, end - start)); + } + return search_paths; +} + +} // namespace holoscan::gxf diff --git a/src/core/gxf/gxf_io_context.cpp b/src/core/gxf/gxf_io_context.cpp index 041aa94c..7d102936 100644 --- a/src/core/gxf/gxf_io_context.cpp +++ b/src/core/gxf/gxf_io_context.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -54,7 +54,7 @@ std::any GXFInputContext::receive_impl(const char* name, bool no_error_message) op_->name(), inputs_.begin()->first, name); - return nullptr; // to cause a bad_any_cast + return -1; // to cause a bad_any_cast } else { auto msg_buf = fmt::memory_buffer(); auto& op_inputs = op_->spec()->inputs(); @@ -69,10 +69,10 @@ std::any GXFInputContext::receive_impl(const char* name, bool no_error_message) "The operator({}) does not have an input port with label '{}'. It should be " "one of ({:.{}}) in receive() method", op_->name(), - name, + input_name, msg_buf.data(), msg_buf.size()); - return nullptr; // to cause a bad_any_cast + return -1; // to cause a bad_any_cast } } @@ -81,7 +81,7 @@ std::any GXFInputContext::receive_impl(const char* name, bool no_error_message) auto gxf_resource = std::dynamic_pointer_cast(resource); if (gxf_resource == nullptr) { HOLOSCAN_LOG_ERROR("Invalid resource type"); - return nullptr; // to cause a bad_any_cast + return -1; // to cause a bad_any_cast } gxf_result_t code; @@ -96,8 +96,7 @@ std::any GXFInputContext::receive_impl(const char* name, bool no_error_message) auto entity = receiver->receive(); if (!entity || entity.value().is_null()) { - HOLOSCAN_LOG_ERROR("Failed to receive entity"); - return nullptr; // to cause a bad_any_cast + return nullptr; // to indicate that there is no data } auto message = entity.value().get(); @@ -154,7 +153,7 @@ void GXFOutputContext::emit_impl(std::any data, const char* name, OutputType out "The operator({}) does not have an output port with label '{}'. It should be " "one of ({:.{}}) in emit() method", op_->name(), - name, + output_name, msg_buf.data(), msg_buf.size()); return; @@ -187,6 +186,7 @@ void GXFOutputContext::emit_impl(std::any data, const char* name, OutputType out // Set the data to the value of the Message object. buffer.value()->set_value(data); // Publish the Entity object. + // TODO(gbae): Check error message static_cast(tx_ptr)->publish(std::move(gxf_entity.value())); break; } @@ -194,6 +194,7 @@ void GXFOutputContext::emit_impl(std::any data, const char* name, OutputType out // Cast to an Entity object and publish it. try { auto gxf_entity = std::any_cast(data); + // TODO(gbae): Check error message static_cast(tx_ptr)->publish(std::move(gxf_entity)); } catch (const std::bad_any_cast& e) { HOLOSCAN_LOG_ERROR("Unable to cast to gxf::Entity: {}", e.what()); diff --git a/src/core/gxf/gxf_resource.cpp b/src/core/gxf/gxf_resource.cpp index 95a0881e..134e40eb 100644 --- a/src/core/gxf/gxf_resource.cpp +++ b/src/core/gxf/gxf_resource.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,6 +30,17 @@ namespace holoscan::gxf { +GXFResource::GXFResource(const std::string& name, nvidia::gxf::Component* component) { + id_ = component->cid(); + name_ = name; + gxf_context_ = component->context(); + gxf_eid_ = component->eid(); + gxf_cid_ = component->cid(); + GxfComponentType(gxf_context_, gxf_cid_, &gxf_tid_); + gxf_cname_ = name; + gxf_cptr_ = component; +} + void GXFResource::initialize() { Resource::initialize(); gxf_context_ = fragment()->executor().context(); diff --git a/src/core/gxf/gxf_wrapper.cpp b/src/core/gxf/gxf_wrapper.cpp index 1c055e5e..dd450e16 100644 --- a/src/core/gxf/gxf_wrapper.cpp +++ b/src/core/gxf/gxf_wrapper.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,63 +18,68 @@ #include "holoscan/core/gxf/gxf_wrapper.hpp" #include "holoscan/core/common.hpp" -#include "holoscan/core/io_context.hpp" #include "holoscan/core/gxf/gxf_execution_context.hpp" +#include "holoscan/core/io_context.hpp" #include "gxf/std/transmitter.hpp" namespace holoscan::gxf { gxf_result_t GXFWrapper::initialize() { - HOLOSCAN_LOG_TRACE("GXFWrapper::initialize()"); - return GXF_SUCCESS; + HOLOSCAN_LOG_TRACE("GXFWrapper::initialize()"); + return GXF_SUCCESS; } gxf_result_t GXFWrapper::deinitialize() { - HOLOSCAN_LOG_TRACE("GXFWrapper::deinitialize()"); - return GXF_SUCCESS; + HOLOSCAN_LOG_TRACE("GXFWrapper::deinitialize()"); + return GXF_SUCCESS; } gxf_result_t GXFWrapper::registerInterface(nvidia::gxf::Registrar* registrar) { - HOLOSCAN_LOG_TRACE("GXFWrapper::registerInterface()"); - (void)registrar; - return GXF_SUCCESS; + HOLOSCAN_LOG_TRACE("GXFWrapper::registerInterface()"); + (void)registrar; + return GXF_SUCCESS; } gxf_result_t GXFWrapper::start() { - HOLOSCAN_LOG_TRACE("GXFWrapper::start()"); - if (op_ == nullptr) { - HOLOSCAN_LOG_ERROR("GXFWrapper::start() - Operator is not set"); - return GXF_FAILURE; - } - op_->start(); - return GXF_SUCCESS; + HOLOSCAN_LOG_TRACE("GXFWrapper::start()"); + if (op_ == nullptr) { + HOLOSCAN_LOG_ERROR("GXFWrapper::start() - Operator is not set"); + return GXF_FAILURE; + } + op_->start(); + return GXF_SUCCESS; } gxf_result_t GXFWrapper::tick() { - HOLOSCAN_LOG_TRACE("GXFWrapper::tick()"); - if (op_ == nullptr) { - HOLOSCAN_LOG_ERROR("GXFWrapper::tick() - Operator is not set"); - return GXF_FAILURE; - } + HOLOSCAN_LOG_TRACE("GXFWrapper::tick()"); + if (op_ == nullptr) { + HOLOSCAN_LOG_ERROR("GXFWrapper::tick() - Operator is not set"); + return GXF_FAILURE; + } - HOLOSCAN_LOG_TRACE("Calling operator: {}", op_->name()); + HOLOSCAN_LOG_TRACE("Calling operator: {}", op_->name()); - GXFExecutionContext exec_context(context(), op_); - InputContext* op_input = exec_context.input(); - OutputContext* op_output = exec_context.output(); + GXFExecutionContext exec_context(context(), op_); + InputContext* op_input = exec_context.input(); + OutputContext* op_output = exec_context.output(); + try { op_->compute(*op_input, *op_output, exec_context); + } catch (const std::exception& e) { + HOLOSCAN_LOG_ERROR("Exception occurred for operator: '{}' - {}", op_->name(), e.what()); + return GXF_FAILURE; + } - return GXF_SUCCESS; + return GXF_SUCCESS; } gxf_result_t GXFWrapper::stop() { - HOLOSCAN_LOG_TRACE("GXFWrapper::stop()"); - if (op_ == nullptr) { - HOLOSCAN_LOG_ERROR("GXFWrapper::stop() - Operator is not set"); - return GXF_FAILURE; - } - op_->stop(); - return GXF_SUCCESS; + HOLOSCAN_LOG_TRACE("GXFWrapper::stop()"); + if (op_ == nullptr) { + HOLOSCAN_LOG_ERROR("GXFWrapper::stop() - Operator is not set"); + return GXF_FAILURE; + } + op_->stop(); + return GXF_SUCCESS; } } // namespace holoscan::gxf diff --git a/src/core/operator.cpp b/src/core/operator.cpp index 7a2fbbcd..c7293067 100644 --- a/src/core/operator.cpp +++ b/src/core/operator.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -38,4 +38,19 @@ void Operator::initialize() { } } +YAML::Node Operator::to_yaml_node() const { + YAML::Node node = Component::to_yaml_node(); + node["type"] = (operator_type_ == OperatorType::kGXF) ? "GXF" : "native"; + node["conditions"] = YAML::Node(YAML::NodeType::Sequence); + for (const auto& c : conditions_) { node["conditions"].push_back(c.second->to_yaml_node()); } + node["resources"] = YAML::Node(YAML::NodeType::Sequence); + for (const auto& r : resources_) { node["resources"].push_back(r.second->to_yaml_node()); } + if (spec_) { + node["spec"] = spec_->to_yaml_node(); + } else { + node["spec"] = YAML::Null; + } + return node; +} + } // namespace holoscan diff --git a/src/core/operator_spec.cpp b/src/core/operator_spec.cpp new file mode 100644 index 00000000..01e71dd0 --- /dev/null +++ b/src/core/operator_spec.cpp @@ -0,0 +1,31 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "holoscan/core/operator_spec.hpp" +#include "holoscan/core/fragment.hpp" + +namespace holoscan { + +YAML::Node OperatorSpec::to_yaml_node() const { + YAML::Node node = ComponentSpec::to_yaml_node(); + node["inputs"] = YAML::Node(YAML::NodeType::Sequence); + for (const auto& i : inputs_) { node["inputs"].push_back(i.first); } + node["outputs"] = YAML::Node(YAML::NodeType::Sequence); + for (const auto& o : outputs_) { node["outputs"].push_back(o.first); } + return node; +} + +} // namespace holoscan diff --git a/src/core/resource.cpp b/src/core/resource.cpp index 490e33b2..6b8df0b4 100644 --- a/src/core/resource.cpp +++ b/src/core/resource.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,5 +21,14 @@ namespace holoscan { +YAML::Node Resource::to_yaml_node() const { + YAML::Node node = Component::to_yaml_node(); + if (spec_) { + node["spec"] = spec_->to_yaml_node(); + } else { + node["spec"] = YAML::Null; + } + return node; +} } // namespace holoscan diff --git a/src/core/resources/gxf/allocator.cpp b/src/core/resources/gxf/allocator.cpp new file mode 100644 index 00000000..75a68b9f --- /dev/null +++ b/src/core/resources/gxf/allocator.cpp @@ -0,0 +1,56 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "holoscan/core/resources/gxf/allocator.hpp" + +namespace holoscan { + +Allocator::Allocator(const std::string& name, nvidia::gxf::Allocator* component) + : GXFResource(name, component) {} + +bool Allocator::is_available(uint64_t size) { + if (gxf_cptr_) { + nvidia::gxf::Allocator* allocator = static_cast(gxf_cptr_); + return allocator->is_available(size); + } + + return false; +} + +nvidia::byte* Allocator::allocate(uint64_t size, MemoryStorageType type) { + if (gxf_cptr_) { + nvidia::gxf::Allocator* allocator = static_cast(gxf_cptr_); + + auto result = allocator->allocate(size, static_cast(type)); + if (result) { return result.value(); } + } + + HOLOSCAN_LOG_ERROR( + "Failed to allocate memory of size {} with type {}", size, static_cast(type)); + + return nullptr; +} + +void Allocator::free(nvidia::byte* pointer) { + if (gxf_cptr_) { + nvidia::gxf::Allocator* allocator = static_cast(gxf_cptr_); + auto result = allocator->free(pointer); + if (!result) { HOLOSCAN_LOG_ERROR("Failed to free memory at {}", static_cast(pointer)); } + } +} + +} // namespace holoscan diff --git a/src/core/resources/gxf/block_memory_pool.cpp b/src/core/resources/gxf/block_memory_pool.cpp index 99b9ec89..e29981dc 100644 --- a/src/core/resources/gxf/block_memory_pool.cpp +++ b/src/core/resources/gxf/block_memory_pool.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,6 +21,19 @@ namespace holoscan { +BlockMemoryPool::BlockMemoryPool(const std::string& name, nvidia::gxf::BlockMemoryPool* component) + : Allocator(name, component) { + int32_t storage_type = 0; + GxfParameterGetInt32(gxf_context_, gxf_cid_, "storage_type", &storage_type); + storage_type_ = storage_type; + uint64_t block_size = 0; + GxfParameterGetUInt64(gxf_context_, gxf_cid_, "block_size", &block_size); + block_size_ = block_size; + uint64_t num_blocks = 0; + GxfParameterGetUInt64(gxf_context_, gxf_cid_, "num_blocks", &num_blocks); + num_blocks_ = num_blocks; +} + void BlockMemoryPool::setup(ComponentSpec& spec) { spec.param(storage_type_, "storage_type", diff --git a/src/core/resources/gxf/cuda_stream_pool.cpp b/src/core/resources/gxf/cuda_stream_pool.cpp index 6e7f5910..ea715a57 100644 --- a/src/core/resources/gxf/cuda_stream_pool.cpp +++ b/src/core/resources/gxf/cuda_stream_pool.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,6 +29,25 @@ constexpr uint32_t kDefaultMaxSize = 0; constexpr int32_t kDefaultDeviceId = 0; } // namespace +CudaStreamPool::CudaStreamPool(const std::string& name, nvidia::gxf::CudaStreamPool* component) + : Allocator(name, component) { + int32_t dev_id = 0; + GxfParameterGetInt32(gxf_context_, gxf_cid_, "dev_id", &dev_id); + dev_id_ = dev_id; + uint32_t stream_flags = 0; + GxfParameterGetUInt32(gxf_context_, gxf_cid_, "stream_flags", &stream_flags); + stream_flags_ = stream_flags; + int32_t stream_priority = 0; + GxfParameterGetInt32(gxf_context_, gxf_cid_, "stream_priority", &stream_priority); + stream_priority_ = stream_priority; + uint32_t reserved_size = 0; + GxfParameterGetUInt32(gxf_context_, gxf_cid_, "reserved_size", &reserved_size); + reserved_size_ = reserved_size; + uint32_t max_size = 0; + GxfParameterGetUInt32(gxf_context_, gxf_cid_, "max_size", &max_size); + max_size_ = max_size; +} + void CudaStreamPool::setup(ComponentSpec& spec) { spec.param( dev_id_, "dev_id", "Device Id", "Create CUDA Stream on which device.", kDefaultDeviceId); diff --git a/src/core/resources/gxf/double_buffer_receiver.cpp b/src/core/resources/gxf/double_buffer_receiver.cpp index be46b08c..037e00d7 100644 --- a/src/core/resources/gxf/double_buffer_receiver.cpp +++ b/src/core/resources/gxf/double_buffer_receiver.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,6 +21,17 @@ namespace holoscan { +DoubleBufferReceiver::DoubleBufferReceiver(const std::string& name, + nvidia::gxf::DoubleBufferReceiver* component) + : Receiver(name, component) { + uint64_t capacity = 0; + GxfParameterGetUInt64(gxf_context_, gxf_cid_, "capacity", &capacity); + capacity_ = capacity; + uint64_t policy = 0; + GxfParameterGetUInt64(gxf_context_, gxf_cid_, "policy", &policy); + policy_ = policy; +} + void DoubleBufferReceiver::setup(ComponentSpec& spec) { spec.param(capacity_, "capacity", "Capacity", "", 1UL); spec.param(policy_, "policy", "Policy", "0: pop, 1: reject, 2: fault", 2UL); diff --git a/src/core/resources/gxf/double_buffer_transmitter.cpp b/src/core/resources/gxf/double_buffer_transmitter.cpp index 930fceaf..f2a8bafa 100644 --- a/src/core/resources/gxf/double_buffer_transmitter.cpp +++ b/src/core/resources/gxf/double_buffer_transmitter.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,6 +21,17 @@ namespace holoscan { +DoubleBufferTransmitter::DoubleBufferTransmitter(const std::string& name, + nvidia::gxf::DoubleBufferTransmitter* component) + : Transmitter(name, component) { + uint64_t capacity = 0; + GxfParameterGetUInt64(gxf_context_, gxf_cid_, "capacity", &capacity); + capacity_ = capacity; + uint64_t policy = 0; + GxfParameterGetUInt64(gxf_context_, gxf_cid_, "policy", &policy); + policy_ = policy; +} + void DoubleBufferTransmitter::setup(ComponentSpec& spec) { spec.param(capacity_, "capacity", "Capacity", "", 1UL); spec.param(policy_, "policy", "Policy", "0: pop, 1: reject, 2: fault", 2UL); diff --git a/gxf_extensions/segmentation_visualizer/glsl/segmentation_mask.vert b/src/core/resources/gxf/receiver.cpp similarity index 65% rename from gxf_extensions/segmentation_visualizer/glsl/segmentation_mask.vert rename to src/core/resources/gxf/receiver.cpp index d8609ddf..00c69513 100644 --- a/gxf_extensions/segmentation_visualizer/glsl/segmentation_mask.vert +++ b/src/core/resources/gxf/receiver.cpp @@ -1,7 +1,5 @@ -#version 460 - /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,11 +15,11 @@ * limitations under the License. */ -layout(location = 0) in vec2 vert_coord; +#include "holoscan/core/resources/gxf/receiver.hpp" + +namespace holoscan { -out vec2 tex_coord; +Receiver::Receiver(const std::string& name, nvidia::gxf::Receiver* component) + : GXFResource(name, component) {} -void main() { - gl_Position = vec4((vert_coord * 2.0) - 1.0, 0.0, 1.0); - tex_coord = vec2(vert_coord.x, 1.0 - vert_coord.y); -} +} // namespace holoscan diff --git a/src/core/resources/gxf/transmitter.cpp b/src/core/resources/gxf/transmitter.cpp new file mode 100644 index 00000000..c9317fdc --- /dev/null +++ b/src/core/resources/gxf/transmitter.cpp @@ -0,0 +1,25 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "holoscan/core/resources/gxf/transmitter.hpp" + +namespace holoscan { + +Transmitter::Transmitter(const std::string& name, nvidia::gxf::Transmitter* component) + : GXFResource(name, component) {} + +} // namespace holoscan diff --git a/src/core/logger.cpp b/src/logger/logger.cpp similarity index 95% rename from src/core/logger.cpp rename to src/logger/logger.cpp index 45dd08cb..eec33e63 100644 --- a/src/core/logger.cpp +++ b/src/logger/logger.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,7 +15,7 @@ * limitations under the License. */ -#include "holoscan/core/logger.hpp" +#include "holoscan/logger/logger.hpp" #include #include diff --git a/apps/CMakeLists.txt b/src/operators/CMakeLists.txt similarity index 56% rename from apps/CMakeLists.txt rename to src/operators/CMakeLists.txt index b40cc93a..d25fdc93 100644 --- a/apps/CMakeLists.txt +++ b/src/operators/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,9 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -add_subdirectory(endoscopy_tool_tracking) -if(HOLOSCAN_BUILD_HI_SPEED_ENDO_APP) - add_subdirectory(hi_speed_endoscopy) -endif() -add_subdirectory(multiai) -add_subdirectory(ultrasound_segmentation) +add_subdirectory(aja_source) +add_subdirectory(bayer_demosaic) +add_subdirectory(format_converter) +add_subdirectory(holoviz) +add_subdirectory(multiai_inference) +add_subdirectory(multiai_postprocessor) +add_subdirectory(ping_rx) +add_subdirectory(ping_tx) +add_subdirectory(segmentation_postprocessor) +add_subdirectory(tensor_rt) +add_subdirectory(video_stream_recorder) +add_subdirectory(video_stream_replayer) diff --git a/src/operators/aja_source/CMakeLists.txt b/src/operators/aja_source/CMakeLists.txt new file mode 100644 index 00000000..b0ba29d5 --- /dev/null +++ b/src/operators/aja_source/CMakeLists.txt @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_holoscan_operator(aja aja_source.cpp) + +# TODO: try making AJA::ajantv2 private, need to remove headers from the operator headers +target_link_libraries(op_aja + PUBLIC + holoscan::core + AJA::ajantv2 + CUDA::cuda_driver + PRIVATE + CUDA::cudart + GXF::multimedia +) diff --git a/src/operators/aja_source/aja_source.cpp b/src/operators/aja_source/aja_source.cpp index 1b3d679e..4e24c776 100644 --- a/src/operators/aja_source/aja_source.cpp +++ b/src/operators/aja_source/aja_source.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,11 +17,25 @@ #include "holoscan/operators/aja_source/aja_source.hpp" +#include +#include + +#include +#include + +#include "gxf/multimedia/video.hpp" +#include "holoscan/core/execution_context.hpp" #include "holoscan/core/gxf/entity.hpp" +#include "holoscan/core/io_spec.hpp" #include "holoscan/core/operator_spec.hpp" namespace holoscan::ops { +// used in more than one function +constexpr uint32_t kNumBuffers = 2; + +AJASourceOp::AJASourceOp() {} + void AJASourceOp::setup(OperatorSpec& spec) { auto& video_buffer_output = spec.output("video_buffer_output"); auto& overlay_buffer_input = @@ -72,10 +86,427 @@ void AJASourceOp::setup(OperatorSpec& spec) { &overlay_buffer_input); } +AJAStatus AJASourceOp::DetermineVideoFormat() { + if (width_ == 1920 && height_ == 1080 && framerate_ == 60) { + video_format_ = NTV2_FORMAT_1080p_6000_A; + } else if (width_ == 3840 && height_ == 2160 && framerate_ == 60) { + video_format_ = NTV2_FORMAT_3840x2160p_6000; + } else { + return AJA_STATUS_UNSUPPORTED; + } + + return AJA_STATUS_SUCCESS; +} + +AJAStatus AJASourceOp::OpenDevice() { + // Get the requested device. + if (!CNTV2DeviceScanner::GetFirstDeviceFromArgument(device_specifier_, device_)) { + HOLOSCAN_LOG_ERROR("Device {} not found.", device_specifier_.get()); + return AJA_STATUS_OPEN; + } + + // Check if the device is ready. + if (!device_.IsDeviceReady(false)) { + HOLOSCAN_LOG_ERROR("Device {} not ready.", device_specifier_.get()); + return AJA_STATUS_INITIALIZE; + } + + // Get the device ID. + device_id_ = device_.GetDeviceID(); + + // Detect Kona HDMI device. + is_kona_hdmi_ = NTV2DeviceGetNumHDMIVideoInputs(device_id_) > 1; + + // Check if a TSI 4x format is needed. + if (is_kona_hdmi_) { use_tsi_ = GetNTV2VideoFormatTSI(&video_format_); } + + // Check device capabilities. + if (!NTV2DeviceCanDoVideoFormat(device_id_, video_format_)) { + HOLOSCAN_LOG_ERROR("AJA device does not support requested video format."); + return AJA_STATUS_UNSUPPORTED; + } + if (!NTV2DeviceCanDoFrameBufferFormat(device_id_, pixel_format_)) { + HOLOSCAN_LOG_ERROR("AJA device does not support requested pixel format."); + return AJA_STATUS_UNSUPPORTED; + } + if (!NTV2DeviceCanDoCapture(device_id_)) { + HOLOSCAN_LOG_ERROR("AJA device cannot capture video."); + return AJA_STATUS_UNSUPPORTED; + } + if (!NTV2_IS_VALID_CHANNEL(channel_)) { + HOLOSCAN_LOG_ERROR("Invalid AJA channel: {}", channel_.get()); + return AJA_STATUS_UNSUPPORTED; + } + + // Check overlay capabilities. + if (enable_overlay_) { + if (!NTV2_IS_VALID_CHANNEL(overlay_channel_)) { + HOLOSCAN_LOG_ERROR("Invalid overlay channel: {}", overlay_channel_.get()); + return AJA_STATUS_UNSUPPORTED; + } + + if (NTV2DeviceGetNumVideoChannels(device_id_) < 2) { + HOLOSCAN_LOG_ERROR("Insufficient number of video channels"); + return AJA_STATUS_UNSUPPORTED; + } + + if (NTV2DeviceGetNumFrameStores(device_id_) < 2) { + HOLOSCAN_LOG_ERROR("Insufficient number of frame stores"); + return AJA_STATUS_UNSUPPORTED; + } + + if (NTV2DeviceGetNumMixers(device_id_) < 1) { + HOLOSCAN_LOG_ERROR("Hardware mixing not supported"); + return AJA_STATUS_UNSUPPORTED; + } + + if (!NTV2DeviceHasBiDirectionalSDI(device_id_)) { + HOLOSCAN_LOG_ERROR("BiDirectional SDI not supported"); + return AJA_STATUS_UNSUPPORTED; + } + } + + return AJA_STATUS_SUCCESS; +} + +AJAStatus AJASourceOp::SetupVideo() { + constexpr size_t kWarmupFrames = 5; + + NTV2InputSourceKinds input_kind = is_kona_hdmi_ ? NTV2_INPUTSOURCES_HDMI : NTV2_INPUTSOURCES_SDI; + NTV2InputSource input_src = ::NTV2ChannelToInputSource(channel_, input_kind); + NTV2Channel tsi_channel = static_cast(channel_ + 1); + + if (!IsRGBFormat(pixel_format_)) { + HOLOSCAN_LOG_ERROR("YUV formats not yet supported"); + return AJA_STATUS_UNSUPPORTED; + } + + // Detect if the source is YUV or RGB (i.e. if CSC is required or not). + bool is_input_rgb(false); + if (input_kind == NTV2_INPUTSOURCES_HDMI) { + NTV2LHIHDMIColorSpace input_color; + device_.GetHDMIInputColor(input_color, channel_); + is_input_rgb = (input_color == NTV2_LHIHDMIColorSpaceRGB); + } + + // Setup the input routing. + device_.ClearRouting(); + device_.EnableChannel(channel_); + if (use_tsi_) { + device_.SetTsiFrameEnable(true, channel_); + device_.EnableChannel(tsi_channel); + } + device_.SetMode(channel_, NTV2_MODE_CAPTURE); + if (NTV2DeviceHasBiDirectionalSDI(device_id_) && NTV2_INPUT_SOURCE_IS_SDI(input_src)) { + device_.SetSDITransmitEnable(channel_, false); + } + device_.SetVideoFormat(video_format_, false, false, channel_); + device_.SetFrameBufferFormat(channel_, pixel_format_); + if (use_tsi_) { device_.SetFrameBufferFormat(tsi_channel, pixel_format_); } + device_.EnableInputInterrupt(channel_); + device_.SubscribeInputVerticalEvent(channel_); + + NTV2OutputXptID input_output_xpt = + GetInputSourceOutputXpt(input_src, /*DS2*/ false, is_input_rgb, /*Quadrant*/ 0); + NTV2InputXptID fb_input_xpt(GetFrameBufferInputXptFromChannel(channel_)); + if (use_tsi_) { + if (!is_input_rgb) { + if (NTV2DeviceGetNumCSCs(device_id_) < 4) { + HOLOSCAN_LOG_ERROR("CSCs not available for TSI input."); + return AJA_STATUS_UNSUPPORTED; + } + device_.Connect(NTV2_XptFrameBuffer1Input, NTV2_Xpt425Mux1ARGB); + device_.Connect(NTV2_XptFrameBuffer1BInput, NTV2_Xpt425Mux1BRGB); + device_.Connect(NTV2_XptFrameBuffer2Input, NTV2_Xpt425Mux2ARGB); + device_.Connect(NTV2_XptFrameBuffer2BInput, NTV2_Xpt425Mux2BRGB); + device_.Connect(NTV2_Xpt425Mux1AInput, NTV2_XptCSC1VidRGB); + device_.Connect(NTV2_Xpt425Mux1BInput, NTV2_XptCSC2VidRGB); + device_.Connect(NTV2_Xpt425Mux2AInput, NTV2_XptCSC3VidRGB); + device_.Connect(NTV2_Xpt425Mux2BInput, NTV2_XptCSC4VidRGB); + device_.Connect(NTV2_XptCSC1VidInput, NTV2_XptHDMIIn1); + device_.Connect(NTV2_XptCSC2VidInput, NTV2_XptHDMIIn1Q2); + device_.Connect(NTV2_XptCSC3VidInput, NTV2_XptHDMIIn1Q3); + device_.Connect(NTV2_XptCSC4VidInput, NTV2_XptHDMIIn1Q4); + } else { + device_.Connect(NTV2_XptFrameBuffer1Input, NTV2_Xpt425Mux1ARGB); + device_.Connect(NTV2_XptFrameBuffer1BInput, NTV2_Xpt425Mux1BRGB); + device_.Connect(NTV2_XptFrameBuffer2Input, NTV2_Xpt425Mux2ARGB); + device_.Connect(NTV2_XptFrameBuffer2BInput, NTV2_Xpt425Mux2BRGB); + device_.Connect(NTV2_Xpt425Mux1AInput, NTV2_XptHDMIIn1RGB); + device_.Connect(NTV2_Xpt425Mux1BInput, NTV2_XptHDMIIn1Q2RGB); + device_.Connect(NTV2_Xpt425Mux2AInput, NTV2_XptHDMIIn1Q3RGB); + device_.Connect(NTV2_Xpt425Mux2BInput, NTV2_XptHDMIIn1Q4RGB); + } + } else if (!is_input_rgb) { + if (NTV2DeviceGetNumCSCs(device_id_) <= static_cast(channel_)) { + HOLOSCAN_LOG_ERROR("No CSC available for NTV2_CHANNEL{}", channel_ + 1); + return AJA_STATUS_UNSUPPORTED; + } + NTV2InputXptID csc_input = GetCSCInputXptFromChannel(channel_); + NTV2OutputXptID csc_output = + GetCSCOutputXptFromChannel(channel_, /*inIsKey*/ false, /*inIsRGB*/ true); + device_.Connect(fb_input_xpt, csc_output); + device_.Connect(csc_input, input_output_xpt); + } else { + device_.Connect(fb_input_xpt, input_output_xpt); + } + + if (enable_overlay_) { + // Setup output channel. + device_.SetReference(NTV2_REFERENCE_INPUT1); + device_.SetMode(overlay_channel_, NTV2_MODE_DISPLAY); + device_.SetSDITransmitEnable(overlay_channel_, true); + device_.SetVideoFormat(video_format_, false, false, overlay_channel_); + device_.SetFrameBufferFormat(overlay_channel_, NTV2_FBF_ABGR); + + // Setup mixer controls. + device_.SetMixerFGInputControl(0, NTV2MIXERINPUTCONTROL_SHAPED); + device_.SetMixerBGInputControl(0, NTV2MIXERINPUTCONTROL_FULLRASTER); + device_.SetMixerCoefficient(0, 0x10000); + device_.SetMixerFGMatteEnabled(0, false); + device_.SetMixerBGMatteEnabled(0, false); + + // Setup routing (overlay frame to CSC, CSC and SDI input to mixer, mixer to SDI output). + NTV2OutputDestination output_dst = ::NTV2ChannelToOutputDestination(overlay_channel_); + device_.Connect(GetCSCInputXptFromChannel(overlay_channel_), + GetFrameBufferOutputXptFromChannel(overlay_channel_, true /*RGB*/)); + device_.Connect(NTV2_XptMixer1FGVidInput, + GetCSCOutputXptFromChannel(overlay_channel_, false /*Key*/)); + device_.Connect(NTV2_XptMixer1FGKeyInput, + GetCSCOutputXptFromChannel(overlay_channel_, true /*Key*/)); + device_.Connect(NTV2_XptMixer1BGVidInput, input_output_xpt); + device_.Connect(GetOutputDestInputXpt(output_dst), NTV2_XptMixer1VidYUV); + + // Set initial output frame (overlay uses HW frames 2 and 3). + current_overlay_hw_frame_ = 2; + device_.SetOutputFrame(overlay_channel_, current_overlay_hw_frame_); + } + + // Wait for a number of frames to acquire video signal. + current_hw_frame_ = 0; + device_.SetInputFrame(channel_, current_hw_frame_); + device_.WaitForInputVerticalInterrupt(channel_, kWarmupFrames); + + return AJA_STATUS_SUCCESS; +} + +bool AJASourceOp::AllocateBuffers(std::vector& buffers, size_t num_buffers, + size_t buffer_size, bool rdma) { + buffers.resize(num_buffers); + for (auto& buf : buffers) { + if (rdma) { + cudaMalloc(&buf, buffer_size); + unsigned int syncFlag = 1; + if (cuPointerSetAttribute( + &syncFlag, CU_POINTER_ATTRIBUTE_SYNC_MEMOPS, reinterpret_cast(buf))) { + HOLOSCAN_LOG_ERROR("Failed to set SYNC_MEMOPS CUDA attribute for RDMA"); + return false; + } + } else { + buf = malloc(buffer_size); + } + + if (!buf) { + HOLOSCAN_LOG_ERROR("Failed to allocate buffer memory"); + return false; + } + + if (!device_.DMABufferLock(static_cast(buf), buffer_size, true, rdma)) { + HOLOSCAN_LOG_ERROR("Failed to map buffer for DMA"); + return false; + } + } + + return true; +} + +void AJASourceOp::FreeBuffers(std::vector& buffers, bool rdma) { + for (auto& buf : buffers) { + if (rdma) { + cudaFree(buf); + } else { + free(buf); + } + } + buffers.clear(); +} + +AJAStatus AJASourceOp::SetupBuffers() { + auto size = GetVideoWriteSize(video_format_, pixel_format_); + + if (!AllocateBuffers(buffers_, kNumBuffers, size, use_rdma_)) { return AJA_STATUS_INITIALIZE; } + + if (enable_overlay_) { + if (!AllocateBuffers(overlay_buffers_, kNumBuffers, size, overlay_rdma_)) { + return AJA_STATUS_INITIALIZE; + } + } + + return AJA_STATUS_SUCCESS; +} + void AJASourceOp::initialize() { register_converter(); - holoscan::ops::GXFOperator::initialize(); + Operator::initialize(); +} + +void AJASourceOp::start() { + HOLOSCAN_LOG_INFO("AJA Source: Capturing from NTV2_CHANNEL{}", (channel_.get() + 1)); + HOLOSCAN_LOG_INFO("AJA Source: RDMA is {}", use_rdma_ ? "enabled" : "disabled"); + if (enable_overlay_) { + HOLOSCAN_LOG_INFO("AJA Source: Outputting overlay to NTV2_CHANNEL{}", + (overlay_channel_.get() + 1)); + HOLOSCAN_LOG_INFO("AJA Source: Overlay RDMA is {}", overlay_rdma_ ? "enabled" : "disabled"); + } else { + HOLOSCAN_LOG_INFO("AJA Source: Overlay output is disabled"); + } + + AJAStatus status = DetermineVideoFormat(); + if (AJA_FAILURE(status)) { + throw std::runtime_error("Video format could not be determined based on parameters."); + } + + status = OpenDevice(); + if (AJA_FAILURE(status)) { + throw std::runtime_error(fmt::format("Failed to open device {}", device_specifier_.get())); + } + + status = SetupVideo(); + if (AJA_FAILURE(status)) { + throw std::runtime_error(fmt::format("Failed to setup device {}", device_specifier_.get())); + } + + status = SetupBuffers(); + if (AJA_FAILURE(status)) { throw std::runtime_error("Failed to setup AJA buffers."); } +} + +void AJASourceOp::compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) { + // holoscan::gxf::Entity + bool have_overlay_in = false; + holoscan::gxf::Entity overlay_in_message; + try { + overlay_in_message = op_input.receive("overlay_buffer_input"); + have_overlay_in = true; + } catch (const std::runtime_error& r_) { + HOLOSCAN_LOG_TRACE("Failed to find overlay_buffer_input: {}", std::string(r_.what())); + } + + if (enable_overlay_ && have_overlay_in) { + nvidia::gxf::Handle overlay_buffer; + try { + overlay_buffer = holoscan::gxf::get_videobuffer(overlay_in_message); + // Overlay uses HW frames 2 and 3. + current_overlay_hw_frame_ = ((current_overlay_hw_frame_ + 1) % 2) + 2; + + const auto& buffer = overlay_buffer.get(); + ULWord* ptr = reinterpret_cast(buffer->pointer()); + device_.DMAWriteFrame(current_overlay_hw_frame_, ptr, buffer->size()); + device_.SetOutputFrame(overlay_channel_, current_overlay_hw_frame_); + device_.SetMixerMode(0, NTV2MIXERMODE_MIX); + } catch (const std::runtime_error& r_) { + HOLOSCAN_LOG_TRACE("Failed to read VideoBuffer with error: {}", std::string(r_.what())); + } + } + + // Update the next input frame and wait until it starts. + uint32_t next_hw_frame = (current_hw_frame_ + 1) % 2; + device_.SetInputFrame(channel_, next_hw_frame); + device_.WaitForInputVerticalInterrupt(channel_); + + // Read the last completed frame. + auto size = GetVideoWriteSize(video_format_, pixel_format_); + auto ptr = static_cast(buffers_[current_buffer_]); + device_.DMAReadFrame(current_hw_frame_, ptr, size); + + // Set the frame to read for the next tick. + current_hw_frame_ = next_hw_frame; + + // Common (output and overlay) buffer info + nvidia::gxf::VideoTypeTraits video_type; + nvidia::gxf::VideoFormatSize color_format; + auto color_planes = color_format.getDefaultColorPlanes(width_, height_); + nvidia::gxf::VideoBufferInfo info{width_, + height_, + video_type.value, + color_planes, + nvidia::gxf::SurfaceLayout::GXF_SURFACE_LAYOUT_PITCH_LINEAR}; + + // Pass an empty overlay buffer downstream. + if (enable_overlay_) { + auto overlay_output = nvidia::gxf::Entity::New(context.context()); + if (!overlay_output) { + HOLOSCAN_LOG_ERROR("Failed to allocate overlay output; terminating."); + return; + } + + auto overlay_buffer = overlay_output.value().add(); + if (!overlay_buffer) { + HOLOSCAN_LOG_ERROR("Failed to allocate overlay buffer; terminating."); + return; + } + + auto overlay_storage_type = overlay_rdma_ ? nvidia::gxf::MemoryStorageType::kDevice + : nvidia::gxf::MemoryStorageType::kHost; + overlay_buffer.value()->wrapMemory( + info, size, overlay_storage_type, overlay_buffers_[current_buffer_], nullptr); + + auto overlay_result = gxf::Entity(std::move(overlay_output.value())); + op_output.emit(overlay_result, "overlay_buffer_output"); + } + + // Pass the video output buffer downstream. + auto video_output = nvidia::gxf::Entity::New(context.context()); + if (!video_output) { + throw std::runtime_error("Failed to allocate video output; terminating."); + return; + } + + auto video_buffer = video_output.value().add(); + if (!video_buffer) { + throw std::runtime_error("Failed to allocate video buffer; terminating."); + return; + } + + auto storage_type = + use_rdma_ ? nvidia::gxf::MemoryStorageType::kDevice : nvidia::gxf::MemoryStorageType::kHost; + video_buffer.value()->wrapMemory(info, size, storage_type, buffers_[current_buffer_], nullptr); + + auto result = gxf::Entity(std::move(video_output.value())); + op_output.emit(result, "video_buffer_output"); + + // Update the current buffer (index shared between video and overlay) + current_buffer_ = (current_buffer_ + 1) % kNumBuffers; +} + +void AJASourceOp::stop() { + device_.UnsubscribeInputVerticalEvent(channel_); + device_.DMABufferUnlockAll(); + + if (enable_overlay_) { device_.SetMixerMode(0, NTV2MIXERMODE_FOREGROUND_OFF); } + + FreeBuffers(buffers_, use_rdma_); + FreeBuffers(overlay_buffers_, overlay_rdma_); +} + +bool AJASourceOp::GetNTV2VideoFormatTSI(NTV2VideoFormat* format) { + switch (*format) { + case NTV2_FORMAT_3840x2160p_2400: + *format = NTV2_FORMAT_4x1920x1080p_2400; + return true; + case NTV2_FORMAT_3840x2160p_6000: + *format = NTV2_FORMAT_4x1920x1080p_6000; + return true; + case NTV2_FORMAT_4096x2160p_2400: + *format = NTV2_FORMAT_4x2048x1080p_2400; + return true; + case NTV2_FORMAT_4096x2160p_6000: + *format = NTV2_FORMAT_4x2048x1080p_6000; + return true; + default: + return false; + } } } // namespace holoscan::ops diff --git a/src/operators/bayer_demosaic/CMakeLists.txt b/src/operators/bayer_demosaic/CMakeLists.txt new file mode 100644 index 00000000..0c4a6229 --- /dev/null +++ b/src/operators/bayer_demosaic/CMakeLists.txt @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_holoscan_operator(bayer_demosaic bayer_demosaic.cpp) + +target_link_libraries(op_bayer_demosaic + PUBLIC holoscan::core +) + +# Wraps the GXF extension +add_dependencies(op_bayer_demosaic gxf_bayer_demosaic) diff --git a/src/operators/custom_lstm_inference/lstm_tensor_rt_inference.cpp b/src/operators/custom_lstm_inference/lstm_tensor_rt_inference.cpp deleted file mode 100644 index d8ce8079..00000000 --- a/src/operators/custom_lstm_inference/lstm_tensor_rt_inference.cpp +++ /dev/null @@ -1,129 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "holoscan/operators/custom_lstm_inference/lstm_tensor_rt_inference.hpp" - -#include "holoscan/core/gxf/entity.hpp" -#include "holoscan/core/operator_spec.hpp" - -#include "holoscan/core/resources/gxf/allocator.hpp" -#include "holoscan/core/resources/gxf/cuda_stream_pool.hpp" - -namespace holoscan::ops { - -void LSTMTensorRTInferenceOp::setup(OperatorSpec& spec) { - auto& in_tensor = spec.input("source_video"); - auto& out_tensor = spec.output("tensor"); - - spec.param( - model_file_path_, "model_file_path", "Model File Path", "Path to ONNX model to be loaded."); - spec.param(engine_cache_dir_, - "engine_cache_dir", - "Engine Cache Directory", - "Path to a folder containing cached engine files to be serialized and loaded from."); - spec.param(plugins_lib_namespace_, - "plugins_lib_namespace", - "Plugins Lib Namespace", - "Namespace used to register all the plugins in this library.", - std::string("")); - spec.param(force_engine_update_, - "force_engine_update", - "Force Engine Update", - "Always update engine regard less of existing engine file. " - "Such conversion may take minutes. Default to false.", - false); - - spec.param(input_tensor_names_, - "input_tensor_names", - "Input Tensor Names", - "Names of input tensors in the order to be fed into the model."); - spec.param(input_binding_names_, - "input_binding_names", - "Input Binding Names", - "Names of input bindings as in the model in the same order of " - "what is provided in input_tensor_names."); - - spec.param(input_state_tensor_names_, - "input_state_tensor_names", - "Input State Tensor Names", - "Names of input state tensors that are used internally by TensorRT.", - std::vector{}); - - spec.param(output_tensor_names_, - "output_tensor_names", - "Output Tensor Names", - "Names of output tensors in the order to be retrieved " - "from the model."); - spec.param(output_state_tensor_names_, - "output_state_tensor_names", - "Output State Tensor Names", - "Names of output state tensors that are used internally by TensorRT.", - std::vector{}); - - spec.param(output_binding_names_, - "output_binding_names", - "Output Binding Names", - "Names of output bindings in the model in the same " - "order of of what is provided in output_tensor_names."); - spec.param(pool_, "pool", "Pool", "Allocator instance for output tensors."); - spec.param(cuda_stream_pool_, - "cuda_stream_pool", - "Cuda Stream Pool", - "Instance of gxf::CudaStreamPool to allocate CUDA stream."); - - spec.param(max_workspace_size_, - "max_workspace_size", - "Max Workspace Size", - "Size of working space in bytes. Default to 64MB", - 67108864l); - spec.param(dla_core_, - "dla_core", - "DLA Core", - "DLA Core to use. Fallback to GPU is always enabled. " - "Default to use GPU only."); - spec.param(max_batch_size_, - "max_batch_size", - "Max Batch Size", - "Maximum possible batch size in case the first dimension is " - "dynamic and used as batch size.", - 1); - spec.param(enable_fp16_, - "enable_fp16_", - "Enable FP16 Mode", - "Enable inference with FP16 and FP32 fallback.", - false); - - spec.param(verbose_, - "verbose", - "Verbose", - "Enable verbose logging on console. Default to false.", - false); - spec.param(relaxed_dimension_check_, - "relaxed_dimension_check", - "Relaxed Dimension Check", - "Ignore dimensions of 1 for input tensor dimension check.", - true); - spec.param(clock_, "clock", "Clock", "Instance of clock for publish time."); - - spec.param(rx_, "rx", "RX", "List of receivers to take input tensors", {&in_tensor}); - spec.param(tx_, "tx", "TX", "Transmitter to publish output tensors", &out_tensor); - - // TODO (gbae): spec object holds an information about errors - // TODO (gbae): incorporate std::expected to not throw exceptions -} - -} // namespace holoscan::ops diff --git a/src/operators/emergent_source/emergent_source.cpp b/src/operators/emergent_source/emergent_source.cpp deleted file mode 100644 index f5cfdc84..00000000 --- a/src/operators/emergent_source/emergent_source.cpp +++ /dev/null @@ -1,44 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "holoscan/operators/emergent_source/emergent_source.hpp" - -#include "holoscan/core/gxf/entity.hpp" -#include "holoscan/core/operator_spec.hpp" - -namespace holoscan::ops { - -void EmergentSourceOp::setup(OperatorSpec& spec) { - auto& signal = spec.output("signal"); - - constexpr uint32_t kDefaultWidth = 4200; - constexpr uint32_t kDefaultHeight = 2160; - constexpr uint32_t kDefaultFramerate = 240; - constexpr bool kDefaultRDMA = false; - - spec.param(signal_, "signal", "Output", "Output channel", &signal); - spec.param(width_, "width", "Width", "Width of the stream.", kDefaultWidth); - spec.param(height_, "height", "Height", "Height of the stream.", kDefaultHeight); - spec.param(framerate_, "framerate", "Framerate", "Framerate of the stream.", kDefaultFramerate); - spec.param(use_rdma_, "rdma", "RDMA", "Enable RDMA.", kDefaultRDMA); -} - -void EmergentSourceOp::initialize() { - holoscan::ops::GXFOperator::initialize(); -} - -} // namespace holoscan::ops diff --git a/src/operators/format_converter/CMakeLists.txt b/src/operators/format_converter/CMakeLists.txt new file mode 100644 index 00000000..07c3e628 --- /dev/null +++ b/src/operators/format_converter/CMakeLists.txt @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_holoscan_operator(format_converter format_converter.cpp) + +target_link_libraries(op_format_converter + PUBLIC + holoscan::core + PRIVATE + CUDA::nppidei + CUDA::nppig + CUDA::nppicc +) diff --git a/src/operators/format_converter/format_converter.cpp b/src/operators/format_converter/format_converter.cpp index a31126c2..15d093ca 100644 --- a/src/operators/format_converter/format_converter.cpp +++ b/src/operators/format_converter/format_converter.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,14 +17,843 @@ #include "holoscan/operators/format_converter/format_converter.hpp" +#include +#include +#include + +#include "holoscan/core/execution_context.hpp" +#include "holoscan/core/executor.hpp" +#include "holoscan/core/fragment.hpp" #include "holoscan/core/gxf/entity.hpp" +#include "holoscan/core/gxf/gxf_tensor.hpp" +#include "holoscan/core/io_context.hpp" +#include "holoscan/core/io_spec.hpp" #include "holoscan/core/operator_spec.hpp" - #include "holoscan/core/resources/gxf/allocator.hpp" #include "holoscan/core/resources/gxf/cuda_stream_pool.hpp" +#define CUDA_TRY(stmt) \ + ({ \ + cudaError_t _holoscan_cuda_err = stmt; \ + if (cudaSuccess != _holoscan_cuda_err) { \ + GXF_LOG_ERROR("CUDA Runtime call %s in line %d of file %s failed with '%s' (%d).\n", \ + #stmt, \ + __LINE__, \ + __FILE__, \ + cudaGetErrorString(_holoscan_cuda_err), \ + _holoscan_cuda_err); \ + } \ + _holoscan_cuda_err; \ + }) + namespace holoscan::ops { +// Note that currently "uint8" for the `input_dtype` or `output_dtype` arguments to the operator +// is identical to using "rgb888" and the format "float32" implies a 32-bit floating point RGB +// image (3 channels).. With the exception of "yuv420" and "nv12", this operator currently operates +// only on three channel (RGB) or four channel (RGBA) inputs stored in a channel-packed format. +// In other words, for a given pixel, the RGB values are adjacent in memory rather than being +// stored as separate planes. There currently is no support for grayscale images. + +static FormatDType toFormatDType(const std::string& str) { + if (str == "rgb888") { + return FormatDType::kRGB888; + } else if (str == "uint8") { + return FormatDType::kUnsigned8; + } else if (str == "float32") { + return FormatDType::kFloat32; + } else if (str == "rgba8888") { + return FormatDType::kRGBA8888; + } else if (str == "yuv420") { + return FormatDType::kYUV420; + } else if (str == "nv12") { + return FormatDType::kNV12; + } else { + return FormatDType::kUnknown; + } +} + +static constexpr FormatConversionType getFormatConversionType(FormatDType from, FormatDType to) { + if (from != FormatDType::kUnknown && to != FormatDType::kUnknown && from == to) { + return FormatConversionType::kNone; + } else if (from == FormatDType::kUnsigned8 && to == FormatDType::kFloat32) { + return FormatConversionType::kUnsigned8ToFloat32; + } else if (from == FormatDType::kFloat32 && to == FormatDType::kUnsigned8) { + return FormatConversionType::kFloat32ToUnsigned8; + } else if (from == FormatDType::kUnsigned8 && to == FormatDType::kRGBA8888) { + return FormatConversionType::kRGB888ToRGBA8888; + } else if (from == FormatDType::kRGBA8888 && to == FormatDType::kUnsigned8) { + return FormatConversionType::kRGBA8888ToRGB888; + } else if (from == FormatDType::kRGBA8888 && to == FormatDType::kFloat32) { + return FormatConversionType::kRGBA8888ToFloat32; + } else if (from == FormatDType::kUnsigned8 && to == FormatDType::kYUV420) { + return FormatConversionType::kRGB888ToYUV420; + } else if (from == FormatDType::kYUV420 && to == FormatDType::kRGBA8888) { + return FormatConversionType::kYUV420ToRGBA8888; + } else if (from == FormatDType::kYUV420 && to == FormatDType::kUnsigned8) { + return FormatConversionType::kYUV420ToRGB888; + } else if (from == FormatDType::kNV12 && to == FormatDType::kUnsigned8) { + return FormatConversionType::kNV12ToRGB888; + } else { + return FormatConversionType::kUnknown; + } +} + +static constexpr FormatDType normalizeFormatDType(FormatDType dtype) { + switch (dtype) { + case FormatDType::kRGB888: + return FormatDType::kUnsigned8; + default: + return dtype; + } +} + +static constexpr nvidia::gxf::PrimitiveType primitiveTypeFromFormatDType(FormatDType dtype) { + switch (dtype) { + case FormatDType::kRGB888: + case FormatDType::kRGBA8888: + case FormatDType::kUnsigned8: + case FormatDType::kYUV420: + case FormatDType::kNV12: + return nvidia::gxf::PrimitiveType::kUnsigned8; + case FormatDType::kFloat32: + return nvidia::gxf::PrimitiveType::kFloat32; + default: + return nvidia::gxf::PrimitiveType::kCustom; + } +} + +static constexpr FormatDType FormatDTypeFromPrimitiveType(nvidia::gxf::PrimitiveType type) { + switch (type) { + case nvidia::gxf::PrimitiveType::kUnsigned8: + return FormatDType::kUnsigned8; + case nvidia::gxf::PrimitiveType::kFloat32: + return FormatDType::kFloat32; + default: + return FormatDType::kUnknown; + } +} + +static gxf_result_t verifyFormatDTypeChannels(FormatDType dtype, int channel_count) { + switch (dtype) { + case FormatDType::kRGB888: + if (channel_count != 3) { + HOLOSCAN_LOG_ERROR("Invalid channel count for RGB888 {} != 3\n", channel_count); + return GXF_FAILURE; + } + break; + case FormatDType::kRGBA8888: + if (channel_count != 4) { + HOLOSCAN_LOG_ERROR("Invalid channel count for RGBA8888 {} != 4\n", channel_count); + return GXF_FAILURE; + } + break; + default: + break; + } + return GXF_SUCCESS; +} + +void FormatConverterOp::initialize() { + auto nppStatus = nppGetStreamContext(&npp_stream_ctx_); + if (NPP_SUCCESS != nppStatus) { + throw std::runtime_error("Failed to get NPP CUDA stream context"); + } + Operator::initialize(); +} + +void FormatConverterOp::start() { + out_dtype_ = toFormatDType(out_dtype_str_.get()); + if (out_dtype_ == FormatDType::kUnknown) { + throw std::runtime_error( + fmt::format("Unsupported output format dtype: {}\n", out_dtype_str_.get())); + } + out_primitive_type_ = primitiveTypeFromFormatDType(out_dtype_); + if (out_primitive_type_ == nvidia::gxf::PrimitiveType::kCustom) { + throw std::runtime_error( + fmt::format("Unsupported output format dtype: {}\n", out_dtype_str_.get())); + } + + if (!in_dtype_str_.get().empty()) { + in_dtype_ = toFormatDType(in_dtype_str_.get()); + if (in_dtype_ == FormatDType::kUnknown) { + throw std::runtime_error( + fmt::format("Unsupported input format dtype: {}\n", in_dtype_str_.get())); + } + format_conversion_type_ = + getFormatConversionType(normalizeFormatDType(in_dtype_), normalizeFormatDType(out_dtype_)); + in_primitive_type_ = primitiveTypeFromFormatDType(in_dtype_); + } + + switch (resize_mode_) { + case 0: + // resize_mode_.set(NPPI_INTER_CUBIC); + resize_mode_ = NPPI_INTER_CUBIC; + break; + case 1: // NPPI_INTER_NN + case 2: // NPPI_INTER_LINEAR + case 4: // NPPI_INTER_CUBIC + case 5: // NPPI_INTER_CUBIC2P_BSPLINE + case 6: // NPPI_INTER_CUBIC2P_CATMULLROM + case 7: // NPPI_INTER_CUBIC2P_B05C03 + case 8: // NPPI_INTER_SUPER + case 16: // NPPI_INTER_LANCZOS + case 17: // NPPI_INTER_LANCZOS3_ADVANCED + case static_cast(0x8000000): // NPPI_SMOOTH_EDGE + break; + default: + throw std::runtime_error(fmt::format("Unsupported resize mode: {}\n", resize_mode_.get())); + } +} + +void FormatConverterOp::stop() { + resize_buffer_.freeBuffer(); + channel_buffer_.freeBuffer(); + device_scratch_buffer_.freeBuffer(); +} + +void FormatConverterOp::compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) { + // Process input message + auto in_message = op_input.receive("source_video"); + + // get the CUDA stream from the input message + gxf_result_t stream_handler_result = + cuda_stream_handler_.fromMessage(context.context(), in_message); + if (stream_handler_result != GXF_SUCCESS) { + throw std::runtime_error("Failed to get the CUDA stream from incoming messages"); + } + + // assign the CUDA stream to the NPP stream context + npp_stream_ctx_.hStream = cuda_stream_handler_.getCudaStream(context.context()); + + nvidia::gxf::Shape out_shape{0, 0, 0}; + void* in_tensor_data = nullptr; + nvidia::gxf::PrimitiveType in_primitive_type = nvidia::gxf::PrimitiveType::kCustom; + nvidia::gxf::MemoryStorageType in_memory_storage_type = nvidia::gxf::MemoryStorageType::kHost; + int32_t rows = 0; + int32_t columns = 0; + int16_t in_channels = 0; + int16_t out_channels = 0; + + // get Handle to underlying nvidia::gxf::Allocator from std::shared_ptr + auto pool = nvidia::gxf::Handle::Create(context.context(), + pool_.get()->gxf_cid()); + + // Get either the Tensor or VideoBuffer attached to the message + bool is_video_buffer; + nvidia::gxf::Handle video_buffer; + try { + video_buffer = holoscan::gxf::get_videobuffer(in_message); + is_video_buffer = true; + } catch (const std::runtime_error& r_) { + HOLOSCAN_LOG_TRACE("Failed to read VideoBuffer with error: {}", std::string(r_.what())); + is_video_buffer = false; + } + + if (is_video_buffer) { + // Convert VideoBuffer to Tensor + auto frame = video_buffer.get(); + + // NOTE: VideoBuffer::moveToTensor() converts VideoBuffer instance to the Tensor instance + // with an unexpected shape: [width, height] or [width, height, num_planes]. + // And, if we use moveToTensor() to convert VideoBuffer to Tensor, we may lose the the original + // video buffer when the VideoBuffer instance is used in other places. For that reason, we + // directly access internal data of VideoBuffer instance to access Tensor data. + const auto& buffer_info = frame->video_frame_info(); + switch (buffer_info.color_format) { + case nvidia::gxf::VideoFormat::GXF_VIDEO_FORMAT_RGBA: + in_primitive_type = nvidia::gxf::PrimitiveType::kUnsigned8; + in_channels = 4; // RGBA + out_channels = in_channels; + out_shape = nvidia::gxf::Shape{ + static_cast(buffer_info.height), static_cast(buffer_info.width), 4}; + break; + case nvidia::gxf::VideoFormat::GXF_VIDEO_FORMAT_NV12: + in_primitive_type = nvidia::gxf::PrimitiveType::kUnsigned8; + in_channels = buffer_info.color_planes.size(); + out_channels = 3; + switch (out_dtype_) { + case FormatDType::kRGB888: + out_channels = 3; + out_shape = nvidia::gxf::Shape{static_cast(buffer_info.height), + static_cast(buffer_info.width), + 3}; + break; + default: + throw std::runtime_error(fmt::format("Unsupported format conversion: {} -> {}\n", + in_dtype_str_.get(), out_dtype_str_.get())); + break; + } + break; + default: + throw std::runtime_error(fmt::format("Unsupported input format: {}\n", + static_cast(buffer_info.color_format))); + } + + // Get needed information from the tensor + in_memory_storage_type = frame->storage_type(); + out_shape = nvidia::gxf::Shape{ + static_cast(buffer_info.height), static_cast(buffer_info.width), 4}; + in_tensor_data = frame->pointer(); + rows = buffer_info.height; + columns = buffer_info.width; + + // If the buffer is in host memory, copy it to a device (GPU) buffer + // as needed for the NPP resize/convert operations. + if (in_memory_storage_type == nvidia::gxf::MemoryStorageType::kHost) { + size_t buffer_size = rows * columns * in_channels; + if (buffer_size > device_scratch_buffer_.size()) { + device_scratch_buffer_.resize( + pool.value(), buffer_size, nvidia::gxf::MemoryStorageType::kDevice); + if (!device_scratch_buffer_.pointer()) { + throw std::runtime_error( + fmt::format("Failed to allocate device scratch buffer ({} bytes)", buffer_size)); + } + } + CUDA_TRY(cudaMemcpy( + device_scratch_buffer_.pointer(), frame->pointer(), buffer_size, cudaMemcpyHostToDevice)); + in_tensor_data = device_scratch_buffer_.pointer(); + in_memory_storage_type = nvidia::gxf::MemoryStorageType::kDevice; + } + } else { + const auto maybe_tensor = in_message.get(in_tensor_name_.get().c_str()); + if (!maybe_tensor) { + throw std::runtime_error( + fmt::format("Tensor '{}' not found in message.\n", in_tensor_name_.get())); + } + // Tensor in_tensor; + auto in_tensor = maybe_tensor; + + // Get needed information from the tensor + // cast Holoscan::Tensor to GXFTensor so attribute access code can remain as-is + holoscan::gxf::GXFTensor in_tensor_gxf{in_tensor->dl_ctx()}; + out_shape = in_tensor_gxf.shape(); + in_tensor_data = in_tensor_gxf.pointer(); + in_primitive_type = in_tensor_gxf.element_type(); + in_memory_storage_type = in_tensor_gxf.storage_type(); + rows = in_tensor_gxf.shape().dimension(0); + columns = in_tensor_gxf.shape().dimension(1); + in_channels = in_tensor_gxf.shape().dimension(2); + out_channels = in_channels; + } + + if (in_memory_storage_type != nvidia::gxf::MemoryStorageType::kDevice) { + throw std::runtime_error(fmt::format( + "Tensor('{}') or VideoBuffer is not allocated on device.\n", in_tensor_name_.get())); + } + + if (in_dtype_ == FormatDType::kUnknown) { + in_primitive_type_ = in_primitive_type; + in_dtype_ = FormatDTypeFromPrimitiveType(in_primitive_type_); + format_conversion_type_ = + getFormatConversionType(normalizeFormatDType(in_dtype_), normalizeFormatDType(out_dtype_)); + } + + // Check if input tensor is consistent over all the frames + if (in_primitive_type != in_primitive_type_) { + throw std::runtime_error("Input tensor element type is inconsistent over all the frames.\n"); + } + + // Check if the input/output tensor is compatible with the format conversion + if (format_conversion_type_ == FormatConversionType::kUnknown) { + throw std::runtime_error(fmt::format("Unsupported format conversion: {} ({}) -> {}\n", + in_dtype_str_.get(), + static_cast(in_dtype_), + out_dtype_str_.get())); + } + + // Check that if the format requires a specific number of channels they are consistent. + // Some formats (e.g. float32) are agnostic to channel count, while others (e.g. RGB888) have a + // specific channel count. + if (GXF_SUCCESS != verifyFormatDTypeChannels(in_dtype_, in_channels)) { + throw std::runtime_error( + fmt::format("Failed to verify the channels for the expected input dtype [{}]: {}.", + static_cast(in_dtype_), + in_channels)); + } + + // Resize the input image before converting data type + if (resize_width_ > 0 && resize_height_ > 0) { + auto resize_result = resizeImage(in_tensor_data, + rows, + columns, + in_channels, + in_primitive_type, + resize_width_, + resize_height_); + if (!resize_result) { throw std::runtime_error("Failed to resize image.\n"); } + + // Update the tensor pointer and shape + out_shape = nvidia::gxf::Shape{resize_height_, resize_width_, in_channels}; + in_tensor_data = resize_result.value(); + rows = resize_height_; + columns = resize_width_; + } + + // Create output message + const uint32_t dst_typesize = nvidia::gxf::PrimitiveTypeSize(out_primitive_type_); + + // Adjust output shape if the conversion involves the change in the channel dimension + switch (format_conversion_type_) { + case FormatConversionType::kRGB888ToRGBA8888: { + out_channels = 4; + out_shape = nvidia::gxf::Shape{out_shape.dimension(0), out_shape.dimension(1), out_channels}; + break; + } + case FormatConversionType::kRGBA8888ToRGB888: + case FormatConversionType::kNV12ToRGB888: + case FormatConversionType::kYUV420ToRGB888: + case FormatConversionType::kRGBA8888ToFloat32: { + out_channels = 3; + out_shape = nvidia::gxf::Shape{out_shape.dimension(0), out_shape.dimension(1), out_channels}; + break; + } + default: + break; + } + + // Check that if the format requires a specific number of channels they are consistent. + // Some formats (e.g. float32) are agnostic to channel count, while others (e.g. RGB888) have a + // specific channel count. + if (GXF_SUCCESS != verifyFormatDTypeChannels(out_dtype_, out_channels)) { + throw std::runtime_error( + fmt::format("Failed to verify the channels for the expected output dtype [{}]: {}.", + static_cast(out_dtype_), + out_channels)); + } + + nvidia::gxf::Expected out_message = + CreateTensorMap(context.context(), + pool.value(), + {{out_tensor_name_.get(), + nvidia::gxf::MemoryStorageType::kDevice, + out_shape, + out_primitive_type_, + 0, + nvidia::gxf::ComputeTrivialStrides(out_shape, dst_typesize)}}, + false); + + if (!out_message) { std::runtime_error("failed to create out_message"); } + const auto out_tensor = out_message.value().get(); + if (!out_tensor) { std::runtime_error("failed to create out_tensor"); } + + // Set tensor to constant using NPP + if (in_channels == 2 || in_channels == 3 || in_channels == 4) { + // gxf_result_t convert_result = convertTensorFormat( + convertTensorFormat( + in_tensor_data, out_tensor.value()->pointer(), rows, columns, in_channels, out_channels); + + } else { + throw std::runtime_error("Only support 3 or 4 channel input tensor"); + } + + // pass the CUDA stream to the output message + stream_handler_result = cuda_stream_handler_.toMessage(out_message); + if (stream_handler_result != GXF_SUCCESS) { + throw std::runtime_error("Failed to add the CUDA stream to the outgoing messages"); + } + + // Emit the tensor + auto result = gxf::Entity(std::move(out_message.value())); + op_output.emit(result); +} + +nvidia::gxf::Expected FormatConverterOp::resizeImage( + const void* in_tensor_data, const int32_t rows, const int32_t columns, const int16_t channels, + const nvidia::gxf::PrimitiveType primitive_type, const int32_t resize_width, + const int32_t resize_height) { + if (resize_buffer_.size() == 0) { + auto frag = fragment(); + + // get Handle to underlying nvidia::gxf::Allocator from std::shared_ptr + auto pool = nvidia::gxf::Handle::Create(frag->executor().context(), + pool_.get()->gxf_cid()); + + uint64_t buffer_size = resize_width * resize_height * channels; + resize_buffer_.resize(pool.value(), buffer_size, nvidia::gxf::MemoryStorageType::kDevice); + } + + const auto converted_tensor_ptr = resize_buffer_.pointer(); + if (converted_tensor_ptr == nullptr) { + GXF_LOG_ERROR("Failed to allocate memory for the resizing image"); + return nvidia::gxf::ExpectedOrCode(GXF_FAILURE, nullptr); + } + + // Resize image + NppStatus status = NPP_ERROR; + const NppiSize src_size = {static_cast(columns), static_cast(rows)}; + const NppiRect src_roi = {0, 0, static_cast(columns), static_cast(rows)}; + const NppiSize dst_size = {static_cast(resize_width), static_cast(resize_height)}; + const NppiRect dst_roi = {0, 0, static_cast(resize_width), static_cast(resize_height)}; + + switch (channels) { + case 3: + switch (primitive_type) { + case nvidia::gxf::PrimitiveType::kUnsigned8: + status = nppiResize_8u_C3R_Ctx(static_cast(in_tensor_data), + columns * channels, + src_size, + src_roi, + converted_tensor_ptr, + resize_width * channels, + dst_size, + dst_roi, + resize_mode_, + npp_stream_ctx_); + break; + default: + GXF_LOG_ERROR("Unsupported input primitive type for resizing image"); + return nvidia::gxf::ExpectedOrCode(GXF_FAILURE, nullptr); + } + break; + case 4: + switch (primitive_type) { + case nvidia::gxf::PrimitiveType::kUnsigned8: + status = nppiResize_8u_C4R_Ctx(static_cast(in_tensor_data), + columns * channels, + src_size, + src_roi, + converted_tensor_ptr, + resize_width * channels, + dst_size, + dst_roi, + resize_mode_, + npp_stream_ctx_); + break; + default: + GXF_LOG_ERROR("Unsupported input primitive type for resizing image"); + return nvidia::gxf::ExpectedOrCode(GXF_FAILURE, nullptr); + } + break; + default: + GXF_LOG_ERROR("Unsupported input primitive type for resizing image (%d, %d)", + channels, + static_cast(primitive_type)); + return nvidia::gxf::ExpectedOrCode(GXF_FAILURE, nullptr); + break; + } + + if (status != NPP_SUCCESS) { return nvidia::gxf::ExpectedOrCode(GXF_FAILURE, nullptr); } + + return nvidia::gxf::ExpectedOrCode(GXF_SUCCESS, converted_tensor_ptr); +} + +// gxf_result_t FormatConverterOp::convertTensorFormat(const void* in_tensor_data, void* +// out_tensor_data, +void FormatConverterOp::convertTensorFormat(const void* in_tensor_data, void* out_tensor_data, + const int32_t rows, const int32_t columns, + const int16_t in_channels, const int16_t out_channels) { + const uint32_t src_typesize = nvidia::gxf::PrimitiveTypeSize(in_primitive_type_); + const uint32_t dst_typesize = nvidia::gxf::PrimitiveTypeSize(out_primitive_type_); + + const int32_t src_step = columns * in_channels * src_typesize; + const int32_t dst_step = columns * out_channels * dst_typesize; + + const auto& out_channel_order = out_channel_order_.get(); + + NppStatus status = NPP_ERROR; + const NppiSize roi = {static_cast(columns), static_cast(rows)}; + + switch (format_conversion_type_) { + case FormatConversionType::kNone: { + const auto in_tensor_ptr = static_cast(in_tensor_data); + auto out_tensor_ptr = static_cast(out_tensor_data); + + cudaError_t cuda_status = CUDA_TRY(cudaMemcpyAsync(out_tensor_ptr, + in_tensor_ptr, + src_step * rows, + cudaMemcpyDeviceToDevice, + npp_stream_ctx_.hStream)); + if (cuda_status) { throw std::runtime_error("Failed to copy GPU data to GPU memory."); } + status = NPP_SUCCESS; + break; + } + case FormatConversionType::kUnsigned8ToFloat32: { + const auto in_tensor_ptr = static_cast(in_tensor_data); + const auto out_tensor_ptr = static_cast(out_tensor_data); + status = nppiScale_8u32f_C3R_Ctx(in_tensor_ptr, + src_step, + out_tensor_ptr, + dst_step, + roi, + scale_min_, + scale_max_, + npp_stream_ctx_); + break; + } + case FormatConversionType::kFloat32ToUnsigned8: { + const auto in_tensor_ptr = static_cast(in_tensor_data); + const auto out_tensor_ptr = static_cast(out_tensor_data); + status = nppiScale_32f8u_C3R_Ctx(in_tensor_ptr, + src_step, + out_tensor_ptr, + dst_step, + roi, + scale_min_, + scale_max_, + npp_stream_ctx_); + break; + } + case FormatConversionType::kRGB888ToRGBA8888: { + const auto in_tensor_ptr = static_cast(in_tensor_data); + const auto out_tensor_ptr = static_cast(out_tensor_data); + // Convert RGB888 to RGBA8888 (3 channels -> 4 channels, uint8_t) + int dst_order[4]{0, 1, 2, 3}; + if (!out_channel_order.empty()) { + if (out_channel_order.size() != 4) { + throw std::runtime_error("Invalid channel order for RGBA8888."); + } + for (int i = 0; i < 4; i++) { dst_order[i] = out_channel_order[i]; } + } + status = nppiSwapChannels_8u_C3C4R_Ctx(in_tensor_ptr, + src_step, + out_tensor_ptr, + out_channels * dst_typesize * columns, + roi, + dst_order, + alpha_value_.get(), + npp_stream_ctx_); + break; + } + case FormatConversionType::kRGBA8888ToRGB888: { + const auto in_tensor_ptr = static_cast(in_tensor_data); + const auto out_tensor_ptr = static_cast(out_tensor_data); + // Convert RGBA8888 to RGB888 (4 channels -> 3 channels, uint8_t) + int dst_order[3]{0, 1, 2}; + if (!out_channel_order.empty()) { + if (out_channel_order.size() != 3) { + throw std::runtime_error("Invalid channel order for RGB888."); + } + for (int i = 0; i < 3; i++) { dst_order[i] = out_channel_order[i]; } + } + status = nppiSwapChannels_8u_C4C3R_Ctx(in_tensor_ptr, + src_step, + out_tensor_ptr, + out_channels * dst_typesize * columns, + roi, + dst_order, + npp_stream_ctx_); + break; + } + case FormatConversionType::kRGBA8888ToFloat32: { + const auto in_tensor_ptr = static_cast(in_tensor_data); + const auto out_tensor_ptr = static_cast(out_tensor_data); + + if (channel_buffer_.size() == 0) { + auto frag = fragment(); + + // get Handle to underlying nvidia::gxf::Allocator from std::shared_ptr + auto pool = nvidia::gxf::Handle::Create(frag->executor().context(), + pool_.get()->gxf_cid()); + + uint64_t buffer_size = rows * columns * 3; // 4 channels -> 3 channels + channel_buffer_.resize(pool.value(), buffer_size, nvidia::gxf::MemoryStorageType::kDevice); + } + + const auto converted_tensor_ptr = channel_buffer_.pointer(); + if (converted_tensor_ptr == nullptr) { + throw std::runtime_error("Failed to allocate memory for the channel conversion"); + } + + int dst_order[3]{0, 1, 2}; + if (!out_channel_order.empty()) { + if (out_channel_order.size() != 3) { + throw std::runtime_error("Invalid channel order for RGB888"); + } + for (int i = 0; i < 3; i++) { dst_order[i] = out_channel_order[i]; } + } + + status = nppiSwapChannels_8u_C4C3R_Ctx(in_tensor_ptr, + src_step, + converted_tensor_ptr, + out_channels * src_typesize * columns, + roi, + dst_order, + npp_stream_ctx_); + + if (status == NPP_SUCCESS) { + const int32_t new_src_step = columns * out_channels * src_typesize; + status = nppiScale_8u32f_C3R_Ctx(converted_tensor_ptr, + new_src_step, + out_tensor_ptr, + dst_step, + roi, + scale_min_, + scale_max_, + npp_stream_ctx_); + } else { + throw std::runtime_error( + fmt::format("Failed to convert channel order (NPP error code: {})", status)); + } + break; + } + case FormatConversionType::kRGB888ToYUV420: { + nvidia::gxf::VideoFormatSize color_format; + auto color_planes = color_format.getDefaultColorPlanes(columns, rows); + + const auto in_tensor_ptr = static_cast(in_tensor_data); + + const auto out_y_ptr = static_cast(out_tensor_data); + const auto out_u_ptr = out_y_ptr + color_planes[0].size; + const auto out_v_ptr = out_u_ptr + color_planes[1].size; + uint8_t* out_yuv_ptrs[3] = {out_y_ptr, out_u_ptr, out_v_ptr}; + + const int32_t out_y_step = color_planes[0].stride; + const int32_t out_u_step = color_planes[1].stride; + const int32_t out_v_step = color_planes[2].stride; + int32_t out_yuv_steps[3] = { out_y_step, out_u_step, out_v_step }; + + status = nppiRGBToYUV420_8u_C3P3R(in_tensor_ptr, src_step, out_yuv_ptrs, + out_yuv_steps, roi); + if (status != NPP_SUCCESS) { + throw std::runtime_error( + fmt::format("rgb888 to yuv420 conversion failed (NPP error code: {})", status)); + } + break; + } + case FormatConversionType::kYUV420ToRGBA8888: { + nvidia::gxf::VideoFormatSize color_format; + auto color_planes = color_format.getDefaultColorPlanes(columns, rows); + const auto in_y_ptr = static_cast(in_tensor_data); + const auto in_u_ptr = in_y_ptr + color_planes[0].size; + const auto in_v_ptr = in_u_ptr + color_planes[1].size; + const uint8_t* in_yuv_ptrs[3] = {in_y_ptr, in_u_ptr, in_v_ptr}; + + const int32_t in_y_step = color_planes[0].stride; + const int32_t in_u_step = color_planes[1].stride; + const int32_t in_v_step = color_planes[2].stride; + int32_t in_yuv_steps[3] = { in_y_step, in_u_step, in_v_step }; + + const auto out_tensor_ptr = static_cast(out_tensor_data); + + status = nppiYUV420ToRGB_8u_P3AC4R(in_yuv_ptrs, in_yuv_steps, out_tensor_ptr, + dst_step, roi); + if (status != NPP_SUCCESS) { + throw std::runtime_error( + fmt::format("yuv420 to rgba8888 conversion failed (NPP error code: {})", status)); + } + break; + } + case FormatConversionType::kYUV420ToRGB888: { + nvidia::gxf::VideoFormatSize color_format; + auto color_planes = color_format.getDefaultColorPlanes(columns, rows); + const auto in_y_ptr = static_cast(in_tensor_data); + const auto in_u_ptr = in_y_ptr + color_planes[0].size; + const auto in_v_ptr = in_u_ptr + color_planes[1].size; + const uint8_t* in_yuv_ptrs[3] = {in_y_ptr, in_u_ptr, in_v_ptr}; + + const int32_t in_y_step = color_planes[0].stride; + const int32_t in_u_step = color_planes[1].stride; + const int32_t in_v_step = color_planes[2].stride; + int32_t in_yuv_steps[3] = { in_y_step, in_u_step, in_v_step }; + + const auto out_tensor_ptr = static_cast(out_tensor_data); + + status = nppiYUV420ToRGB_8u_P3C3R(in_yuv_ptrs, in_yuv_steps, out_tensor_ptr, + dst_step, roi); + if (status != NPP_SUCCESS) { + throw std::runtime_error( + fmt::format("yuv420 to rgb888 conversion failed (NPP error code: {})", status)); + } + break; + } + case FormatConversionType::kNV12ToRGB888: { + nvidia::gxf::VideoFormatSize color_format; + const auto in_y_ptr = static_cast(in_tensor_data); + auto color_planes = color_format.getDefaultColorPlanes(columns, rows); + const auto in_uv_ptr = in_y_ptr + color_planes[0].size; + const uint8_t* in_y_uv_ptrs[2] = {in_y_ptr, in_uv_ptr}; + + const int32_t in_y_uv_step = color_planes[0].stride; + + const auto out_tensor_ptr = static_cast(out_tensor_data); + + status = nppiNV12ToRGB_709HDTV_8u_P2C3R(in_y_uv_ptrs, in_y_uv_step, out_tensor_ptr, + dst_step, roi); + if (status != NPP_SUCCESS) { + throw std::runtime_error( + fmt::format("NV12 to rgb888 conversion failed (NPP error code: {})", status)); + } + break; + } + default: + throw std::runtime_error(fmt::format("Unsupported format conversion: {} ({}) -> {}\n", + in_dtype_str_.get(), + static_cast(in_dtype_), + out_dtype_str_.get())); + } + + // Reorder channels in the output tensor (inplace) if needed. + switch (format_conversion_type_) { + case FormatConversionType::kNone: + case FormatConversionType::kUnsigned8ToFloat32: + case FormatConversionType::kFloat32ToUnsigned8: { + if (!out_channel_order.empty()) { + switch (out_channels) { + case 3: { + int dst_order[3]{0, 1, 2}; + if (out_channel_order.size() != 3) { + throw std::runtime_error( + fmt::format("Invalid channel order for {}", out_dtype_str_.get())); + } + for (int i = 0; i < 3; i++) { dst_order[i] = out_channel_order[i]; } + switch (out_primitive_type_) { + case nvidia::gxf::PrimitiveType::kUnsigned8: { + auto out_tensor_ptr = static_cast(out_tensor_data); + status = nppiSwapChannels_8u_C3IR_Ctx( + out_tensor_ptr, dst_step, roi, dst_order, npp_stream_ctx_); + break; + } + case nvidia::gxf::PrimitiveType::kFloat32: { + auto out_tensor_ptr = static_cast(out_tensor_data); + status = nppiSwapChannels_32f_C3IR_Ctx( + out_tensor_ptr, dst_step, roi, dst_order, npp_stream_ctx_); + break; + } + default: + throw std::runtime_error( + fmt::format("Unsupported output data type for reordering channels: {}", + out_dtype_str_.get())); + } + break; + } + case 4: { + int dst_order[4]{0, 1, 2, 3}; + if (out_channel_order.size() != 4) { + throw std::runtime_error( + fmt::format("Invalid channel order for {}", out_dtype_str_.get())); + } + for (int i = 0; i < 4; i++) { dst_order[i] = out_channel_order[i]; } + switch (out_primitive_type_) { + case nvidia::gxf::PrimitiveType::kUnsigned8: { + auto out_tensor_ptr = static_cast(out_tensor_data); + status = nppiSwapChannels_8u_C4IR_Ctx( + out_tensor_ptr, dst_step, roi, dst_order, npp_stream_ctx_); + break; + } + case nvidia::gxf::PrimitiveType::kFloat32: { + auto out_tensor_ptr = static_cast(out_tensor_data); + status = nppiSwapChannels_32f_C4IR_Ctx( + out_tensor_ptr, dst_step, roi, dst_order, npp_stream_ctx_); + break; + } + default: + throw std::runtime_error( + fmt::format("Unsupported output data type for reordering channels: {}", + out_dtype_str_.get())); + } + break; + } + } + if (status != NPP_SUCCESS) { throw std::runtime_error("Failed to convert channel order"); } + } + } + default: + break; + } +} + void FormatConverterOp::setup(OperatorSpec& spec) { auto& in_tensor = spec.input("source_video"); auto& out_tensor = spec.output("tensor"); @@ -75,6 +904,8 @@ void FormatConverterOp::setup(OperatorSpec& spec) { spec.param(pool_, "pool", "Pool", "Pool to allocate the output message."); + cuda_stream_handler_.defineParams(spec); + // TODO (gbae): spec object holds an information about errors // TODO (gbae): incorporate std::expected to not throw exceptions } diff --git a/setup.cfg b/src/operators/holoviz/CMakeLists.txt similarity index 70% rename from setup.cfg rename to src/operators/holoviz/CMakeLists.txt index 32b264ec..0815c970 100644 --- a/setup.cfg +++ b/src/operators/holoviz/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,9 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +add_holoscan_operator(holoviz holoviz.cpp) -[flake8] -max-line-length=100 -exclude = _deps,build*,.cache,html,_build,_static,generated,latex,install*,.git,xml -per-file-ignores = - scripts/convert_video_to_gxf_entities.py:E203 +target_link_libraries(op_holoviz + PUBLIC + holoscan::core + holoscan::viz + PRIVATE + CUDA::cudart + GXF::multimedia +) diff --git a/src/operators/holoviz/holoviz.cpp b/src/operators/holoviz/holoviz.cpp index f06294d7..29286b89 100644 --- a/src/operators/holoviz/holoviz.cpp +++ b/src/operators/holoviz/holoviz.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,17 +17,177 @@ #include "holoscan/operators/holoviz/holoviz.hpp" +#include + +#include +#include + +#include "holoscan/core/conditions/gxf/boolean.hpp" +#include "holoscan/core/execution_context.hpp" +#include "holoscan/core/executor.hpp" #include "holoscan/core/fragment.hpp" #include "holoscan/core/gxf/entity.hpp" +#include "holoscan/core/io_context.hpp" #include "holoscan/core/operator_spec.hpp" - -#include "holoscan/core/conditions/gxf/boolean.hpp" #include "holoscan/core/resources/gxf/allocator.hpp" +#include "holoscan/core/resources/gxf/cuda_stream_pool.hpp" + +#include "gxf/multimedia/video.hpp" +#include "gxf/std/scheduling_terms.hpp" +#include "gxf/std/tensor.hpp" +#include "holoviz/holoviz.hpp" // holoviz module + +#define CUDA_TRY(stmt) \ + ({ \ + cudaError_t _holoscan_cuda_err = stmt; \ + if (cudaSuccess != _holoscan_cuda_err) { \ + GXF_LOG_ERROR("CUDA Runtime call %s in line %d of file %s failed with '%s' (%d).\n", \ + #stmt, \ + __LINE__, \ + __FILE__, \ + cudaGetErrorString(_holoscan_cuda_err), \ + _holoscan_cuda_err); \ + } \ + _holoscan_cuda_err; \ + }) + +namespace viz = holoscan::viz; namespace { +/// Buffer information, can be initialized either with a tensor or a video buffer +struct BufferInfo { + /** + * Initialize with tensor + * + * @returns error code + */ + gxf_result_t init(const nvidia::gxf::Handle& tensor) { + rank = tensor->rank(); + shape = tensor->shape(); + element_type = tensor->element_type(); + name = tensor.name(); + buffer_ptr = tensor->pointer(); + storage_type = tensor->storage_type(); + bytes_size = tensor->bytes_size(); + for (uint32_t i = 0; i < rank; ++i) { stride[i] = tensor->stride(i); } + + return GXF_SUCCESS; + } + + /** + * Initialize with video buffer + * + * @returns error code + */ + gxf_result_t init(const nvidia::gxf::Handle& video) { + // NOTE: VideoBuffer::moveToTensor() converts VideoBuffer instance to the Tensor instance + // with an unexpected shape: [width, height] or [width, height, num_planes]. + // And, if we use moveToTensor() to convert VideoBuffer to Tensor, we may lose the original + // video buffer when the VideoBuffer instance is used in other places. For that reason, we + // directly access internal data of VideoBuffer instance to access Tensor data. + const auto& buffer_info = video->video_frame_info(); + + int32_t channels; + switch (buffer_info.color_format) { + case nvidia::gxf::VideoFormat::GXF_VIDEO_FORMAT_GRAY: + element_type = nvidia::gxf::PrimitiveType::kUnsigned8; + channels = 1; + break; + case nvidia::gxf::VideoFormat::GXF_VIDEO_FORMAT_GRAY16: + element_type = nvidia::gxf::PrimitiveType::kUnsigned16; + channels = 1; + break; + case nvidia::gxf::VideoFormat::GXF_VIDEO_FORMAT_GRAY32: + element_type = nvidia::gxf::PrimitiveType::kUnsigned32; + channels = 1; + break; + case nvidia::gxf::VideoFormat::GXF_VIDEO_FORMAT_RGB: + element_type = nvidia::gxf::PrimitiveType::kUnsigned8; + channels = 3; + break; + case nvidia::gxf::VideoFormat::GXF_VIDEO_FORMAT_RGBA: + element_type = nvidia::gxf::PrimitiveType::kUnsigned8; + channels = 4; + break; + default: + GXF_LOG_ERROR("Unsupported input format: %" PRId64 "\n", + static_cast(buffer_info.color_format)); + return GXF_FAILURE; + } + + rank = 3; + shape = nvidia::gxf::Shape{static_cast(buffer_info.height), + static_cast(buffer_info.width), + channels}; + name = video.name(); + buffer_ptr = video->pointer(); + storage_type = video->storage_type(); + bytes_size = video->size(); + stride[0] = buffer_info.color_planes[0].stride; + stride[1] = channels; + stride[2] = PrimitiveTypeSize(element_type); + + return GXF_SUCCESS; + } + + uint32_t rank; + nvidia::gxf::Shape shape; + nvidia::gxf::PrimitiveType element_type; + std::string name; + const nvidia::byte* buffer_ptr; + nvidia::gxf::MemoryStorageType storage_type; + uint64_t bytes_size; + nvidia::gxf::Tensor::stride_array_t stride; +}; + +/** + * Get the Holoviz image format for a given buffer. + * + * @param buffer_info buffer info + * @return Holoviz image format + */ +nvidia::gxf::Expected getImageFormatFromTensor(const BufferInfo& buffer_info) { + if (buffer_info.rank != 3) { + GXF_LOG_ERROR("Invalid tensor rank count, expected 3, got %u", buffer_info.rank); + return nvidia::gxf::Unexpected{GXF_INVALID_DATA_FORMAT}; + } + + struct Format { + nvidia::gxf::PrimitiveType type_; + int32_t channels_; + viz::ImageFormat format_; + }; + constexpr Format kGFXToHolovizFormats[] = { + {nvidia::gxf::PrimitiveType::kUnsigned8, 1, viz::ImageFormat::R8_UINT}, + {nvidia::gxf::PrimitiveType::kUnsigned16, 1, viz::ImageFormat::R16_UINT}, + {nvidia::gxf::PrimitiveType::kUnsigned32, 1, viz::ImageFormat::R32_UINT}, + {nvidia::gxf::PrimitiveType::kFloat32, 1, viz::ImageFormat::R32_SFLOAT}, + {nvidia::gxf::PrimitiveType::kUnsigned8, 3, viz::ImageFormat::R8G8B8_UNORM}, + {nvidia::gxf::PrimitiveType::kUnsigned8, 4, viz::ImageFormat::R8G8B8A8_UNORM}, + {nvidia::gxf::PrimitiveType::kUnsigned16, 4, viz::ImageFormat::R16G16B16A16_UNORM}, + {nvidia::gxf::PrimitiveType::kFloat32, 4, viz::ImageFormat::R32G32B32A32_SFLOAT}}; + + viz::ImageFormat image_format = static_cast(-1); + for (auto&& format : kGFXToHolovizFormats) { + if ((format.type_ == buffer_info.element_type) && + (format.channels_ == buffer_info.shape.dimension(2))) { + image_format = format.format_; + break; + } + } + if (image_format == static_cast(-1)) { + GXF_LOG_ERROR("Element type %d and channel count %d not supported", + static_cast(buffer_info.element_type), + buffer_info.shape.dimension(3)); + return nvidia::gxf::Unexpected{GXF_INVALID_DATA_FORMAT}; + } + + return image_format; +} + /// table to convert input type to string -static const std::array, 11> +static const std::array, 13> kInputTypeToStr{{{holoscan::ops::HolovizOp::InputType::UNKNOWN, "unknown"}, {holoscan::ops::HolovizOp::InputType::COLOR, "color"}, {holoscan::ops::HolovizOp::InputType::COLOR_LUT, "color_lut"}, @@ -38,7 +198,9 @@ static const std::array inputTypeFromS [&string](const auto& v) { return v.second == string; }); if (it != std::cend(kInputTypeToStr)) { return it->first; } - GXF_LOG_ERROR("Unsupported tensor type '%s'", string.c_str()); + HOLOSCAN_LOG_ERROR("Unsupported input type '{}'", string); return nvidia::gxf::Unexpected{GXF_FAILURE}; } @@ -72,6 +234,122 @@ static std::string inputTypeToString(holoscan::ops::HolovizOp::InputType input_t return "invalid"; } +/// table to convert depth map render mode to string +static const std::array, 3> + kDepthMapRenderModeToStr{ + {{holoscan::ops::HolovizOp::DepthMapRenderMode::POINTS, "points"}, + {holoscan::ops::HolovizOp::DepthMapRenderMode::LINES, "lines"}, + {holoscan::ops::HolovizOp::DepthMapRenderMode::TRIANGLES, "triangles"}}}; + +/** + * Convert a string to a depth map render mode enum + * + * @param string depth map render mode string + * @return depth map render mode enum + */ +static nvidia::gxf::Expected +depthMapRenderModeFromString(const std::string& string) { + const auto it = std::find_if(std::cbegin(kDepthMapRenderModeToStr), + std::cend(kDepthMapRenderModeToStr), + [&string](const auto& v) { return v.second == string; }); + if (it != std::cend(kDepthMapRenderModeToStr)) { return it->first; } + + HOLOSCAN_LOG_ERROR("Unsupported depth map render mode '{}'", string); + return nvidia::gxf::Unexpected{GXF_FAILURE}; +} + +/** + * Convert a depth map render mode enum to a string + * + * @param depth_map_render_mode depth map render mode enum + * @return depth map render mode string + */ +static std::string depthMapRenderModeToString( + holoscan::ops::HolovizOp::DepthMapRenderMode depth_map_render_mode) { + const auto it = std::find_if( + std::cbegin(kDepthMapRenderModeToStr), + std::cend(kDepthMapRenderModeToStr), + [&depth_map_render_mode](const auto& v) { return v.first == depth_map_render_mode; }); + if (it != std::cend(kDepthMapRenderModeToStr)) { return it->second; } + + return "invalid"; +} + +/** + * Try to detect the input type enum for given buffer properties. + * + * @param buffer_info buffer info + * @param has_lut true if the user specified a LUT + * + * @return input type enum + */ +nvidia::gxf::Expected detectInputType( + const BufferInfo& buffer_info, bool has_lut) { + // auto detect type + if (buffer_info.rank == 3) { + if ((buffer_info.shape.dimension(2) == 2) && (buffer_info.shape.dimension(0) == 1) && + (buffer_info.element_type == nvidia::gxf::PrimitiveType::kFloat32)) { + // array of 2D coordinates, draw crosses + return holoscan::ops::HolovizOp::InputType::CROSSES; + } else if ((buffer_info.shape.dimension(2) == 1) && has_lut) { + // color image with lookup table + return holoscan::ops::HolovizOp::InputType::COLOR_LUT; + } else if ((buffer_info.shape.dimension(2) == 3) || (buffer_info.shape.dimension(2) == 4)) { + // color image (RGB or RGBA) + return holoscan::ops::HolovizOp::InputType::COLOR; + } else { + HOLOSCAN_LOG_ERROR("Can't auto detect type of input '{}'", buffer_info.name); + } + } + return nvidia::gxf::Unexpected{GXF_FAILURE}; +} + +/** + * Log the input spec + * + * @param input_specs input spec to log + */ +void logInputSpec(const std::vector& input_specs) { + std::stringstream ss; + ss << "Input spec:" << std::endl; + for (auto&& input_spec : input_specs) { + ss << "- name: '" << input_spec.tensor_name_ << "'" << std::endl; + ss << " type: '" << inputTypeToString(input_spec.type_) << "'" << std::endl; + ss << " opacity: " << input_spec.opacity_ << std::endl; + ss << " priority: " << input_spec.priority_ << std::endl; + if ((input_spec.type_ == holoscan::ops::HolovizOp::InputType::POINTS) || + (input_spec.type_ == holoscan::ops::HolovizOp::InputType::LINES) || + (input_spec.type_ == holoscan::ops::HolovizOp::InputType::LINE_STRIP) || + (input_spec.type_ == holoscan::ops::HolovizOp::InputType::TRIANGLES) || + (input_spec.type_ == holoscan::ops::HolovizOp::InputType::CROSSES) || + (input_spec.type_ == holoscan::ops::HolovizOp::InputType::RECTANGLES) || + (input_spec.type_ == holoscan::ops::HolovizOp::InputType::OVALS) || + (input_spec.type_ == holoscan::ops::HolovizOp::InputType::TEXT)) { + ss << " color: ["; + for (auto it = input_spec.color_.cbegin(); it < input_spec.color_.cend(); ++it) { + ss << *it; + if (it + 1 != input_spec.color_.cend()) { ss << ", "; } + } + ss << "]" << std::endl; + ss << " line_width: " << input_spec.line_width_ << std::endl; + ss << " point_size: " << input_spec.point_size_ << std::endl; + if (input_spec.type_ == holoscan::ops::HolovizOp::InputType::TEXT) { + ss << " text: ["; + for (auto it = input_spec.text_.cbegin(); it < input_spec.text_.cend(); ++it) { + ss << *it; + if (it + 1 != input_spec.text_.cend()) { ss << ", "; } + } + ss << "]" << std::endl; + } + if (input_spec.type_ == holoscan::ops::HolovizOp::InputType::DEPTH_MAP) { + ss << " depth_map_render_mode: '" + << depthMapRenderModeToString(input_spec.depth_map_render_mode_) << "'" << std::endl; + } + } + } + HOLOSCAN_LOG_INFO(ss.str()); +} + } // namespace /** @@ -89,6 +367,7 @@ struct YAML::convert { node["line_width"] = std::to_string(input_spec.line_width_); node["point_size"] = std::to_string(input_spec.point_size_); node["text"] = input_spec.text_; + node["depth_map_render_mode"] = depthMapRenderModeToString(input_spec.depth_map_render_mode_); return node; } @@ -112,6 +391,14 @@ struct YAML::convert { input_spec.point_size_ = node["point_size"].as(input_spec.point_size_); input_spec.text_ = node["text"].as>(input_spec.text_); + if (node["depth_map_render_mode"]) { + const auto maybe_depth_map_render_mode = + depthMapRenderModeFromString(node["depth_map_render_mode"].as()); + if (maybe_depth_map_render_mode) { + input_spec.depth_map_render_mode_ = maybe_depth_map_render_mode.value(); + } + } + return true; } catch (const std::exception& e) { GXF_LOG_ERROR(e.what()); @@ -125,7 +412,7 @@ namespace holoscan::ops { void HolovizOp::setup(OperatorSpec& spec) { constexpr uint32_t DEFAULT_WIDTH = 1920; constexpr uint32_t DEFAULT_HEIGHT = 1080; - constexpr uint32_t DEFAULT_FRAMERATE = 60; + constexpr float DEFAULT_FRAMERATE = 60.f; static const std::string DEFAULT_WINDOW_TITLE("Holoviz"); static const std::string DEFAULT_DISPLAY_NAME("DP-0"); constexpr bool DEFAULT_EXCLUSIVE_DISPLAY = false; @@ -133,7 +420,6 @@ void HolovizOp::setup(OperatorSpec& spec) { constexpr bool DEFAULT_HEADLESS = false; spec.param(receivers_, "receivers", "Input Receivers", "List of input receivers.", {}); - auto& render_buffer_input = spec.input("render_buffer_input").condition(ConditionType::kNone); spec.param(render_buffer_input_, @@ -187,7 +473,7 @@ void HolovizOp::setup(OperatorSpec& spec) { spec.param(framerate_, "framerate", "Framerate", - "Display framerate if in exclusive mode.", + "Display framerate in Hz if in exclusive mode.", DEFAULT_FRAMERATE); spec.param(use_exclusive_display_, "use_exclusive_display", @@ -212,6 +498,28 @@ void HolovizOp::setup(OperatorSpec& spec) { spec.param( allocator_, "allocator", "Allocator", "Allocator used to allocate render buffer output."); + + spec.param(font_path_, + "font_path", + "FontPath", + "File path for the font used for rendering text", + std::string()); + + cuda_stream_handler_.defineParams(spec); +} + +bool HolovizOp::check_port_enabled(const std::string& port_name) { + const std::string enable_port_name = std::string("enable_") + port_name; + + // Check if the boolean argument with the name "enable_(port_name)" is present. + auto enable_port = + std::find_if(args().begin(), args().end(), [&enable_port_name](const auto& arg) { + return (arg.name() == enable_port_name); + }); + const bool disable_port = + (enable_port == args().end()) || + (enable_port->has_value() && (std::any_cast(enable_port->value()) == false)); + return !disable_port; } void HolovizOp::enable_conditional_port(const std::string& port_name) { @@ -241,15 +549,687 @@ void HolovizOp::initialize() { // Set up prerequisite parameters before calling GXFOperator::initialize() auto frag = fragment(); - auto window_close_scheduling_term = - frag->make_condition("window_close_scheduling_term"); - add_arg(Arg("window_close_scheduling_term") = window_close_scheduling_term); + + // Find if there is an argument for 'window_close_scheduling_term' + auto has_window_close_scheduling_term = + std::find_if(args().begin(), args().end(), [](const auto& arg) { + return (arg.name() == "window_close_scheduling_term"); + }); + // Create the BooleanCondition if there is no argument provided. + if (has_window_close_scheduling_term == args().end()) { + window_close_scheduling_term_ = + frag->make_condition("window_close_scheduling_term"); + add_arg(window_close_scheduling_term_.get()); + } + + // TODO: avoid duplicate computations between check_port_enabled and enable_conditional_port + render_buffer_input_enabled_ = check_port_enabled("render_buffer_input"); + render_buffer_output_enabled_ = check_port_enabled("render_buffer_output"); // Conditional inputs and outputs are enabled using a boolean argument enable_conditional_port("render_buffer_input"); enable_conditional_port("render_buffer_output"); - GXFOperator::initialize(); + // parent class initialize() call must be after the argument additions above + Operator::initialize(); +} + +void HolovizOp::start() { + // set the font to be used + if (!font_path_.get().empty()) { viz::SetFont(font_path_.get().c_str(), 25.f); } + + // initialize Holoviz + viz::InitFlags init_flags = viz::InitFlags::NONE; + if (fullscreen_ && headless_) { + throw std::runtime_error("Headless and fullscreen are mutually exclusive."); + } + if (fullscreen_) { init_flags = viz::InitFlags::FULLSCREEN; } + if (headless_) { init_flags = viz::InitFlags::HEADLESS; } + + if (use_exclusive_display_) { + viz::Init( + display_name_.get().c_str(), width_, height_, uint32_t(framerate_ * 1000.f), init_flags); + } else { + viz::Init(width_, height_, window_title_.get().c_str(), init_flags); + } + + // get the color lookup table + const auto& color_lut = color_lut_.get(); + lut_.reserve(color_lut.size() * 4); + for (auto&& color : color_lut) { + if (color.size() != 4) { + std::string msg = fmt::format( + "Expected four components in color lookup table element, but got {}", color.size()); + throw std::runtime_error(msg); + } + lut_.insert(lut_.end(), color.begin(), color.end()); + } + + // cast Condition to BooleanCondition + auto bool_cond = window_close_scheduling_term_.get(); + bool_cond->enable_tick(); + + // Copy the user defined input spec list to the internal input spec list. If there is no user + // defined input spec it will be generated from the first messages received. + if (!tensors_.get().empty()) { + input_spec_.reserve(tensors_.get().size()); + input_spec_.insert(input_spec_.begin(), tensors_.get().begin(), tensors_.get().end()); + } +} + +void HolovizOp::stop() { + viz::Shutdown(); +} + +void HolovizOp::compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) { + std::vector messages_h = op_input.receive>("receivers"); + + // create vector of nvidia::gxf::Entity as expected by the code below + std::vector messages; + messages.reserve(messages_h.size()); + for (auto& message_h : messages_h) { + // cast each holoscan::gxf:Entity to its base class + nvidia::gxf::Entity message = static_cast(message_h); + messages.push_back(message); + } + + // cast Condition to BooleanCondition + auto bool_cond = window_close_scheduling_term_.get(); + if (viz::WindowShouldClose()) { + bool_cond->disable_tick(); + return; + } + + // nothing to do if minimized + if (viz::WindowIsMinimized()) { return; } + + // if user provided it, we have a input spec which had been copied at start(). + // else we build the input spec automatically by inspecting the tensors/videobuffers of all + // messages + if (input_spec_.empty()) { + // get all tensors and video buffers of all messages and build the input spec + for (auto&& message : messages) { + const auto tensors = message.findAll(); + for (auto&& tensor : tensors.value()) { + BufferInfo buffer_info; + if (buffer_info.init(tensor.value()) != GXF_FAILURE) { + // try to detect the input type, if we can't detect it, ignore the tensor + const auto maybe_input_type = detectInputType(buffer_info, !lut_.empty()); + if (maybe_input_type) { + input_spec_.emplace_back(tensor->name(), maybe_input_type.value()); + } + } + } + const auto video_buffers = message.findAll(); + for (auto&& video_buffer : video_buffers.value()) { + BufferInfo buffer_info; + if (buffer_info.init(video_buffer.value()) != GXF_FAILURE) { + // try to detect the input type, if we can't detect it, ignore the tensor + const auto maybe_input_type = detectInputType(buffer_info, !lut_.empty()); + if (maybe_input_type) { + input_spec_.emplace_back(video_buffer->name(), maybe_input_type.value()); + } + } + } + } + } + + // get the CUDA stream from the input message + const gxf_result_t result = cuda_stream_handler_.fromMessages(context.context(), messages); + if (result != GXF_SUCCESS) { + throw std::runtime_error("Failed to get the CUDA stream from incoming messages"); + } + viz::SetCudaStream(cuda_stream_handler_.getCudaStream(context.context())); + + // Depth maps have two tensors, the depth map itself and the depth map color values. Therefore + // collect the information in the first pass through the input specs and then render the depth + // map. + InputSpec* input_spec_depth_map = nullptr; + BufferInfo buffer_info_depth_map; + InputSpec* input_spec_depth_map_color = nullptr; + BufferInfo buffer_info_depth_map_color; + + // begin visualization + viz::Begin(); + + // get the tensors attached to the messages by the tensor names defined by the input spec and + // display them + for (auto& input_spec : input_spec_) { + nvidia::gxf::Expected> maybe_input_tensor = + nvidia::gxf::Unexpected{GXF_UNINITIALIZED_VALUE}; + nvidia::gxf::Expected> maybe_input_video = + nvidia::gxf::Unexpected{GXF_UNINITIALIZED_VALUE}; + auto message = messages.begin(); + while (message != messages.end()) { + maybe_input_tensor = message->get(input_spec.tensor_name_.c_str()); + if (maybe_input_tensor) { + // pick the first one with that name + break; + } + // check for video if no tensor found + maybe_input_video = message->get(input_spec.tensor_name_.c_str()); + if (maybe_input_video) { // pick the first one with that name + break; + } + ++message; + } + if (message == messages.end()) { + throw std::runtime_error( + fmt::format("Failed to retrieve input '{}'", input_spec.tensor_name_)); + } + + BufferInfo buffer_info; + gxf_result_t result; + if (maybe_input_tensor) { + result = buffer_info.init(maybe_input_tensor.value()); + } else { + result = buffer_info.init(maybe_input_video.value()); + } + if (result != GXF_SUCCESS) { + throw std::runtime_error(fmt::format("Unsupported buffer format tensor/video buffer '{}'", + input_spec.tensor_name_)); + } + + // if the input type is unknown it now can be detected using the image properties + if (input_spec.type_ == InputType::UNKNOWN) { + const auto maybe_input_type = detectInputType(buffer_info, !lut_.empty()); + if (!maybe_input_type) { + auto code = nvidia::gxf::ToResultCode(maybe_input_type); + throw std::runtime_error(fmt::format("failed setting input type with code {}", code)); + } + input_spec.type_ = maybe_input_type.value(); + } + + switch (input_spec.type_) { + case InputType::COLOR: + case InputType::COLOR_LUT: { + // 2D color image + + // sanity checks + if (buffer_info.rank != 3) { + throw std::runtime_error( + fmt::format("Expected rank 3 for tensor '{}', type '{}', but got {}", + buffer_info.name, + inputTypeToString(input_spec.type_), + buffer_info.rank)); + } + + /// @todo this is assuming HWC, should either auto-detect (if possible) or make user + /// configurable + const auto height = buffer_info.shape.dimension(0); + const auto width = buffer_info.shape.dimension(1); + const auto channels = buffer_info.shape.dimension(2); + + if (input_spec.type_ == InputType::COLOR_LUT) { + if (channels != 1) { + throw std::runtime_error(fmt::format( + "Expected one channel for tensor '{}' when using lookup table, but got {}", + buffer_info.name, + channels)); + } + if (lut_.empty()) { + throw std::runtime_error(fmt::format( + "Type of tensor '{}' is '{}', but a color lookup table has not been specified", + buffer_info.name, + inputTypeToString(input_spec.type_))); + } + } + + auto maybe_image_format = getImageFormatFromTensor(buffer_info); + if (!maybe_image_format) { + auto code = nvidia::gxf::ToResultCode(maybe_image_format); + throw std::runtime_error(fmt::format("failed setting input format with code {}", code)); + } + const viz::ImageFormat image_format = maybe_image_format.value(); + + // start an image layer + viz::BeginImageLayer(); + viz::LayerPriority(input_spec.priority_); + viz::LayerOpacity(input_spec.opacity_); + + if (input_spec.type_ == InputType::COLOR_LUT) { + viz::LUT(lut_.size() / 4, + viz::ImageFormat::R32G32B32A32_SFLOAT, + lut_.size() * sizeof(float), + lut_.data()); + } + + if (buffer_info.storage_type == nvidia::gxf::MemoryStorageType::kDevice) { + // if it's the device convert to `CUDeviceptr` + const auto cu_buffer_ptr = reinterpret_cast(buffer_info.buffer_ptr); + viz::ImageCudaDevice(width, height, image_format, cu_buffer_ptr); + } else { + // convert to void * if using the system/host + const auto host_buffer_ptr = reinterpret_cast(buffer_info.buffer_ptr); + viz::ImageHost(width, height, image_format, host_buffer_ptr); + } + viz::EndLayer(); + } break; + + case InputType::POINTS: + case InputType::LINES: + case InputType::LINE_STRIP: + case InputType::TRIANGLES: + case InputType::CROSSES: + case InputType::RECTANGLES: + case InputType::OVALS: + case InputType::TEXT: { + // geometry layer + if (buffer_info.element_type != nvidia::gxf::PrimitiveType::kFloat32) { + throw std::runtime_error(fmt::format( + "Expected gxf::PrimitiveType::kFloat32 element type for coordinates, but got " + "element type {}", + static_cast(buffer_info.element_type))); + } + + // get pointer to tensor buffer + std::vector host_buffer; + if (buffer_info.storage_type == nvidia::gxf::MemoryStorageType::kDevice) { + host_buffer.resize(buffer_info.bytes_size); + + // copy from device to host + CUDA_TRY(cudaMemcpyAsync(static_cast(host_buffer.data()), + static_cast(buffer_info.buffer_ptr), + buffer_info.bytes_size, + cudaMemcpyDeviceToHost, + cuda_stream_handler_.getCudaStream(context.context()))); + // wait for the CUDA memory copy to finish + CUDA_TRY(cudaStreamSynchronize(cuda_stream_handler_.getCudaStream(context.context()))); + + buffer_info.buffer_ptr = host_buffer.data(); + } + + // start a geometry layer + viz::BeginGeometryLayer(); + viz::LayerPriority(input_spec.priority_); + viz::LayerOpacity(input_spec.opacity_); + std::array color{1.f, 1.f, 1.f, 1.f}; + for (size_t index = 0; index < std::min(input_spec.color_.size(), color.size()); ++index) { + color[index] = input_spec.color_[index]; + } + viz::Color(color[0], color[1], color[2], color[3]); + + /// @todo this is assuming NHW, should either auto-detect (if possible) or make user + /// configurable + const auto coordinates = buffer_info.shape.dimension(1); + const auto components = buffer_info.shape.dimension(2); + + if (input_spec.type_ == InputType::TEXT) { + // text is defined by the top left coordinate and the size (x, y, s) per string, text + // strings are define by InputSpec::text_ + if ((components < 2) || (components > 3)) { + throw std::runtime_error( + fmt::format("Expected two or three values per text, but got '{}'", components)); + } + const float* src_coord = reinterpret_cast(buffer_info.buffer_ptr); + for (int32_t index = 0; index < coordinates; ++index) { + viz::Text( + src_coord[0], + src_coord[1], + (components == 3) ? src_coord[2] : 0.05f, + input_spec.text_[std::min(index, static_cast(input_spec.text_.size()) - 1)] + .c_str()); + src_coord += components; + } + } else { + viz::LineWidth(input_spec.line_width_); + + std::vector coords; + viz::PrimitiveTopology topology; + uint32_t primitive_count; + uint32_t coordinate_count; + uint32_t values_per_coordinate; + std::vector default_coord; + if (input_spec.type_ == InputType::POINTS) { + // point primitives, one coordinate (x, y) per primitive + if (components != 2) { + throw std::runtime_error( + fmt::format("Expected two values per point, but got '{}'", components)); + } + + viz::PointSize(input_spec.point_size_); + + topology = viz::PrimitiveTopology::POINT_LIST; + primitive_count = coordinates; + coordinate_count = primitive_count; + values_per_coordinate = 2; + default_coord = {0.f, 0.f}; + } else if (input_spec.type_ == InputType::LINES) { + // line primitives, two coordinates (x0, y0) and (x1, y1) per primitive + if (components != 2) { + throw std::runtime_error( + fmt::format("Expected two values per line vertex, but got '{}'", components)); + } + topology = viz::PrimitiveTopology::LINE_LIST; + primitive_count = coordinates / 2; + coordinate_count = primitive_count * 2; + values_per_coordinate = 2; + default_coord = {0.f, 0.f}; + } else if (input_spec.type_ == InputType::LINE_STRIP) { + // line primitives, two coordinates (x0, y0) and (x1, y1) per primitive + if (components != 2) { + throw std::runtime_error(fmt::format( + "Expected two values per line strip vertex, but got '{}'", components)); + } + topology = viz::PrimitiveTopology::LINE_STRIP; + primitive_count = coordinates - 1; + coordinate_count = coordinates; + values_per_coordinate = 2; + default_coord = {0.f, 0.f}; + } else if (input_spec.type_ == InputType::TRIANGLES) { + // triangle primitive, three coordinates (x0, y0), (x1, y1) and (x2, y2) per primitive + if (components != 2) { + throw std::runtime_error( + fmt::format("Expected two values per triangle vertex, but got '{}'", components)); + } + topology = viz::PrimitiveTopology::TRIANGLE_LIST; + primitive_count = coordinates / 3; + coordinate_count = primitive_count * 3; + values_per_coordinate = 2; + default_coord = {0.f, 0.f}; + } else if (input_spec.type_ == InputType::CROSSES) { + // cross primitive, a cross is defined by the center coordinate and the size (xi, yi, + // si) + if ((components < 2) || (components > 3)) { + throw std::runtime_error( + fmt::format("Expected two or three values per cross, but got '{}'", components)); + } + + topology = viz::PrimitiveTopology::CROSS_LIST; + primitive_count = coordinates; + coordinate_count = primitive_count; + values_per_coordinate = 3; + default_coord = {0.f, 0.f, 0.05f}; + } else if (input_spec.type_ == InputType::RECTANGLES) { + // axis aligned rectangle primitive, each rectangle is defined by two coordinates (xi, + // yi) and (xi+1, yi+1) + if (components != 2) { + throw std::runtime_error(fmt::format( + "Expected two values per rectangle vertex, but got '{}'", components)); + } + topology = viz::PrimitiveTopology::RECTANGLE_LIST; + primitive_count = coordinates / 2; + coordinate_count = primitive_count * 2; + values_per_coordinate = 2; + default_coord = {0.f, 0.f}; + } else if (input_spec.type_ == InputType::OVALS) { + // oval primitive, an oval primitive is defined by the center coordinate and the axis + // sizes (xi, yi, sxi, syi) + if ((components < 2) || (components > 4)) { + throw std::runtime_error(fmt::format( + "Expected two, three or four values per oval, but got '{}'", components)); + } + topology = viz::PrimitiveTopology::OVAL_LIST; + primitive_count = coordinates; + coordinate_count = primitive_count; + values_per_coordinate = 4; + default_coord = {0.f, 0.f, 0.05f, 0.05f}; + } else { + throw std::runtime_error( + fmt::format("Unhandled tensor type '{}'", inputTypeToString(input_spec.type_))); + } + + // copy coordinates + const float* src_coord = reinterpret_cast(buffer_info.buffer_ptr); + coords.reserve(coordinate_count * values_per_coordinate); + for (int32_t index = 0; index < static_cast(coordinate_count); ++index) { + int32_t component_index = 0; + // copy from source array + while (component_index < std::min(components, int32_t(values_per_coordinate))) { + coords.push_back(src_coord[component_index]); + ++component_index; + } + // fill from default array + while (component_index < static_cast(values_per_coordinate)) { + coords.push_back(default_coord[component_index]); + ++component_index; + } + src_coord += buffer_info.stride[1] / sizeof(float); + } + + if (primitive_count) { + viz::Primitive(topology, primitive_count, coords.size(), coords.data()); + } + } + + viz::EndLayer(); + } break; + case InputType::DEPTH_MAP: { + // 2D depth map + + // sanity checks + if (buffer_info.rank != 3) { + throw std::runtime_error( + fmt::format("Expected rank 3 for tensor '{}', type '{}', but got {}", + buffer_info.name, + inputTypeToString(input_spec.type_), + buffer_info.rank)); + } + if (buffer_info.element_type != nvidia::gxf::PrimitiveType::kUnsigned8) { + throw std::runtime_error(fmt::format( + "Expected gxf::PrimitiveType::kUnsigned8 element type for tensor '{}', but got " + "element type {}", + buffer_info.name, + static_cast(buffer_info.element_type))); + } + if (buffer_info.storage_type != nvidia::gxf::MemoryStorageType::kDevice) { + throw std::runtime_error( + fmt::format("Only device storage is supported for tensor '{}'", buffer_info.name)); + } + /// @todo this is assuming HWC, should either auto-detect (if possible) or make user + /// configurable + const auto channels = buffer_info.shape.dimension(2); + if (buffer_info.shape.dimension(2) != 1) { + throw std::runtime_error(fmt::format( + "Expected one channel for tensor '{}', but got {}", buffer_info.name, channels)); + } + + // Store the depth map information, we render after the end of the input spec loop when + // we also have the (optional) depth map color information. + input_spec_depth_map = &input_spec; + buffer_info_depth_map = buffer_info; + } break; + case InputType::DEPTH_MAP_COLOR: { + // 2D depth map color + + // sanity checks + if (buffer_info.rank != 3) { + throw std::runtime_error( + fmt::format("Expected rank 3 for tensor '{}', type '{}', but got {}", + buffer_info.name, + inputTypeToString(input_spec.type_), + buffer_info.rank)); + } + if (buffer_info.element_type != nvidia::gxf::PrimitiveType::kUnsigned8) { + throw std::runtime_error(fmt::format( + "Expected gxf::PrimitiveType::kUnsigned8 element type for tensor '{}', but got " + "element type {}", + buffer_info.name, + static_cast(buffer_info.element_type))); + } + if (buffer_info.storage_type != nvidia::gxf::MemoryStorageType::kDevice) { + throw std::runtime_error( + fmt::format("Only device storage is supported for tensor '{}'", buffer_info.name)); + } + /// @todo this is assuming HWC, should either auto-detect (if possible) or make user + /// configurable + const auto channels = buffer_info.shape.dimension(2); + if (buffer_info.shape.dimension(2) != 4) { + throw std::runtime_error(fmt::format( + "Expected four channels for tensor '{}', but got {}", buffer_info.name, channels)); + } + + // Store the depth map color information, we render after the end of the input spec loop + // when we have both the depth and color information + input_spec_depth_map_color = &input_spec; + buffer_info_depth_map_color = buffer_info; + } break; + default: + throw std::runtime_error( + fmt::format("Unhandled input type '{}'", inputTypeToString(input_spec.type_))); + } + } + + // we now have both tensors to render a depth map + if (input_spec_depth_map) { + /// @todo this is assuming HWC, should either auto-detect (if possible) or make user + /// configurable + const auto height = buffer_info_depth_map.shape.dimension(0); + const auto width = buffer_info_depth_map.shape.dimension(1); + + viz::ImageFormat depth_map_color_fmt = viz::ImageFormat::R8G8B8A8_UNORM; + CUdeviceptr depth_map_color_device_ptr = 0; + if (input_spec_depth_map_color) { + // if there is a color buffer, the size has to match + + /// @todo this is assuming HWC, should either auto-detect (if possible) or make user + /// configurable + const auto color_height = buffer_info_depth_map_color.shape.dimension(0); + const auto color_width = buffer_info_depth_map_color.shape.dimension(1); + if ((width != color_width) || (height != color_height)) { + throw std::runtime_error( + fmt::format("The buffer dimensions {}x{} of the depth map color buffer '{}' need to " + "match the depth map '{}' dimensions {}x{}", + color_width, + color_height, + buffer_info_depth_map_color.name, + buffer_info_depth_map.name, + width, + height)); + } + auto maybe_image_format = getImageFormatFromTensor(buffer_info_depth_map_color); + if (!maybe_image_format) { + auto code = nvidia::gxf::ToResultCode(maybe_image_format); + throw std::runtime_error(fmt::format("failed setting input format with code {}", code)); + } + depth_map_color_fmt = maybe_image_format.value(); + + depth_map_color_device_ptr = + reinterpret_cast(buffer_info_depth_map_color.buffer_ptr); + } + + viz::DepthMapRenderMode depth_map_render_mode; + switch (input_spec_depth_map->depth_map_render_mode_) { + case DepthMapRenderMode::POINTS: + depth_map_render_mode = viz::DepthMapRenderMode::POINTS; + break; + case DepthMapRenderMode::LINES: + depth_map_render_mode = viz::DepthMapRenderMode::LINES; + break; + case DepthMapRenderMode::TRIANGLES: + depth_map_render_mode = viz::DepthMapRenderMode::TRIANGLES; + break; + default: + throw std::runtime_error( + fmt::format("Unhandled depth map render mode {}", + depthMapRenderModeToString(input_spec_depth_map->depth_map_render_mode_))); + } + + // start a geometry layer containing the depth map + viz::BeginGeometryLayer(); + viz::LayerPriority(input_spec_depth_map->priority_); + viz::LayerOpacity(input_spec_depth_map->opacity_); + std::array color{1.f, 1.f, 1.f, 1.f}; + for (size_t index = 0; index < std::min(input_spec_depth_map->color_.size(), color.size()); + ++index) { + color[index] = input_spec_depth_map->color_[index]; + } + viz::Color(color[0], color[1], color[2], color[3]); + + if (depth_map_render_mode == viz::DepthMapRenderMode::POINTS) { + viz::PointSize(input_spec_depth_map->point_size_); + } else if (depth_map_render_mode == viz::DepthMapRenderMode::LINES) { + viz::LineWidth(input_spec_depth_map->line_width_); + } + + const auto cu_buffer_ptr = reinterpret_cast(buffer_info_depth_map.buffer_ptr); + viz::DepthMap(depth_map_render_mode, + width, + height, + viz::ImageFormat::R8_UNORM, + cu_buffer_ptr, + depth_map_color_fmt, + depth_map_color_device_ptr); + viz::EndLayer(); + } + + viz::End(); + + // check if the render buffer should be output + if (render_buffer_output_enabled_) { + auto entity = nvidia::gxf::Entity::New(context.context()); + if (!entity) { + throw std::runtime_error("Failed to allocate message for the render buffer output."); + } + + auto video_buffer = entity.value().add("render_buffer_output"); + if (!video_buffer) { + throw std::runtime_error("Failed to allocate the video buffer for the render buffer output."); + } + + if (render_buffer_input_enabled_) { + // check if there is a input buffer given to copy the output into + auto render_buffer_input = op_input.receive("render_buffer_input"); + if (!render_buffer_input) { + throw std::runtime_error("No message available at 'render_buffer_input'."); + } + + // Get the empty input buffer + const auto& video_buffer_in = + static_cast(render_buffer_input).get(); + if (!video_buffer_in) { + throw std::runtime_error("No video buffer attached to message on 'render_buffer_input'."); + } + + const nvidia::gxf::VideoBufferInfo info = video_buffer_in.value()->video_frame_info(); + + if ((info.color_format != nvidia::gxf::VideoFormat::GXF_VIDEO_FORMAT_RGBA)) { + throw std::runtime_error("Invalid render buffer input, expected an RGBA buffer."); + } + + video_buffer.value()->wrapMemory(info, + video_buffer_in.value()->size(), + video_buffer_in.value()->storage_type(), + video_buffer_in.value()->pointer(), + nullptr); + } else { + // if there is no input buffer given, allocate one + if (!allocator_.get()) { + throw std::runtime_error("No render buffer input specified and no allocator set."); + } + + // get Handle to underlying nvidia::gxf::Allocator from std::shared_ptr + auto allocator = nvidia::gxf::Handle::Create( + context.context(), allocator_.get()->gxf_cid()); + + video_buffer.value()->resize( + width_, + height_, + nvidia::gxf::SurfaceLayout::GXF_SURFACE_LAYOUT_BLOCK_LINEAR, + nvidia::gxf::MemoryStorageType::kDevice, + allocator.value()); + if (!video_buffer.value()->pointer()) { + throw std::runtime_error("Failed to allocate render output buffer."); + } + } + + // read the framebuffer + viz::ReadFramebuffer(viz::ImageFormat::R8G8B8A8_UNORM, + width_, + height_, + video_buffer.value()->size(), + reinterpret_cast(video_buffer.value()->pointer())); + + // Output the filled render buffer object + auto result = gxf::Entity(std::move(entity.value())); + op_output.emit(result); + } + + if (is_first_tick_) { + logInputSpec(input_spec_); + is_first_tick_ = false; + } } } // namespace holoscan::ops diff --git a/src/operators/multiai_inference/CMakeLists.txt b/src/operators/multiai_inference/CMakeLists.txt new file mode 100644 index 00000000..88470b03 --- /dev/null +++ b/src/operators/multiai_inference/CMakeLists.txt @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_holoscan_operator(multiai_inference multiai_inference.cpp) + +target_link_libraries(op_multiai_inference + PUBLIC + holoscan::core + holoscan::infer + holoscan::infer_utils +) diff --git a/src/operators/multiai_inference/multiai_inference.cpp b/src/operators/multiai_inference/multiai_inference.cpp index 693e6551..f2146804 100644 --- a/src/operators/multiai_inference/multiai_inference.cpp +++ b/src/operators/multiai_inference/multiai_inference.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,9 +16,13 @@ */ #include "holoscan/operators/multiai_inference/multiai_inference.hpp" + +#include "holoscan/core/execution_context.hpp" #include "holoscan/core/gxf/entity.hpp" +#include "holoscan/core/io_context.hpp" #include "holoscan/core/operator_spec.hpp" #include "holoscan/core/resources/gxf/allocator.hpp" +#include "holoscan/utils/holoinfer_utils.hpp" /** * Custom YAML parser for DataMap class @@ -89,6 +93,8 @@ struct YAML::convert { namespace holoscan::ops { void MultiAIInferenceOp::setup(OperatorSpec& spec) { + register_converter(); + register_converter(); auto& transmitter = spec.output("transmitter"); spec.param(backend_, "backend", "Supported backend"); spec.param(model_path_map_, @@ -127,7 +133,121 @@ void MultiAIInferenceOp::setup(OperatorSpec& spec) { void MultiAIInferenceOp::initialize() { register_converter(); register_converter(); - GXFOperator::initialize(); + Operator::initialize(); +} + +void MultiAIInferenceOp::start() { + try { + // Check for the validity of parameters from configuration + auto status = HoloInfer::multiai_inference_validity_check(model_path_map_.get().get_map(), + pre_processor_map_.get().get_map(), + inference_map_.get().get_map(), + in_tensor_names_.get(), + out_tensor_names_.get()); + if (status.get_code() != HoloInfer::holoinfer_code::H_SUCCESS) { + HoloInfer::raise_error(module_, "Parameter Validation failed: " + status.get_message()); + } + + bool is_aarch64 = HoloInfer::is_platform_aarch64(); + if (is_aarch64 && backend_.get().compare("onnxrt") == 0 && !infer_on_cpu_.get()) { + HoloInfer::raise_error(module_, "Onnxruntime with CUDA not supported on aarch64."); + } + + // Create multiai specification structure + multiai_specs_ = std::make_shared(backend_.get(), + model_path_map_.get().get_map(), + inference_map_.get().get_map(), + is_engine_path_.get(), + infer_on_cpu_.get(), + parallel_inference_.get(), + enable_fp16_.get(), + input_on_cuda_.get(), + output_on_cuda_.get()); + + // Create holoscan inference context + holoscan_infer_context_ = std::make_unique(); + + // Set and transfer inference specification to inference context + // Multi AI specifications are updated with memory allocations + status = holoscan_infer_context_->set_inference_params(multiai_specs_); + if (status.get_code() != HoloInfer::holoinfer_code::H_SUCCESS) { + HoloInfer::raise_error(module_, "Start, Parameters setup, " + status.get_message()); + } + } catch (const std::bad_alloc& b_) { + HoloInfer::raise_error(module_, "Start, Memory allocation, Message: " + std::string(b_.what())); + } catch (const std::runtime_error& rt_) { + HOLOSCAN_LOG_ERROR(rt_.what()); + throw; + } catch (...) { HoloInfer::raise_error(module_, "Start, Unknown exception"); } +} + +void MultiAIInferenceOp::stop() { + holoscan_infer_context_.reset(); +} + +gxf_result_t timer_check(HoloInfer::TimePoint& start, HoloInfer::TimePoint& end, + const std::string& module) { + HoloInfer::timer_init(end); + int64_t delta = std::chrono::duration_cast(end - start).count(); + HOLOSCAN_LOG_DEBUG("{} : {} ms", module, delta); + return GXF_SUCCESS; +} + +void MultiAIInferenceOp::compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) { + // get Handle to underlying nvidia::gxf::Allocator from std::shared_ptr + auto allocator = nvidia::gxf::Handle::Create(context.context(), + allocator_.get()->gxf_cid()); + + try { + // Extract relevant data from input GXF Receivers, and update multiai specifications + gxf_result_t stat = + holoscan::utils::multiai_get_data_per_model(op_input, + in_tensor_names_.get(), + multiai_specs_->data_per_tensor_, + dims_per_tensor_, + input_on_cuda_.get(), + module_); + + if (stat != GXF_SUCCESS) { HoloInfer::raise_error(module_, "Tick, Data extraction"); } + + auto status = HoloInfer::map_data_to_model_from_tensor(pre_processor_map_.get().get_map(), + multiai_specs_->data_per_model_, + multiai_specs_->data_per_tensor_); + if (status.get_code() != HoloInfer::holoinfer_code::H_SUCCESS) { + HoloInfer::raise_error(module_, "Tick, Data mapping, " + status.get_message()); + } + + // Execute inference and populate output buffer in multiai specifications + status = holoscan_infer_context_->execute_inference(multiai_specs_->data_per_model_, + multiai_specs_->output_per_model_); + if (status.get_code() != HoloInfer::holoinfer_code::H_SUCCESS) { + HoloInfer::raise_error(module_, "Tick, Inference execution, " + status.get_message()); + } + HOLOSCAN_LOG_DEBUG(status.get_message()); + + // Get output dimensions + auto model_out_dims_map = holoscan_infer_context_->get_output_dimensions(); + + auto cont = context.context(); + + // Transmit output buffers via a single GXF transmitter + stat = holoscan::utils::multiai_transmit_data_per_model(cont, + inference_map_.get().get_map(), + multiai_specs_->output_per_model_, + op_output, + out_tensor_names_.get(), + model_out_dims_map, + output_on_cuda_.get(), + transmit_on_cuda_.get(), + data_type_, + allocator.value(), + module_); + if (stat != GXF_SUCCESS) { HoloInfer::raise_error(module_, "Tick, Data Transmission"); } + } catch (const std::runtime_error& r_) { + HoloInfer::raise_error(module_, + "Tick, Inference execution, Message->" + std::string(r_.what())); + } catch (...) { HoloInfer::raise_error(module_, "Tick, unknown exception"); } } } // namespace holoscan::ops diff --git a/src/operators/multiai_postprocessor/CMakeLists.txt b/src/operators/multiai_postprocessor/CMakeLists.txt new file mode 100644 index 00000000..fcd786ee --- /dev/null +++ b/src/operators/multiai_postprocessor/CMakeLists.txt @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_holoscan_operator(multiai_postprocessor multiai_postprocessor.cpp) + +target_link_libraries(op_multiai_postprocessor + PUBLIC + holoscan::core + holoscan::infer + holoscan::infer_utils +) diff --git a/src/operators/multiai_postprocessor/multiai_postprocessor.cpp b/src/operators/multiai_postprocessor/multiai_postprocessor.cpp index 105cd4ff..b5f0ecd9 100644 --- a/src/operators/multiai_postprocessor/multiai_postprocessor.cpp +++ b/src/operators/multiai_postprocessor/multiai_postprocessor.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,9 +16,12 @@ */ #include "holoscan/operators/multiai_postprocessor/multiai_postprocessor.hpp" +#include "holoscan/core/execution_context.hpp" #include "holoscan/core/gxf/entity.hpp" +#include "holoscan/core/io_context.hpp" #include "holoscan/core/operator_spec.hpp" #include "holoscan/core/resources/gxf/allocator.hpp" +#include "holoscan/utils/holoinfer_utils.hpp" template <> struct YAML::convert { @@ -93,12 +96,15 @@ void MultiAIPostprocessorOp::setup(OperatorSpec& spec) { spec.param(process_operations_, "process_operations", "Operations per tensor", - "Operations in sequence on tensors."); - spec.param(processed_map_, "processed_map", "In to out tensor", "Input-output tensor mapping."); - spec.param( - in_tensor_names_, "in_tensor_names", "Input Tensors", "Input tensors", {std::string("")}); - spec.param( - out_tensor_names_, "out_tensor_names", "Output Tensors", "Output tensors", {std::string("")}); + "Operations in sequence on tensors.", + DataVecMap()); + spec.param(processed_map_, + "processed_map", + "In to out tensor", + "Input-output tensor mapping.", + DataMap()); + spec.param(in_tensor_names_, "in_tensor_names", "Input Tensors", "Input tensors", {}); + spec.param(out_tensor_names_, "out_tensor_names", "Output Tensors", "Output tensors", {}); spec.param(input_on_cuda_, "input_on_cuda", "Input buffer on CUDA", "", false); spec.param(output_on_cuda_, "output_on_cuda", "Output buffer on CUDA", "", false); spec.param(transmit_on_cuda_, "transmit_on_cuda", "Transmit message on CUDA", "", false); @@ -107,10 +113,124 @@ void MultiAIPostprocessorOp::setup(OperatorSpec& spec) { spec.param(transmitter_, "transmitter", "Transmitter", "Transmitter", {&transmitter}); } +void MultiAIPostprocessorOp::conditional_disable_output_port(const std::string& port_name) { + const std::string disable_port_name = std::string("disable_") + port_name; + + // Check if the boolean argument with the name "disable_(port_name)" is present. + auto disable_port = + std::find_if(args().begin(), args().end(), [&disable_port_name](const auto& arg) { + return (arg.name() == disable_port_name); + }); + + // If the argument exists without value or is set to true, we unset(nullify) the port parameter. + const bool need_disabled = + (disable_port != args().end() && + (!disable_port->has_value() || + (disable_port->has_value() && (std::any_cast(disable_port->value()) == true)))); + + if (disable_port != args().end()) { + // If the argument is present, we just remove it from the arguments. + args().erase(disable_port); + } + + if (need_disabled) { + // Set the condition of the port to kNone, which means that the port doesn't have any condition. + spec()->outputs()[port_name]->condition(ConditionType::kNone); + add_arg(Arg(port_name) = std::vector()); + } +} + void MultiAIPostprocessorOp::initialize() { register_converter(); register_converter(); - GXFOperator::initialize(); + + // Output port is conditionally disabled using a boolean argument + conditional_disable_output_port("transmitter"); + + Operator::initialize(); +} + +void MultiAIPostprocessorOp::start() { + try { + // Check for the validity of parameters from configuration + if (input_on_cuda_.get() || output_on_cuda_.get() || transmit_on_cuda_.get()) { + HoloInfer::raise_error(module_, "CUDA based data not supported in Multi AI post processor"); + } + auto status = HoloInfer::multiai_processor_validity_check( + processed_map_.get().get_map(), in_tensor_names_.get(), out_tensor_names_.get()); + if (status.get_code() != HoloInfer::holoinfer_code::H_SUCCESS) { + status.display_message(); + HoloInfer::raise_error(module_, "Parameter Validation failed: " + status.get_message()); + } + + // Create holoscan processing context + holoscan_postprocess_context_ = std::make_unique(); + } catch (const std::bad_alloc& b_) { + HoloInfer::raise_error(module_, "Start, Memory allocation, Message: " + std::string(b_.what())); + } catch (const std::runtime_error& rt_) { + HOLOSCAN_LOG_ERROR(rt_.what()); + throw; + } catch (...) { HoloInfer::raise_error(module_, "Start, Unknown exception"); } + + // Initialize holoscan processing context + auto status = holoscan_postprocess_context_->initialize(process_operations_.get().get_map()); + if (status.get_code() != HoloInfer::holoinfer_code::H_SUCCESS) { + status.display_message(); + HoloInfer::raise_error(module_, "Start, Out data setup"); + } +} + +void MultiAIPostprocessorOp::compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) { + // get Handle to underlying nvidia::gxf::Allocator from std::shared_ptr + auto allocator = nvidia::gxf::Handle::Create(context.context(), + allocator_.get()->gxf_cid()); + + try { + // Extract relevant data from input GXF Receivers, and update multiai specifications + gxf_result_t stat = holoscan::utils::multiai_get_data_per_model(op_input, + in_tensor_names_.get(), + data_per_tensor_, + dims_per_tensor_, + input_on_cuda_.get(), + module_); + + if (stat != GXF_SUCCESS) { HoloInfer::raise_error(module_, "Tick, Data extraction"); } + + // Execute processing + auto status = holoscan_postprocess_context_->process(process_operations_.get().get_map(), + processed_map_.get().get_map(), + data_per_tensor_, + dims_per_tensor_); + if (status.get_code() != HoloInfer::holoinfer_code::H_SUCCESS) { + status.display_message(); + HoloInfer::report_error(module_, "Tick, post_process"); + } + + // Get processed data and dimensions (currently only on host) + auto processed_data_map = holoscan_postprocess_context_->get_processed_data(); + auto processed_dims_map = holoscan_postprocess_context_->get_processed_data_dims(); + + if (out_tensor_names_.get().size() != 0) { + auto cont = context.context(); + // Transmit output buffers via a single GXF transmitter + stat = holoscan::utils::multiai_transmit_data_per_model(cont, + processed_map_.get().get_map(), + processed_data_map, + op_output, + out_tensor_names_.get(), + processed_dims_map, + output_on_cuda_.get(), + transmit_on_cuda_.get(), + nvidia::gxf::PrimitiveType::kFloat32, + allocator.value(), + module_); + + if (stat != GXF_SUCCESS) { HoloInfer::report_error(module_, "Tick, Data Transmission"); } + } + } catch (const std::runtime_error& r_) { + HoloInfer::report_error(module_, "Tick, Message->" + std::string(r_.what())); + } catch (...) { HoloInfer::report_error(module_, "Tick, unknown exception"); } } } // namespace holoscan::ops diff --git a/apps/endoscopy_tool_tracking/CMakeLists.txt b/src/operators/ping_rx/CMakeLists.txt similarity index 77% rename from apps/endoscopy_tool_tracking/CMakeLists.txt rename to src/operators/ping_rx/CMakeLists.txt index 221c7a5a..cdd7b7f7 100644 --- a/apps/endoscopy_tool_tracking/CMakeLists.txt +++ b/src/operators/ping_rx/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,5 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -add_subdirectory(cpp) -add_subdirectory(python) +add_holoscan_operator(ping_rx ping_rx.cpp) + +target_link_libraries(op_ping_rx + PUBLIC + holoscan::core +) diff --git a/src/operators/ping_rx/ping_rx.cpp b/src/operators/ping_rx/ping_rx.cpp new file mode 100644 index 00000000..e25558db --- /dev/null +++ b/src/operators/ping_rx/ping_rx.cpp @@ -0,0 +1,31 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "holoscan/operators/ping_rx/ping_rx.hpp" + +namespace holoscan::ops { + +void PingRxOp::setup(OperatorSpec& spec) { + spec.input("in"); +} + +void PingRxOp::compute(InputContext& op_input, OutputContext&, ExecutionContext&) { + auto value = op_input.receive("in"); + std::cout << "Rx message value: " << *(value.get()) << std::endl; +} + +} // namespace holoscan::ops diff --git a/apps/hi_speed_endoscopy/CMakeLists.txt b/src/operators/ping_tx/CMakeLists.txt similarity index 77% rename from apps/hi_speed_endoscopy/CMakeLists.txt rename to src/operators/ping_tx/CMakeLists.txt index 221c7a5a..cab9c6b4 100644 --- a/apps/hi_speed_endoscopy/CMakeLists.txt +++ b/src/operators/ping_tx/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,5 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -add_subdirectory(cpp) -add_subdirectory(python) +add_holoscan_operator(ping_tx ping_tx.cpp) + +target_link_libraries(op_ping_tx + PUBLIC + holoscan::core +) diff --git a/src/operators/ping_tx/ping_tx.cpp b/src/operators/ping_tx/ping_tx.cpp new file mode 100644 index 00000000..ee459f39 --- /dev/null +++ b/src/operators/ping_tx/ping_tx.cpp @@ -0,0 +1,31 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "holoscan/operators/ping_tx/ping_tx.hpp" + +namespace holoscan::ops { + +void PingTxOp::setup(OperatorSpec& spec) { + spec.output("out"); +} + +void PingTxOp::compute(InputContext&, OutputContext& op_output, ExecutionContext&) { + auto value = std::make_shared(index_++); + op_output.emit(value, "out"); +} + +} // namespace holoscan::ops diff --git a/src/operators/segmentation_postprocessor/CMakeLists.txt b/src/operators/segmentation_postprocessor/CMakeLists.txt new file mode 100644 index 00000000..ef0812c5 --- /dev/null +++ b/src/operators/segmentation_postprocessor/CMakeLists.txt @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_holoscan_operator(segmentation_postprocessor + segmentation_postprocessor.cpp + segmentation_postprocessor.cu +) + +target_link_libraries(op_segmentation_postprocessor + PUBLIC holoscan::core +) diff --git a/src/operators/segmentation_postprocessor/segmentation_postprocessor.cpp b/src/operators/segmentation_postprocessor/segmentation_postprocessor.cpp index 54c49101..c4451432 100644 --- a/src/operators/segmentation_postprocessor/segmentation_postprocessor.cpp +++ b/src/operators/segmentation_postprocessor/segmentation_postprocessor.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,10 +17,25 @@ #include "holoscan/operators/segmentation_postprocessor/segmentation_postprocessor.hpp" +#include +#include +#include + +#include "gxf/std/tensor.hpp" + +#include "holoscan/core/domain/tensor.hpp" +#include "holoscan/core/execution_context.hpp" #include "holoscan/core/gxf/entity.hpp" +#include "holoscan/core/io_context.hpp" #include "holoscan/core/operator_spec.hpp" - #include "holoscan/core/resources/gxf/allocator.hpp" +#include "holoscan/core/resources/gxf/cuda_stream_pool.hpp" + +using holoscan::ops::segmentation_postprocessor::cuda_postprocess; +using holoscan::ops::segmentation_postprocessor::DataFormat; +using holoscan::ops::segmentation_postprocessor::NetworkOutputType; +using holoscan::ops::segmentation_postprocessor::output_type_t; +using holoscan::ops::segmentation_postprocessor::Shape; namespace holoscan::ops { @@ -48,8 +63,127 @@ void SegmentationPostprocessorOp::setup(OperatorSpec& spec) { std::string("hwc")); spec.param(allocator_, "allocator", "Allocator", "Output Allocator"); + cuda_stream_handler_.defineParams(spec); + // TODO (gbae): spec object holds an information about errors // TODO (gbae): incorporate std::expected to not throw exceptions } +void SegmentationPostprocessorOp::compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) { + constexpr size_t kMaxChannelCount = std::numeric_limits::max(); + + // Process input message + // The type of `in_message` is 'holoscan::gxf::Entity'. + auto in_message = op_input.receive("in_tensor"); + + // if (!in_message || in_message.value().is_null()) { return GXF_CONTRACT_MESSAGE_NOT_AVAILABLE; } + + const std::string in_tensor_name = in_tensor_name_.get(); + + // Get tensor attached to the message + // The type of `maybe_tensor` is 'std::shared_ptr'. + auto maybe_tensor = in_message.get(in_tensor_name.c_str()); + if (!maybe_tensor) { + maybe_tensor = in_message.get(); + if (!maybe_tensor) { + throw std::runtime_error(fmt::format("Tensor '{}' not found in message", in_tensor_name)); + } + } + auto in_tensor = maybe_tensor; + + // get the CUDA stream from the input message + gxf_result_t stream_handler_result = + cuda_stream_handler_.fromMessage(context.context(), in_message); + if (stream_handler_result != GXF_SUCCESS) { + throw std::runtime_error("Failed to get the CUDA stream from incoming messages"); + } + + segmentation_postprocessor::Shape shape = {}; + switch (data_format_value_) { + case DataFormat::kHWC: { + shape.height = in_tensor->shape()[0]; + shape.width = in_tensor->shape()[1]; + shape.channels = in_tensor->shape()[2]; + } break; + case DataFormat::kNCHW: { + shape.channels = in_tensor->shape()[1]; + shape.height = in_tensor->shape()[2]; + shape.width = in_tensor->shape()[3]; + } break; + case DataFormat::kNHWC: { + shape.height = in_tensor->shape()[1]; + shape.width = in_tensor->shape()[2]; + shape.channels = in_tensor->shape()[3]; + } break; + } + + if (static_cast(shape.channels) > kMaxChannelCount) { + throw std::runtime_error(fmt::format( + "Input channel count larger than allowed: {} > {}", shape.channels, kMaxChannelCount)); + } + + // Create a new message (nvidia::gxf::Entity) + auto out_message = nvidia::gxf::Entity::New(context.context()); + + auto out_tensor = out_message.value().add("out_tensor"); + if (!out_tensor) { throw std::runtime_error("Failed to allocate output tensor"); } + + // Allocate and convert output buffer on the device. + nvidia::gxf::Shape output_shape{shape.height, shape.width, 1}; + + // get Handle to underlying nvidia::gxf::Allocator from std::shared_ptr + auto allocator = nvidia::gxf::Handle::Create(context.context(), + allocator_.get()->gxf_cid()); + out_tensor.value()->reshape( + output_shape, nvidia::gxf::MemoryStorageType::kDevice, allocator.value()); + if (!out_tensor.value()->pointer()) { + throw std::runtime_error("Failed to allocate output tensor buffer."); + } + + const float* in_tensor_data = static_cast(in_tensor->data()); + + nvidia::gxf::Expected out_tensor_data = out_tensor.value()->data(); + if (!out_tensor_data) { throw std::runtime_error("Failed to get out tensor data!"); } + + cuda_postprocess(network_output_type_value_, + data_format_value_, + shape, + in_tensor_data, + out_tensor_data.value(), + cuda_stream_handler_.getCudaStream(context.context())); + + // pass the CUDA stream to the output message + stream_handler_result = cuda_stream_handler_.toMessage(out_message); + if (stream_handler_result != GXF_SUCCESS) { + throw std::runtime_error("Failed to add the CUDA stream to the outgoing messages"); + } + + auto result = gxf::Entity(std::move(out_message.value())); + op_output.emit(result); +} + +void SegmentationPostprocessorOp::start() { + const std::string network_output_type = network_output_type_.get(); + if (network_output_type == "sigmoid") { + network_output_type_value_ = NetworkOutputType::kSigmoid; + } else if (network_output_type == "softmax") { + network_output_type_value_ = NetworkOutputType::kSoftmax; + } else { + throw std::runtime_error( + fmt::format("Unsupported network output type {}", network_output_type)); + } + + const std::string data_format = data_format_.get(); + if (data_format == "nchw") { + data_format_value_ = DataFormat::kNCHW; + } else if (data_format == "hwc") { + data_format_value_ = DataFormat::kHWC; + } else if (data_format == "nhwc") { + data_format_value_ = DataFormat::kNHWC; + } else { + throw std::runtime_error(fmt::format("Unsupported data format type {}", data_format)); + } +} + } // namespace holoscan::ops diff --git a/gxf_extensions/segmentation_postprocessor/segmentation_postprocessor.cu.cpp b/src/operators/segmentation_postprocessor/segmentation_postprocessor.cu similarity index 87% rename from gxf_extensions/segmentation_postprocessor/segmentation_postprocessor.cu.cpp rename to src/operators/segmentation_postprocessor/segmentation_postprocessor.cu index 1b08e141..f13d93e9 100644 --- a/gxf_extensions/segmentation_postprocessor/segmentation_postprocessor.cu.cpp +++ b/src/operators/segmentation_postprocessor/segmentation_postprocessor.cu @@ -14,10 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#include "segmentation_postprocessor.cu.hpp" -namespace nvidia { -namespace holoscan { +#include "holoscan/operators/segmentation_postprocessor/segmentation_postprocessor.cuh" + +namespace holoscan::ops { namespace segmentation_postprocessor { __forceinline__ __device__ uint32_t hwc_to_index(Shape shape, uint32_t y, uint32_t x, uint32_t c) { @@ -38,7 +38,7 @@ __forceinline__ __device__ uint32_t data_format_to_index(Shape return hwc_to_index(shape, y, x, c); } -template<> +template <> __forceinline__ __device__ uint32_t data_format_to_index(Shape shape, uint32_t y, uint32_t x, uint32_t c) { @@ -91,7 +91,8 @@ uint16_t ceil_div(uint16_t numerator, uint16_t denominator) { } void cuda_postprocess(enum NetworkOutputType network_output_type, enum DataFormat data_format, - Shape shape, const float* input, output_type_t* output) { + Shape shape, const float* input, output_type_t* output, + cudaStream_t cuda_stream) { dim3 block(32, 32, 1); dim3 grid(ceil_div(shape.width, block.x), ceil_div(shape.height, block.y), 1); switch (network_output_type) { @@ -99,15 +100,15 @@ void cuda_postprocess(enum NetworkOutputType network_output_type, enum DataForma switch (data_format) { case DataFormat::kNCHW: postprocessing_kernel - <<>>(shape, input, output); + <<>>(shape, input, output); break; case DataFormat::kHWC: postprocessing_kernel - <<>>(shape, input, output); + <<>>(shape, input, output); break; case DataFormat::kNHWC: postprocessing_kernel - <<>>(shape, input, output); + <<>>(shape, input, output); break; } break; @@ -115,15 +116,15 @@ void cuda_postprocess(enum NetworkOutputType network_output_type, enum DataForma switch (data_format) { case DataFormat::kNCHW: postprocessing_kernel - <<>>(shape, input, output); + <<>>(shape, input, output); break; case DataFormat::kHWC: postprocessing_kernel - <<>>(shape, input, output); + <<>>(shape, input, output); break; case DataFormat::kNHWC: postprocessing_kernel - <<>>(shape, input, output); + <<>>(shape, input, output); break; } break; @@ -131,5 +132,4 @@ void cuda_postprocess(enum NetworkOutputType network_output_type, enum DataForma } } // namespace segmentation_postprocessor -} // namespace holoscan -} // namespace nvidia +} // namespace holoscan::ops diff --git a/src/operators/stream_playback/video_stream_recorder.cpp b/src/operators/stream_playback/video_stream_recorder.cpp deleted file mode 100644 index 3d2262b3..00000000 --- a/src/operators/stream_playback/video_stream_recorder.cpp +++ /dev/null @@ -1,57 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "holoscan/operators/stream_playback/video_stream_recorder.hpp" - -#include "holoscan/core/fragment.hpp" -#include "holoscan/core/gxf/entity.hpp" -#include "holoscan/core/operator_spec.hpp" - -#include "holoscan/core/conditions/gxf/boolean.hpp" -#include "holoscan/core/conditions/gxf/message_available.hpp" -#include "holoscan/core/resources/gxf/video_stream_serializer.hpp" - -namespace holoscan::ops { - -void VideoStreamRecorderOp::setup(OperatorSpec& spec) { - auto& input = spec.input("input"); - - spec.param(receiver_, "receiver", "Entity receiver", "Receiver channel to log", &input); - spec.param(entity_serializer_, - "entity_serializer", - "Entity serializer", - "Serializer for serializing entities"); - spec.param(directory_, "directory", "Directory path", "Directory path for storing files"); - spec.param(basename_, "basename", "Base file name", "User specified file name without extension"); - spec.param(flush_on_tick_, - "flush_on_tick", - "Flush on tick", - "Flushes output buffer on every tick when true", - false); -} - -void VideoStreamRecorderOp::initialize() { - // Set up prerequisite parameters before calling GXFOperator::initialize() - auto frag = fragment(); - auto entity_serializer = - frag->make_resource("entity_serializer"); - add_arg(Arg("entity_serializer") = entity_serializer); - - GXFOperator::initialize(); -} - -} // namespace holoscan::ops diff --git a/src/operators/stream_playback/video_stream_replayer.cpp b/src/operators/stream_playback/video_stream_replayer.cpp deleted file mode 100644 index dc66c7f3..00000000 --- a/src/operators/stream_playback/video_stream_replayer.cpp +++ /dev/null @@ -1,94 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "holoscan/operators/stream_playback/video_stream_replayer.hpp" - -#include "holoscan/core/fragment.hpp" -#include "holoscan/core/gxf/entity.hpp" -#include "holoscan/core/operator_spec.hpp" - -#include "holoscan/core/conditions/gxf/boolean.hpp" -#include "holoscan/core/resources/gxf/video_stream_serializer.hpp" - -namespace holoscan::ops { - -void VideoStreamReplayerOp::setup(OperatorSpec& spec) { - auto& output = spec.output("output"); - - spec.param(transmitter_, - "transmitter", - "Entity transmitter", - "Transmitter channel for replaying entities", - &output); - - spec.param(entity_serializer_, - "entity_serializer", - "Entity serializer", - "Serializer for serializing entities"); - spec.param( - boolean_scheduling_term_, - "boolean_scheduling_term", - "BooleanSchedulingTerm", - "BooleanSchedulingTerm to stop the codelet from ticking after all messages are published."); - spec.param(directory_, "directory", "Directory path", "Directory path for storing files"); - spec.param(basename_, "basename", "Base file name", "User specified file name without extension"); - spec.param(batch_size_, - "batch_size", - "Batch Size", - "Number of entities to read and publish for one tick", - 1UL); - spec.param( - ignore_corrupted_entities_, - "ignore_corrupted_entities", - "Ignore Corrupted Entities", - "If an entity could not be deserialized, it is ignored by default; otherwise a failure is " - "generated.", - true); - spec.param(frame_rate_, - "frame_rate", - "Frame rate", - "Frame rate to replay. If zero value is specified, it follows timings in timestamps.", - 0.f); - spec.param(realtime_, - "realtime", - "Realtime playback", - "Playback video in realtime, based on frame_rate or timestamps (default: true).", - true); - spec.param(repeat_, "repeat", "RepeatVideo", "Repeat video stream (default: false)", false); - spec.param(count_, - "count", - "Number of frame counts to playback", - "Number of frame counts to playback. If zero value is specified, it is ignored. If " - "the count " - "is less than the number of frames in the video, it would be finished early.", - 0UL); -} - -void VideoStreamReplayerOp::initialize() { - // Set up prerequisite parameters before calling GXFOperator::initialize() - auto frag = fragment(); - auto entity_serializer = - frag->make_resource("entity_serializer"); - auto boolean_scheduling_term = - frag->make_condition("boolean_scheduling_term"); - add_arg(Arg("entity_serializer") = entity_serializer); - add_arg(Arg("boolean_scheduling_term") = boolean_scheduling_term); - - GXFOperator::initialize(); -} - -} // namespace holoscan::ops diff --git a/src/operators/tensor_rt/CMakeLists.txt b/src/operators/tensor_rt/CMakeLists.txt new file mode 100644 index 00000000..98ba5f07 --- /dev/null +++ b/src/operators/tensor_rt/CMakeLists.txt @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_holoscan_operator(tensor_rt tensor_rt_inference.cpp) + +target_link_libraries(op_tensor_rt + PUBLIC holoscan::core +) + +# Wraps the GXF extension +add_dependencies(op_tensor_rt gxf_tensor_rt) diff --git a/src/operators/tensor_rt/tensor_rt_inference.cpp b/src/operators/tensor_rt/tensor_rt_inference.cpp index 7af817d6..53d474a4 100644 --- a/src/operators/tensor_rt/tensor_rt_inference.cpp +++ b/src/operators/tensor_rt/tensor_rt_inference.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/src/operators/tool_tracking_postprocessor/tool_tracking_postprocessor.cpp b/src/operators/tool_tracking_postprocessor/tool_tracking_postprocessor.cpp deleted file mode 100644 index f2f6eab3..00000000 --- a/src/operators/tool_tracking_postprocessor/tool_tracking_postprocessor.cpp +++ /dev/null @@ -1,64 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "holoscan/operators/tool_tracking_postprocessor/tool_tracking_postprocessor.hpp" - -#include "holoscan/core/fragment.hpp" -#include "holoscan/core/gxf/entity.hpp" -#include "holoscan/core/operator_spec.hpp" - -#include "holoscan/core/conditions/gxf/boolean.hpp" -#include "holoscan/core/resources/gxf/allocator.hpp" - -namespace holoscan::ops { - -void ToolTrackingPostprocessorOp::setup(OperatorSpec& spec) { - constexpr float DEFAULT_MIN_PROB = 0.5f; - // 12 qualitative classes color scheme from colorbrewer2 - static const std::vector> DEFAULT_COLORS = {{0.12f, 0.47f, 0.71f}, - {0.20f, 0.63f, 0.17f}, - {0.89f, 0.10f, 0.11f}, - {1.00f, 0.50f, 0.00f}, - {0.42f, 0.24f, 0.60f}, - {0.69f, 0.35f, 0.16f}, - {0.65f, 0.81f, 0.89f}, - {0.70f, 0.87f, 0.54f}, - {0.98f, 0.60f, 0.60f}, - {0.99f, 0.75f, 0.44f}, - {0.79f, 0.70f, 0.84f}, - {1.00f, 1.00f, 0.60f}}; - - auto& in_tensor = spec.input("in"); - auto& out_tensor = spec.output("out"); - - spec.param(in_, "in", "Input", "Input channel.", &in_tensor); - spec.param(out_, "out", "Output", "Output channel.", &out_tensor); - - spec.param(min_prob_, "min_prob", "Minimum probability", "Minimum probability.", - DEFAULT_MIN_PROB); - - spec.param(overlay_img_colors_, - "overlay_img_colors", - "Overlay Image Layer Colors", - "Color of the image overlays, a list of RGB values with components between 0 and 1", - DEFAULT_COLORS); - - spec.param(host_allocator_, "host_allocator", "Allocator", "Output Allocator"); - spec.param(device_allocator_, "device_allocator", "Allocator", "Output Allocator"); -} - -} // namespace holoscan::ops diff --git a/src/operators/video_stream_recorder/CMakeLists.txt b/src/operators/video_stream_recorder/CMakeLists.txt new file mode 100644 index 00000000..f429f40a --- /dev/null +++ b/src/operators/video_stream_recorder/CMakeLists.txt @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_holoscan_operator(video_stream_recorder video_stream_recorder.cpp) + +target_link_libraries(op_video_stream_recorder + PUBLIC + holoscan::core + GXF::serialization +) + +# Uses the VideoStreamSerializer from the GXF extension +add_dependencies(op_video_stream_recorder gxf_stream_playback) diff --git a/src/operators/video_stream_recorder/video_stream_recorder.cpp b/src/operators/video_stream_recorder/video_stream_recorder.cpp new file mode 100644 index 00000000..4bf37069 --- /dev/null +++ b/src/operators/video_stream_recorder/video_stream_recorder.cpp @@ -0,0 +1,171 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "holoscan/operators/video_stream_recorder/video_stream_recorder.hpp" + +#include +#include + +#include "gxf/core/expected.hpp" +#include "gxf/serialization/entity_serializer.hpp" + +#include "holoscan/core/execution_context.hpp" +#include "holoscan/core/fragment.hpp" +#include "holoscan/core/io_context.hpp" +#include "holoscan/core/gxf/entity.hpp" +#include "holoscan/core/operator_spec.hpp" + +#include "holoscan/core/conditions/gxf/boolean.hpp" +#include "holoscan/core/conditions/gxf/message_available.hpp" +#include "holoscan/core/resources/gxf/video_stream_serializer.hpp" + +namespace holoscan::ops { + +void VideoStreamRecorderOp::setup(OperatorSpec& spec) { + auto& input = spec.input("input"); + + spec.param(receiver_, "receiver", "Entity receiver", "Receiver channel to log", &input); + spec.param(entity_serializer_, + "entity_serializer", + "Entity serializer", + "Serializer for serializing entities"); + spec.param(directory_, "directory", "Directory path", "Directory path for storing files"); + spec.param(basename_, "basename", "Base file name", "User specified file name without extension"); + spec.param(flush_on_tick_, + "flush_on_tick", + "Flush on tick", + "Flushes output buffer on every tick when true", + false); +} + +void VideoStreamRecorderOp::initialize() { + // Set up prerequisite parameters before calling GXFOperator::initialize() + auto frag = fragment(); + auto entity_serializer = + frag->make_resource("entity_serializer"); + add_arg(Arg("entity_serializer") = entity_serializer); + + // Operator::initialize must occur after all arguments have been added + Operator::initialize(); + + // Create path by appending receiver name to directory path if basename is not provided + std::string path = directory_.get() + '/'; + + // Note: basename was optional in the GXF operator, but not yet in the native operator, + // so in practice, this should always have a value. + if (basename_.has_value()) { + path += basename_.get(); + } else { + path += receiver_.get()->name(); + } + + // Initialize index file stream as write-only + index_file_stream_ = + nvidia::gxf::FileStream("", path + nvidia::gxf::FileStream::kIndexFileExtension); + + // Initialize binary file stream as write-only + binary_file_stream_ = + nvidia::gxf::FileStream("", path + nvidia::gxf::FileStream::kBinaryFileExtension); + + // Open index file stream + nvidia::gxf::Expected result = index_file_stream_.open(); + if (!result) { + auto code = nvidia::gxf::ToResultCode(result); + throw std::runtime_error(fmt::format("Failed to open index_file_stream_ with code: {}", code)); + } + + // Open binary file stream + result = binary_file_stream_.open(); + if (!result) { + auto code = nvidia::gxf::ToResultCode(result); + throw std::runtime_error(fmt::format("Failed to open binary_file_stream_ with code: {}", code)); + } + binary_file_offset_ = 0; +} + +VideoStreamRecorderOp::~VideoStreamRecorderOp() { + // for the GXF codelet, this code is in a deinitialize() method + + // Close binary file stream + nvidia::gxf::Expected result = binary_file_stream_.close(); + if (!result) { + auto code = nvidia::gxf::ToResultCode(result); + HOLOSCAN_LOG_ERROR("Failed to close binary_file_stream_ with code: {}", code); + } + + // Close index file stream + result = index_file_stream_.close(); + if (!result) { + auto code = nvidia::gxf::ToResultCode(result); + HOLOSCAN_LOG_ERROR("Failed to close index_file_stream_ with code: {}", code); + } +} + +void VideoStreamRecorderOp::compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) { + // avoid warning about unused variable + (void)op_output; + + auto entity = op_input.receive("input"); + + // dynamic cast from holoscan::Resource to holoscan::VideoStreamSerializer + auto vs_serializer = + std::dynamic_pointer_cast(entity_serializer_.get()); + // get the Handle to the underlying GXF EntitySerializer + auto entity_serializer = nvidia::gxf::Handle::Create( + context.context(), vs_serializer.get()->gxf_cid()); + nvidia::gxf::Expected size = + entity_serializer.value()->serializeEntity(entity, &binary_file_stream_); + if (!size) { + auto code = nvidia::gxf::ToResultCode(size); + throw std::runtime_error(fmt::format("Failed to serialize entity with code {}", code)); + } + + // Create entity index + nvidia::gxf::EntityIndex index; + index.log_time = std::chrono::system_clock::now().time_since_epoch().count(); + index.data_size = size.value(); + index.data_offset = binary_file_offset_; + + // Write entity index to index file + nvidia::gxf::Expected result = index_file_stream_.writeTrivialType(&index); + if (!result) { + auto code = nvidia::gxf::ToResultCode(size); + throw std::runtime_error(fmt::format("Failed writing to index file stream with code {}", code)); + } + binary_file_offset_ += size.value(); + + if (flush_on_tick_) { + // Flush binary file output stream + nvidia::gxf::Expected result = binary_file_stream_.flush(); + if (!result) { + auto code = nvidia::gxf::ToResultCode(size); + throw std::runtime_error( + fmt::format("Failed writing to index file stream with code {}", code)); + } + + // Flush index file output stream + result = index_file_stream_.flush(); + if (!result) { + auto code = nvidia::gxf::ToResultCode(size); + throw std::runtime_error( + fmt::format("Failed writing to index file stream with code {}", code)); + } + } +} + +} // namespace holoscan::ops diff --git a/src/operators/video_stream_replayer/CMakeLists.txt b/src/operators/video_stream_replayer/CMakeLists.txt new file mode 100644 index 00000000..b657dccc --- /dev/null +++ b/src/operators/video_stream_replayer/CMakeLists.txt @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_holoscan_operator(video_stream_replayer video_stream_replayer.cpp) + +target_link_libraries(op_video_stream_replayer + PUBLIC + holoscan::core + GXF::serialization +) + +# Uses the VideoStreamSerializer from the GXF extension +add_dependencies(op_video_stream_replayer gxf_stream_playback) diff --git a/src/operators/video_stream_replayer/video_stream_replayer.cpp b/src/operators/video_stream_replayer/video_stream_replayer.cpp new file mode 100644 index 00000000..f2b458a1 --- /dev/null +++ b/src/operators/video_stream_replayer/video_stream_replayer.cpp @@ -0,0 +1,280 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "holoscan/operators/video_stream_replayer/video_stream_replayer.hpp" + +#include +#include +#include +#include + +#include "gxf/core/expected.hpp" +#include "gxf/serialization/entity_serializer.hpp" + +#include "holoscan/core/conditions/gxf/boolean.hpp" +#include "holoscan/core/execution_context.hpp" +#include "holoscan/core/executor.hpp" +#include "holoscan/core/fragment.hpp" +#include "holoscan/core/gxf/entity.hpp" +#include "holoscan/core/operator_spec.hpp" +#include "holoscan/core/resources/gxf/video_stream_serializer.hpp" + + +namespace holoscan::ops { + +void VideoStreamReplayerOp::setup(OperatorSpec& spec) { + auto& output = spec.output("output"); + + spec.param(transmitter_, + "transmitter", + "Entity transmitter", + "Transmitter channel for replaying entities", + &output); + + spec.param(entity_serializer_, + "entity_serializer", + "Entity serializer", + "Serializer for serializing entities"); + spec.param( + boolean_scheduling_term_, + "boolean_scheduling_term", + "BooleanSchedulingTerm", + "BooleanSchedulingTerm to stop the codelet from ticking after all messages are published."); + spec.param(directory_, "directory", "Directory path", "Directory path for storing files"); + spec.param(basename_, "basename", "Base file name", "User specified file name without extension"); + spec.param(batch_size_, + "batch_size", + "Batch Size", + "Number of entities to read and publish for one tick", + 1UL); + spec.param( + ignore_corrupted_entities_, + "ignore_corrupted_entities", + "Ignore Corrupted Entities", + "If an entity could not be deserialized, it is ignored by default; otherwise a failure is " + "generated.", + true); + spec.param(frame_rate_, + "frame_rate", + "Frame rate", + "Frame rate to replay. If zero value is specified, it follows timings in timestamps.", + 0.f); + spec.param(realtime_, + "realtime", + "Realtime playback", + "Playback video in realtime, based on frame_rate or timestamps (default: true).", + true); + spec.param(repeat_, "repeat", "RepeatVideo", "Repeat video stream (default: false)", false); + spec.param(count_, + "count", + "Number of frame counts to playback", + "Number of frame counts to playback. If zero value is specified, it is ignored. If " + "the count " + "is less than the number of frames in the video, it would be finished early.", + 0UL); +} + +void VideoStreamReplayerOp::initialize() { + // Set up prerequisite parameters before calling GXFOperator::initialize() + auto frag = fragment(); + auto entity_serializer = + frag->make_resource("entity_serializer"); + add_arg(Arg("entity_serializer") = entity_serializer); + + // Find if there is an argument for 'boolean_scheduling_term' + auto has_boolean_scheduling_term = + std::find_if(args().begin(), args().end(), [](const auto& arg) { + return (arg.name() == "boolean_scheduling_term"); + }); + // Create the BooleanCondition if there is no argument provided. + if (has_boolean_scheduling_term == args().end()) { + boolean_scheduling_term_ = + frag->make_condition("boolean_scheduling_term"); + add_arg(boolean_scheduling_term_.get()); + } + + // Operator::initialize must occur after all arguments have been added + Operator::initialize(); + + // Create path by appending component name to directory path if basename is not provided + std::string path = directory_.get() + '/'; + + // Note: basename was optional in the GXF operator, but not yet in the native operator, + // so in practice, this should always have a value. + if (basename_.has_value()) { + path += basename_.get(); + } else { + path += name(); + } + + // Filenames for index and data + const std::string index_filename = path + nvidia::gxf::FileStream::kIndexFileExtension; + const std::string entity_filename = path + nvidia::gxf::FileStream::kBinaryFileExtension; + + // Open index file stream as read-only + index_file_stream_ = nvidia::gxf::FileStream(index_filename, ""); + nvidia::gxf::Expected result = index_file_stream_.open(); + if (!result) { + HOLOSCAN_LOG_WARN("Could not open index file: {}", index_filename); + auto code = nvidia::gxf::ToResultCode(result); + throw std::runtime_error(fmt::format("File open failed with code: {}", code)); + } + + // Open entity file stream as read-only + entity_file_stream_ = nvidia::gxf::FileStream(entity_filename, ""); + result = entity_file_stream_.open(); + if (!result) { + HOLOSCAN_LOG_WARN("Could not open entity file: {}", entity_filename); + auto code = nvidia::gxf::ToResultCode(result); + throw std::runtime_error(fmt::format("File open failed with code: {}", code)); + } + + boolean_scheduling_term_.get()->enable_tick(); + + playback_index_ = 0; + playback_count_ = 0; + index_start_timestamp_ = 0; + index_last_timestamp_ = 0; + index_timestamp_duration_ = 0; + index_frame_count_ = 1; + playback_start_timestamp_ = 0; +} + +VideoStreamReplayerOp::~VideoStreamReplayerOp() { + // for the GXF codelet, this code is in a deinitialize() method + + // Close binary file stream + nvidia::gxf::Expected result = entity_file_stream_.close(); + if (!result) { + auto code = nvidia::gxf::ToResultCode(result); + HOLOSCAN_LOG_ERROR("Failed to close entity_file_stream_ with code: {}", code); + } + + // Close index file stream + result = index_file_stream_.close(); + if (!result) { + auto code = nvidia::gxf::ToResultCode(result); + HOLOSCAN_LOG_ERROR("Failed to close index_file_stream_ with code: {}", code); + } +} + +void VideoStreamReplayerOp::compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) { + // avoid warning about unused variable + (void)op_input; + + for (size_t i = 0; i < batch_size_; i++) { + // Read entity index from index file + // Break if index not found and clear stream errors + nvidia::gxf::EntityIndex index; + nvidia::gxf::Expected size = index_file_stream_.readTrivialType(&index); + if (!size) { + if (repeat_) { + // Rewind index stream + index_file_stream_.clear(); + if (!index_file_stream_.setReadOffset(0)) { + HOLOSCAN_LOG_ERROR("Could not rewind index file"); + } + size = index_file_stream_.readTrivialType(&index); + + // Rewind entity stream + entity_file_stream_.clear(); + if (!entity_file_stream_.setReadOffset(0)) { + HOLOSCAN_LOG_ERROR("Could not rewind entity file"); + } + + // Initialize the frame index + playback_index_ = 0; + } + } + if ((!size && !repeat_) || (count_ > 0 && playback_count_ >= count_)) { + HOLOSCAN_LOG_INFO("Reach end of file or playback count reaches to the limit. Stop ticking."); + auto bool_cond = boolean_scheduling_term_.get(); + bool_cond->disable_tick(); + index_file_stream_.clear(); + break; + } + + // dynamic cast from holoscan::Resource to holoscan::VideoStreamSerializer + auto vs_serializer = + std::dynamic_pointer_cast(entity_serializer_.get()); + // get underlying GXF EntitySerializer + auto entity_serializer = nvidia::gxf::Handle::Create( + context.context(), vs_serializer.get()->gxf_cid()); + // Read entity from binary file + nvidia::gxf::Expected entity = + entity_serializer.value()->deserializeEntity(context.context(), &entity_file_stream_); + if (!entity) { + if (ignore_corrupted_entities_) { + continue; + } else { + auto code = nvidia::gxf::ToResultCode(entity); + throw std::runtime_error( + fmt::format("failed reading entity from entity_file_stream with code {}", code)); + } + } + + int64_t time_to_delay = 0; + + if (playback_count_ == 0) { + playback_start_timestamp_ = std::chrono::system_clock::now().time_since_epoch().count(); + index_start_timestamp_ = index.log_time; + } + // Update last timestamp + if (index.log_time > index_last_timestamp_) { + index_last_timestamp_ = index.log_time; + index_frame_count_ = playback_count_ + 1; + index_timestamp_duration_ = index_last_timestamp_ - index_start_timestamp_; + } + + // Delay if realtime is specified + if (realtime_) { + // Calculate the delay time based on frame rate or timestamps. + uint64_t current_timestamp = std::chrono::system_clock::now().time_since_epoch().count(); + int64_t time_delta = static_cast(current_timestamp - playback_start_timestamp_); + if (frame_rate_ > 0.f) { + time_to_delay = + static_cast(1000000000 / frame_rate_) * playback_count_ - time_delta; + } else { + // Get timestamp from entity + uint64_t timestamp = index.log_time; + time_to_delay = static_cast((timestamp - index_start_timestamp_) + + index_timestamp_duration_ * + (playback_count_ / index_frame_count_)) - + time_delta; + } + if (time_to_delay < 0 && (playback_count_ % index_frame_count_ != 0)) { + HOLOSCAN_LOG_INFO( + fmt::format("Playing video stream is lagging behind (count: % {} , delay: {} ns)", + playback_count_, + time_to_delay)); + } + } + + if (time_to_delay > 0) { std::this_thread::sleep_for(std::chrono::nanoseconds(time_to_delay)); } + + // emit the entity + auto result = gxf::Entity(std::move(entity.value())); + op_output.emit(result); + + // Increment frame counter and index + ++playback_count_; + ++playback_index_; + } +} + +} // namespace holoscan::ops diff --git a/src/operators/visualizer_icardio/visualizer_icardio.cpp b/src/operators/visualizer_icardio/visualizer_icardio.cpp deleted file mode 100644 index f799411c..00000000 --- a/src/operators/visualizer_icardio/visualizer_icardio.cpp +++ /dev/null @@ -1,56 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "holoscan/operators/visualizer_icardio/visualizer_icardio.hpp" -#include "holoscan/core/gxf/entity.hpp" -#include "holoscan/core/operator_spec.hpp" -#include "holoscan/core/resources/gxf/allocator.hpp" - -namespace holoscan::ops { - -void VisualizerICardioOp::setup(OperatorSpec& spec) { - auto& out_tensor_1 = spec.output("keypoints"); - auto& out_tensor_2 = spec.output("keyarea_1"); - auto& out_tensor_3 = spec.output("keyarea_2"); - auto& out_tensor_4 = spec.output("keyarea_3"); - auto& out_tensor_5 = spec.output("keyarea_4"); - auto& out_tensor_6 = spec.output("keyarea_5"); - auto& out_tensor_7 = spec.output("lines"); - auto& out_tensor_8 = spec.output("logo"); - - spec.param( - in_tensor_names_, "in_tensor_names", "Input Tensors", "Input tensors", {std::string("")}); - spec.param( - out_tensor_names_, "out_tensor_names", "Output Tensors", "Output tensors", {std::string("")}); - spec.param(input_on_cuda_, "input_on_cuda", "Input buffer on CUDA", "", false); - spec.param(allocator_, "allocator", "Allocator", "Output Allocator"); - spec.param(receivers_, "receivers", "Receivers", "List of receivers", {}); - spec.param(transmitters_, - "transmitters", - "Transmitters", - "List of transmitters", - {&out_tensor_1, - &out_tensor_2, - &out_tensor_3, - &out_tensor_4, - &out_tensor_5, - &out_tensor_6, - &out_tensor_7, - &out_tensor_8}); -} - -} // namespace holoscan::ops diff --git a/src/operators/visualizer_tool_tracking/visualizer_tool_tracking.cpp b/src/operators/visualizer_tool_tracking/visualizer_tool_tracking.cpp deleted file mode 100644 index 585c263f..00000000 --- a/src/operators/visualizer_tool_tracking/visualizer_tool_tracking.cpp +++ /dev/null @@ -1,175 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "holoscan/operators/visualizer_tool_tracking/visualizer_tool_tracking.hpp" - -#include "holoscan/core/fragment.hpp" -#include "holoscan/core/gxf/entity.hpp" -#include "holoscan/core/operator_spec.hpp" - -#include "holoscan/core/conditions/gxf/boolean.hpp" -#include "holoscan/core/resources/gxf/allocator.hpp" - -namespace holoscan::ops { - -void ToolTrackingVizOp::setup(OperatorSpec& spec) { - constexpr int32_t DEFAULT_SRC_WIDTH = 640; - constexpr int32_t DEFAULT_SRC_HEIGHT = 480; - constexpr int16_t DEFAULT_SRC_CHANNELS = 3; - constexpr uint8_t DEFAULT_SRC_BYTES_PER_PIXEL = 1; - // 12 qualitative classes color scheme from colorbrewer2 - static const std::vector> DEFAULT_COLORS = {{0.12f, 0.47f, 0.71f}, - {0.20f, 0.63f, 0.17f}, - {0.89f, 0.10f, 0.11f}, - {1.00f, 0.50f, 0.00f}, - {0.42f, 0.24f, 0.60f}, - {0.69f, 0.35f, 0.16f}, - {0.65f, 0.81f, 0.89f}, - {0.70f, 0.87f, 0.54f}, - {0.98f, 0.60f, 0.60f}, - {0.99f, 0.75f, 0.44f}, - {0.79f, 0.70f, 0.84f}, - {1.00f, 1.00f, 0.60f}}; - - auto& in_source_video = spec.input("source_video"); - auto& in_tensor = spec.input("tensor"); - auto& overlay_buffer_input = spec.input("overlay_buffer_input") - .condition(ConditionType::kNone); - auto& overlay_buffer_output = spec.output("overlay_buffer_output") - .condition(ConditionType::kNone); - - spec.param(in_, "in", "Input", "List of input channels", {&in_source_video, &in_tensor}); - - spec.param(videoframe_vertex_shader_path_, - "videoframe_vertex_shader_path", - "Videoframe GLSL Vertex Shader File Path", - "Path to vertex shader to be loaded"); - spec.param(videoframe_fragment_shader_path_, - "videoframe_fragment_shader_path", - "Videoframe GLSL Fragment Shader File Path", - "Path to fragment shader to be loaded"); - - spec.param(tooltip_vertex_shader_path_, - "tooltip_vertex_shader_path", - "Tool tip GLSL Vertex Shader File Path", - "Path to vertex shader to be loaded"); - spec.param(tooltip_fragment_shader_path_, - "tooltip_fragment_shader_path", - "Tool tip GLSL Fragment Shader File Path", - "Path to fragment shader to be loaded"); - spec.param( - num_tool_classes_, "num_tool_classes", "Tool Classes", "Number of different tool classes"); - spec.param(num_tool_pos_components_, - "num_tool_pos_components", - "Position Components", - "Number of components of the tool position vector", - 2); - spec.param(tool_tip_colors_, - "tool_tip_colors", - "Tool Tip Colors", - "Color of the tool tips, a list of RGB values with components between 0 and 1", - DEFAULT_COLORS); - - spec.param(overlay_img_vertex_shader_path_, - "overlay_img_vertex_shader_path", - "Overlay Image GLSL Vertex Shader File Path", - "Path to vertex shader to be loaded"); - spec.param(overlay_img_fragment_shader_path_, - "overlay_img_fragment_shader_path", - "Overlay Image GLSL Fragment Shader File Path", - "Path to fragment shader to be loaded"); - spec.param( - overlay_img_width_, "overlay_img_width", "Overlay Image Width", "Width of overlay image"); - spec.param( - overlay_img_height_, "overlay_img_height", "Overlay Image Height", "Height of overlay image"); - spec.param(overlay_img_channels_, - "overlay_img_channels", - "Number of Overlay Image Channels", - "Number of Overlay Image Channels"); - spec.param(overlay_img_layers_, - "overlay_img_layers", - "Number of Overlay Image Layers", - "Number of Overlay Image Layers"); - spec.param(overlay_img_colors_, - "overlay_img_colors", - "Overlay Image Layer Colors", - "Color of the image overlays, a list of RGB values with components between 0 and 1", - DEFAULT_COLORS); - - spec.param( - tool_labels_, - "tool_labels", - "Tool Labels", - "List of tool names.", - {}); // Default handled in instrument_label to dynamically adjust for the number of tools - spec.param(label_sans_font_path_, - "label_sans_font_path", - "File path for sans font for displaying tool name", - "Path for sans font to be loaded"); - spec.param(label_sans_bold_font_path_, - "label_sans_bold_font_path", - "File path for sans bold font for displaying tool name", - "Path for sans bold font to be loaded"); - - spec.param(in_tensor_names_, - "in_tensor_names", - "Input Tensor Names", - "Names of input tensors.", - {std::string("")}); - spec.param(in_width_, "in_width", "InputWidth", "Width of the image.", DEFAULT_SRC_WIDTH); - spec.param(in_height_, "in_height", "InputHeight", "Height of the image.", DEFAULT_SRC_HEIGHT); - spec.param( - in_channels_, "in_channels", "InputChannels", "Number of channels.", DEFAULT_SRC_CHANNELS); - spec.param(in_bytes_per_pixel_, - "in_bytes_per_pixel", - "InputBytesPerPixel", - "Number of bytes per pexel of the image.", - DEFAULT_SRC_BYTES_PER_PIXEL); - spec.param(alpha_value_, - "alpha_value", - "Alpha value", - "Alpha value that can be used when converting RGB888 to RGBA8888.", - static_cast(255)); - spec.param(pool_, "pool", "Pool", "Pool to allocate the output message."); - spec.param( - window_close_scheduling_term_, - "window_close_scheduling_term", - "WindowCloseSchedulingTerm", - "BooleanSchedulingTerm to stop the codelet from ticking after all messages are published."); - spec.param(overlay_buffer_input_, - "overlay_buffer_input", - "OverlayBufferInput", - "Input for an empty overlay buffer.", - &overlay_buffer_input); - spec.param(overlay_buffer_output_, - "overlay_buffer_output", - "OverlayBufferOutput", - "Output for a filled overlay buffer.", - &overlay_buffer_output); -} - -void ToolTrackingVizOp::initialize() { - // Set up prerequisite parameters before calling GXFOperator::initialize() - auto frag = fragment(); - auto window_close_scheduling_term = - frag->make_condition("window_close_scheduling_term"); - add_arg(Arg("window_close_scheduling_term") = window_close_scheduling_term); - - GXFOperator::initialize(); -} - -} // namespace holoscan::ops diff --git a/src/utils/holoinfer_utils.cpp b/src/utils/holoinfer_utils.cpp new file mode 100644 index 00000000..95f3fec4 --- /dev/null +++ b/src/utils/holoinfer_utils.cpp @@ -0,0 +1,321 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HOLOSCAN_UTILS_HOLOINFER_HPP +#define HOLOSCAN_UTILS_HOLOINFER_HPP + +#include +#include +#include +#include + +#include "holoscan/utils/holoinfer_utils.hpp" +#include "holoscan/core/io_context.hpp" + +#include "gxf/std/tensor.hpp" // TODO: remove? + +#include +#include +#include + +namespace HoloInfer = holoscan::inference; + +namespace holoscan::utils { + +gxf_result_t multiai_get_data_per_model(InputContext& op_input, + const std::vector& in_tensors, + HoloInfer::DataMap& data_per_input_tensor, + std::map>& dims_per_tensor, + bool cuda_buffer_out, const std::string& module) { + try { + HoloInfer::TimePoint s_time, e_time; + HoloInfer::timer_init(s_time); + + nvidia::gxf::MemoryStorageType to = nvidia::gxf::MemoryStorageType::kHost; + + if (cuda_buffer_out) { to = nvidia::gxf::MemoryStorageType::kDevice; } + + auto messages = op_input.receive>("receivers"); + for (unsigned int i = 0; i < in_tensors.size(); ++i) { + // nvidia::gxf::Handle in_tensor; + std::shared_ptr in_tensor; + for (unsigned int j = 0; j < messages.size(); ++j) { + const auto& in_message = messages[j]; + const auto maybe_tensor = in_message.get(in_tensors[i].c_str(), false); + if (maybe_tensor) { + // break out if the expected tensor name was found in this message + in_tensor = maybe_tensor; + break; + } + } + if (!in_tensor) + return HoloInfer::report_error(module, + "Data_per_tensor, Tensor " + in_tensors[i] + " not found"); + + // convert from Tensor to GXFTensor so code below can be re-used as-is. + // (otherwise cannot easily get element_type, storage_type) + holoscan::gxf::GXFTensor in_tensor_gxf{in_tensor.get()->dl_ctx()}; + void* in_tensor_data = in_tensor_gxf.pointer(); + + auto element_type = in_tensor_gxf.element_type(); + auto storage_type = in_tensor_gxf.storage_type(); + + if (element_type != nvidia::gxf::PrimitiveType::kFloat32) { + return HoloInfer::report_error(module, "Data extraction, element type not supported"); + } + if (!(storage_type != nvidia::gxf::MemoryStorageType::kHost || + storage_type != nvidia::gxf::MemoryStorageType::kDevice)) { + return HoloInfer::report_error(module, + "Data extraction, memory not resident on CPU or GPU"); + } + if (to != nvidia::gxf::MemoryStorageType::kHost && + to != nvidia::gxf::MemoryStorageType::kDevice) { + return HoloInfer::report_error(module, + "Input storage type in data extraction not supported"); + } + + std::vector dims; + for (unsigned int i = 0; i < in_tensor_gxf.shape().rank(); ++i) + dims.push_back(in_tensor_gxf.shape().dimension(i)); + + size_t buffer_size = std::accumulate(dims.begin(), dims.end(), 1, std::multiplies()); + + if (data_per_input_tensor.find(in_tensors[i]) == data_per_input_tensor.end()) { + auto db = std::make_shared(); + db->host_buffer.resize(buffer_size); + db->device_buffer->resize(buffer_size); + + data_per_input_tensor.insert({in_tensors[i], std::move(db)}); + } + + dims_per_tensor[in_tensors[i]] = std::move(dims); + + if (to == nvidia::gxf::MemoryStorageType::kHost) { + std::vector in_tensor_ptr(buffer_size, 0); + + if (storage_type == nvidia::gxf::MemoryStorageType::kDevice) { + cudaError_t cuda_result = cudaMemcpy(static_cast(in_tensor_ptr.data()), + static_cast(in_tensor_data), + buffer_size * sizeof(float), + cudaMemcpyDeviceToHost); + if (cuda_result != cudaSuccess) { + return HoloInfer::report_error(module, + "Data extraction, error in DtoH cudaMemcpy::" + + std::string(cudaGetErrorString(cuda_result))); + } + + } else if (storage_type == nvidia::gxf::MemoryStorageType::kHost) { + memcpy(in_tensor_ptr.data(), + static_cast(in_tensor_data), + buffer_size * sizeof(float)); + } + data_per_input_tensor.at(in_tensors[i])->host_buffer = std::move(in_tensor_ptr); + + } else { + if (to == nvidia::gxf::MemoryStorageType::kDevice) { + void* device_buff = data_per_input_tensor.at(in_tensors[i])->device_buffer->data(); + if (storage_type == nvidia::gxf::MemoryStorageType::kDevice) { + cudaError_t cuda_result = cudaMemcpy(static_cast(device_buff), + static_cast(in_tensor_data), + buffer_size * sizeof(float), + cudaMemcpyDeviceToDevice); + + if (cuda_result != cudaSuccess) { + return HoloInfer::report_error(module, + "Data extraction, error in DtoD cudaMemcpy::" + + std::string(cudaGetErrorString(cuda_result))); + } + } else { + return HoloInfer::report_error(module, "Data extraction parameters not supported."); + } + } + } + } + + HoloInfer::timer_init(e_time); + } catch (std::exception& _ex) { + return HoloInfer::report_error(module, "Data_per_tensor, Message: " + std::string(_ex.what())); + } catch (...) { return HoloInfer::report_error(module, "Data_per_tensor, Unknown exception"); } + return GXF_SUCCESS; +} + +gxf_result_t multiai_transmit_data_per_model( + gxf_context_t& cont, const HoloInfer::Mappings& model_to_tensor_map, + HoloInfer::DataMap& input_data_map, OutputContext& op_output, + const std::vector& out_tensors, HoloInfer::DimType& tensor_out_dims_map, + bool cuda_buffer_in, bool cuda_buffer_out, const nvidia::gxf::PrimitiveType& element_type, + const nvidia::gxf::Handle& allocator_, const std::string& module) { + try { + if (element_type != nvidia::gxf::PrimitiveType::kFloat32) { + return HoloInfer::report_error(module, "Element type to be transmitted not supported"); + } + + nvidia::gxf::MemoryStorageType from = nvidia::gxf::MemoryStorageType::kHost; + nvidia::gxf::MemoryStorageType to = nvidia::gxf::MemoryStorageType::kHost; + + if (cuda_buffer_in) { from = nvidia::gxf::MemoryStorageType::kDevice; } + if (cuda_buffer_out) { to = nvidia::gxf::MemoryStorageType::kDevice; } + + HoloInfer::TimePoint s_time, e_time; + HoloInfer::timer_init(s_time); + + // single transmitter used + auto out_message = nvidia::gxf::Entity::New(cont); + if (!out_message) { + return HoloInfer::report_error(module, "Inference Toolkit, Out message allocation"); + } + + for (unsigned int i = 0; i < out_tensors.size(); ++i) { + if (input_data_map.find(out_tensors[i]) == input_data_map.end()) + return HoloInfer::report_error( + module, "Inference Toolkit, Mapped data not found for " + out_tensors[i]); + + std::string key_name{""}; + + for (const auto& key_to_tensor : model_to_tensor_map) { + if (key_to_tensor.second.compare(out_tensors[i]) == 0) key_name = key_to_tensor.first; + } + if (key_name.length() == 0) + return HoloInfer::report_error(module, "Tensor mapping not found in model to tensor map"); + + if (tensor_out_dims_map.find(key_name) == tensor_out_dims_map.end()) + return HoloInfer::report_error(module, + "Tensor mapping not found in dimension map for " + key_name); + + auto out_tensor = out_message.value().add(out_tensors[i].c_str()); + if (!out_tensor) + return HoloInfer::report_error(module, "Inference Toolkit, Out tensor allocation"); + + std::vector dims = tensor_out_dims_map.at(key_name); + + nvidia::gxf::Shape output_shape; + if (dims.size() == 4) { + std::array dimarray; + for (size_t i = 0; i < 4; ++i) { dimarray[i] = (static_cast(dims[i])); } + nvidia::gxf::Shape out_shape(dimarray); + output_shape = std::move(out_shape); + } + if (dims.size() == 2) { + nvidia::gxf::Shape out_shape{static_cast(dims[0]), static_cast(dims[1])}; + output_shape = std::move(out_shape); + } + + size_t buffer_size = std::accumulate(dims.begin(), dims.end(), 1, std::multiplies()); + + if (from == nvidia::gxf::MemoryStorageType::kHost) { + if (to == nvidia::gxf::MemoryStorageType::kHost) { + out_tensor.value()->reshape( + output_shape, nvidia::gxf::MemoryStorageType::kHost, allocator_); + if (!out_tensor.value()->pointer()) + return HoloInfer::report_error(module, + "Inference Toolkit, Out tensor buffer allocation"); + + nvidia::gxf::Expected out_tensor_data = out_tensor.value()->data(); + if (!out_tensor_data) + return HoloInfer::report_error(module, "Inference Toolkit, Getting out tensor data"); + + auto current_model_output = input_data_map.at(out_tensors[i]); + memcpy(out_tensor_data.value(), + current_model_output->host_buffer.data(), + buffer_size * sizeof(float)); + } else { // to is on device + out_tensor.value()->reshape( + output_shape, nvidia::gxf::MemoryStorageType::kDevice, allocator_); + if (!out_tensor.value()->pointer()) + return HoloInfer::report_error(module, + "Inference Toolkit, Out tensor buffer allocation"); + + nvidia::gxf::Expected out_tensor_data = out_tensor.value()->data(); + if (!out_tensor_data) + return HoloInfer::report_error(module, "Inference Toolkit, Getting out tensor data"); + + auto current_model_dev_buff = input_data_map.at(out_tensors[i])->host_buffer.data(); + cudaError_t cuda_result = cudaMemcpy(static_cast(out_tensor_data.value()), + static_cast(current_model_dev_buff), + buffer_size * sizeof(float), + cudaMemcpyHostToDevice); + if (cuda_result != cudaSuccess) { + return HoloInfer::report_error("Inference Toolkit", + "Data extraction, error in DtoD cudaMemcpy::" + + std::string(cudaGetErrorString(cuda_result))); + } + } + } else { + if (from == nvidia::gxf::MemoryStorageType::kDevice) { + if (to == nvidia::gxf::MemoryStorageType::kDevice) { + out_tensor.value()->reshape( + output_shape, nvidia::gxf::MemoryStorageType::kDevice, allocator_); + if (!out_tensor.value()->pointer()) + return HoloInfer::report_error(module, + "Inference Toolkit, Out tensor buffer allocation"); + + nvidia::gxf::Expected out_tensor_data = out_tensor.value()->data(); + if (!out_tensor_data) + return HoloInfer::report_error(module, "Inference Toolkit, Getting out tensor data"); + + void* current_model_dev_buff = input_data_map.at(out_tensors[i])->device_buffer->data(); + cudaError_t cuda_result = cudaMemcpy(static_cast(out_tensor_data.value()), + static_cast(current_model_dev_buff), + buffer_size * sizeof(float), + cudaMemcpyDeviceToDevice); + if (cuda_result != cudaSuccess) { + return HoloInfer::report_error("Inference Toolkit", + "Data extraction, error in DtoD cudaMemcpy::" + + std::string(cudaGetErrorString(cuda_result))); + } + } else { // to is on host + out_tensor.value()->reshape( + output_shape, nvidia::gxf::MemoryStorageType::kHost, allocator_); + if (!out_tensor.value()->pointer()) + return HoloInfer::report_error(module, + "Inference Toolkit, Out tensor buffer allocation"); + + nvidia::gxf::Expected out_tensor_data = out_tensor.value()->data(); + if (!out_tensor_data) + return HoloInfer::report_error(module, "Inference Toolkit, Getting out tensor data"); + + void* current_model_dev_buff = input_data_map.at(out_tensors[i])->device_buffer->data(); + cudaError_t cuda_result = cudaMemcpy(static_cast(out_tensor_data.value()), + static_cast(current_model_dev_buff), + buffer_size * sizeof(float), + cudaMemcpyDeviceToHost); + if (cuda_result != cudaSuccess) { + return HoloInfer::report_error("Inference Toolkit", + "Data extraction, error in DtoH cudaMemcpy::" + + std::string(cudaGetErrorString(cuda_result))); + } + } + } + } + } + + // single transmitter used + auto result = gxf::Entity(std::move(out_message.value())); + op_output.emit(result); + HoloInfer::timer_init(e_time); + } catch (std::exception& _ex) { + return HoloInfer::report_error(module, + "Data_transmission_tensor, Message: " + std::string(_ex.what())); + } catch (...) { + return HoloInfer::report_error(module, "Data_transmission_tensor, Unknown exception"); + } + return GXF_SUCCESS; +} + +} // namespace holoscan::utils + +#endif /* HOLOSCAN_UTILS_HOLOINFER_HPP */ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 40268ccf..252935e9 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,9 +19,9 @@ # This function takes in a test name and test source and handles setting all of the associated # properties and linking to build the test function(ConfigureTest CMAKE_TEST_NAME) - + add_executable(${CMAKE_TEST_NAME} ${ARGN}) - + set(BIN_DIR ${${HOLOSCAN_PACKAGE_NAME}_BINARY_DIR}) set_target_properties( @@ -36,7 +36,7 @@ function(ConfigureTest CMAKE_TEST_NAME) target_link_libraries(${CMAKE_TEST_NAME} PRIVATE - ${HOLOSCAN_PACKAGE_NAME} + holoscan::core GTest::gmock_main GTest::gtest_main ) @@ -44,7 +44,7 @@ function(ConfigureTest CMAKE_TEST_NAME) # Run the test from the main bin directory to access data as needed add_test(NAME ${CMAKE_TEST_NAME} COMMAND ${CMAKE_TEST_NAME} WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) set_property(TEST ${CMAKE_TEST_NAME} PROPERTY ENVIRONMENT - "LD_LIBRARY_PATH=${BIN_DIR}/lib:${BIN_DIR}/gxf_extensions/aja:${BIN_DIR}/gxf_extensions/custom_lstm_inference:${BIN_DIR}/gxf_extensions/format_converter:${BIN_DIR}/gxf_extensions/holoviz:${BIN_DIR}/gxf_extensions/segmentation_postprocessor:${BIN_DIR}/gxf_extensions/stream_playback:${BIN_DIR}/gxf_extensions/tensor_rt:${BIN_DIR}/gxf_extensions/tool_tracking_postprocessor:${BIN_DIR}/gxf_extensions/visualizer_tool_tracking:$ENV{LD_LIBRARY_PATH}") + "LD_LIBRARY_PATH=${BIN_DIR}/lib:${BIN_DIR}/gxf_extensions/aja:${BIN_DIR}/gxf_extensions/format_converter:${BIN_DIR}/gxf_extensions/holoviz:${BIN_DIR}/gxf_extensions/stream_playback:${BIN_DIR}/gxf_extensions/tensor_rt:$ENV{LD_LIBRARY_PATH}") install( TARGETS ${CMAKE_TEST_NAME} @@ -58,7 +58,6 @@ endfunction() # * core tests ---------------------------------------------------------------------------------- ConfigureTest( CORE_TEST - core/application.cpp core/arg.cpp core/argument_setter.cpp core/component.cpp @@ -69,22 +68,54 @@ ConfigureTest( core/fragment.cpp core/io_spec.cpp core/logger.cpp - core/operator_classes.cpp + core/operator_spec.cpp core/parameter.cpp core/resource.cpp core/resource_classes.cpp - config.hpp - utils.hpp + ) + +# ################################################################################################## +# * operator classes tests ---------------------------------------------------------------------------------- +ConfigureTest(OPERATORS_CLASSES_TEST + operators/operator_classes.cpp +) +target_link_libraries(OPERATORS_CLASSES_TEST + PRIVATE + holoscan::ops::aja + holoscan::ops::bayer_demosaic + holoscan::ops::format_converter + holoscan::ops::holoviz + holoscan::ops::multiai_inference + holoscan::ops::multiai_postprocessor + holoscan::ops::segmentation_postprocessor + holoscan::ops::tensor_rt + holoscan::ops::video_stream_recorder + holoscan::ops::video_stream_replayer ) +add_dependencies(OPERATORS_CLASSES_TEST endoscopy_data) + +# ################################################################################################## +# * system tests ---------------------------------------------------------------------------------- +ConfigureTest( + SYSTEM_TEST + system/exception_handling.cpp + system/native_operator_minimal_app.cpp + system/native_operator_multibroadcasts_app.cpp + system/native_operator_ping_app.cpp + system/ping_rx_op.cpp + system/ping_rx_op.hpp + system/ping_tx_op.cpp + system/ping_tx_op.hpp + ) + # ####### ConfigureTest(SEGMENTATION_POSTPROCESSOR_TEST - gxf_extensions/segmentation_postprocessor/test_postprocessor.cpp + operators/segmentation_postprocessor/test_postprocessor.cpp ) target_link_libraries(SEGMENTATION_POSTPROCESSOR_TEST PRIVATE - CUDA::cudart - segmentation_postprocessor + holoscan::ops::segmentation_postprocessor ) # ####### @@ -101,3 +132,4 @@ target_link_libraries(HOLOINFER_TEST PRIVATE holoinfer ) +add_dependencies(HOLOINFER_TEST multiai_ultrasound_data) diff --git a/tests/core/application.cpp b/tests/core/application.cpp index 98e1894b..511ca3b0 100644 --- a/tests/core/application.cpp +++ b/tests/core/application.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +22,6 @@ #include "../config.hpp" #include -#include #include "common/assert.hpp" using namespace std::string_literals; diff --git a/tests/core/arg.cpp b/tests/core/arg.cpp index ea5d7160..210a5825 100644 --- a/tests/core/arg.cpp +++ b/tests/core/arg.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -46,21 +47,36 @@ TEST(Arg, TestArgType) { ArgType A = ArgType(); EXPECT_EQ(A.element_type(), ArgElementType::kCustom); EXPECT_EQ(A.container_type(), ArgContainerType::kNative); + EXPECT_EQ(A.dimension(), 0); + EXPECT_EQ(A.to_string(), "CustomType"); // constructor - A = ArgType(ArgElementType::kInt8, ArgContainerType::kArray); + A = ArgType(ArgElementType::kInt8, ArgContainerType::kArray, 1); EXPECT_EQ(A.element_type(), ArgElementType::kInt8); EXPECT_EQ(A.container_type(), ArgContainerType::kArray); + EXPECT_EQ(A.dimension(), 1); + EXPECT_EQ(A.to_string(), "std::array"); // initialize a vector of float via ArgType::create A = ArgType::create>(); EXPECT_EQ(A.element_type(), ArgElementType::kFloat32); EXPECT_EQ(A.container_type(), ArgContainerType::kVector); + EXPECT_EQ(A.dimension(), 1); + EXPECT_EQ(A.to_string(), "std::vector"); + + // initialize a vector of vector of string via ArgType::create + A = ArgType::create>>(); + EXPECT_EQ(A.element_type(), ArgElementType::kString); + EXPECT_EQ(A.container_type(), ArgContainerType::kVector); + EXPECT_EQ(A.dimension(), 2); + EXPECT_EQ(A.to_string(), "std::vector>"); // initialize an array of int8_t via ArgType::create A = ArgType::create>(); EXPECT_EQ(A.element_type(), ArgElementType::kInt8); EXPECT_EQ(A.container_type(), ArgContainerType::kArray); + EXPECT_EQ(A.dimension(), 1); + EXPECT_EQ(A.to_string(), "std::array"); // Verify get_element_type returns the expected type EXPECT_EQ(ArgType::get_element_type(std::type_index(typeid(5.0))), ArgElementType::kFloat64); @@ -75,40 +91,43 @@ class ArgParameterizedTestFixture : public ::testing::TestWithParam Arg A = Arg("alpha"); }; -TEST_P(ArgParameterizedTestFixture, ArgElementAndContainerTypes) { - auto [value, ExpectedElementType, ExpectedContainerType] = GetParam(); - - if (value.type() == typeid(double)) A = std::any_cast(value); +void any_to_arg(const std::any& value, Arg& arg) { + if (value.type() == typeid(double)) arg = std::any_cast(value); else if (value.type() == typeid(float)) - A = std::any_cast(value); + arg = std::any_cast(value); else if (value.type() == typeid(bool)) - A = std::any_cast(value); + arg = std::any_cast(value); else if (value.type() == typeid(uint8_t)) - A = std::any_cast(value); + arg = std::any_cast(value); else if (value.type() == typeid(uint16_t)) - A = std::any_cast(value); + arg = std::any_cast(value); else if (value.type() == typeid(uint32_t)) - A = std::any_cast(value); + arg = std::any_cast(value); else if (value.type() == typeid(uint64_t)) - A = std::any_cast(value); + arg = std::any_cast(value); else if (value.type() == typeid(int8_t)) - A = std::any_cast(value); + arg = std::any_cast(value); else if (value.type() == typeid(int16_t)) - A = std::any_cast(value); + arg = std::any_cast(value); else if (value.type() == typeid(int32_t)) - A = std::any_cast(value); + arg = std::any_cast(value); else if (value.type() == typeid(int64_t)) - A = std::any_cast(value); + arg = std::any_cast(value); else if (value.type() == typeid(YAML::Node)) - A = std::any_cast(value); + arg = std::any_cast(value); else if (value.type() == typeid(std::string)) - A = std::any_cast(value); + arg = std::any_cast(value); else if (value.type() == typeid(CustomType)) - A = std::any_cast(value); + arg = std::any_cast(value); else if (value.type() == typeid(std::vector)) - A = std::any_cast>(value); + arg = std::any_cast>(value); else if (value.type() == typeid(std::array)) - A = std::any_cast>(value); + arg = std::any_cast>(value); +} + +TEST_P(ArgParameterizedTestFixture, ArgElementAndContainerTypes) { + auto [value, ExpectedElementType, ExpectedContainerType] = GetParam(); + any_to_arg(value, A); check_arg_types(A, ExpectedElementType, ExpectedContainerType); } @@ -136,6 +155,28 @@ INSTANTIATE_TEST_CASE_P( ArgElementType::kUnsigned16, ArgContainerType::kArray})); +// name, value, value as string +static std::vector>>> + arg_params = { + {"A", 7.0, {{"7"}}}, + {"B", 7.0F, {{"7"}}}, + {"C", false, {{"false"}}}, + {"D", int8_t{1}, {{"\"\\x01\""}}}, // hex + {"E", int16_t{1}, {{"1"}}}, + {"F", int32_t{1}, {{"1"}}}, + {"G", int64_t{1}, {{"1"}}}, + {"H", uint8_t{1}, {{"\"\\x01\""}}}, // hex + {"I", uint16_t{1}, {{"1"}}}, + {"J", uint32_t{1}, {{"1"}}}, + {"K", uint64_t{1}, {{"1"}}}, + {"L", YAML::Node(YAML::NodeType::Null), {{"~"}}}, + {"M", std::string{"abcd"}, {{"abcd"}}}, + {"N", std::vector{"abcd", "ef", "ghijklm"}, {{"abcd", "ef", "ghijklm"}}}, + {"O", std::array{1, 8, 9, 15}, std::nullopt}, // can't guess size + {"Z", CustomType(), std::nullopt}, // Unknown type + {"Z", std::any(), std::nullopt}, // Unknown type +}; + TEST(Arg, TestArgHandleType) { // initialize without any value Arg A = Arg("alpha"); @@ -168,6 +209,25 @@ TEST(Arg, TestOtherEnums) { T = ArgElementType::kResource; } +TEST(Arg, TestArgDescription) { + for (const auto& [name, value, str_list] : arg_params) { + Arg arg = Arg(name); + any_to_arg(value, arg); + std::string description = fmt::format("name: {}\ntype: {}", name, arg.arg_type().to_string()); + if (str_list) { + description = fmt::format("{}\nvalue:", description); + if (str_list->size() == 1) { + description = fmt::format("{} {}", description, str_list->at(0)); + } else { + for (const std::string& val_str : str_list.value()) { + description = fmt::format("{}\n - {}", description, val_str); + } + } + } + EXPECT_EQ(arg.description(), description); + } +} + TEST(Arg, TestArgList) { ArgList args; EXPECT_EQ(args.size(), 0); @@ -243,4 +303,18 @@ TEST(Arg, TestArgList2) { EXPECT_EQ(std::any_cast(a2.value()), s1); } +TEST(Arg, TestArgListDescription) { + ArgList args; + std::string description = "name: arglist\nargs:"; + for (const auto& [name, value, _] : arg_params) { + Arg arg = Arg(name); + any_to_arg(value, arg); + std::string indented_arg_description = + std::regex_replace(arg.description(), std::regex("\n"), "\n "); + args.add(arg); + description = fmt::format("{}\n - {}", description, indented_arg_description); + } + EXPECT_EQ(args.description(), description); +} + } // namespace holoscan diff --git a/tests/core/component_spec.cpp b/tests/core/component_spec.cpp index 021a2070..571aab1d 100644 --- a/tests/core/component_spec.cpp +++ b/tests/core/component_spec.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,9 +28,12 @@ #include "holoscan/core/arg.hpp" #include "holoscan/core/argument_setter.hpp" #include "holoscan/core/parameter.hpp" +#include "../utils.hpp" namespace holoscan { +using ComponentSpecWithGXFContext = TestWithGXFContext; + TEST(ComponentSpec, TestComponentSpec) { ComponentSpec spec; EXPECT_EQ(spec.fragment(), nullptr); @@ -88,4 +91,61 @@ TEST(ComponentSpec, TestComponentSpecDefaultValue) { EXPECT_EQ((int)p, default_val); } +TEST(ComponentSpec, TestComponentSpecDescriptionWithoutFragment) { + ComponentSpec spec; + Parameter b; + Parameter> i; + Parameter>> d; + Parameter> s; + spec.param(b, "bool_scalar", "Boolean parameter", "true or false"); + spec.param(i, "int_array", "Int array parameter", "5 integers"); + spec.param( + d, "double_vec_of_vec", "Double 2D vector parameter", "double floats in double vector"); + spec.param(s, "string_vector", "String vector parameter", ""); + + constexpr auto description = R"(fragment: ~ +params: + - name: string_vector + type: std::vector + description: "" + - name: double_vec_of_vec + type: std::vector> + description: double floats in double vector + - name: int_array + type: std::array + description: 5 integers + - name: bool_scalar + type: bool + description: true or false)"; + EXPECT_EQ(spec.description(), description); +} + +TEST_F(ComponentSpecWithGXFContext, TestComponentSpecDescription) { + auto spec = std::make_shared(&F); + Parameter b; + Parameter> i; + Parameter>> d; + Parameter> s; + spec->param(b, "bool_scalar", "Boolean parameter", "true or false"); + spec->param(i, "int_array", "Int array parameter", "5 integers"); + spec->param( + d, "double_vec_of_vec", "Double 2D vector parameter", "double floats in double vector"); + spec->param(s, "string_vector", "String vector parameter", ""); + + constexpr auto description = R"(fragment: "" +params: + - name: string_vector + type: std::vector + description: "" + - name: double_vec_of_vec + type: std::vector> + description: double floats in double vector + - name: int_array + type: std::array + description: 5 integers + - name: bool_scalar + type: bool + description: true or false)"; + EXPECT_EQ(spec->description(), description); +} } // namespace holoscan diff --git a/tests/core/fragment.cpp b/tests/core/fragment.cpp index e56b76e7..ef8f3ecc 100644 --- a/tests/core/fragment.cpp +++ b/tests/core/fragment.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -41,7 +41,7 @@ #include "holoscan/core/operator_spec.hpp" #include "holoscan/core/resource.hpp" #include "holoscan/core/resources/gxf/unbounded_allocator.hpp" -#include "holoscan/operators/stream_playback/video_stream_recorder.hpp" +#include "holoscan/operators/video_stream_recorder/video_stream_recorder.hpp" using namespace std::string_literals; @@ -115,13 +115,8 @@ TEST(Fragment, TestFragmentConfigNestedArgs) { TEST(Fragment, TestConfigUninitializedWarning) { Fragment F; - // verify that calling config() file to passing a config_file raises a warning - testing::internal::CaptureStderr(); Config C = F.config(); - std::string log_output = testing::internal::GetCapturedStderr(); - EXPECT_TRUE(log_output.find("warning") != std::string::npos); - EXPECT_TRUE(log_output.find("Config object was not created. Call " - "config(config_file, prefix) first.") != std::string::npos); + EXPECT_EQ(C.config_file(), ""); } TEST(Fragment, TestFragmentFromConfigNonexistentKey) { diff --git a/tests/core/logger.cpp b/tests/core/logger.cpp index 811428a0..bf9fc4e1 100644 --- a/tests/core/logger.cpp +++ b/tests/core/logger.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,7 +20,6 @@ #include #include -#include #include "../config.hpp" #include diff --git a/tests/core/native_operator.cpp b/tests/core/native_operator.cpp new file mode 100644 index 00000000..6a046bfa --- /dev/null +++ b/tests/core/native_operator.cpp @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include + +#include "../config.hpp" +#include +#include "common/assert.hpp" + +using namespace std::string_literals; + +static HoloscanTestConfig test_config; + +namespace holoscan { + +namespace ops { + +class PingTxOp : public Operator { + public: + HOLOSCAN_OPERATOR_FORWARD_ARGS(PingTxOp) + + PingTxOp() = default; + + void setup(OperatorSpec& spec) override { + spec.output("out1"); + spec.output("out2"); + } + + void compute(InputContext&, OutputContext& op_output, ExecutionContext&) override { + auto value1 = std::make_shared(1); + op_output.emit(value1, "out1"); + + auto value2 = std::make_shared(100); + op_output.emit(value2, "out2"); + }; +}; + +class PingRxOp : public Operator { + public: + HOLOSCAN_OPERATOR_FORWARD_ARGS(PingRxOp) + + PingRxOp() = default; + + void setup(OperatorSpec& spec) override { + spec.param(receivers_, "receivers", "Input Receivers", "List of input receivers.", {}); + } + + void compute(InputContext& op_input, OutputContext&, ExecutionContext&) override { + auto value_vector = op_input.receive>("receivers"); + + HOLOSCAN_LOG_INFO("Rx message received (count: {}, size: {})", count_++, value_vector.size()); + + HOLOSCAN_LOG_INFO("Rx message value1: {}", *(value_vector[0].get())); + HOLOSCAN_LOG_INFO("Rx message value2: {}", *(value_vector[1].get())); + }; + + private: + Parameter> receivers_; + int count_ = 1; +}; +} // namespace ops + + +class NativeOpApp : public holoscan::Application { + public: + void compose() override { + using namespace holoscan; + auto tx = make_operator("tx", make_condition(10)); + auto rx = make_operator("rx"); + + add_flow(tx, rx, {{"out1", "receivers"}, {"out2", "receivers"}}); + } +}; + +TEST(NativeOperatorApp, TestNativeOperatorApp) { + load_env_log_level(); + + auto app = make_application(); + + const std::string config_file = test_config.get_test_data_file("minimal.yaml"); + app->config(config_file); + + // capture output so that we can check that the expected value is present + testing::internal::CaptureStderr(); + + app->run(); + + std::string log_output = testing::internal::GetCapturedStderr(); + EXPECT_TRUE(log_output.find("value1: 1") != std::string::npos); + EXPECT_TRUE(log_output.find("value2: 100") != std::string::npos); +} + +} // namespace holoscan diff --git a/tests/core/operator_spec.cpp b/tests/core/operator_spec.cpp new file mode 100644 index 00000000..d5b7f8bd --- /dev/null +++ b/tests/core/operator_spec.cpp @@ -0,0 +1,215 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include // unordered map find +#include +#include + +#include +#include +#include +#include +#include + +#include "holoscan/core/arg.hpp" +#include "holoscan/core/parameter.hpp" +#include "holoscan/core/gxf/entity.hpp" +#include "holoscan/core/operator_spec.hpp" // must be before argument_setter import +#include "holoscan/core/argument_setter.hpp" + +namespace holoscan { + +TEST(OperatorSpec, TestOperatorSpecInput) { + testing::internal::CaptureStderr(); + + OperatorSpec spec = OperatorSpec(); + spec.input("a"); + + // check if key "a" exist + EXPECT_TRUE(spec.inputs().find("a") != spec.inputs().end()); + + // check size + EXPECT_EQ(spec.inputs().size(), 1); + + // duplicate name + spec.input("a"); + std::string log_output = testing::internal::GetCapturedStderr(); + EXPECT_TRUE(log_output.find("already exists") != std::string::npos); +} + +TEST(OperatorSpec, TestOperatorSpectOutput) { + testing::internal::CaptureStderr(); + + OperatorSpec spec = OperatorSpec(); + spec.output("a"); + + // check if key "a" exist + EXPECT_TRUE(spec.outputs().find("a") != spec.outputs().end()); + + // check size + EXPECT_EQ(spec.outputs().size(), 1); + + // duplicate name + spec.output("a"); + std::string log_output = testing::internal::GetCapturedStderr(); + EXPECT_TRUE(log_output.find("already exists") != std::string::npos); +} + +TEST(OperatorSpec, TestOperatorSpecParam) { + testing::internal::CaptureStderr(); + + OperatorSpec spec = OperatorSpec(); + EXPECT_EQ(spec.fragment(), nullptr); + + // add one parameter named "beta" + IOSpec in{&spec, "a", IOSpec::IOType::kInput, &typeid(gxf::Entity)}; + IOSpec* in_ptr = ∈ + MetaParameter meta_params = Parameter(in_ptr); + + spec.param(meta_params, "beta", "headline1", "description1"); + + // check the stored values + EXPECT_EQ(spec.params().size(), 1); + ParameterWrapper w = spec.params()["beta"]; + std::any& val = w.value(); + auto& p = *std::any_cast*>(val); + EXPECT_EQ(p.key(), "beta"); + EXPECT_EQ(p.get(), in_ptr); + + // repeating a key will not add an additional parameter + spec.param(p, "beta", "headline1", "description4"); + std::string log_output = testing::internal::GetCapturedStderr(); + EXPECT_TRUE(log_output.find("already exists") != std::string::npos); + EXPECT_EQ(spec.params().size(), 1); +} + +TEST(OperatorSpec, TestOperatorSpecParamDefault) { + OperatorSpec spec = OperatorSpec(); + + // initialize + IOSpec default_input{&spec, "a", IOSpec::IOType::kInput, &typeid(gxf::Entity)}; + IOSpec* default_input_ptr = &default_input; + MetaParameter empty_p = Parameter(); // set val + + // add one parameter + spec.param(empty_p, "beta", "headline1", "description1", default_input_ptr); + EXPECT_EQ(spec.params().size(), 1); + + // verify that the extracted parameter has no value + ParameterWrapper w = spec.params()["beta"]; + std::any& val = w.value(); + auto& p = *std::any_cast*>(val); + EXPECT_EQ(p.has_value(), false); + + // set to the default value + p.set_default_value(); + EXPECT_EQ(p.get(), default_input_ptr); + EXPECT_EQ((IOSpec*)p, default_input_ptr); +} + +TEST(OperatorSpec, TestOperatorSpecParamVector) { + testing::internal::CaptureStderr(); + + // initialize + OperatorSpec spec = OperatorSpec(); + + IOSpec in1{&spec, "a", IOSpec::IOType::kInput, &typeid(gxf::Entity)}; + IOSpec* in1_ptr = &in1; + IOSpec in2{&spec, "b", IOSpec::IOType::kInput, &typeid(gxf::Entity)}; + IOSpec* in2_ptr = &in2; + IOSpec in3{&spec, "c", IOSpec::IOType::kInput, &typeid(gxf::Entity)}; + IOSpec* in3_ptr = &in3; + std::vector v = {in1_ptr, in2_ptr, in3_ptr}; + + // add one parameter named "beta" + MetaParameter meta_params = Parameter>(v); + spec.param(meta_params, "beta", "headline1", "description1"); + + // check the stored values + EXPECT_EQ(spec.params().size(), 1); + ParameterWrapper w = spec.params()["beta"]; + std::any& val = w.value(); + auto& p = *std::any_cast>*>(val); + EXPECT_EQ(p.key(), "beta"); + EXPECT_EQ(p.get(), v); + + // repeating a key will not add an additional parameter + spec.param(p, "beta", "headline1", "description1"); + EXPECT_EQ(spec.params().size(), 1); + + std::string log_output = testing::internal::GetCapturedStderr(); + EXPECT_TRUE(log_output.find("already exists") != std::string::npos); + EXPECT_EQ(spec.params().size(), 1); +} + +TEST(OperatorSpec, TestOperatorSpecParamVectorDefault) { + OperatorSpec spec = OperatorSpec(); + + // initialize + IOSpec in1{&spec, "a", IOSpec::IOType::kInput, &typeid(gxf::Entity)}; + IOSpec* in1_ptr = &in1; + IOSpec in2{&spec, "b", IOSpec::IOType::kInput, &typeid(gxf::Entity)}; + IOSpec* in2_ptr = &in2; + IOSpec in3{&spec, "c", IOSpec::IOType::kInput, &typeid(gxf::Entity)}; + IOSpec* in3_ptr = &in3; + std::vector default_v = {in1_ptr, in2_ptr, in3_ptr}; + MetaParameter empty_p = Parameter>(); + + // add one parameter + spec.param(empty_p, "beta", "headline1", "description1", default_v); + EXPECT_EQ(spec.params().size(), 1); + + // verify that the extracted parameter has no value + ParameterWrapper w = spec.params()["beta"]; + std::any& val = w.value(); + auto& p = *std::any_cast>*>(val); + EXPECT_EQ(p.has_value(), false); + + // set to the default value + p.set_default_value(); + EXPECT_EQ(p.get(), default_v); + EXPECT_EQ((std::vector)p, default_v); +} + +TEST(OperatorSpec, TestOperatorSpecDescription) { + OperatorSpec spec; + spec.input("gxf_entity_in"); + spec.output("gxf_entity_out"); + spec.input("holoscan_tensor_in"); + spec.output("holoscan_tensor_out"); + + Parameter b; + Parameter> i; + Parameter>> d; + Parameter> s; + spec.param(b, "bool_scalar", "Boolean parameter", "true or false"); + spec.param(i, "int_array", "Int array parameter", "5 integers"); + spec.param( + d, "double_vec_of_vec", "Double 2D vector parameter", "double floats in double vector"); + spec.param(s, "string_vector", "String vector parameter", ""); + + std::string description = fmt::format(R"({} +inputs: + - holoscan_tensor_in + - gxf_entity_in +outputs: + - holoscan_tensor_out + - gxf_entity_out)", + ComponentSpec(spec).description()); + EXPECT_EQ(spec.description(), description); +} +} // namespace holoscan diff --git a/tests/core/resource_classes.cpp b/tests/core/resource_classes.cpp index 76f98ce3..21405091 100644 --- a/tests/core/resource_classes.cpp +++ b/tests/core/resource_classes.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -205,8 +205,8 @@ TEST_F(ResourceClassesWithGXFContext, TestAllocator) { EXPECT_EQ(typeid(resource), typeid(std::make_shared())); EXPECT_EQ(std::string(resource->gxf_typename()), "nvidia::gxf::Allocator"s); - // For the base Allocator, this always returns true - EXPECT_EQ(resource->is_available(1024), true); + // For the base Allocator, this always returns false + EXPECT_EQ(resource->is_available(1024), false); // For the base Allocator, allocate and free exist but don't do anything EXPECT_EQ(resource->allocate(1024, MemoryStorageType::kHost), nullptr); diff --git a/tests/data/app_config.yaml b/tests/data/app_config.yaml index bc8b286e..e8d22028 100644 --- a/tests/data/app_config.yaml +++ b/tests/data/app_config.yaml @@ -1,5 +1,5 @@ %YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,24 +14,18 @@ # See the License for the specific language governing permissions and # limitations under the License. --- -extensions: - - libgxf_std.so - - libgxf_cuda.so - - libgxf_multimedia.so - - libgxf_serialization.so - - libbayer_demosaic.so - - libaja_source.so - - libcustom_lstm_inference.so - - libformat_converter.so - - libholoviz.so - - libmultiai_inference.so - - libmultiai_postprocessor.so - - libsegmentation_postprocessor.so - - libstream_playback.so - - libtensor_rt.so - - libtool_tracking_postprocessor.so - - libvisualizer_icardio.so - - libvisualizer_tool_tracking.so +# The following extensions are loaded by default so we don't need to specify these. +# Default extension loaded by framework is listed in src/core/executors/gxf/gxf_executor.cpp +# (`kDefaultGXFExtensions` and `kDefaultHoloscanGXFExtensions`) + +# extensions: +# - libgxf_std.so +# - libgxf_cuda.so +# - libgxf_multimedia.so +# - libgxf_serialization.so +# - libgxf_bayer_demosaic.so +# - libgxf_stream_playback.so +# - libgxf_tensor_rt.so source: "replayer" # or "aja" do_record: false # or 'true' if you want to record input video stream. @@ -74,40 +68,6 @@ format_converter_aja: resize_width: 854 resize_height: 480 -lstm_inference: - model_file_path: ../data/endoscopy/model/tool_loc_convlstm.onnx - engine_cache_dir: ../data/endoscopy/model/tool_loc_convlstm_engines - input_tensor_names: - - source_video - - cellstate_in - - hiddenstate_in - input_state_tensor_names: - - cellstate_in - - hiddenstate_in - input_binding_names: - - data_ph:0 # (shape=[1, 480, 854, 3], dtype=float32) <==> source_video - - cellstate_ph:0 # (shape=[1, 60, 107, 7], dtype=float32) == internal state - - hiddenstate_ph:0 # (shape=[1, 60, 107, 7], dtype=float32) == internal state - output_tensor_names: - - cellstate_out - - hiddenstate_out - - probs - - scaled_coords - - binary_masks - output_state_tensor_names: - - cellstate_out - - hiddenstate_out - output_binding_names: - - Model/net_states:0 # (shape=[ 1, 60, 107, 7], dtype=float32) - - Model/net_hidden:0 # (shape=[ 1, 60, 107, 7], dtype=float32) - - probs:0 # (shape=[1, 7], dtype=float32) - - Localize/scaled_coords:0 # (shape=[1, 7, 2], dtype=float32) - - Localize_1/binary_masks:0 # (shape=[1, 7, 60, 107], dtype=float32) - force_engine_update: false - verbose: true - max_workspace_size: 2147483648 - enable_fp16_: true - visualizer_format_converter_replayer: in_dtype: "rgb888" out_dtype: "rgba8888" @@ -119,70 +79,6 @@ visualizer_format_converter_aja: resize_width: 854 resize_height: 480 -visualizer: - videoframe_vertex_shader_path: gxf_extensions/visualizer_tool_tracking/glsl/viewport_filling_triangle.vert - videoframe_fragment_shader_path: gxf_extensions/visualizer_tool_tracking/glsl/video_frame.frag - tooltip_vertex_shader_path: gxf_extensions/visualizer_tool_tracking/glsl/instrument_tip.vert - tooltip_fragment_shader_path: gxf_extensions/visualizer_tool_tracking/glsl/instrument_tip.frag - overlay_img_vertex_shader_path: gxf_extensions/visualizer_tool_tracking/glsl/viewport_filling_triangle.vert - overlay_img_fragment_shader_path: gxf_extensions/visualizer_tool_tracking/glsl/instrument_mask.frag - overlay_img_width: 107 - overlay_img_height: 60 - overlay_img_channels: 1 - overlay_img_layers: 7 - # overlay_img_colors: - # - [1.0, 0.0, 0.0] - # - [0.0, 1.0, 0.0] - # - [0.0, 0.0, 1.0] - # - [1.0, 1.0, 0.0] - # - [1.0, 0.0, 1.0] - # - [0.0, 1.0, 1.0] - # - [0.5, 1.0, 0.0] - num_tool_classes: 7 - num_tool_pos_components: 2 - # tool_tip_colors: - # - [1.0, 0.0, 0.0] - # - [0.0, 1.0, 0.0] - # - [0.0, 0.0, 1.0] - # - [1.0, 1.0, 0.0] - # - [1.0, 0.0, 1.0] - # - [0.0, 1.0, 1.0] - # - [0.5, 1.0, 0.0] - tool_labels: - - Grasper - - Bipolar - - Hook - - Scissors - - Clipper - - Irrigator - - Spec.Bag - label_sans_font_path: gxf_extensions/visualizer_tool_tracking/fonts/Roboto-Regular.ttf - label_sans_bold_font_path: gxf_extensions/visualizer_tool_tracking/fonts/Roboto-Bold.ttf - in_tensor_names: - - "" - - probs - - net_states - - binary_masks - - scaled_coords - in_width: 854 - in_height: 480 - in_channels: 4 - in_bytes_per_pixel: 1 - # tool_tip_colors: [[0.0, 0.0, 0.0], - # [0.0, 0.0, 0.0], - # [0.0, 0.0, 0.0], - # [0.0, 0.0, 0.0], - # [0.0, 0.0, 0.0], - # [0.0, 0.0, 0.0], - # [0.0, 0.0, 0.0], - # [0.0, 0.0, 0.0], - # [0.0, 0.0, 0.0], - # [0.0, 0.0, 0.0], - # [0.0, 0.0, 0.0], - # [0.0, 0.0, 0.0]] - -tool_tracking_postprocessor: - holoviz: width: 854 height: 480 @@ -233,18 +129,18 @@ segmentation_inference: # TensorRtInference multiai_inference: backend: "trt" - model_path_map: + model_path_map: "icardio_plax_chamber": "../data/multiai_ultrasound/models/plax_chamber.onnx" "icardio_aortic_stenosis": "../data/multiai_ultrasound/models/aortic_stenosis.onnx" "icardio_bmode_perspective": "../data/multiai_ultrasound/models/bmode_perspective.onnx" - pre_processor_map: + pre_processor_map: "icardio_plax_chamber": ["plax_cham_pre_proc"] "icardio_aortic_stenosis": ["aortic_pre_proc"] - "icardio_bmode_perspective": ["bmode_pre_proc"] - inference_map: + "icardio_bmode_perspective": ["bmode_pre_proc"] + inference_map: "icardio_plax_chamber": "plax_cham_infer" "icardio_aortic_stenosis": "aortic_infer" - "icardio_bmode_perspective": "bmode_infer" + "icardio_bmode_perspective": "bmode_infer" in_tensor_names: ["plax_cham_pre_proc", "aortic_pre_proc", "bmode_pre_proc"] out_tensor_names: ["plax_cham_infer", "aortic_infer", "bmode_infer"] parallel_inference: true @@ -260,24 +156,17 @@ multiai_postprocessor: "plax_cham_infer": ["max_per_channel_scaled"] processed_map: "plax_cham_infer": "plax_chamber_processed" - in_tensor_names: ["plax_cham_infer", + in_tensor_names: ["plax_cham_infer", "aortic_infer", "bmode_infer"] - out_tensor_names : ["plax_chamber_processed"] + out_tensor_names : ["plax_chamber_processed"] input_on_cuda: false output_on_cuda: false transmit_on_cuda: false -visualizer_icardio: - in_tensor_names: ["plax_chamber_processed"] - out_tensor_names: ["keypoints", "keyarea_1", "keyarea_2", - "keyarea_3", "keyarea_4", "keyarea_5", "lines"] - input_on_cuda: false - demosaic: generate_alpha: false bayer_grid_pos: 2 interpolation_mode: 0 # this is the only interpolation mode supported by NPP currently video_composer: - diff --git a/tests/data/minimal.yaml b/tests/data/minimal.yaml index 06d0f233..e39afef6 100644 --- a/tests/data/minimal.yaml +++ b/tests/data/minimal.yaml @@ -1,5 +1,5 @@ %YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,10 +14,4 @@ # See the License for the specific language governing permissions and # limitations under the License. --- -extensions: - - libgxf_std.so - - libgxf_cuda.so - - libgxf_multimedia.so - - libgxf_serialization.so - value: 5.3 diff --git a/tests/data/multi-ai-ultrasound/app_config_test_1.yaml b/tests/data/multi-ai-ultrasound/app_config_test_1.yaml deleted file mode 100644 index ac69f7a8..00000000 --- a/tests/data/multi-ai-ultrasound/app_config_test_1.yaml +++ /dev/null @@ -1,161 +0,0 @@ -%YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -extensions: - - libgxf_std.so - - libgxf_cuda.so - - libgxf_multimedia.so - - libgxf_serialization.so - - libformat_converter.so - - libholoviz.so - - libstream_playback.so - - libmultiai_postprocessor.so - - libmultiai_inference.so - - libvisualizer_icardio.so - -source: "replayer" # or "aja" -do_record: false # or 'true' if you want to record input video stream. - -replayer: - directory: "../data/multiai_ultrasound/video" - basename: "tensor" - frame_rate: 0 # as specified in timestamps - repeat: true # default: false - realtime: true # default: true - count: 0 # default: 0 (no frame count restriction) - -broadcast: - -plax_cham_resized: - out_tensor_name: plax_cham_resize - out_dtype: "rgb888" - resize_width: 320 - resize_height: 320 - -plax_cham_pre: - out_tensor_name: plax_cham_pre_proc - out_dtype: "float32" - resize_width: 320 - resize_height: 320 - -aortic_ste_pre: - out_tensor_name: aortic_pre_proc - out_dtype: "float32" - resize_width: 300 - resize_height: 300 - -b_mode_pers_pre: - out_tensor_name: bmode_pre_proc - out_dtype: "float32" - resize_width: 320 - resize_height: 240 - -multiai_inference: - backend: "trt" - model_path_map: - "plax_chamber": "../data/multiai_ultrasound/models/plax_chamber.onnx" - "aortic_stenosis": "../data/multiai_ultrasound/models/aortic_stenosis.onnx" - "bmode_perspective": "../data/multiai_ultrasound/models/bmode_perspective.onnx" - pre_processor_map: - "plax_chamber": ["plax_cham_pre_proc"] - "aortic_stenosis": ["aortic_pre_proc"] - "bmode_perspective": ["bmode_pre_proc"] - inference_map: - "plax_chamber": "plax_cham_infer" - "aortic_stenosis": "aortic_infer" - "bmode_perspective": "bmode_infer" - in_tensor_names: ["plax_cham_pre_proc", "aortic_pre_proc", "bmode_pre_proc"] - out_tensor_names: ["plax_cham_infer", "aortic_infer", "bmode_infer"] - parallel_inference: true - infer_on_cpu: false - enable_fp16: false - input_on_cuda: true - output_on_cuda: true - transmit_on_cuda: true - is_engine_path: false - -multiai_postprocessor: - process_operations: - "plax_cham_infer": ["max_per_channel_scaled"] - processed_map: - "plax_cham_infer": "plax_chamber_processed" - in_tensor_names: ["plax_cham_infer", - "aortic_infer", - "bmode_infer"] - out_tensor_names : ["plax_chamber_processed"] - input_on_cuda: false - output_on_cuda: false - transmit_on_cuda: false - -visualizer_icardio: - in_tensor_names: ["plax_chamber_processed"] - out_tensor_names: ["keypoints", "keyarea_1", "keyarea_2", - "keyarea_3", "keyarea_4", "keyarea_5", "lines","logo"] - input_on_cuda: false - -holoviz: - tensors: - - name: plax_cham_resize - type: color - priority: 0 - - name: logo - type: color - priority: 0 - opacity: 0.5 - - name: keyarea_1 - type: ovals - color: [1.0, 0.0, 0.0, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keyarea_2 - type: ovals - color: [0.0, 1.0, 0.0, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keyarea_3 - type: ovals - color: [0.0, 1.0, 1.0, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keyarea_4 - type: ovals - color: [1.0, 0.5, 0.5, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keyarea_5 - type: ovals - color: [1.0, 0.0, 1.0, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keypoints - type: crosses - line_width: 4 - color: [1.0, 1.0, 1.0, 1.0] - priority: 3 - - name: lines - type: line_strip - line_width: 1 - color: [1.0, 1.0, 0.0, 1.0] - priority: 1 - window_title: "Multi AI Inference" - width: 320 - height: 320 - use_exclusive_display: false \ No newline at end of file diff --git a/tests/data/multi-ai-ultrasound/app_config_test_2.yaml b/tests/data/multi-ai-ultrasound/app_config_test_2.yaml deleted file mode 100644 index b522f455..00000000 --- a/tests/data/multi-ai-ultrasound/app_config_test_2.yaml +++ /dev/null @@ -1,161 +0,0 @@ -%YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -extensions: - - libgxf_std.so - - libgxf_cuda.so - - libgxf_multimedia.so - - libgxf_serialization.so - - libformat_converter.so - - libholoviz.so - - libstream_playback.so - - libmultiai_postprocessor.so - - libmultiai_inference.so - - libvisualizer_icardio.so - -source: "replayer" # or "aja" -do_record: false # or 'true' if you want to record input video stream. - -replayer: - directory: "../data/multiai_ultrasound/video" - basename: "tensor" - frame_rate: 0 # as specified in timestamps - repeat: true # default: false - realtime: true # default: true - count: 0 # default: 0 (no frame count restriction) - -broadcast: - -plax_cham_resized: - out_tensor_name: plax_cham_resize - out_dtype: "rgb888" - resize_width: 320 - resize_height: 320 - -plax_cham_pre: - out_tensor_name: plax_cham_pre_proc - out_dtype: "float32" - resize_width: 320 - resize_height: 320 - -aortic_ste_pre: - out_tensor_name: aortic_pre_proc - out_dtype: "float32" - resize_width: 300 - resize_height: 300 - -b_mode_pers_pre: - out_tensor_name: bmode_pre_proc - out_dtype: "float32" - resize_width: 320 - resize_height: 240 - -multiai_inference: - backend: "trt" - model_path_map: - "plax_chamber": "../data/multiai_ultrasound/models/plax_chamber.onnx" - "aortic_stenosis": "../data/multiai_ultrasound/models/aortic_stenosis.onnx" - "bmode_perspective": "../data/multiai_ultrasound/models/bmode_perspective.onnx" - pre_processor_map: - "plax_chamber": ["plax_cham_pre_proc"] - "aortic_stenosis": ["aortic_pre_proc"] - "bmode_perspective": ["bmode_pre_proc"] - inference_map: - "plax_chamber": "plax_cham_infer" - "aortic_stenosis": "aortic_infer" - "bmode_perspective": "bmode_infer" - in_tensor_names: ["plax_cham_pre_proc", "aortic_pre_proc", "bmode_pre_proc"] - out_tensor_names: ["plax_cham_infer", "aortic_infer", "bmode_infer"] - parallel_inference: true - infer_on_cpu: false - enable_fp16: true - input_on_cuda: true - output_on_cuda: true - transmit_on_cuda: true - is_engine_path: false - -multiai_postprocessor: - process_operations: - "plax_cham_infer": ["max_per_channel_scaled"] - processed_map: - "plax_cham_infer": "plax_chamber_processed" - in_tensor_names: ["plax_cham_infer", - "aortic_infer", - "bmode_infer"] - out_tensor_names : ["plax_chamber_processed"] - input_on_cuda: false - output_on_cuda: false - transmit_on_cuda: false - -visualizer_icardio: - in_tensor_names: ["plax_chamber_processed"] - out_tensor_names: ["keypoints", "keyarea_1", "keyarea_2", - "keyarea_3", "keyarea_4", "keyarea_5", "lines","logo"] - input_on_cuda: false - -holoviz: - tensors: - - name: plax_cham_resize - type: color - priority: 0 - - name: logo - type: color - priority: 0 - opacity: 0.5 - - name: keyarea_1 - type: ovals - color: [1.0, 0.0, 0.0, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keyarea_2 - type: ovals - color: [0.0, 1.0, 0.0, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keyarea_3 - type: ovals - color: [0.0, 1.0, 1.0, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keyarea_4 - type: ovals - color: [1.0, 0.5, 0.5, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keyarea_5 - type: ovals - color: [1.0, 0.0, 1.0, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keypoints - type: crosses - line_width: 4 - color: [1.0, 1.0, 1.0, 1.0] - priority: 3 - - name: lines - type: line_strip - line_width: 1 - color: [1.0, 1.0, 0.0, 1.0] - priority: 1 - window_title: "Multi AI Inference" - width: 320 - height: 320 - use_exclusive_display: false \ No newline at end of file diff --git a/tests/data/multi-ai-ultrasound/app_config_test_3.yaml b/tests/data/multi-ai-ultrasound/app_config_test_3.yaml deleted file mode 100644 index dcdfb478..00000000 --- a/tests/data/multi-ai-ultrasound/app_config_test_3.yaml +++ /dev/null @@ -1,161 +0,0 @@ -%YAML 1.2 -# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. ---- -extensions: - - libgxf_std.so - - libgxf_cuda.so - - libgxf_multimedia.so - - libgxf_serialization.so - - libformat_converter.so - - libholoviz.so - - libstream_playback.so - - libmultiai_postprocessor.so - - libmultiai_inference.so - - libvisualizer_icardio.so - -source: "replayer" # or "aja" -do_record: false # or 'true' if you want to record input video stream. - -replayer: - directory: "../data/multiai_ultrasound/video" - basename: "tensor" - frame_rate: 0 # as specified in timestamps - repeat: true # default: false - realtime: true # default: true - count: 0 # default: 0 (no frame count restriction) - -broadcast: - -plax_cham_resized: - out_tensor_name: plax_cham_resize - out_dtype: "rgb888" - resize_width: 320 - resize_height: 320 - -plax_cham_pre: - out_tensor_name: plax_cham_pre_proc - out_dtype: "float32" - resize_width: 320 - resize_height: 320 - -aortic_ste_pre: - out_tensor_name: aortic_pre_proc - out_dtype: "float32" - resize_width: 300 - resize_height: 300 - -b_mode_pers_pre: - out_tensor_name: bmode_pre_proc - out_dtype: "float32" - resize_width: 320 - resize_height: 240 - -multiai_inference: - backend: "onnxrt" - model_path_map: - "plax_chamber": "../data/multiai_ultrasound/models/plax_chamber.onnx" - "aortic_stenosis": "../data/multiai_ultrasound/models/aortic_stenosis.onnx" - "bmode_perspective": "../data/multiai_ultrasound/models/bmode_perspective.onnx" - pre_processor_map: - "plax_chamber": ["plax_cham_pre_proc"] - "aortic_stenosis": ["aortic_pre_proc"] - "bmode_perspective": ["bmode_pre_proc"] - inference_map: - "plax_chamber": "plax_cham_infer" - "aortic_stenosis": "aortic_infer" - "bmode_perspective": "bmode_infer" - in_tensor_names: ["plax_cham_pre_proc", "aortic_pre_proc", "bmode_pre_proc"] - out_tensor_names: ["plax_cham_infer", "aortic_infer", "bmode_infer"] - parallel_inference: true - infer_on_cpu: true - enable_fp16: false - input_on_cuda: false - output_on_cuda: false - transmit_on_cuda: false - is_engine_path: false - -multiai_postprocessor: - process_operations: - "plax_cham_infer": ["max_per_channel_scaled"] - processed_map: - "plax_cham_infer": "plax_chamber_processed" - in_tensor_names: ["plax_cham_infer", - "aortic_infer", - "bmode_infer"] - out_tensor_names : ["plax_chamber_processed"] - input_on_cuda: false - output_on_cuda: false - transmit_on_cuda: false - -visualizer_icardio: - in_tensor_names: ["plax_chamber_processed"] - out_tensor_names: ["keypoints", "keyarea_1", "keyarea_2", - "keyarea_3", "keyarea_4", "keyarea_5", "lines","logo"] - input_on_cuda: false - -holoviz: - tensors: - - name: plax_cham_resize - type: color - priority: 0 - - name: logo - type: color - priority: 0 - opacity: 0.5 - - name: keyarea_1 - type: ovals - color: [1.0, 0.0, 0.0, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keyarea_2 - type: ovals - color: [0.0, 1.0, 0.0, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keyarea_3 - type: ovals - color: [0.0, 1.0, 1.0, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keyarea_4 - type: ovals - color: [1.0, 0.5, 0.5, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keyarea_5 - type: ovals - color: [1.0, 0.0, 1.0, 1.0] - line_width: 4 - opacity: 0.7 - priority: 2 - - name: keypoints - type: crosses - line_width: 4 - color: [1.0, 1.0, 1.0, 1.0] - priority: 3 - - name: lines - type: line_strip - line_width: 1 - color: [1.0, 1.0, 0.0, 1.0] - priority: 1 - window_title: "Multi AI Inference" - width: 320 - height: 320 - use_exclusive_display: false \ No newline at end of file diff --git a/tests/holoinfer/multiai_tests.cpp b/tests/holoinfer/multiai_tests.cpp index 70e6d9ac..3215e5ca 100644 --- a/tests/holoinfer/multiai_tests.cpp +++ b/tests/holoinfer/multiai_tests.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -160,6 +160,7 @@ void parameter_setup_test() { holoinfer_assert(status, test_module, test_name, HoloInfer::holoinfer_code::H_ERROR); test_name = "ONNX backend, Default"; + if (!is_x86_64) { infer_on_cpu = true; } is_engine_path = false; input_on_cuda = false; output_on_cuda = false; @@ -168,6 +169,7 @@ void parameter_setup_test() { holoinfer_assert(status, test_module, test_name, HoloInfer::holoinfer_code::H_SUCCESS); backend = "trt"; + if (!is_x86_64) { infer_on_cpu = false; } input_on_cuda = true; output_on_cuda = true; test_name = "TRT backend, Default check 1"; @@ -335,33 +337,40 @@ void inference_tests() { status = do_inference(); holoinfer_assert(status, test_module, test_name, HoloInfer::holoinfer_code::H_SUCCESS); - test_name = "ONNX backend, Basic sequential inference on GPU"; - infer_on_cpu = false; - status = prepare_for_inference(); - status = do_mapping(); - status = do_inference(); - holoinfer_assert(status, test_module, test_name, HoloInfer::holoinfer_code::H_SUCCESS); - - test_name = "ONNX backend, Basic parallel inference on GPU"; - parallel_inference = true; - status = prepare_for_inference(); - status = do_mapping(); - status = do_inference(); - holoinfer_assert(status, test_module, test_name, HoloInfer::holoinfer_code::H_SUCCESS); - - test_name = "ONNX backend, Empty host input"; - dbs = multiai_specs_->data_per_model_.at("plax_chamber")->host_buffer.size(); - multiai_specs_->data_per_model_.at("plax_chamber")->host_buffer.resize(0); - status = do_inference(); - holoinfer_assert(status, test_module, test_name, HoloInfer::holoinfer_code::H_ERROR); - multiai_specs_->data_per_model_.at("plax_chamber")->host_buffer.resize(dbs); - - test_name = "ONNX backend, Empty host output"; - dbs = multiai_specs_->output_per_model_.at("aortic_infer")->host_buffer.size(); - multiai_specs_->output_per_model_.at("aortic_infer")->host_buffer.resize(0); - status = do_inference(); - holoinfer_assert(status, test_module, test_name, HoloInfer::holoinfer_code::H_ERROR); - multiai_specs_->output_per_model_.at("aortic_infer")->host_buffer.resize(dbs); + if (is_x86_64) { + test_name = "ONNX backend, Basic sequential inference on GPU"; + infer_on_cpu = false; + status = prepare_for_inference(); + status = do_mapping(); + status = do_inference(); + holoinfer_assert(status, test_module, test_name, HoloInfer::holoinfer_code::H_SUCCESS); + + test_name = "ONNX backend, Basic parallel inference on GPU"; + parallel_inference = true; + status = prepare_for_inference(); + status = do_mapping(); + status = do_inference(); + holoinfer_assert(status, test_module, test_name, HoloInfer::holoinfer_code::H_SUCCESS); + + test_name = "ONNX backend, Empty host input"; + dbs = multiai_specs_->data_per_model_.at("plax_chamber")->host_buffer.size(); + multiai_specs_->data_per_model_.at("plax_chamber")->host_buffer.resize(0); + status = do_inference(); + holoinfer_assert(status, test_module, test_name, HoloInfer::holoinfer_code::H_ERROR); + multiai_specs_->data_per_model_.at("plax_chamber")->host_buffer.resize(dbs); + + test_name = "ONNX backend, Empty host output"; + dbs = multiai_specs_->output_per_model_.at("aortic_infer")->host_buffer.size(); + multiai_specs_->output_per_model_.at("aortic_infer")->host_buffer.resize(0); + status = do_inference(); + holoinfer_assert(status, test_module, test_name, HoloInfer::holoinfer_code::H_ERROR); + multiai_specs_->output_per_model_.at("aortic_infer")->host_buffer.resize(dbs); + } else { + test_name = "ONNX backend on ARM, Basic sequential inference on GPU"; + infer_on_cpu = false; + status = prepare_for_inference(); + holoinfer_assert(status, test_module, test_name, HoloInfer::holoinfer_code::H_ERROR); + } } int main() { diff --git a/tests/holoinfer/test_infer_settings.hpp b/tests/holoinfer/test_infer_settings.hpp index d251e66e..764c0916 100644 --- a/tests/holoinfer/test_infer_settings.hpp +++ b/tests/holoinfer/test_infer_settings.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,9 +29,10 @@ namespace HoloInfer = holoscan::inference; unsigned int test_count = 0; +bool is_x86_64 = !HoloInfer::is_platform_aarch64(); /// Default parameters for inference -std::string backend = "trt"; // NOLINT +std::string backend = "trt"; // NOLINT std::vector in_tensor_names = { "plax_cham_pre_proc", "aortic_pre_proc", "bmode_pre_proc"}; std::vector out_tensor_names = {"plax_cham_infer", "aortic_infer", "bmode_infer"}; diff --git a/tests/core/operator_classes.cpp b/tests/operators/operator_classes.cpp similarity index 59% rename from tests/core/operator_classes.cpp rename to tests/operators/operator_classes.cpp index 55e729bc..be332dd7 100644 --- a/tests/core/operator_classes.cpp +++ b/tests/operators/operator_classes.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,6 +19,7 @@ #include #include +#include #include #include "../config.hpp" @@ -35,9 +36,19 @@ #include "holoscan/core/resources/gxf/block_memory_pool.hpp" #include "holoscan/core/resources/gxf/cuda_stream_pool.hpp" #include "holoscan/core/resources/gxf/unbounded_allocator.hpp" -#include "holoscan/std_ops.hpp" #include "common/assert.hpp" +#include "holoscan/operators/aja_source/aja_source.hpp" +#include "holoscan/operators/bayer_demosaic/bayer_demosaic.hpp" +#include "holoscan/operators/format_converter/format_converter.hpp" +#include "holoscan/operators/holoviz/holoviz.hpp" +#include "holoscan/operators/multiai_inference/multiai_inference.hpp" +#include "holoscan/operators/multiai_postprocessor/multiai_postprocessor.hpp" +#include "holoscan/operators/segmentation_postprocessor/segmentation_postprocessor.hpp" +#include "holoscan/operators/tensor_rt/tensor_rt_inference.hpp" +#include "holoscan/operators/video_stream_recorder/video_stream_recorder.hpp" +#include "holoscan/operators/video_stream_replayer/video_stream_replayer.hpp" + using namespace std::string_literals; namespace holoscan { @@ -62,7 +73,17 @@ TEST_F(OperatorClassesWithGXFContext, TestAJASourceOpChannelFromYAML) { EXPECT_EQ(typeid(op), typeid(std::make_shared(args))); std::string log_output = testing::internal::GetCapturedStderr(); - EXPECT_TRUE(log_output.find("[error]") == std::string::npos); + auto error_pos = log_output.find("[error]"); + if (error_pos != std::string::npos) { + // Initializing a native operator outside the context of app.run() will result in the + // following error being logged because the GXFWrapper will not yet have been created for + // the operator: + // [error] [gxf_executor.cpp:452] Unable to get GXFWrapper for Operator 'aja-source' + + // GXFWrapper was mentioned and no additional error was logged + EXPECT_TRUE(log_output.find("GXFWrapper", error_pos + 1) != std::string::npos); + EXPECT_TRUE(log_output.find("[error]", error_pos + 1) == std::string::npos); + } } TEST_F(TestWithGXFContext, TestAJASourceOpChannelFromEnum) { @@ -83,7 +104,17 @@ TEST_F(TestWithGXFContext, TestAJASourceOpChannelFromEnum) { EXPECT_EQ(typeid(op), typeid(std::make_shared(args))); std::string log_output = testing::internal::GetCapturedStderr(); - EXPECT_TRUE(log_output.find("[error]") == std::string::npos); + auto error_pos = log_output.find("[error]"); + if (error_pos != std::string::npos) { + // Initializing a native operator outside the context of app.run() will result in the + // following error being logged because the GXFWrapper will not yet have been created for + // the operator: + // [error] [gxf_executor.cpp:452] Unable to get GXFWrapper for Operator 'aja-source' + + // GXFWrapper was mentioned and no additional error was logged + EXPECT_TRUE(log_output.find("GXFWrapper", error_pos + 1) != std::string::npos); + EXPECT_TRUE(log_output.find("[error]", error_pos + 1) == std::string::npos); + } } TEST_F(OperatorClassesWithGXFContext, TestFormatConverterOp) { @@ -116,26 +147,17 @@ TEST_F(OperatorClassesWithGXFContext, TestFormatConverterOp) { EXPECT_EQ(typeid(op), typeid(std::make_shared(args))); std::string log_output = testing::internal::GetCapturedStderr(); - EXPECT_TRUE(log_output.find("[error]") == std::string::npos); -} - -TEST_F(OperatorClassesWithGXFContext, TestLSTMTensorRTInferenceOp) { - const std::string name{"lstm_inferer"}; - - // load most arguments from the YAML file - ArgList kwargs = F.from_config("lstm_inference"); - kwargs.add(Arg{"pool", F.make_resource("pool")}); - kwargs.add( - Arg{"cuda_stream_pool", F.make_resource("cuda_stream", 0, 0, 0, 1, 5)}); - - testing::internal::CaptureStderr(); - - auto op = F.make_operator(name, kwargs); - EXPECT_EQ(op->name(), name); - EXPECT_EQ(typeid(op), typeid(std::make_shared(kwargs))); - - std::string log_output = testing::internal::GetCapturedStderr(); - EXPECT_TRUE(log_output.find("[error]") == std::string::npos); + auto error_pos = log_output.find("[error]"); + if (error_pos != std::string::npos) { + // Initializing a native operator outside the context of app.run() will result in the + // following error being logged because the GXFWrapper will not yet have been created for + // the operator: + // [error] [gxf_executor.cpp:452] Unable to get GXFWrapper for Operator 'format_converter' + + // GXFWrapper was mentioned and no additional error was logged + EXPECT_TRUE(log_output.find("GXFWrapper", error_pos + 1) != std::string::npos); + EXPECT_TRUE(log_output.find("[error]", error_pos + 1) == std::string::npos); + } } TEST_F(OperatorClassesWithGXFContext, TestTensorRTInferenceOp) { @@ -157,23 +179,6 @@ TEST_F(OperatorClassesWithGXFContext, TestTensorRTInferenceOp) { EXPECT_TRUE(log_output.find("[error]") == std::string::npos); } -TEST_F(OperatorClassesWithGXFContext, TestToolTrackingVizOp) { - const std::string name{"visualizer"}; - - // load most arguments from the YAML file - ArgList kwargs = F.from_config("visualizer"); - kwargs.add(Arg{"pool", F.make_resource("pool", 1, 854 * 480 * 4 * 4, 2)}); - - testing::internal::CaptureStderr(); - - auto op = F.make_operator(name, kwargs); - EXPECT_EQ(op->name(), name); - EXPECT_EQ(typeid(op), typeid(std::make_shared(kwargs))); - - std::string log_output = testing::internal::GetCapturedStderr(); - EXPECT_TRUE(log_output.find("[error]") == std::string::npos); -} - TEST_F(OperatorClassesWithGXFContext, TestVideoStreamRecorderOp) { const std::string name{"recorder"}; ArgList args{ @@ -187,14 +192,25 @@ TEST_F(OperatorClassesWithGXFContext, TestVideoStreamRecorderOp) { EXPECT_EQ(typeid(op), typeid(std::make_shared(args))); std::string log_output = testing::internal::GetCapturedStderr(); - EXPECT_TRUE(log_output.find("[error]") == std::string::npos); + auto error_pos = log_output.find("[error]"); + if (error_pos != std::string::npos) { + // Initializing a native operator outside the context of app.run() will result in the + // following error being logged because the GXFWrapper will not yet have been created for + // the operator: + // [error] [gxf_executor.cpp:452] Unable to get GXFWrapper for Operator 'recorder' + + // GXFWrapper was mentioned and no additional error was logged + EXPECT_TRUE(log_output.find("GXFWrapper", error_pos + 1) != std::string::npos); + EXPECT_TRUE(log_output.find("[error]", error_pos + 1) == std::string::npos); + } } TEST_F(OperatorClassesWithGXFContext, TestVideoStreamReplayerOp) { const std::string name{"replayer"}; + const std::string sample_data_path = std::string(std::getenv("HOLOSCAN_SAMPLE_DATA_PATH")); ArgList args{ - Arg{"directory", "/tmp"s}, - Arg{"basename", "video_in"s}, + Arg{"directory", sample_data_path + "/endoscopy/video"s}, + Arg{"basename", "surgical_video"s}, Arg{"batch_size", static_cast(1UL)}, Arg{"ignore_corrupted_entities", true}, Arg{"frame_rate", 0.f}, @@ -208,7 +224,17 @@ TEST_F(OperatorClassesWithGXFContext, TestVideoStreamReplayerOp) { EXPECT_EQ(typeid(op), typeid(std::make_shared(args))); std::string log_output = testing::internal::GetCapturedStderr(); - EXPECT_TRUE(log_output.find("[error]") == std::string::npos); + auto error_pos = log_output.find("[error]"); + if (error_pos != std::string::npos) { + // Initializing a native operator outside the context of app.run() will result in the + // following error being logged because the GXFWrapper will not yet have been created for + // the operator: + // [error] [gxf_executor.cpp:452] Unable to get GXFWrapper for Operator 'replayer' + + // GXFWrapper was mentioned and no additional error was logged + EXPECT_TRUE(log_output.find("GXFWrapper", error_pos + 1) != std::string::npos); + EXPECT_TRUE(log_output.find("[error]", error_pos + 1) == std::string::npos); + } } TEST_F(OperatorClassesWithGXFContext, TestSegmentationPostprocessorOp) { @@ -227,7 +253,18 @@ TEST_F(OperatorClassesWithGXFContext, TestSegmentationPostprocessorOp) { EXPECT_EQ(typeid(op), typeid(std::make_shared(args))); std::string log_output = testing::internal::GetCapturedStderr(); - EXPECT_TRUE(log_output.find("[error]") == std::string::npos); + auto error_pos = log_output.find("[error]"); + if (error_pos != std::string::npos) { + // Initializing a native operator outside the context of app.run() will result in the + // following error being logged because the GXFWrapper will not yet have been created for + // the operator: + // [error] [gxf_executor.cpp:452] Unable to get GXFWrapper for Operator + // 'segmentation_postprocessor' + + // GXFWrapper was mentioned and no additional error was logged + EXPECT_TRUE(log_output.find("GXFWrapper", error_pos + 1) != std::string::npos); + EXPECT_TRUE(log_output.find("[error]", error_pos + 1) == std::string::npos); + } } TEST_F(OperatorClassesWithGXFContext, TestHolovizOp) { @@ -253,23 +290,18 @@ TEST_F(OperatorClassesWithGXFContext, TestHolovizOp) { EXPECT_EQ(typeid(op), typeid(std::make_shared(kwargs))); std::string log_output = testing::internal::GetCapturedStderr(); - EXPECT_TRUE(log_output.find("[error]") == std::string::npos); -} - -TEST_F(OperatorClassesWithGXFContext, TestToolTrackingPostprocessorOp) { - const std::string name{"tool_tracking_post"}; - - // load most arguments from the YAML file - ArgList kwargs{Arg{"min_prob", 0.5f}}; - - testing::internal::CaptureStderr(); - - auto op = F.make_operator(name, kwargs); - EXPECT_EQ(op->name(), name); - EXPECT_EQ(typeid(op), typeid(std::make_shared(kwargs))); - - std::string log_output = testing::internal::GetCapturedStderr(); - EXPECT_TRUE(log_output.find("[error]") == std::string::npos); + // EXPECT_TRUE(log_output.find("[error]") == std::string::npos); + auto error_pos = log_output.find("[error]"); + if (error_pos != std::string::npos) { + // Initializing a native operator outside the context of app.run() will result in the + // following error being logged because the GXFWrapper will not yet have been created for + // the operator: + // [error] [gxf_executor.cpp:452] Unable to get GXFWrapper for Operator 'holoviz' + + // GXFWrapper was mentioned and no additional error was logged + EXPECT_TRUE(log_output.find("GXFWrapper", error_pos + 1) != std::string::npos); + EXPECT_TRUE(log_output.find("[error]", error_pos + 1) == std::string::npos); + } } TEST_F(OperatorClassesWithGXFContext, TestMultiAIInferenceOp) { @@ -286,7 +318,17 @@ TEST_F(OperatorClassesWithGXFContext, TestMultiAIInferenceOp) { EXPECT_EQ(typeid(op), typeid(std::make_shared(kwargs))); std::string log_output = testing::internal::GetCapturedStderr(); - EXPECT_TRUE(log_output.find("[error]") == std::string::npos); + auto error_pos = log_output.find("[error]"); + if (error_pos != std::string::npos) { + // Initializing a native operator outside the context of app.run() will result in the + // following error being logged because the GXFWrapper will not yet have been created for + // the operator: + // [error] [gxf_executor.cpp:452] Unable to get GXFWrapper for Operator 'multiai_inference' + + // GXFWrapper was mentioned and no additional error was logged + EXPECT_TRUE(log_output.find("GXFWrapper", error_pos + 1) != std::string::npos); + EXPECT_TRUE(log_output.find("[error]", error_pos + 1) == std::string::npos); + } } TEST_F(OperatorClassesWithGXFContext, TestMultiAIPostprocessorOp) { @@ -303,24 +345,18 @@ TEST_F(OperatorClassesWithGXFContext, TestMultiAIPostprocessorOp) { EXPECT_EQ(typeid(op), typeid(std::make_shared(kwargs))); std::string log_output = testing::internal::GetCapturedStderr(); - EXPECT_TRUE(log_output.find("[error]") == std::string::npos); -} - -TEST_F(OperatorClassesWithGXFContext, TestVisualizerICardioOp) { - const std::string name{"visualizer_icardio"}; - - // load most arguments from the YAML file - ArgList kwargs = F.from_config("visualizer_icardio"); - kwargs.add(Arg{"allocator", F.make_resource("pool")}); - - testing::internal::CaptureStderr(); - - auto op = F.make_operator(name, kwargs); - EXPECT_EQ(op->name(), name); - EXPECT_EQ(typeid(op), typeid(std::make_shared(kwargs))); - - std::string log_output = testing::internal::GetCapturedStderr(); - EXPECT_TRUE(log_output.find("[error]") == std::string::npos); + auto error_pos = log_output.find("[error]"); + if (error_pos != std::string::npos) { + // Initializing a native operator outside the context of app.run() will result in the + // following error being logged because the GXFWrapper will not yet have been created for + // the operator: + // [error] [gxf_executor.cpp:452] Unable to get GXFWrapper for Operator + // 'multiai_postprocessor' + + // GXFWrapper was mentioned and no additional error was logged + EXPECT_TRUE(log_output.find("GXFWrapper", error_pos + 1) != std::string::npos); + EXPECT_TRUE(log_output.find("[error]", error_pos + 1) == std::string::npos); + } } TEST_F(OperatorClassesWithGXFContext, TestBayerDemosaicOp) { @@ -353,26 +389,6 @@ TEST_F(OperatorClassesWithGXFContext, TestBayerDemosaicOpDefaultConstructor) { EXPECT_TRUE(log_output.find("[error]") == std::string::npos); } -#if HOLOSCAN_BUILD_EMERGENT == 1 -using OperatorClassesWithGXFEmergentContext = TestWithGXFEmergentContext; - -TEST_F(OperatorClassesWithGXFEmergentContext, TestEmergentSourceOp) { - const std::string name{"emergent"}; - - // load most arguments from the YAML file - ArgList kwargs = F.from_config("emergent"); - - testing::internal::CaptureStderr(); - - auto op = F.make_operator(name, kwargs); - EXPECT_EQ(op->name(), name); - EXPECT_EQ(typeid(op), typeid(std::make_shared(kwargs))); - - std::string log_output = testing::internal::GetCapturedStderr(); - EXPECT_TRUE(log_output.find("[error]") == std::string::npos); -} -#endif - TEST(Operator, TestNativeOperatorWithoutFragment) { Operator op; EXPECT_EQ(op.name(), ""s); diff --git a/tests/gxf_extensions/segmentation_postprocessor/test_postprocessor.cpp b/tests/operators/segmentation_postprocessor/test_postprocessor.cpp similarity index 91% rename from tests/gxf_extensions/segmentation_postprocessor/test_postprocessor.cpp rename to tests/operators/segmentation_postprocessor/test_postprocessor.cpp index 4d47e8d3..6d65ff5a 100644 --- a/tests/gxf_extensions/segmentation_postprocessor/test_postprocessor.cpp +++ b/tests/operators/segmentation_postprocessor/test_postprocessor.cpp @@ -20,8 +20,7 @@ #include #include -#include - +#include // This test data is generated in python as follows: // @@ -125,7 +124,7 @@ static const uint8_t kArgmaxOutputData[] = { 2, 1, 0, 4, 3, 1, 0, 4, 2, 4, 4, 1, 4, 3, 3, 0, 4, 2, 4, 1, 1, 1, 3, 1, 4, 1, 3, 0, 2, 0}; TEST(SegmentationPostprocessor, Argmax) { - nvidia::holoscan::segmentation_postprocessor::Shape shape; + holoscan::ops::segmentation_postprocessor::Shape shape; shape.height = 19; shape.width = 10; shape.channels = 5; @@ -136,28 +135,31 @@ TEST(SegmentationPostprocessor, Argmax) { ASSERT_EQ(input_data_size, shape.height * shape.width * shape.channels * sizeof(float)); ASSERT_EQ(output_data_size, shape.height * shape.width * - sizeof(nvidia::holoscan::segmentation_postprocessor::output_type_t)); + sizeof(holoscan::ops::segmentation_postprocessor::output_type_t)); - nvidia::holoscan::segmentation_postprocessor::output_type_t host_output_data[output_data_size] = - {}; + holoscan::ops::segmentation_postprocessor::output_type_t host_output_data[output_data_size] = {}; float* device_input_data = nullptr; - nvidia::holoscan::segmentation_postprocessor::output_type_t* device_output_data = nullptr; + holoscan::ops::segmentation_postprocessor::output_type_t* device_output_data = nullptr; ASSERT_EQ(cudaSuccess, cudaMalloc(reinterpret_cast(&device_input_data), input_data_size)); ASSERT_EQ(cudaSuccess, cudaMalloc(reinterpret_cast(&device_output_data), output_data_size)); - ASSERT_EQ(cudaSuccess, cudaMemcpy(device_input_data, kArgmaxInputData, input_data_size, - cudaMemcpyHostToDevice)); + ASSERT_EQ( + cudaSuccess, + cudaMemcpy(device_input_data, kArgmaxInputData, input_data_size, cudaMemcpyHostToDevice)); ASSERT_EQ(cudaSuccess, cudaMemset(device_output_data, 0, output_data_size)); - nvidia::holoscan::segmentation_postprocessor::cuda_postprocess( - nvidia::holoscan::segmentation_postprocessor::NetworkOutputType::kSoftmax, - nvidia::holoscan::segmentation_postprocessor::DataFormat::kHWC, shape, device_input_data, + holoscan::ops::segmentation_postprocessor::cuda_postprocess( + holoscan::ops::segmentation_postprocessor::NetworkOutputType::kSoftmax, + holoscan::ops::segmentation_postprocessor::DataFormat::kHWC, + shape, + device_input_data, device_output_data); - ASSERT_EQ(cudaSuccess, cudaMemcpy(host_output_data, device_output_data, output_data_size, - cudaMemcpyDeviceToHost)); + ASSERT_EQ( + cudaSuccess, + cudaMemcpy(host_output_data, device_output_data, output_data_size, cudaMemcpyDeviceToHost)); ASSERT_EQ(cudaSuccess, cudaFree(device_input_data)); ASSERT_EQ(cudaSuccess, cudaFree(device_output_data)); diff --git a/tests/system/exception_handling.cpp b/tests/system/exception_handling.cpp new file mode 100644 index 00000000..1965aad5 --- /dev/null +++ b/tests/system/exception_handling.cpp @@ -0,0 +1,67 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include + +#include +#include "../config.hpp" +#include "common/assert.hpp" + +static HoloscanTestConfig test_config; + +namespace holoscan { + +namespace ops { + +class MinimalThrowOp : public Operator { + public: + HOLOSCAN_OPERATOR_FORWARD_ARGS(MinimalThrowOp) + + MinimalThrowOp() = default; + + void compute(InputContext& op_input, OutputContext&, ExecutionContext&) override { + throw std::runtime_error("Exception occurred in MinimalThrowOp::compute"); + }; +}; + +} // namespace ops + +class MinimalThrowApp : public holoscan::Application { + public: + void compose() override { + using namespace holoscan; + auto op = make_operator("min_op", make_condition(3)); + add_operator(op); + } +}; + +TEST(MinimalNativeOperatorApp, TestComputeMethodExceptionHandling) { + load_env_log_level(); + + auto app = make_application(); + + const std::string config_file = test_config.get_test_data_file("minimal.yaml"); + app->config(config_file); + + EXPECT_EXIT( + app->run(), testing::ExitedWithCode(1), "Exception occurred in MinimalThrowOp::compute"); +} + +} // namespace holoscan diff --git a/tests/system/native_operator_minimal_app.cpp b/tests/system/native_operator_minimal_app.cpp new file mode 100644 index 00000000..400d25ec --- /dev/null +++ b/tests/system/native_operator_minimal_app.cpp @@ -0,0 +1,85 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include + +#include + +#include "../config.hpp" +#include "common/assert.hpp" + +using namespace std::string_literals; + +static HoloscanTestConfig test_config; + +namespace holoscan { + +namespace ops { + +class MinimalOp : public Operator { + public: + HOLOSCAN_OPERATOR_FORWARD_ARGS(MinimalOp) + + MinimalOp() = default; + + void setup(OperatorSpec& spec) override { + spec.param(value_, "value", "value", "value stored by the operator", 2.5); + } + + void compute(InputContext& op_input, OutputContext&, ExecutionContext&) override { + HOLOSCAN_LOG_INFO("MinimalOp: count: {}", count_++); + HOLOSCAN_LOG_INFO("MinimalOp: value: {}", value_.get()); + }; + + private: + int count_ = 0; + Parameter value_; +}; + +} // namespace ops + +class MinimalApp : public holoscan::Application { + public: + void compose() override { + using namespace holoscan; + auto op = make_operator( + "min_op", make_condition(3), from_config("value")); + add_operator(op); + } +}; + +TEST(MinimalNativeOperatorApp, TestMinimalNativeOperatorApp) { + load_env_log_level(); + + auto app = make_application(); + + const std::string config_file = test_config.get_test_data_file("minimal.yaml"); + app->config(config_file); + + // capture output so that we can check that the expected value is present + testing::internal::CaptureStderr(); + + app->run(); + + std::string log_output = testing::internal::GetCapturedStderr(); + EXPECT_TRUE(log_output.find("value: 5.3") != std::string::npos); +} + +} // namespace holoscan diff --git a/tests/system/native_operator_multibroadcasts_app.cpp b/tests/system/native_operator_multibroadcasts_app.cpp new file mode 100644 index 00000000..669c4b61 --- /dev/null +++ b/tests/system/native_operator_multibroadcasts_app.cpp @@ -0,0 +1,81 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include + +#include +#include "../config.hpp" +#include "common/assert.hpp" + +#include "ping_rx_op.hpp" +#include "ping_tx_op.hpp" + +using namespace std::string_literals; + +static HoloscanTestConfig test_config; + +namespace holoscan { + +class NativeMultiBroadcastsApp : public holoscan::Application { + public: + void compose() override { + using namespace holoscan; + auto tx = make_operator("tx", make_condition(10)); + auto rx11 = make_operator("rx11"); + auto rx12 = make_operator("rx12"); + auto rx21 = make_operator("rx21"); + auto rx22 = make_operator("rx22"); + + add_flow(tx, rx11, {{"out1", "receivers"}}); + add_flow(tx, rx12, {{"out1", "receivers"}}); + + add_flow(tx, rx21, {{"out2", "receivers"}}); + add_flow(tx, rx22, {{"out2", "receivers"}}); + } +}; + +TEST(NativeOperatorMultiBroadcastsApp, TestNativeOperatorMultiBroadcastsApp) { + load_env_log_level(); + + auto app = make_application(); + + const std::string config_file = test_config.get_test_data_file("minimal.yaml"); + app->config(config_file); + + // capture output so that we can check that the expected value is present + testing::internal::CaptureStderr(); + + app->run(); + + std::string log_output = testing::internal::GetCapturedStderr(); + + // Check if 'log_output' has 'Rx message received (count: 10, size: 1)' four times in it. + // (from rx11, rx12, rx21, rx22) + int count = 0; + std::string recv_string{"Rx message received (count: 10, size: 1)"}; + auto pos = log_output.find(recv_string); + while (pos != std::string::npos) { + count++; + pos = log_output.find(recv_string, pos + recv_string.size()); + } + EXPECT_EQ(count, 4); +} + +} // namespace holoscan diff --git a/tests/system/native_operator_ping_app.cpp b/tests/system/native_operator_ping_app.cpp new file mode 100644 index 00000000..06ea1f60 --- /dev/null +++ b/tests/system/native_operator_ping_app.cpp @@ -0,0 +1,66 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include + +#include + +#include "../config.hpp" +#include "common/assert.hpp" + +#include "ping_rx_op.hpp" +#include "ping_tx_op.hpp" + +using namespace std::string_literals; + +static HoloscanTestConfig test_config; + +namespace holoscan { + +class NativeOpApp : public holoscan::Application { + public: + void compose() override { + using namespace holoscan; + auto tx = make_operator("tx", make_condition(10)); + auto rx = make_operator("rx"); + + add_flow(tx, rx, {{"out1", "receivers"}, {"out2", "receivers"}}); + } +}; + +TEST(NativeOperatorPingApp, TestNativeOperatorPingApp) { + load_env_log_level(); + + auto app = make_application(); + + const std::string config_file = test_config.get_test_data_file("minimal.yaml"); + app->config(config_file); + + // capture output so that we can check that the expected value is present + testing::internal::CaptureStderr(); + + app->run(); + + std::string log_output = testing::internal::GetCapturedStderr(); + EXPECT_TRUE(log_output.find("value1: 1") != std::string::npos); + EXPECT_TRUE(log_output.find("value2: 100") != std::string::npos); +} + +} // namespace holoscan diff --git a/tests/system/ping_rx_op.cpp b/tests/system/ping_rx_op.cpp new file mode 100644 index 00000000..60253172 --- /dev/null +++ b/tests/system/ping_rx_op.cpp @@ -0,0 +1,37 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ping_rx_op.hpp" + +namespace holoscan { +namespace ops { + +void PingRxOp::setup(OperatorSpec& spec) { + spec.param(receivers_, "receivers", "Input Receivers", "List of input receivers.", {}); +} + +void PingRxOp::compute(InputContext& op_input, OutputContext&, ExecutionContext&) { + auto value_vector = op_input.receive>("receivers"); + + HOLOSCAN_LOG_INFO("Rx message received (count: {}, size: {})", count_++, value_vector.size()); + for (int i = 0; i < value_vector.size(); ++i) { + HOLOSCAN_LOG_INFO("Rx message value{}: {}", i + 1, *(value_vector[i].get())); + } +} + +} // namespace ops +} // namespace holoscan diff --git a/gxf_extensions/sample/ping_rx.cpp b/tests/system/ping_rx_op.hpp similarity index 52% rename from gxf_extensions/sample/ping_rx.cpp rename to tests/system/ping_rx_op.hpp index d12113e0..ef8b5912 100644 --- a/gxf_extensions/sample/ping_rx.cpp +++ b/tests/system/ping_rx_op.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,26 +15,32 @@ * limitations under the License. */ -#include "ping_rx.hpp" +#ifndef TESTS_CORE_PING_RX_OP_HPP +#define TESTS_CORE_PING_RX_OP_HPP + +#include + +#include -namespace nvidia { namespace holoscan { -namespace sample { - -gxf_result_t PingRx::registerInterface(gxf::Registrar* registrar) { - gxf::Expected result; - result &= registrar->parameter(signal_, "signal"); - return gxf::ToResultCode(result); -} - -gxf_result_t PingRx::tick() { - auto message = signal_->receive(); - GXF_LOG_INFO("Message Received: %d", this->count); - this->count = this->count + 1; - if (!message || message.value().is_null()) { return GXF_CONTRACT_MESSAGE_NOT_AVAILABLE; } - return GXF_SUCCESS; -} - -} // namespace sample +namespace ops { + +class PingRxOp : public Operator { + public: + HOLOSCAN_OPERATOR_FORWARD_ARGS(PingRxOp) + + PingRxOp() = default; + + void setup(OperatorSpec& spec) override; + + void compute(InputContext& op_input, OutputContext&, ExecutionContext&) override; + + private: + Parameter> receivers_; + int count_ = 1; +}; + +} // namespace ops } // namespace holoscan -} // namespace nvidia + +#endif /* TESTS_CORE_PING_RX_OP_HPP */ diff --git a/gxf_extensions/tool_tracking_postprocessor/tool_tracking_postprocessor.cu.hpp b/tests/system/ping_tx_op.cpp similarity index 59% rename from gxf_extensions/tool_tracking_postprocessor/tool_tracking_postprocessor.cu.hpp rename to tests/system/ping_tx_op.cpp index 21b0b549..d730852a 100644 --- a/gxf_extensions/tool_tracking_postprocessor/tool_tracking_postprocessor.cu.hpp +++ b/tests/system/ping_tx_op.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,18 +15,23 @@ * limitations under the License. */ -#include +#include "ping_tx_op.hpp" -#include -#include - -namespace nvidia { namespace holoscan { -namespace tool_tracking_postprocessor { +namespace ops { + +void PingTxOp::setup(OperatorSpec& spec) { + spec.output("out1"); + spec.output("out2"); +} + +void PingTxOp::compute(InputContext&, OutputContext& op_output, ExecutionContext&) { + auto value1 = std::make_shared(1); + op_output.emit(value1, "out1"); -void cuda_postprocess(uint32_t width, uint32_t height, const std::array& color, - bool first, const float* input, float4* output); + auto value2 = std::make_shared(100); + op_output.emit(value2, "out2"); +} -} // namespace tool_tracking_postprocessor +} // namespace ops } // namespace holoscan -} // namespace nvidia diff --git a/gxf_extensions/visualizer_tool_tracking/frame_data.hpp b/tests/system/ping_tx_op.hpp similarity index 56% rename from gxf_extensions/visualizer_tool_tracking/frame_data.hpp rename to tests/system/ping_tx_op.hpp index c957abf5..8ea6dc4d 100644 --- a/gxf_extensions/visualizer_tool_tracking/frame_data.hpp +++ b/tests/system/ping_tx_op.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,25 +14,27 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#ifndef NVIDIA_CLARA_HOLOSCAN_GXF_FRAME_DATA_HPP_ -#define NVIDIA_CLARA_HOLOSCAN_GXF_FRAME_DATA_HPP_ -#include +#ifndef TESTS_CORE_PING_TX_OP_HPP +#define TESTS_CORE_PING_TX_OP_HPP + +#include -namespace nvidia { namespace holoscan { -namespace visualizer_tool_tracking { - -struct FrameData { - // host memory buffers - std::vector confidence_host_; - std::vector position_host_; - // OpenGL buffers - GLuint confidence_; - GLuint position_; +namespace ops { + +class PingTxOp : public Operator { + public: + HOLOSCAN_OPERATOR_FORWARD_ARGS(PingTxOp) + + PingTxOp() = default; + + void setup(OperatorSpec& spec) override; + + void compute(InputContext&, OutputContext& op_output, ExecutionContext&) override; }; -} // namespace visualizer_tool_tracking +} // namespace ops } // namespace holoscan -} // namespace nvidia -#endif // NVIDIA_CLARA_HOLOSCAN_GXF_FRAME_DATA_HPP_ + +#endif /* TESTS_CORE_PING_TX_OP_HPP */ diff --git a/tests/utils.hpp b/tests/utils.hpp index fcef6200..3d820c9e 100644 --- a/tests/utils.hpp +++ b/tests/utils.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -35,39 +35,19 @@ class TestWithGXFContext : public ::testing::Test { protected: void SetUp() override { F.config(config_file); - auto context = F.executor().context(); + auto& executor = F.executor(); + auto context = executor.context(); + auto extension_manager = executor.extension_manager(); // Load GXF extensions included in the config file // We should do this before we can initialized GXF-based Conditions, Resources or Operators const char* manifest_filename = F.config().config_file().c_str(); - const GxfLoadExtensionsInfo load_ext_info{nullptr, 0, &manifest_filename, 1, - nullptr}; - GXF_ASSERT_SUCCESS(GxfLoadExtensions(context, &load_ext_info)); + auto node = YAML::LoadFile(manifest_filename); + ASSERT_TRUE(extension_manager->load_extensions_from_yaml(node)); } Fragment F; const std::string config_file = test_config.get_test_data_file("app_config.yaml"); }; -#if HOLOSCAN_BUILD_EMERGENT == 1 - // Fixture that creates a Fragment and initializes a GXF context for it loading emergent extension -class TestWithGXFEmergentContext : public ::testing::Test { - protected: - void SetUp() override { - F.config(config_file); - auto context = F.executor().context(); - - // Load GXF extensions included in the config file - // We should do this before we can initialized GXF-based Conditions, Resources or Operators - const char* manifest_filename = F.config().config_file().c_str(); - const GxfLoadExtensionsInfo load_ext_info{nullptr, 0, &manifest_filename, 1, - nullptr}; - GXF_ASSERT_SUCCESS(GxfLoadExtensions(context, &load_ext_info)); - } - - Fragment F; - const std::string config_file = test_config.get_test_data_file("emergent.yaml"); -}; -#endif - } // namespace holoscan