diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 72e26f87..981d90fb 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -30,8 +30,13 @@ "--security-opt=seccomp=unconfined", ], "containerEnv": { + // X11 support "DISPLAY": "${localEnv:DISPLAY}", - "NVIDIA_DRIVER_CAPABILITIES": "graphics,video,compute,utility,display", + // Wayland support + "WAYLAND_DISPLAY": "${localEnv:WAYLAND_DISPLAY}", + "XDG_RUNTIME_DIR": "${localEnv:XDG_RUNTIME_DIR}", + "XDG_SESSION_TYPE": "${localEnv:XDG_SESSION_TYPE}", + "NVIDIA_DRIVER_CAPABILITIES": "all", // Set the following environment variables to use the same folder name as the host machine. // This is needed to launch container from the workspace folder that is not same as the SDK source root folder. "HOLOSCAN_PUBLIC_FOLDER": "${localEnv:HOLOSCAN_PUBLIC_FOLDER}", @@ -40,7 +45,10 @@ "CMAKE_BUILD_PARALLEL_LEVEL": "${localEnv:CMAKE_BUILD_PARALLEL_LEVEL}", }, "mounts": [ + // X11 support "source=/tmp/.X11-unix,target=/tmp/.X11-unix,type=bind,consistency=cached", + // Wayland support + "source=${localEnv:XDG_RUNTIME_DIR},target=${localEnv:XDG_RUNTIME_DIR},type=bind,consistency=cached", ], "workspaceMount": "source=${localWorkspaceFolder},target=/workspace/holoscan-sdk,type=bind,consistency=cached", "workspaceFolder": "/workspace/holoscan-sdk", @@ -74,4 +82,4 @@ // "postCreateCommand": "gcc -v", // Comment out this line to run as root instead. "remoteUser": "holoscan-sdk" -} \ No newline at end of file +} diff --git a/.vscode/launch.json b/.vscode/launch.json index ee2d522d..3724ee49 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -372,6 +372,24 @@ } ] }, + { + "name": "(gdb) examples/holoviz/cpp/holoviz_camera", + "type": "cppdbg", + "request": "launch", + "program": "${command:cmake.buildDirectory}/examples/holoviz/cpp/holoviz_camera", + "args": [], + "stopAtEntry": false, + "cwd": "${command:cmake.buildDirectory}", + "environment": [], + "MIMode": "gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] + }, { "name": "(gdb) examples/holoviz/cpp/holoviz_geometry", "type": "cppdbg", @@ -485,6 +503,55 @@ } ] }, + { + "name": "(gdb) examples/import_gxf_components/cpp", + "type": "cppdbg", + "request": "launch", + "program": "${command:cmake.buildDirectory}/examples/import_gxf_components/cpp/import_gxf_components", + "args": [], + "stopAtEntry": false, + "cwd": "${command:cmake.buildDirectory}", + "environment": [ + { + "name": "HOLOSCAN_LOG_LEVEL", + "value": "DEBUG" + }, + ], + "MIMode": "gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ] + }, + { + "name": "(gdb) examples/import_gxf_components/python", + "type": "cppdbg", + "request": "launch", + "program": "/usr/bin/bash", + "args": [ + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/.vscode/debug_python", + "${workspaceFolder}/${env:HOLOSCAN_PUBLIC_FOLDER}/examples/import_gxf_components/python/import_gxf_components.py", + ], + "stopAtEntry": false, + "cwd": "${command:cmake.buildDirectory}", + "environment": [ + { + "name": "HOLOSCAN_LOG_LEVEL", + "value": "DEBUG" + }, + { + "name": "PYTHONPATH", + "value": "${command:cmake.buildDirectory}/python/lib" + }, + { + "name": "HOLOSCAN_INPUT_PATH", + "value": "${command:cmake.buildDirectory}/../data" + }, + ], + }, { "name": "(gdb) examples/multithread/cpp/multithread", "type": "cppdbg", diff --git a/CMakeLists.txt b/CMakeLists.txt index 2ced8e43..b47a95c4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -188,6 +188,7 @@ list(APPEND HOLOSCAN_INSTALL_TARGETS op_async_ping_tx op_bayer_demosaic op_format_converter + op_gxf_codelet op_holoviz op_inference op_inference_processor diff --git a/Dockerfile b/Dockerfile index df8f1971..5366c8ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,6 +38,7 @@ FROM nvcr.io/nvidia/tensorrt:23.12-py3-igpu AS igpu_base FROM ${GPU_TYPE}_base AS base ARG DEBIAN_FRONTEND=noninteractive +ENV NVIDIA_DRIVER_CAPABILITIES=all ############################################################ # Variables @@ -328,12 +329,14 @@ RUN install -m 0755 -d /etc/apt/keyrings \ # libx* - X packages # libvulkan1 - for Vulkan apps (Holoviz) # vulkan-validationlayers, spirv-tools - for Vulkan validation layer (enabled for Holoviz in debug mode) +# libwayland-dev, libxkbcommon-dev, pkg-config - GLFW compile dependency for Wayland support +# libdecor-0-plugin-1-cairo - GLFW runtime dependency for Wayland window decorations # libegl1 - to run headless Vulkan apps # libopenblas0 - libtorch dependency # libv4l-dev - V4L2 operator dependency # v4l-utils - V4L2 operator utility # libpng-dev - torchvision dependency -# libjpeg-dev - torchvision dependency +# libjpeg-dev - torchvision, v4l2 mjpeg dependency # docker-ce-cli - enable Docker DooD for CLI # docker-buildx-plugin - enable Docker DooD for CLI RUN apt-get update \ @@ -349,6 +352,10 @@ RUN apt-get update \ libvulkan1="1.3.204.1-*" \ vulkan-validationlayers="1.3.204.1-*" \ spirv-tools="2022.1+1.3.204.0-*" \ + libwayland-dev="1.20.0-*" \ + libxkbcommon-dev="1.4.0-*" \ + pkg-config="0.29.2-*" \ + libdecor-0-plugin-1-cairo="0.1.0-*" \ libegl1="1.4.0-*" \ libopenblas0="0.3.20+ds-*" \ libv4l-dev="1.22.1-*" \ diff --git a/NOTICE.txt b/NOTICE.txt index 7d02d28c..cf7751bd 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -47,8 +47,8 @@ Licensed under MIT (https://github.com/fmtlib/fmt/blob/8.1.1/LICENSE.rst) GLFW (https://www.glfw.org/) Copyright (c) 2002-2006 Marcus Geelnard -Copyright (c) 2006-2016 Camilla Berglund -Licensed under Zlib (https://github.com/glfw/glfw/blob/3.3.7/COPYING.txt) +Copyright (c) 2006-2019 Camilla Löwy +Licensed under Zlib (https://github.com/glfw/glfw/blob/3.4/LICENSE.md) gRPC (https://github.com/grpc/grpc) Copyright 2014 gRPC authors. @@ -81,6 +81,20 @@ Copyright © 2015 Research Organization for Information Science and Technol Copyright © 2015-2016 Intel, Inc. All rights reserved. Licensed under BSD-3-clause (https://github.com/open-mpi/hwloc/blob/hwloc-2.9.0/COPYING) +AJA NTV2 SDK (https://github.com/nvidia-holoscan/libajantv2) +Copyright (c) 2023 AJA Video Systems +Licensed under MIT (https://github.com/nvidia-holoscan/libajantv2/blob/holoscan/LICENSE) + +libdecor-0-plugin-1-cairo (https://packages.ubuntu.com/jammy/libdecor-0-plugin-1-cairo) +2010 Intel Corporation +2011 Benjamin Franzke +2017-2018 Red Hat Inc +2018-2019 Jonas Ã…dahl +2019 Christian Rauch +2021 Christian Rauch +2021 Marco Trevisan +Licensed under MIT (http://changelogs.ubuntu.com/changelogs/pool/main/libd/libdecor-0/libdecor-0_0.1.0-3build1/copyright) + libegl1 (https://packages.ubuntu.com/jammy/libegl1) 2013-2017 NVIDIA Corporation 2007-2013 VMware, Inc @@ -144,6 +158,63 @@ libvulkan1 (https://packages.ubuntu.com/jammy/libvulkan1) 2015-2016 LunarG, Inc Licensed under Apache-2.0 (http://changelogs.ubuntu.com/changelogs/pool/main/v/vulkan-loader/vulkan-loader_1.3.204.1-2/copyright) +libwayland-client (https://packages.ubuntu.com/jammy/libwayland-client++0) +Copyright: 2014-2019, Nils Christopher Brause +Copyright: 2014-2019, Nils Christopher Brause, Philipp Kerling +Copyright: 2014-2019, Philipp Kerling, Nils Christopher Brause +Copyright: 2015-2016, Red Hat Inc. +Copyright: 2013-2014, Collabora, Ltd. +Copyright: 2017-2019, Philipp Kerling +Copyright: 2014-2019, Nils Christopher Brause, Philipp Kerling, Zsolt Bölöny +Copyright: 2012-2013, Intel Corporation +Copyright: 2014, Jonas Ã…dahl +Copyright: 2014, Stephen "Lyude" Chandler Paul +Copyright: 2014-2019, Philipp Kerling, Nils Christopher Brause, Craig Andrews, Tobias Kortkamp, Balint Reczey +Copyright: 2014-2019, Nils Christopher Brause, Philipp Kerling, Bernd Kuhls +Copyright: 2012-2013, Intel Corporation +Copyright: 2016, The Chromium Authors. +Copyright: 2008-2011, Kristian Høgsberg +Copyright: 2008-2013, Kristian Høgsberg +Copyright: 2008-2013, Kristian Høgsberg +Copyright: 2015, Jason Ekstrand +Copyright: 2015-2016, Red Hat +Copyright: 2015, Samsung Electronics Co., Ltd +Copyright: 2018, Simon Ser +Copyright: 2017 wsnipex 2019 Balint Reczey +Licensed under MIT, BSD-2 (https://changelogs.ubuntu.com/changelogs/pool/universe/w/waylandpp/waylandpp_0.2.8-2/copyright) + +libwayland-dev (https://packages.ubuntu.com/jammy/libwayland-dev) +Copyright: © 2011 Cyril Brulebois +Copyright: © 1999 SuSE, Inc. © 2002 Keith Packard © 2006, 2008 Junio C Hamano © 2008-2012 Kristian Høgsberg © 2010-2012 Intel Corporation © 2011 Benjamin Franzke © 2012-2013, 2016 Collabora, Ltd © 2012-2013 Jason Ekstrand © 2012-2014 Jonas Ã…dahl © 2013 Marek Chalupa © 2014-2015 Red Hat, Inc. © 2015 Giulio Camuffo © 2016 Klarälvdalens Datakonsult AB © 2016 Yong Bakos © 2017 NVIDIA CORPORATION © 2017 Samsung Electronics Co., Ltd +Licensed under MIT (https://changelogs.ubuntu.com/changelogs/pool/main/w/wayland/wayland_1.20.0-1ubuntu0.1/copyright) + +libwayland-egl1 (https://packages.ubuntu.com/jammy/libwayland-egl1) +Copyright 2011 Cyril Brulebois +Copyright: 1999 SuSE, Inc. +Copyright: 2002 Keith Packard +Copyright: 2006, 2008 Junio C Hamano +Copyright: 2008-2012 Kristian Høgsberg +Copyright: 2010-2012 Intel Corporation +Copyright: 2011 Benjamin Franzke +Copyright: 2012-2013, 2016 Collabora, Ltd +Copyright: 2012-2013 Jason Ekstrand +Copyright: 2012-2014 Jonas Ã…dahl +Copyright: 2013 Marek Chalupa +Copyright: 2014-2015 Red Hat, Inc. +Copyright: 2015 Giulio Camuffo +Copyright: 2016 Klarälvdalens Datakonsult AB +Copyright: 2016 Yong Bakos +Copyright: 2017 NVIDIA CORPORATION +Copyright: 2017 Samsung Electronics Co., Ltd +Licensed under X11 (https://changelogs.ubuntu.com/changelogs/pool/main/w/wayland/wayland_1.20.0-1ubuntu0.1/copyright) + +libxkbcommon-dev (https://packages.ubuntu.com/jammy/libxkbcommon-dev) +Copyright 1985, 1987, 1988, 1990, 1998 The Open Group +Copyright 2008, 2009 Dan Nicholson +Copyright (c) 1993, 1994, 1995, 1996 by Silicon Graphics Computer Systems, Inc. +Copyright 1987, 1988 by Digital Equipment Corporation, Maynard, Massachusetts. +Licensed under MIT (https://changelogs.ubuntu.com/changelogs/pool/main/libx/libxkbcommon/libxkbcommon_1.4.0-1/copyright) + magic_enum (https://github.com/Neargye/magic_enum) Copyright (c) 2019 - 2023 Daniil Goncharov Licensed under MIT (https://github.com/Neargye/magic_enum/blob/v0.9.3/LICENSE) @@ -156,10 +227,6 @@ Ninja (https://packages.ubuntu.com/jammy/ninja-build) 2011-2014 Google Licensed under Apache 2 License (http://changelogs.ubuntu.com/changelogs/pool/universe/n/ninja-build/ninja-build_1.10.1-1/copyright) -AJA NTV2 SDK (https://github.com/ibstewart/ntv2) -Copyright (c) 2021 AJA Video Systems -Licensed under MIT (https://github.com/ibstewart/ntv2/blob/holoscan-v0.2.0/LICENSE) - nvpro_core (https://github.com/nvpro-samples/nvpro_core) Copyright (c) 2014-2023, NVIDIA CORPORATION. All rights reserved. Licensed under Apache 2.0 (https://github.com/nvpro-samples/nvpro_core/blob/master/LICENSE) @@ -175,6 +242,11 @@ openblas (https://packages.ubuntu.com/jammy/libopenblas0) 2020 IBM Corporation Licensed under BSD (http://changelogs.ubuntu.com/changelogs/pool/universe/o/openblas/openblas_0.3.20+ds-1/copyright) +pkg-config (https://packages.ubuntu.com/jammy/pkg-config) +Copyright (C) 2001, 2002 Red Hat Inc. +Copyright (C) 2004, 2005 Tollef Fog Heen +Licensed under GPLv2 (http://changelogs.ubuntu.com/changelogs/pool/main/p/pkg-config/pkg-config_0.29.2-1ubuntu3/copyright) + pybind11 (https://github.com/pybind/pybind11) Copyright (c) 2016 Wenzel Jakob , All rights reserved. Licensed under BSD-3-clause (https://github.com/pybind/pybind11/blob/v2.11.1/LICENSE) diff --git a/README.md b/README.md index 32136618..2ff38d1c 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ We appreciate community discussion and feedback in support of Holoscan platform ### Relation to NVIDIA Clara -In previous releases, the prefix [`Clara`](https://developer.nvidia.com/industries/healthcare) was used to define Holoscan as a platform designed initially for [medical devices](https://www.nvidia.com/en-us/clara/developer-kits/). As Holoscan has grown, its potential to serve other areas has become apparent. With version 0.4.0, we're proud to announce that the Holoscan SDK is now officially built to be domain-agnostic and can be used to build sensor AI applications in multiple domains. Note that some of the content of the SDK (sample applications) or the documentation might still appear to be healthcare-specific pending additional updates. Going forward, domain specific content will be hosted on the [HoloHub](https://nvidia-holoscan.github.io/holohub) repository. +In previous releases, the prefix [`Clara`](https://developer.nvidia.com/industries/healthcare) was used to define Holoscan as a platform designed initially for [medical devices](https://www.nvidia.com/en-us/clara/developer-kits/). Starting with version 0.4.0, the Holoscan SDK is built to be domain-agnostic and can be used to build sensor AI applications in multiple domains. Domain specific content will be hosted on the [HoloHub](https://nvidia-holoscan.github.io/holohub) repository. ### Repository structure diff --git a/VERSION b/VERSION index 359a5b95..50aea0e7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.0 \ No newline at end of file +2.1.0 \ No newline at end of file diff --git a/cmake/deps/ajantv2_rapids.cmake b/cmake/deps/ajantv2_rapids.cmake index 5cd2da9b..59778b75 100644 --- a/cmake/deps/ajantv2_rapids.cmake +++ b/cmake/deps/ajantv2_rapids.cmake @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,21 +16,22 @@ # https://docs.rapids.ai/api/rapids-cmake/stable/command/rapids_find_package.html# include(${rapids-cmake-dir}/cpm/find.cmake) -rapids_cpm_find(ajantv2 16.2.0 +# Setting NTV2_VERSION_BUILD environment variable to avoid CMake warning +set(ENV{NTV2_VERSION_BUILD} 1) + +rapids_cpm_find(ajantv2 17.0.1 GLOBAL_TARGETS AJA::ajantv2 CPM_ARGS - GITHUB_REPOSITORY nvidia-holoscan/ntv2 - GIT_TAG 1321a8d4c1a8de696c996d05b65e5aa2934f89d1 + GITHUB_REPOSITORY nvidia-holoscan/libajantv2 + GIT_TAG d4250c556bcf1ebade627a3ef7a2027de7dc85ee OPTIONS - "AJA_BUILD_APPS OFF" - "AJA_BUILD_DOCS OFF" - "AJA_BUILD_DRIVER OFF" - "AJA_BUILD_LIBS ON" - "AJA_BUILD_PLUGINS OFF" - "AJA_BUILD_QA OFF" - "AJA_BUILD_TESTS OFF" + "AJANTV2_DISABLE_DEMOS ON" + "AJANTV2_DISABLE_DRIVER ON" + "AJANTV2_DISABLE_PLUGINS ON" + "AJANTV2_DISABLE_TESTS ON" + "AJANTV2_DISABLE_TOOLS ON" "AJA_INSTALL_HEADERS OFF" "AJA_INSTALL_SOURCES OFF" EXCLUDE_FROM_ALL @@ -43,8 +44,8 @@ if(ajantv2_ADDED) add_library(AJA::ajantv2 ALIAS ajantv2) # Install the headers needed for development with the SDK - install(DIRECTORY ${ajantv2_SOURCE_DIR}/ajalibraries - DESTINATION "include" + install(DIRECTORY ${ajantv2_SOURCE_DIR}/ajantv2 ${ajantv2_SOURCE_DIR}/ajabase + DESTINATION "include/libajantv2" COMPONENT "holoscan-dependencies" FILES_MATCHING PATTERN "*.h" PATTERN "*.hh" ) diff --git a/cmake/deps/glfw_rapids.cmake b/cmake/deps/glfw_rapids.cmake index ab2861a4..14a0707f 100644 --- a/cmake/deps/glfw_rapids.cmake +++ b/cmake/deps/glfw_rapids.cmake @@ -28,36 +28,59 @@ if(NOT X11_Xrandr_INCLUDE_PATH) endif() # Check for Xinerama (legacy multi-monitor support) -if(NOT X11_Xrandr_INCLUDE_PATH) +if(NOT X11_Xinerama_INCLUDE_PATH) message(FATAL_ERROR "Xinerama headers not found. Please install Xinerama ('sudo apt-get install libxinerama-dev') and try again.") endif() +# Check for Xkb (X keyboard extension) +if(NOT X11_Xkb_INCLUDE_PATH) + message(FATAL_ERROR "Xkb headers not found. Please install Xkb ('sudo apt-get install libxkbcommon-dev' or 'sudo apt-get install libx11-dev') and try again.") +endif() + # Check for Xcursor (cursor creation from RGBA images) if(NOT X11_Xcursor_INCLUDE_PATH) message(FATAL_ERROR "Xcursor headers not found. Please install Xcursor ('sudo apt-get install libxcursor-dev') and try again.") endif() -# Check for Xkb (X keyboard extension) +# Check for XInput (modern HID input) +if(NOT X11_Xi_INCLUDE_PATH) + message(FATAL_ERROR "XInput headers not found. Please install XInput ('sudo apt-get install libxi-dev') and try again.") +endif() + +# Check for X Shape (custom window input shape) +if(NOT X11_Xshape_INCLUDE_PATH) + message(FATAL_ERROR "X Shape headers not found. Please install Xext ('sudo apt-get install libxext-dev') and try again.") +endif() + +# Check for XKB compiler if(NOT X11_Xkb_INCLUDE_PATH) - message(FATAL_ERROR "Xkb headers not found. Please install Xkb ('sudo apt-get install libxkbcommon-dev' or 'sudo apt-get install libx11-dev') and try again.") + message(FATAL_ERROR "Xkb compiler not found. Please install XKB compiler ('sudo apt-get install libxkbcommon-dev') and try again.") endif() -# Check for XInput (modern HID input) -if(NOT X11_Xinput_INCLUDE_PATH) - message(FATAL_ERROR "XInput headers not found. Please install XInput ('sudo apt-get install libxi-dev') and try again.") +# Check for Wayland +include(FindPkgConfig) +pkg_check_modules(Wayland + wayland-client>=0.2.7 + wayland-cursor>=0.2.7 + wayland-egl>=0.2.7 + ) +if(NOT Wayland_FOUND) + message(FATAL_ERROR "Wayland not found. Please install Wayland development files ('sudo apt-get install libwayland-dev') and try again.") endif() -rapids_cpm_find(GLFW 3.3.7 +rapids_cpm_find(GLFW 3.4 GLOBAL_TARGETS glfw CPM_ARGS GITHUB_REPOSITORY glfw/glfw - GIT_TAG 3.3.7 + GIT_TAG 3.4 OPTIONS "BUILD_SHARED_LIBS OFF" "CXXOPTS_BUILD_EXAMPLES OFF" "CXXOPTS_BUILD_TESTS OFF" + "GLFW_BUILD_X11 ON" + "GLFW_BUILD_WAYLAND ON" "GLFW_BUILD_TESTS OFF" "GLFW_BUILD_EXAMPLES OFF" "GLFW_BULID_DOCS OFF" diff --git a/cmake/modules/HoloscanCPack.cmake b/cmake/modules/HoloscanCPack.cmake index 0e689b8e..c4b0a1fe 100644 --- a/cmake/modules/HoloscanCPack.cmake +++ b/cmake/modules/HoloscanCPack.cmake @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -96,7 +96,7 @@ set(CPACK_DEBIAN_PACKAGE_DEPENDS # Note: only libnpp (non dev) needed at runtime # - libnvjitlink: needed by cupy # - libgomb1: needed by cupy -# - libvulkan1, libx...: needed for holoviz operator +# - libvulkan1 : needed for holoviz operator # - libegl1: needed for holoviz operator in headless mode # - libv4l-0: needed for v4l2 operator # - python3-cloudpickle: needed for python distributed applications @@ -120,13 +120,14 @@ libcusparse.so.${CUDA_MAJOR} | ${CUSPARSE_PACKAGES}, \ libnpp.so.${CUDA_MAJOR}-dev | ${NPP_DEV_PACKAGES}, \ libnvJitLink.so.${CUDA_MAJOR} | ${NVJITLINK_PACKAGES}, \ libgomb1, \ -libvulkan1, libx11-6, libxcb-glx0, libxcb-glx0, libxcursor1, libxi6, libxinerama1, libxrandr2, \ +libvulkan1, \ libegl1, \ libv4l-0, \ python3-cloudpickle, \ python3-pip" ) -# - libpng, libjpeg, libopenblas: needed for Torch inference backend +# - libpng, libjpeg, libopenblas: needed for Torch inference backend. +# - libjpeg needed by v4l2 for mjpeg support set(CPACK_DEBIAN_PACKAGE_SUGGESTS "libpng16-16, libjpeg-turbo8, libopenblas0") include(CPack) diff --git a/cmake/modules/cpack/NOTICE.txt b/cmake/modules/cpack/NOTICE.txt index 38355b6b..9acf490c 100644 --- a/cmake/modules/cpack/NOTICE.txt +++ b/cmake/modules/cpack/NOTICE.txt @@ -31,8 +31,8 @@ Licensed under MIT (https://github.com/fmtlib/fmt/blob/8.1.1/LICENSE.rst) GLFW (https://www.glfw.org/) Copyright (c) 2002-2006 Marcus Geelnard -Copyright (c) 2006-2016 Camilla Berglund -Licensed under Zlib (https://github.com/glfw/glfw/blob/3.3.7/COPYING.txt) +Copyright (c) 2006-2019 Camilla Löwy +Licensed under Zlib (https://github.com/glfw/glfw/blob/3.4/LICENSE.md) gRPC (https://github.com/grpc/grpc) Copyright 2014 gRPC authors. @@ -69,14 +69,69 @@ jq (https://github.com/jqlang/jq) Copyright (C) 2012 Stephen Dolan authors. Licensed under MIT (https://github.com/jqlang/jq/raw/master/COPYING) +AJA NTV2 SDK (https://github.com/nvidia-holoscan/libajantv2) +Copyright (c) 2023 AJA Video Systems +Licensed under MIT (https://github.com/nvidia-holoscan/libajantv2/blob/holoscan/LICENSE) + +libdecor-0-plugin-1-cairo (https://packages.ubuntu.com/jammy/libdecor-0-plugin-1-cairo) +2010 Intel Corporation +2011 Benjamin Franzke +2017-2018 Red Hat Inc +2018-2019 Jonas Ã…dahl +2019 Christian Rauch +2021 Christian Rauch +2021 Marco Trevisan +Licensed under MIT (http://changelogs.ubuntu.com/changelogs/pool/main/libd/libdecor-0/libdecor-0_0.1.0-3build1/copyright) + +libwayland-client (https://packages.ubuntu.com/jammy/libwayland-client++0) +Copyright: 2014-2019, Nils Christopher Brause +Copyright: 2014-2019, Nils Christopher Brause, Philipp Kerling +Copyright: 2014-2019, Philipp Kerling, Nils Christopher Brause +Copyright: 2015-2016, Red Hat Inc. +Copyright: 2013-2014, Collabora, Ltd. +Copyright: 2017-2019, Philipp Kerling +Copyright: 2014-2019, Nils Christopher Brause, Philipp Kerling, Zsolt Bölöny +Copyright: 2012-2013, Intel Corporation +Copyright: 2014, Jonas Ã…dahl +Copyright: 2014, Stephen "Lyude" Chandler Paul +Copyright: 2014-2019, Philipp Kerling, Nils Christopher Brause, Craig Andrews, Tobias Kortkamp, Balint Reczey +Copyright: 2014-2019, Nils Christopher Brause, Philipp Kerling, Bernd Kuhls +Copyright: 2012-2013, Intel Corporation +Copyright: 2016, The Chromium Authors. +Copyright: 2008-2011, Kristian Høgsberg +Copyright: 2008-2013, Kristian Høgsberg +Copyright: 2008-2013, Kristian Høgsberg +Copyright: 2015, Jason Ekstrand +Copyright: 2015-2016, Red Hat +Copyright: 2015, Samsung Electronics Co., Ltd +Copyright: 2018, Simon Ser +Copyright: 2017 wsnipex 2019 Balint Reczey +Licensed under MIT, BSD-2 (https://changelogs.ubuntu.com/changelogs/pool/universe/w/waylandpp/waylandpp_0.2.8-2/copyright) + +libwayland-egl1 (https://packages.ubuntu.com/jammy/libwayland-egl1) +Copyright 2011 Cyril Brulebois +Copyright: 1999 SuSE, Inc. +Copyright: 2002 Keith Packard +Copyright: 2006, 2008 Junio C Hamano +Copyright: 2008-2012 Kristian Høgsberg +Copyright: 2010-2012 Intel Corporation +Copyright: 2011 Benjamin Franzke +Copyright: 2012-2013, 2016 Collabora, Ltd +Copyright: 2012-2013 Jason Ekstrand +Copyright: 2012-2014 Jonas Ã…dahl +Copyright: 2013 Marek Chalupa +Copyright: 2014-2015 Red Hat, Inc. +Copyright: 2015 Giulio Camuffo +Copyright: 2016 Klarälvdalens Datakonsult AB +Copyright: 2016 Yong Bakos +Copyright: 2017 NVIDIA CORPORATION +Copyright: 2017 Samsung Electronics Co., Ltd +Licensed under X11 (https://changelogs.ubuntu.com/changelogs/pool/main/w/wayland/wayland_1.20.0-1ubuntu0.1/copyright) + magic_enum (https://github.com/Neargye/magic_enum) Copyright (c) 2019 - 2023 Daniil Goncharov Licensed under MIT (https://github.com/Neargye/magic_enum/blob/v0.9.3/LICENSE) -AJA NTV2 SDK (https://github.com/ibstewart/ntv2) -Copyright (c) 2021 AJA Video Systems -Licensed under MIT (https://github.com/ibstewart/ntv2/blob/holoscan-v0.2.0/LICENSE) - nvpro_core (https://github.com/nvpro-samples/nvpro_core) Copyright (c) 2014-2023, NVIDIA CORPORATION. All rights reserved. Licensed under Apache 2.0 (https://github.com/nvpro-samples/nvpro_core/blob/master/LICENSE) diff --git a/docs/aja_setup.rst b/docs/aja_setup.rst index c16184f7..b2985558 100644 --- a/docs/aja_setup.rst +++ b/docs/aja_setup.rst @@ -4,10 +4,10 @@ AJA Video Systems ================= `AJA`_ provides a wide range of proven, professional video I/O devices, and thanks to a -partnership between NVIDIA and AJA, Holoscan supports the AJA NTV2 SDK and device -drivers as of the NTV2 SDK 16.1 release. +partnership between NVIDIA and AJA, Holoscan provides ongoing support for the AJA NTV2 +SDK and device drivers. -The AJA drivers and SDK now offer RDMA support for NVIDIA GPUs. This feature allows +The AJA drivers and SDK offer RDMA support for NVIDIA GPUs. This feature allows video data to be captured directly from the AJA card to GPU memory, which significantly reduces latency and system PCI bandwidth for GPU video processing applications as sysmem to GPU copies are eliminated from the processing @@ -17,7 +17,7 @@ The following instructions describe the steps required to setup and use an AJA device with RDMA support on NVIDIA Developer Kits with a PCIe slot. Note that the AJA NTV2 SDK support for Holoscan includes all of the `AJA Developer Products`_, though the following instructions have only been verified for the `Corvid 44 -12G BNC`_ and `KONA HDMI`_ products, specifically. +12G BNC`_, `KONA XM`, and `KONA HDMI`_ products, specifically. .. Note:: @@ -32,6 +32,7 @@ though the following instructions have only been verified for the `Corvid 44 .. _AJA: https://www.aja.com/ .. _AJA Developer Products: https://www.aja.com/family/developer .. _Corvid 44 12G BNC: https://www.aja.com/products/corvid-44-12g-bnc +.. _KONA XM: https://www.aja.com/products/kona-xm .. _KONA HDMI: https://www.aja.com/products/kona-hdmi @@ -115,8 +116,8 @@ then perform the following to clone the NTV2 SDK source code. .. code-block:: sh - $ git clone https://github.com/nvidia-holoscan/ntv2.git - $ export NTV2=$(pwd)/ntv2 + $ git clone https://github.com/nvidia-holoscan/libajantv2.git + $ export NTV2=$(pwd)/libajantv2 .. Note:: @@ -126,9 +127,38 @@ then perform the following to clone the NTV2 SDK source code. repository whenever possible with the goal to minimize or eliminate divergence between the two repositories. -.. _AJA NTV2 Repository: https://github.com/aja-video/ntv2 +.. _AJA NTV2 Repository: https://github.com/aja-video/libajantv2 +.. _nvidia_open_driver_install: + +Installing the NVIDIA Open Kernel Modules for RDMA Support +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If the AJA NTV2 drivers are going to be built with RDMA support, the open-source +NVIDIA kernel modules must be installed instead of the default proprietary drivers. +If the drivers were installed from an NVIDIA driver installer package then follow +the directions on the `NVIDIA Open GPU Kernel Module Source GitHub`_ page. If the +NVIDIA drivers were installed using an Ubuntu package via `apt`, then replace the +installed `nvidia-kernel-source` package with the corresponding `nvidia-kernel-open` +package. For example, the following shows that the `545` version drivers are installed: + + .. code-block:: sh + + S dpkg --list | grep nvidia-kernel-source + ii nvidia-kernel-source-545 545.23.08-0ubuntu1 amd64 NVIDIA kernel source package + +And the following will replace those with the corresponding `nvidia-kernel-open` drivers: + + .. code-block:: sh + + S sudo apt install -y nvidia-kernel-open-545 + $ sudo dpkg-reconfigure nvidia-dkms-545 + +The system must then be rebooted to load the new open kernel modules. + +.. _NVIDIA Open GPU Kernel Module Source GitHub: https://github.com/NVIDIA/open-gpu-kernel-modules + .. _aja_driver_build: Building the AJA NTV2 Drivers @@ -137,13 +167,13 @@ Building the AJA NTV2 Drivers The following will build the AJA NTV2 drivers with RDMA support enabled. Once built, the kernel module (**ajantv2.ko**) and load/unload scripts (**load_ajantv2** and **unload_ajantv2**) will be output to the -:code:`${NTV2}/bin` directory. +:code:`${NTV2}/driver/bin` directory. .. code-block:: sh $ export AJA_RDMA=1 # Or unset AJA_RDMA to disable RDMA support $ unset AJA_IGPU # Or export AJA_IGPU=1 to run on the integrated GPU of the IGX Orin Devkit (L4T >= 35.4) - $ make -j --directory ${NTV2}/ajadriver/linux + $ make -j --directory ${NTV2}/driver/linux .. _aja_driver_load: @@ -164,9 +194,8 @@ The AJA drivers must be manually loaded every time the machine is rebooted using .. code-block:: sh - $ sudo sh ${NTV2}/bin/load_ajantv2 + $ sudo sh ${NTV2}/driver/bin/load_ajantv2 loaded ajantv2 driver module - created node /dev/ajantv20 .. Note:: @@ -239,15 +268,17 @@ previous step, :ref:`aja_sdk_install`. If any errors occur, see the 2Kp47.95a, 2Kp48a 2. To ensure that RDMA support has been compiled into the AJA driver and is - functioning correctly, the :code:`testrdma` utility can be used: + functioning correctly, the :code:`rdmawhacker` utility can be used (use + `` to terminate): .. code-block:: sh - $ testrdma -t500 - - test device 0 start 0 end 7 size 8388608 count 500 + $ rdmawhacker - frames/errors 500/0 + DMA engine 1 WRITE 8388608 bytes rate: 3975.63 MB/sec 496.95 xfers/sec + Max rate: 4010.03 MB/sec + Min rate: 3301.69 MB/sec + Avg rate: 3923.94 MB/sec .. _aja_use_in_containers: @@ -265,7 +296,7 @@ the :code:`--device` docker argument, such as `--device /dev/ajantv20:/dev/ajant Troubleshooting --------------- -1. **Problem:** The :code:`sudo sh ${NTV2}/bin/load_ajantv2` command returns +1. **Problem:** The :code:`sudo sh ${NTV2}/driver/bin/load_ajantv2` command returns an error. **Solutions:** @@ -327,11 +358,11 @@ Troubleshooting ib_core 211721 1 mlx5_ib nvidia 34655210 315 nvidia_modeset -3. **Problem:** The :code:`testrdma` command outputs the following error: +3. **Problem:** The :code:`rdmawhacker` command outputs the following error: .. code-block:: sh - error - GPU buffer lock failed + ## ERROR: GPU buffer lock failed **Solution:** The AJA drivers need to be compiled with RDMA support enabled. Follow the instructions in :ref:`aja_driver_build`, making sure not to skip diff --git a/docs/api/holoscan_cpp_api.md b/docs/api/holoscan_cpp_api.md index 86f08ee7..019004d6 100644 --- a/docs/api/holoscan_cpp_api.md +++ b/docs/api/holoscan_cpp_api.md @@ -37,11 +37,13 @@ - {ref}`exhale_define_operator_8hpp_1ab2c635a927962650e72a33623f2f9ca1` - {ref}`exhale_define_operator_8hpp_1af59d84ffa537c4b1186e2a1ae2be30ad` +- {ref}`exhale_define_gxf__codelet_8hpp_1adb58640018e9787efd52475fc95a958e` ### Resource Definition - {ref}`exhale_define_resource_8hpp_1a4c671dac9ff91b8ef6f9b5a5a168941f` - {ref}`exhale_define_resource_8hpp_1a94bcc7c12f51de26c6873cf1e7be9ea9` +- {ref}`exhale_define_gxf__component__resource_8hpp_1a269b593e54aca3766ff5b26f780f3e35` ### Condition Definition @@ -102,6 +104,7 @@ - {ref}`exhale_class_classholoscan_1_1ops_1_1AsyncPingTxOp` - {ref}`exhale_class_classholoscan_1_1ops_1_1BayerDemosaicOp` - {ref}`exhale_class_classholoscan_1_1ops_1_1FormatConverterOp` +- {ref}`exhale_class_classholoscan_1_1ops_1_1GXFCodeletOp` - {ref}`exhale_class_classholoscan_1_1ops_1_1HolovizOp` - {ref}`exhale_class_classholoscan_1_1ops_1_1InferenceOp` - {ref}`exhale_class_classholoscan_1_1ops_1_1InferenceProcessorOp` @@ -144,6 +147,7 @@ - {ref}`exhale_class_classholoscan_1_1CudaStreamPool` - {ref}`exhale_class_classholoscan_1_1DoubleBufferReceiver` - {ref}`exhale_class_classholoscan_1_1DoubleBufferTransmitter` +- {ref}`exhale_class_classholoscan_1_1GXFComponentResource` - {ref}`exhale_class_classholoscan_1_1ManualClock` - {ref}`exhale_class_classholoscan_1_1RealtimeClock` - {ref}`exhale_class_classholoscan_1_1Receiver` @@ -172,6 +176,10 @@ - {ref}`exhale_class_classholoscan_1_1Message` +### Analytics + +- {ref}`exhale_class_classholoscan_1_1CsvDataExporter` +- {ref}`exhale_class_classholoscan_1_1DataExporter` ### Domain Objects diff --git a/docs/components/analytics.md b/docs/components/analytics.md new file mode 100644 index 00000000..6bbf83ba --- /dev/null +++ b/docs/components/analytics.md @@ -0,0 +1,49 @@ +# Analytics + +## Data Exporter API +The new Data Exporter C++ API (`DataExporter` and `CsvDataExporter`) is now available. This API can be used to export output from Holoscan applications to comma separated value (CSV) files for Holoscan Federated Analytics applications. `DataExporter` is a base class to support exporting Holoscan application output in different formats. `CsvDataExporter` is a class derived from `DataExporter` to support exporting Holoscan application output to CSV files. + +The data root directory can be specified using the environment variable `HOLOSCAN_ANALYTICS_DATA_DIRECTORY`. If not specified, it defaults to the current directory. The data file name can be specified using the environment variable `HOLOSCAN_ANALYTICS_DATA_FILE_NAME`. If not specified, it defaults to the name `data.csv`. All the generated data will be stored inside a directory with the same name as the application name that is passed to the `DataExporter` constructor. On each run, a new directory inside the `\\` will be created and a new data file will be created inside it. Each new data directory will be named with the current timestamp. This timestamp convention prevents a given run of the application from overwriting any data stored previously by an earlier run of that same application. + + +### Sample usage of the API +```{code-block} c++ +// Include Header +#include + +// Define CsvDataExporter member variable +CsvDataExporter exporter + +// Initialize CsvDataExporter +exporter("app_name", std::vector({"column1", "column2", "column3"})) + +// Export data (typically called within an Operator::compute method) +exporter.export_data(std::vector({"value1", "value2", "value3"})) +``` + +## Using Data Exporter API with DataProcessor + +The Holoscan applications like `Endoscopy Out of Body Detection` uses Inference Processor operator (`InferenceProcessorOp`) to output the binary classification results. The `DataProcessor` class used by the inference processor operator (`InferenceProcessorOp`) is now updated to support writing output to CSV files which can then be used as input to analytics applications. Also any other application using `InferenceProcessorOp` can now export the binary classification output to the CSV files. + +Below is an example application config using the new export operation: + +```yaml +inference_processor_op: + process_operations: + "out_of_body_inferred": ["export_results_to_csv, + out_of_body_detection, + In-body, + Out-of-body, + Confidence Score"] + in_tensor_names: ["out_of_body_inferred"] +``` + +This will create a folder named `out_of_body_detection` in the specified root directory, creates another folder inside it with current timestamp on each run, and creates a `.csv` file with specified name and three columns - `In-body`, `Out-of-body`, and `Confidence Score`. The lines in the `data.csv` file will look like: + + In-body,Out-of-body,Confidence Score + 1,0,0.972435 + 1,0,0.90207 + 1,0,0.897973 + 0,1,0.939281 + 0,1,0.948691 + 0,1,0.94994 diff --git a/docs/components/conditions.md b/docs/components/conditions.md index a35348de..03aea753 100644 --- a/docs/components/conditions.md +++ b/docs/components/conditions.md @@ -32,18 +32,13 @@ Detailed APIs can be found here: {ref}`C++ `/{p **Conditions are AND-combined** -An Operator can be associated with multiple conditions which define -it's execution behavior. Conditions are AND combined to describe -the current state of an operator. For an operator to be executed by the -scheduler, all the conditions must be in `READY` state and -conversely, the operator is unscheduled from execution whenever any one of -the scheduling term reaches `NEVER` state. The priority of various states -during AND combine follows the -order `NEVER`, `WAIT_EVENT`, `WAIT`, `WAIT_TIME`, and `READY`. +An Operator can be associated with multiple conditions which define its execution behavior. Conditions are AND combined to describe +the current state of an operator. For an operator to be executed by the scheduler, all the conditions must be in `READY` state and +conversely, the operator is unscheduled from execution whenever any one of the scheduling terms reaches `NEVER` state. The priority of various states during AND combine follows the order `NEVER`, `WAIT_EVENT`, `WAIT`, `WAIT_TIME`, and `READY`. ## MessageAvailableCondition -An operator associated with `MessageAvailableCondition` is executed when the associated queue of the input port has at least a certain number of elements. +An operator associated with `MessageAvailableCondition` ({cpp:class}`C++ `/{py:class}`Python `) is executed when the associated queue of the input port has at least a certain number of elements. This condition is associated with a specific input port of an operator through the `condition()` method on the return value (IOSpec) of the OperatorSpec's `input()` method. The minimum number of messages that permits the execution of the operator is specified by `min_size` parameter (default: `1`). @@ -53,19 +48,19 @@ It can be used for operators which do not consume all messages from the queue. ## DownstreamMessageAffordableCondition -This condition specifies that an operator shall be executed if the input port of the downstream operator for a given output port can accept new messages. +The `DownstreamMessageAffordableCondition` ({cpp:class}`C++ `/{py:class}`Python `) condition specifies that an operator shall be executed if the input port of the downstream operator for a given output port can accept new messages. This condition is associated with a specific output port of an operator through the `condition()` method on the return value (IOSpec) of the OperatorSpec's `output()` method. The minimum number of messages that permits the execution of the operator is specified by `min_size` parameter (default: `1`). ## CountCondition -An operator associated with `CountCondition` is executed for a specific number of times specified using its `count` parameter. +An operator associated with `CountCondition` ({cpp:class}`C++ `/{py:class}`Python `) is executed for a specific number of times specified using its `count` parameter. The scheduling status of the operator associated with this condition can either be in `READY` or `NEVER` state. The scheduling status reaches the `NEVER` state when the operator has been executed `count` number of times. ## BooleanCondition -An operator associated with `BooleanCondition` is executed when the associated boolean variable is set to `true`. +An operator associated with `BooleanCondition` ({cpp:class}`C++ `/{py:class}`Python `) is executed when the associated boolean variable is set to `true`. The boolean variable is set to `true`/`false` by calling the `enable_tick()`/`disable_tick()` methods on the `BooleanCondition` object. The `check_tick_enabled()` method can be used to check if the boolean variable is set to `true`/`false`. The scheduling status of the operator associated with this condition can either be in `READY` or `NEVER` state. @@ -99,16 +94,12 @@ def compute(self, op_input, op_output, context): ## PeriodicCondition -An operator associated with `PeriodicCondition` is executed after periodic time intervals specified using its `recess_period` parameter. -The scheduling status of the operator associated with this condition can either be in `READY` or `WAIT_TIME` state. -For the first time or after periodic time intervals, the scheduling status of the operator associated with this condition is set to `READY` and the operator is executed. -After the operator is executed, the scheduling status is set to `WAIT_TIME` and the operator is not executed until the `recess_period` time interval. +An operator associated with `PeriodicCondition` ({cpp:class}`C++ `/{py:class}`Python `) is executed after periodic time intervals specified using its `recess_period` parameter. The scheduling status of the operator associated with this condition can either be in `READY` or `WAIT_TIME` state. +For the first time or after periodic time intervals, the scheduling status of the operator associated with this condition is set to `READY` and the operator is executed. After the operator is executed, the scheduling status is set to `WAIT_TIME`, and the operator is not executed until the `recess_period` time interval. ## AsynchronousCondition -AsynchronousCondition is primarily associated with operators which are working with asynchronous events happening outside of their regular execution performed by the scheduler. -Since these events are non-periodic in nature, AsynchronousCondition prevents the scheduler from polling the operator for its status regularly and reduces CPU utilization. -The scheduling status of the operator associated with this condition can either be in `READY`, `WAIT`, `WAIT_EVENT` or `NEVER` states based on the asynchronous event it's waiting on. +`AsynchronousCondition` ({cpp:class}`C++ `/{py:class}`Python `) is primarily associated with operators which are working with asynchronous events happening outside of their regular execution performed by the scheduler. Since these events are non-periodic in nature, `AsynchronousCondition` prevents the scheduler from polling the operator for its status regularly and reduces CPU utilization. The scheduling status of the operator associated with this condition can either be in `READY`, `WAIT`, `WAIT_EVENT`, or `NEVER` states based on the asynchronous event it's waiting on. The state of an asynchronous event is described using `AsynchronousEventState` and is updated using the `event_state()` API. @@ -120,8 +111,4 @@ The state of an asynchronous event is described using `AsynchronousEventState` a | EVENT\_DONE | Event done notification received, operator ready to be ticked | | EVENT\_NEVER | Operator does not want to be executed again, end of execution | -Operators associated with this scheduling term most likely have an asynchronous thread which can update the state of the condition outside of it's regular execution cycle performed by the scheduler. -When the asynchronous event state is in `WAIT` state, the scheduler regularly polls for the scheduling state of the operator. -When the asynchronous event state is in `EVENT_WAITING` state, schedulers will not check the scheduling status of the operator again until they receive an event notification. -Setting the state of the asynchronous event to `EVENT_DONE` automatically sends the event notification to the scheduler. -Operators can use the `EVENT_NEVER` state to indicate the end of its execution cycle. +Operators associated with this scheduling term most likely have an asynchronous thread which can update the state of the condition outside of its regular execution cycle performed by the scheduler. When the asynchronous event state is in `WAIT` state, the scheduler regularly polls for the scheduling state of the operator. When the asynchronous event state is in `EVENT_WAITING` state, schedulers will not check the scheduling status of the operator again until they receive an event notification. Setting the state of the asynchronous event to `EVENT_DONE` automatically sends the event notification to the scheduler. Operators can use the `EVENT_NEVER` state to indicate the end of its execution cycle. As for all of the condition types, the condition type can be used with any of the schedulers. diff --git a/docs/components/resources.md b/docs/components/resources.md index afe46506..7bbb0d76 100644 --- a/docs/components/resources.md +++ b/docs/components/resources.md @@ -10,11 +10,11 @@ There are a number of other resources classes used internally which are not docu ### UnboundedAllocator -An allocator that uses dynamic host or device memory allocation without an upper bound. This allocator does not take any user-specified parameters. +An allocator that uses dynamic host or device memory allocation without an upper bound. This allocator does not take any user-specified parameters. This memory pool is easy to use and is recommended for initial prototyping. Once an application is working, switching to a `BlockMemoryPool` instead may help provide additional performance. ### BlockMemoryPool -This is a memory pool which provides a user-specified number of equally sized blocks of memory. +This is a memory pool which provides a user-specified number of equally sized blocks of memory. Using this memory pool provides a way to allocate memory blocks once and reuse the blocks on each subsequent call to an Operator's `compute` method. This saves overhead relative to allocating memory again each time `compute` is called. For the built-in operators which accept a memory pool parameer, there is a section in it's API docstrings titled "Device Memory Requirements" which provides guidance on the `num_blocks` and `block_size` needed for use with this memory pool. - The `storage_type` parameter can be set to determine the memory storage type used by the operator. This can be 0 for page-locked host memory (allocated with `cudaMallocHost`), 1 for device memory (allocated with `cudaMalloc`) or 2 for system memory (allocated with C++ `new`). - The `block_size` parameter determines the size of a single block in the memory pool in bytes. Any allocation requests made of this allocator must fit into this block size. diff --git a/docs/getting_started.md b/docs/getting_started.md index b27d9b6e..3648bd79 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -36,5 +36,5 @@ The steps above cover what is required to write your own application and run it. ## 6. Master the details -- Expand your understanding of the framework with details on the [logging utility](./holoscan_logging.md) or the [data flow tracking](./flow_tracking.md) benchmarking tool. +- Expand your understanding of the framework with details on the [logging utility](./holoscan_logging.md) or the [data flow tracking](./flow_tracking.md) benchmarking tool and [job statistics](./gxf_job_statistics) measurements. - Learn more details on the configurable components that control the execution of your application, like [Schedulers], [Conditions], and [Resources]. (Advanced) These components are part on the GXF execution backend, hence the **Graph Execution Framework** section at the bottom of this guide if deep understanding of the application execution is needed. diff --git a/docs/gxf_job_statistics.md b/docs/gxf_job_statistics.md new file mode 100644 index 00000000..43e77fb0 --- /dev/null +++ b/docs/gxf_job_statistics.md @@ -0,0 +1,25 @@ +(gxf-job-satistics)= +# GXF job statistics + +Holoscan can have the underlying graph execution framework (GXF) collect job statistics during application execution. Collection of these statistics causes a small amount of runtime overhead, so they are disabled by default, but can be enabled on request via the environment variables documented below. The job statistics will appear in the console on application shutdown, but can optionally also be saved to a JSON file. + +The statistics collected via this method correspond to individual entities (operators) in isolation. To track execution times along specific paths through the computation graph, see the documentation on [flow tracking](./flow_tracking.md) instead. + +:::{note} +The job statistics will be collected by the underlying Graph Execution Framework (GXF) runtime. Given that, the terms used in the report correspond to GXF concepts (entity and codelet) rather than Holoscan classes. +::: + +From the GXF perspective, each Holoscan Operator is a unique entity which contains a single codelet as well as its associated components (corresponding to Holoscan Condition or Resource classes). Any additional entities and codelets that get implicitly created by Holoscan will also appear in the report. For example, if an output port of an operator connects to multiple downstream operators, you will see a corresponding implicit "broadcast" codelet appearing in the report). + + +#### Holoscan SDK environment variables related to GXF job statistics + +Collection of GXF job statistics can be enabled by setting HOLOSCAN_ENABLE_GXF_JOB_STATISTICS. + +- **HOLOSCAN_ENABLE_GXF_JOB_STATISTICS** : Determines if job statistics should be collected. Interprets values like "true", "1", or "on" (case-insensitive) as true (to enable job statistics). It defaults to false if left unspecified. + +- **HOLOSCAN_GXF_JOB_STATISTICS_CODELET** : Determines if a codelet statistics summary table should be created in addition to the entitty stastics. Interprets values like "true", "1", or "on" (case-insensitive) as true (to enable codelet statistics). It defaults to false if left unspecified. + +- **HOLOSCAN_GXF_JOB_STATISTICS_COUNT** : Count of the number of events to be maintained in history per entity. Statistics such as median and maximum correspond to a history of this length. If unspecified, it defaults to 100. + +- **HOLOSCAN_GXF_JOB_STATISTICS_PATH** : Output JSON file name where statistics should be stored. The default if unspecified (or given an empty string) is to output the statistics only to the console. Statistics will still be shown in the console when a file path is specified. diff --git a/docs/holoscan_create_app.md b/docs/holoscan_create_app.md index 61d8850e..161775b1 100644 --- a/docs/holoscan_create_app.md +++ b/docs/holoscan_create_app.md @@ -459,12 +459,6 @@ def compose(self): # Pass directly to the operator constructor my_op = MyOp(self, c1, c2, c3, name="my_op") - - # Built-in operators that wrap an underlying C++ operator class currently do not accept - # Condition classes as positional arguments from the Python API. Instead, one should add the - # condition via the add_arg method of the class - postproc_op = SegmentationPostprocessorOp(self, allocator=UnboundedAllocator(self), name="post") - postproc_op.add_arg(CountCondition(self, count=10)) ``` ```` @@ -476,10 +470,6 @@ This is also illustrated in the [conditions](https://github.com/nvidia-holoscan/ You'll need to specify a unique name for the conditions if there are multiple conditions applied to an operator. ::: -:::{note} -Python operators that wrap an underlying C++ operator currently do not accept conditions as positional arguments. Instead one needs to call the {py:func}`add_arg()` method after the object has been constructed to add the condition. -::: - (configuring-app-operator-resources)= #### Configuring operator resources @@ -520,10 +510,6 @@ def compose(self): ``` ```` -:::{note} -Python operators that wrap an underlying C++ operator currently do not accept resources as positional arguments. Instead one needs to call the {py:func}`add_arg()` method after the object has been constructed to add the resource. -::: - (configuring-app-scheduler)= ### Configuring the scheduler diff --git a/docs/holoscan_create_distributed_app.md b/docs/holoscan_create_distributed_app.md index 2e698de0..a92483b0 100644 --- a/docs/holoscan_create_distributed_app.md +++ b/docs/holoscan_create_distributed_app.md @@ -122,7 +122,7 @@ if __name__ == "__main__": ### Serialization of Custom Data Types for Distributed Applications -Transmission of data between fragments of a multi-fragment application is done via the [Unified Communications X (UCX)](https://openucx.org/) library. In order to transmit data, it must be serialized into a binary form suitable for transmission over a network. For Tensors ({ref}{cpp:class}`C++ `/{py:class}`Python `), strings and various scalar and vector numeric types, serialization is already built in. For more details on concrete examples of how to extend the data serialization support to additional user-defined classes, see the separate page on {ref}`serialization`. +Transmission of data between fragments of a multi-fragment application is done via the [Unified Communications X (UCX)](https://openucx.org/) library. In order to transmit data, it must be serialized into a binary form suitable for transmission over a network. For Tensors ({cpp:class}`C++ `/{py:class}`Python `), strings and various scalar and vector numeric types, serialization is already built in. For more details on concrete examples of how to extend the data serialization support to additional user-defined classes, see the separate page on {ref}`serialization`. (building-and-running-a-distributed-application)= diff --git a/docs/holoscan_create_operator.md b/docs/holoscan_create_operator.md index f7a9f485..ca5473c8 100644 --- a/docs/holoscan_create_operator.md +++ b/docs/holoscan_create_operator.md @@ -98,7 +98,7 @@ class MyOp : public Operator { To create a custom operator in C++ it is necessary to create a subclass of {cpp:class}`holoscan::Operator`. The following example demonstrates how to use native operators (the operators that do not have an underlying, pre-compiled GXF Codelet). -**Code Snippet:** [**examples/ping_multi_port/cpp/ping_multi_port.cpp**](https:://links-need-to-be-corrected.com) +**Code Snippet:** [**examples/ping_multi_port/cpp/ping_multi_port.cpp**](https://github.com/nvidia-holoscan/holoscan-sdk/blob/main/examples/ping_multi_port/cpp/ping_multi_port.cpp) ```{code-block} cpp :caption: examples/ping_multi_port/cpp/ping_multi_port.cpp @@ -551,6 +551,10 @@ With the Holoscan C++ API, we can also wrap {ref}`GXF Codelets` or {ref}`Python` APIs to skip the need for wrapping gxf codelets as operators. If you do need to create a GXF Extension, follow the {ref}`Creating a GXF Extension ` section for a detailed explanation of the GXF extension development process. ::: +:::{tip} +The manual codelet wrapping mechanism described below is no longer necessary in order to make use of a GXF Codelet as a Holoscan operator. There is a new {cpp:class}`~holoscan::ops::GXFCodeletOp` which allows directly using an existing GXF codelet via {cpp:func}`Fragment::make_operator ` without having to first create a wrapper class for it. Similarly there is now also a {cpp:class}`~holoscan::GXFComponentResource` class which allows a GXF Component to be used as a Holoscan resource via {cpp:func}`Fragment::make_resource `. A detailed example of how to use each of these is provided for both C++ and Python applications in the [**examples/import_gxf_components**](https://github.com/nvidia-holoscan/holoscan-sdk/tree/main/examples/import_gxf_components) folder. +::: + Given an existing GXF extension, we can create a simple "identity" application consisting of a replayer, which reads contents from a file on disk, and our recorder from the last section, which will store the output of the replayer exactly in the same format. This allows us to see whether the output of the recorder matches the original input files. The `MyRecorderOp` Holoscan Operator implementation below will wrap the `MyRecorder` GXF Codelet shown {ref}`here`. @@ -1311,249 +1315,11 @@ Here, `values` as returned by ``op_input.receive("receivers")`` will be a tuple (python-wrapped-operators)= ### Python wrapping of a C++ operator -:::{note} -While we provide some utilities to simplify part of the process, this section is designed for advanced developers, since the wrapping of the C++ class using pybind11 is mostly manual and can vary greatly between each operator. -::: - -For convenience while maintaining highest performance, {ref}`operators written in C++` can be wrapped in Python. In the Holoscan SDK, we've used pybind11 to wrap all the built-in operators in [`python/holoscan/operators`](https://github.com/nvidia-holoscan/holoscan-sdk/tree/v0.6.0/python/holoscan/operators). We'll highlight the main components below: - -#### Trampoline classes for handling Python kwargs - -In a C++ file (`my_op_pybind.cpp` in our skeleton code below), create a subclass of the C++ Operator class to wrap. In the subclass, define a new constructor which takes a `Fragment`, an explicit list of parameters with potential default values (`argA`, `argB` below..), and an operator name to fully initialize the operator similar to what is done in [`Fragment::make_operator`](https://github.com/nvidia-holoscan/holoscan-sdk/blob/v0.5.0/include/holoscan/core/fragment.hpp#L207): - -```{code-block} cpp -:caption: my_op_python/my_op_pybind.cpp - -#include -#include -#include - -#include "my_op.hpp" - -class PyMyOp : public MyOp { - public: - using MyOp::MyOp; - - PyMyOp( - Fragment* fragment, - TypeA argA, TypeB argB = 0, ..., - const std::string& name = "my_op" - ) : MyOp(ArgList{ - Arg{"argA", argA}, - Arg{"argB", argB}, - ... - }) { - # If you have arguments you can't pass directly to the `MyOp` constructor as an `Arg`, do - # the conversion and pass the result to `this->add_arg` before setting up the spec below. - name_ = name; - fragment_ = fragment; - spec_ = std::make_shared(fragment); - setup(*spec_.get()); - } -} -``` - -**Example**: Look at the implementation of `PyLSTMTensorRTInferenceOp` on [HoloHub](https://github.com/nvidia-holoscan/holohub/blob/main/operators/lstm_tensor_rt_inference/python/lstm_tensor_rt_inference.cpp) for a specific example, or any of the `Py*Op` classes used for the SDK built-in operators [here](https://github.com/nvidia-holoscan/holoscan-sdk/blob/v0.6.0/python/holoscan/operators/operators.cpp). In the latter, you can find examples of `add_arg` used for less straightforward arguments. - -#### Documentation strings - -Prepare documentation strings (`const char*`) for your python class and its parameters, which we'll use in the next step. - -:::{note} -Below we use a `PYDOC` macro defined in the [SDK](https://github.com/nvidia-holoscan/holoscan-sdk/blob/v0.6.0/python/holoscan/macros.hpp) and available in [HoloHub](https://github.com/nvidia-holoscan/holohub/blob/main/cmake/pydoc/macros.hpp) as a utility to remove leading spaces. In this skeleton example, the documentation code is located in a header file named `my_op_pybind_docs.hpp`, under a custom `doc::MyOp` namespace. None of this is required, you just need to make the strings available in some way for the next section. -::: - -```{code-block} cpp -:caption: my_op_python/my_op_pybind_docs.hpp - -#include "../macros.hpp" - -namespace doc::MyOp { - - PYDOC(cls, R"doc( - My operator. - )doc") - - PYDOC(constructor, R"doc( - Create the operator. - - Parameters - ---------- - fragment : holoscan.core.Fragment - The fragment that the operator belongs to. - argA : TypeA - argA description - argB : TypeB, optional - argB description - name : str, optional - The name of the operator. - )doc") - - PYDOC(initialize, R"doc( - Initialize the operator. - - This method is called only once when the operator is created for the first time, - and uses a light-weight initialization. - )doc") - - PYDOC(setup, R"doc( - Define the operator specification. - - Parameters - ---------- - spec : holoscan.core.OperatorSpec - The operator specification. - )doc") - -} -``` - -**Examples**: Continuing with the `LSTMTensorRTInferenceOp` example on HoloHub, the documentation strings are defined in [lstm_tensor_rt_inference_pydoc.hpp](https://github.com/nvidia-holoscan/holohub/blob/main/operators/lstm_tensor_rt_inference/python/lstm_tensor_rt_inference_pydoc.hpp). The documentation strings for the SDK built-in operators are located in [operators_pydoc.hpp](https://github.com/nvidia-holoscan/holoscan-sdk/blob/v0.6.0/python/holoscan/operators/operators_pydoc.hpp). - -#### Writing glue code - -In the same C++ file as the first section, call `py::class_` within `PYBIND11_MODULE` to define your operator python class. - -:::{note} -- If you are implementing the python wrapping in Holohub, the `` passed to `PYBIND_11_MODULE` **must** match `_` (covered in more details in the next section), in this case, `_my_op`. -- If you are implementing the python wrapping in a standalone CMake project,the `` passed to `PYBIND_11_MODULE` **must** match the name of the module passed to the [pybind11-add-module](https://pybind11.readthedocs.io/en/stable/compiling.html#pybind11-add-module) CMake function. -::: - -```{code-block} cpp -:caption: my_op_python/my_op_pybind.cpp (continued) -:emphasize-lines: 11-12, 29 - -#include - -#include "my_op_pybind_docs.hpp" - -using pybind11::literals::operator""_a; -namespace py = pybind11; - -#define STRINGIFY(x) #x -#define MACRO_STRINGIFY(x) STRINGIFY(x) - -// See notes above, value of `` is important -PYBIND11_MODULE(, m) { - m.doc() = R"pbdoc( - My Module Python Bindings - --------------------------------------- - .. currentmodule:: - .. autosummary:: - :toctree: _generate - add - subtract - )pbdoc"; - -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif - - py::class_>( - m, "MyOp", doc::MyOp::doc_cls) - .def(py::init(), - "fragment"_a, - "argA"_a, - "argB"_a = 0, - ..., - "name"_a = "my_op", - doc::MyOp::doc_constructor) - .def("initialize", - &MyOp::initialize, - doc::MyOp::doc_initialize) - .def("setup", - &MyOp::setup, - "spec"_a, - doc::MyOp::doc_setup); -} -``` - -**Examples**: Like the trampoline class, the `PYBIND11_MODULE` implementation of the `LSTMTensorRTInferenceOp` example on HoloHub is located in [lstm_tensor_rt_inference.cpp](https://github.com/nvidia-holoscan/holohub/blob/main/operators/lstm_tensor_rt_inference/python/lstm_tensor_rt_inference.cpp#L104). For the SDK built-in operators, their class bindings are all implemented within a single `PYBIND11_MODULE` in [operators.cpp](https://github.com/nvidia-holoscan/holoscan-sdk/blob/v0.6.0/python/holoscan/operators/operators.cpp#L469). - -#### Configuring with CMake - -We use CMake to configure pybind11 and build the bindings for the C++ operator you wish to wrap. There are two approaches detailed below, one for HoloHub (recommended), one for standalone CMake projects. - -:::{tip} -To have your bindings built, ensure the CMake code below is executed as part of a CMake project which already defines the C++ operator as a CMake target, either built in your project (with `add_library`) or imported (with `find_package` or `find_library`). -::: - -`````{tab-set} -````{tab-item} In HoloHub -We provide a CMake utility function named `pybind11_add_holohub_module` in HoloHub to facilitate configuring and building your python bindings. - -In our skeleton code below, a top-level CMakeLists.txt which already defined the `my_op` target for the C++ operator would need to do `add_subdirectory(my_op_python)` to include the following CMakeLists.txt. The `pybind11_add_holohub_module` lists that C++ operator target, the C++ class to wrap, and the path to the C++ binding source code we implemented above. Note how the `` from the previous section would need to match `_` i.e. `_my_op`. - -```{code-block} cmake -:caption: my_op_python/CMakeLists.txt - -include(pybind11_add_holohub_module) -pybind11_add_holohub_module( - CPP_CMAKE_TARGET my_op - CLASS_NAME "MyOp" - SOURCES my_op_pybind.cpp -) -``` - -**Example**: the cmake configuration for the `LSTMTensorRTInferenceOp` python bindings on HoloHub can be found [here](https://github.com/nvidia-holoscan/holohub/blob/main/operators/lstm_tensor_rt_inference/python/CMakeLists.txt). This directory is reachable thanks to the `add_subdirectory(python)` in the CMakeLists.txt one folder above, but that's an arbitrary opinionated location and not a required directory structure. - -```` -````{tab-item} Standalone CMake - -Follow the [pybind11 documentation](https://pybind11.readthedocs.io/en/stable/compiling.html#building-with-cmake) to configure your CMake project to use pybind11. Then, use the [pybind11_add_module](https://pybind11.readthedocs.io/en/stable/compiling.html#pybind11-add-module) function with the cpp files containing the code above, and link against `holoscan::core` and the library that exposes your C++ operator to wrap. - -```{code-block} cmake -:caption: my_op_python/CMakeLists.txt - -pybind11_add_module(my_python_module my_op_pybind.cpp) -target_link_libraries(my_python_module - PRIVATE holoscan::core - PUBLIC my_op -) -``` - -**Example**: in the SDK, this is done [here](https://github.com/nvidia-holoscan/holoscan-sdk/blob/v0.6.0/python/holoscan/CMakeLists.txt). - -```` -````` - -#### Importing the class in Python - -`````{tab-set} -````{tab-item} In HoloHub - -When building your project, two files will be generated inside `/python/lib/holohub/my_op`: -1. the shared library for your bindings (`_my_op.cpython---linux-gnu.so`) -2. an `__init__.py` file that makes the necessary imports to expose this in python - -Assuming you have `export PYTHONPATH=/python/lib/`, you should then be able to create an application in Holohub that imports your class via: - -```python -from holohub.my_op import MyOp -``` -**Example**: `LSTMTensorRTInferenceOp` is imported in the Endoscopy Tool Tracking application on HoloHub [here](https://github.com/nvidia-holoscan/holohub/blob/06365894c7231c312e1217461f9014e3b50425e8/applications/endoscopy_tool_tracking/python/endoscopy_tool_tracking.py#L35). - -```` -````{tab-item} Standalone CMake - -When building your project, a shared library file holding the python bindings and named `my_python_module.cpython---linux-gnu.so` will be generated inside `/my_op_python` (configurable with `OUTPUT_NAME` and `LIBRARY_OUTPUT_DIRECTORY` respectively in CMake). - -From there, you can import it in python via: - -```py -import holoscan.core -import holoscan.gxf # if your c++ operator uses gxf extensions - -from .my_op_python import MyOp -``` +Wrapping an operator developed in C++ for use from Python is covered in a separate section on {ref}`creating C++ operator Python bindings`. :::{tip} -To imitate HoloHub's behavior, you can also place that file alongside the .so file, name it `__init__.py`, and replace `` by `.`. It can then be imported as a python module, assuming `` is a module under the `PYTHONPATH` environment variable. +As of Holoscan 2.1, there is a {py:class}`~holoscan.operators.GXFCodeletOp` class which can be used to easily wrap an existing GXF codelet from Python without having to first write an underlying C++ wrapper class for it. Similarly there is now also a {py:class}`~holoscan.resources.GXFComponentResource` class which allows a GXF Component to be used as a Holoscan resource from Python applications. A detailed example of how to use each of these is provided for Python applications in the [**examples/import_gxf_components**](https://github.com/nvidia-holoscan/holoscan-sdk/tree/main/examples/import_gxf_components/python) folder. ::: -```` -````` (interoperability-with-wrapped-operators-python)= ### Interoperability between wrapped and native Python operators @@ -1585,13 +1351,13 @@ The following code shows how to implement `ImageProcessingOp`'s `compute()` meth :lineno-start: 62 :emphasize-lines: 1,3,9,11,15,18,20 def compute(self, op_input, op_output, context): - # in_message is of dict + # in_message is a dict of tensors in_message = op_input.receive("input_tensor") # smooth along first two axes, but not the color channels sigma = (self.sigma, self.sigma, 0) - # out_message is of dict + # out_message will be a dict of tensors out_message = dict() for key, value in in_message.items(): @@ -1659,3 +1425,66 @@ There is a special serialization code for tensor types for emit/receive of tenso This avoids NumPy or CuPy arrays being serialized to a string via cloudpickle so that they can efficiently be transmitted and the same type is returned again on the opposite side. Worth mentioning is that ,if the type emitted was e.g. a PyTorch host/device tensor on emit, the received value will be a numpy/cupy array since ANY object implementing the interfaces returns those types. ::: + +## Advanced Topics + +### Further customizing inputs and outputs + +This section complements the information above on basic input and output port configuration given separately in the C++ and Python operator creation guides. The concepts described here are the same for either the C++ or Python APIs. + +By default, both the input and output ports of an Operator will use a double-buffered queue that has a capacity of one message and a policy that is set to error if a message arrives while the queue is already full. A single `MessageAvailableCondition` ({cpp:class}`C++ `/{py:class}`Python `)) condition is automatically placed on the operator for each input port so that the `compute` method will not be called until a single message is available at each port. Similarly each output port has a `DownstreamMessageAffordableCondition` ({cpp:class}`C++ `/{py:class}`Python `) condition that does not let the operator call `compute` until any operators connected downstream have space in their receiver queue for a single message. These default conditions ensure that messages never arrive at a queue when it is already full and that a message has already been received whenever the `compute` method is called. These default conditions make it relatively easy to connect a pipeline where each operator calls compute in turn, but may not be suitable for all applications. This section covers how the default behavior can be overridden on request. + +:::{note} +Overriding operator port properties is an advanced topic. Developers may want to skip this section until they come across a case where the default behavior is not sufficient for their application. +::: + + +To override the properties of the queue used for a given port, the `connector` ({cpp:func}`C++ `/{py:func}`Python `) method can be used as shown in the example below. This example also shows how the `condition` ({cpp:func}`C++ `/{py:func}`Python `) method can be used to change the condition type placed on the Operator by a port. In general, when an operator has multiple conditions, they are AND combined, so the conditions on **all** ports must be satisfied before an operator can call `compute`. + + +`````{tab-set} +````{tab-item} C++ Example +Consider the following code from within the {cpp:func}`holoscan::Operator::setup` method of an operator. +```{code-block} cpp +spec.output("out1") + +spec.output("out2").condition(ConditionType::kNone); + +spec.output("in") + .connector(IOSpec::ConnectorType::kDoubleBuffer, + Arg("capacity", static_cast(2)), + Arg("policy", static_cast(1))) // 0=pop, 1=reject, 2=fault (default) + .condition(ConditionType::kMessageAvailable, + Arg("min_size", static_cast(2)), + Arg("front_stage_max_size", static_cast(2))); +``` +This would define +- an output port named "out1" with the default properties +- an output port named "out2" that still has the default connector (a {cpp:class}`holoscan::gxf::DoubleBufferTransmitter`), but the default condition of `ConditionType::kDownstreamMessageAffordable` is removed by setting `ConditionType::kNone`. This indicates that the operator will not check if any port downstream of "out2" has space available in its receiver queue before calling `compute`. +- an input port named "in" where both the connector and condition have different parameters than the default. For example, the queue size is increased to 2 and `policy=1` is "reject", indicating that if a message arrives when the queue is already full, that message will be rejected in favor of the message already in the queue. + +```` +````{tab-item} Python Example +Consider the following code from within the {cpp:func}`holoscan::Operator::setup` method of an operator. +```{code-block} python +spec.output("out1") + +spec.output("out2").condition(ConditionType.NONE) + +spec.input("in").connector( + IOSpec.ConnectorType.DOUBLE_BUFFER, + capacity=2, + policy=1, # 0=pop, 1=reject, 2=fault (default) +).condition(ConditionType.MESSAGE_AVAILABLE, min_size=2, front_stage_max_size=2) + +``` +This would define +- an output port named "out1" with the default properties +- an output port named "out2" that still has the default connector (a {py:class}`holoscan.resources.DoubleBufferTransmitter`), but the default condition of `ConditionType.DOWNSTREAM_MESSAGE_AFFORDABLE` is removed by setting `ConditionType.NONE`. This indicates that the operator will not check if any port downstream of "out2" has space available in its receiver queue before calling `compute`. +- an input port named "in1" where both the connector and condition have different parameters than the default. For example, the queue size is increased to 2 and `policy=1` is "reject", indicating that if a message arrives when the queue is already full, that message will be rejected in favor of the message already in the queue. + +```` +````` + +To learn more about overriding connectors and/or conditions there is a [multi_branch_pipeline](https://github.com/nvidia-holoscan/holoscan-sdk/blob/main/examples/multi_branch_pipeline) example which overrides default conditions to allow two branches of a pipeline to run at different frame rates. There is also an example of increasing the queue sizes available in [this Python queue policy test application](https://github.com/nvidia-holoscan/holoscan-sdk/blob/main/python/tests/system/test_application_with_repeated_emit_on_same_port.py). + diff --git a/docs/holoscan_create_operator_python_bindings.md b/docs/holoscan_create_operator_python_bindings.md new file mode 100644 index 00000000..d9a02204 --- /dev/null +++ b/docs/holoscan_create_operator_python_bindings.md @@ -0,0 +1,554 @@ +(holoscan-create-operators-python-bindings)= +# Writing Python bindings for a C++ Operator + + +For convenience while maintaining high performance, {ref}`operators written in C++` can be wrapped in Python. The general approach uses [Pybind11](https://pybind11.readthedocs.io/en/stable/index.html) to concisely create bindings that provide a familiar, Pythonic experience to application authors. + +:::{note} +While we provide some utilities to simplify part of the process, this section is designed for advanced developers, since the wrapping of the C++ class using pybind11 is mostly manual and can vary between each operator. +::: + +The existing Pybind11 documentation is good and it is recommended to read at least the basics on wrapping [functions](https://pybind11.readthedocs.io/en/stable/basics.html#creating-bindings-for-a-simple-function) and [classes](https://pybind11.readthedocs.io/en/stable/classes.html#object-oriented-code). The material below will assume some basic familiarity with Pybind11, covering the details of creation of the bindings of a C++ {cpp:class}`~holoscan::Operator`. As a concrete example, we will cover creation of the bindings for [ToolTrackingPostprocessorOp](https://github.com/nvidia-holoscan/holohub/tree/main/operators/tool_tracking_postprocessor) from [Holohub](https://github.com/nvidia-holoscan/holohub) as a simple case and then highlight additional scenarios that might be encountered. + +:::{tip} +There are several examples of bindings on Holohub in the [operators folder](https://github.com/nvidia-holoscan/holohub/tree/main/operators). The subset of operators that provide a Python wrapper on top of a C++ implementation will have any C++ headers and sources together in a common folder, while any corresponding Python bindings will be in a "python" subfolder (see the [tool_tracking_postprocessor](https://github.com/nvidia-holoscan/holohub/tree/main/operators/tool_tracking_postprocessor) folder layout, for example). + +There are also several [examples of bindings](https://github.com/nvidia-holoscan/holoscan-sdk/tree/main/python/holoscan/operators) for the built-in operators of the SDK. Unlike on Holohub, for the SDK, the corresponding C++ [headers](https://github.com/nvidia-holoscan/holoscan-sdk/tree/main/include/holoscan/operators) and [sources](https://github.com/nvidia-holoscan/holoscan-sdk/tree/main/src/operators) of an operator are stored under separate directory trees. +::: + +(pybind11-operator-tutorial)= +## Tutorial: binding the ToolTrackingPostprocessorOp class + +(pybind11-operator-trampoline)= +### Creating a PyToolTrackingPostprocessorOp trampoline class + +In a C++ file ([tool_tracking_postprocessor.cpp](https://github.com/nvidia-holoscan/holohub/blob/main/operators/tool_tracking_postprocessor/python/tool_tracking_postprocessor.cpp) in this case), create a subclass of the C++ Operator class to wrap. The general approach taken is to create a Python-specific class that provides a constructor that takes a `Fragment*`, an explicit list of the operators parameters with default values for any that are optional, and an operator name. This constructor needs to setup the operator as done in [`Fragment::make_operator`](https://github.com/nvidia-holoscan/holoscan-sdk/blob/v1.0.3/include/holoscan/core/fragment.hpp#L284), so that it is ready for initialization by the GXF executor. We use the convention of prepending "Py" to the C++ class name for this (so, `PyToolTrackingPostprocessorOp` in this case). : + + +```{code-block} cpp +:caption: tool_tracking_post_processor/python/tool_tracking_post_processor.cpp + +class PyToolTrackingPostprocessorOp : public ToolTrackingPostprocessorOp { + public: + /* Inherit the constructors */ + using ToolTrackingPostprocessorOp::ToolTrackingPostprocessorOp; + + // Define a constructor that fully initializes the object. + PyToolTrackingPostprocessorOp( + Fragment* fragment, const py::args& args, std::shared_ptr device_allocator, + std::shared_ptr host_allocator, float min_prob = 0.5f, + std::vector> overlay_img_colors = VIZ_TOOL_DEFAULT_COLORS, + std::shared_ptr cuda_stream_pool = nullptr, + const std::string& name = "tool_tracking_postprocessor") + : ToolTrackingPostprocessorOp(ArgList{Arg{"device_allocator", device_allocator}, + Arg{"host_allocator", host_allocator}, + Arg{"min_prob", min_prob}, + Arg{"overlay_img_colors", overlay_img_colors}, + }) { + if (cuda_stream_pool) { this->add_arg(Arg{"cuda_stream_pool", cuda_stream_pool}); } + add_positional_condition_and_resource_args(this, args); + name_ = name; + fragment_ = fragment; + spec_ = std::make_shared(fragment); + setup(*spec_.get()); + } +}; +``` + +This constructor will allow providing a Pythonic experience for creating the operator. Specifically, the user can pass Python objects for any of the parameters without having to explicitly create any {cpp:class}`holoscan::Arg` objects via {py:class}`holoscan.core.Arg`. For example, a standard Python float can be passed to `min_prob` and a Python `list[list[float]]` can be passed for `overlay_img_colors` (Pybind11 handles conversion between the C++ and Python types). Pybind11 will also take care of conversion of a Python allocator class like `holoscan.resources.UnboundedAllocator` or `holoscan.resources.BlockMemoryPool` to the underlying C++ `std::shared_ptr` type. The arguments `device_allocator` and `host_allocator` correspond to required Parameters of the C++ class and can be provided from Python either positionally or via keyword while the Parameters `min_prob` and `overlay_img_colors` will be optional keyword arguments. `cuda_stream_pool` is also optional, but is only conditionally passed as an argument to the underlying `ToolTrackingPostprocessorOp` constructor when it is not a `nullptr`. + +- For all operators, the first argument should be `Fragment* fragment` and is the fragment the operator will be assigned to. In the case of a single fragment application (i.e. not a distributed application), the fragment is just the application itself. +- An (optional) `const std::string& name` argument should be provided to enable the application author to set the operator's name. +- The `const py::args& args` argument corresponds to the `*args` notation in Python. It is a set of 0 or more positional arguments. It is not required to provide this in the function signature, but is recommended in order to enable passing additional conditions such as a `CountCondition` or `PeriodicCondtion` as positional arguments to the operator. The call below to + + ```cpp + add_positional_condition_and_resource_args(this, args); + ``` + + uses a helper function defined in [operator_util.hpp](https://github.com/nvidia-holoscan/holohub/blob/main/operators/operator_util.hpp) to add any {py:class}`~holoscan.core.Condition` or {py:class}`~holoscan.core.Resource` arguments found in the list of positional arguments. +- The other arguments all correspond to the various parameters ({cpp:class}`holoscan::Parameter`) that are defined for the C++ `ToolTrackingPostProcessorOp` class. + - All other parameters except `cuda_stream_pool` are passed directly in the argument list to the parent `ToolTrackingPostProcessorOp` class. The parameters present on the C++ operator can be seen in its header [here](https://github.com/grlee77/holohub/blob/3adbba16baafb5958950b261a0d6521f7544cfeb/operators/tool_tracking_postprocessor/tool_tracking_postprocessor.hpp#L46-L52) with default values taken from the `setup` method of the source file [here](https://github.com/grlee77/holohub/blob/3adbba16baafb5958950b261a0d6521f7544cfeb/operators/tool_tracking_postprocessor/tool_tracking_postprocessor.cpp#L77-L89). Note that {cpp:class}`CudaStreamHandler` is a utility that will add a parameter of type `Parameter>`. + - The `cuda_stream_pool` argument is only conditionally added if it was not `nullptr` (Python's `None`). This is done via + ```cpp + if (cuda_stream_pool) { this->add_arg(Arg{"cuda_stream_pool", cuda_stream_pool}); } + ``` + instead of passing it as part of the {cpp:class}`holoscan::ArgList` provided to the `ToolTrackingPostprocessorOp` constructor call above. + +The remaining lines of the constructor +```cpp + name_ = name; + fragment_ = fragment; + spec_ = std::make_shared(fragment); + setup(*spec_.get()); +``` +are required to properly initialize it and should be the same across all operators. These [correspond to equivalent code within the Fragment::make_operator method](https://github.com/nvidia-holoscan/holoscan-sdk/blob/v1.0.3/include/holoscan/core/fragment.hpp#L287-L291). + +(pybind11-operator-module-definition)= +### Defining the Python module + +For this operator, there are no other custom classes aside from the operator itself, so we define a module using `PYBIND11_MODULE` as shown below with only a single class definition. This is done in the same [tool_tracking_postprocessor.cpp](https://github.com/nvidia-holoscan/holohub/blob/main/operators/tool_tracking_postprocessor/python/tool_tracking_postprocessor.cpp) file where we defined the `PyToolTrackingPostprocessorOp` trampoline class. + +The following header will always be needed. +```cpp +#include + +namespace py = pybind11; +using pybind11::literals::operator""_a; +``` +Here, we typically also add defined the `py` namespace as a shorthand for `pybind11` and indicated that we will use the `_a` literal (it provides a shorthand notation when [defining keyword arguments](https://pybind11.readthedocs.io/en/stable/basics.html#keyword-arguments)). + +Often it will be necessary to include the following header if any parameters to the operator involve C++ standard library containers such as `std::vector` or `std::unordered_map`. +```cpp +#include +``` +This allows pybind11 to cast between the C++ container types and corresponding Python types (Python `dict` / C++ `std::unordered_map`, for example). + + +```{code-block} cpp +:caption: tool_tracking_post_processor/python/tool_tracking_post_processor.cpp + +PYBIND11_MODULE(_tool_tracking_postprocessor, m) { + py::class_>( + m, + "ToolTrackingPostprocessorOp", + doc::ToolTrackingPostprocessorOp::doc_ToolTrackingPostprocessorOp_python) + .def(py::init, + std::shared_ptr, + float, + std::vector>, + std::shared_ptr, + const std::string&>(), + "fragment"_a, + "device_allocator"_a, + "host_allocator"_a, + "min_prob"_a = 0.5f, + "overlay_img_colors"_a = VIZ_TOOL_DEFAULT_COLORS, + "cuda_stream_pool"_a = py::none(), + "name"_a = "tool_tracking_postprocessor"s, + doc::ToolTrackingPostprocessorOp::doc_ToolTrackingPostprocessorOp_python); +} // PYBIND11_MODULE NOLINT +``` + + +:::{note} +- If you are implementing the python wrapping in Holohub, the `` passed to `PYBIND_11_MODULE` **must** match `_` as [covered above](#pybind11-module_name_warning). +- If you are implementing the python wrapping in a standalone CMake project,the `` passed to `PYBIND_11_MODULE` **must** match the name of the module passed to the [pybind11-add-module](https://pybind11.readthedocs.io/en/stable/compiling.html#pybind11-add-module) CMake function. + +Using a mismatched name in `PYBIND_11_MODULE` will result in failure to import the module from Python. +::: + +The order in which the classes are specified in the `py::class_<>` template call is important and should follow the convention shown here. The first in the list is the C++ class name (`ToolTrackingPostprocessorOp`) and second is the `PyToolTrackingPostprocessorOp` class we defined above with the additional, explicit constructor. We also need to list the parent `Operator` class so that all of the methods such as `start`, `stop`, `compute`, `add_arg`, etc. that were already wrapped for the parent class don't need to be redefined here. + +The single `.def(py::init<...` call wraps the `PyToolTrackingPostprocessorOp` constructor we wrote above. As such, the argument types provided to `py::init<>` must exactly match the order and types of arguments in that constructor's function signature. The subsequent arguments to `def` are the names and default values (if any) for the named arguments in the same order as the function signature. Note that the `const py::args& args` (Python `*args`) argument is not listed as these are positional arguments that don't have a corresponding name. The use of `py::none()` (Python's `None`) as the default for `cuda_stream_pool` corresponds to the `nullptr` in the C++ function signature. The "_a" literal used in the definition is enabled by the following declaration earlier in the file. + +The final argument to `.def` here is a documentation string that will serve as the Python docstring for the function. It is optional and we chose here to define it in a separate header as described in the next section. + +(pybind11-operator-docstrings)= +### Documentation strings + +Prepare documentation strings (`const char*`) for your python class and its parameters. + +:::{note} +Below we use a `PYDOC` macro defined in the [SDK](https://github.com/nvidia-holoscan/holoscan-sdk/blob/v1.0.3/python/holoscan/macros.hpp) and available in [HoloHub](https://github.com/nvidia-holoscan/holohub/blob/main/cmake/pydoc/macros.hpp) as a utility to remove leading spaces. In this case, the documentation code is located in header file [tool_tracking_post_processor_pydoc.hpp](https://github.com/nvidia-holoscan/holohub/blob/main/operators/tool_tracking_postprocessor/python/tool_tracking_postprocessor_pydoc.hpp), under a custom `holoscan::doc::ToolTrackingPostprocessorOp` namespace. None of this is required, you just need to make any documentation strings available for use as an argument to the `py::class_` constructor or method definition calls. +::: + +```{code-block} cpp +:caption: tool_tracking_post_processor/python/tool_tracking_post_processor_pydoc.hpp + +#include "../macros.hpp" + +namespace holoscan::doc { + +namespace ToolTrackingPostprocessorOp { + +// PyToolTrackingPostprocessorOp Constructor +PYDOC(ToolTrackingPostprocessorOp_python, R"doc( +Operator performing post-processing for the endoscopy tool tracking demo. + +**==Named Inputs==** + + in : nvidia::gxf::Entity containing multiple nvidia::gxf::Tensor + Must contain input tensors named "probs", "scaled_coords" and "binary_masks" that + correspond to the output of the LSTMTensorRTInfereceOp as used in the endoscopy + tool tracking example applications. + +**==Named Outputs==** + + out_coords : nvidia::gxf::Tensor + Coordinates tensor, stored on the host (CPU). + + out_mask : nvidia::gxf::Tensor + Binary mask tensor, stored on device (GPU). + +Parameters +---------- +fragment : Fragment + The fragment that the operator belongs to. +device_allocator : ``holoscan.resources.Allocator`` + Output allocator used on the device side. +host_allocator : ``holoscan.resources.Allocator`` + Output allocator used on the host side. +min_prob : float, optional + Minimum probability (in range [0, 1]). Default value is 0.5. +overlay_img_colors : sequence of sequence of float, optional + Color of the image overlays, a list of RGB values with components between 0 and 1. + The default value is a qualitative colormap with a sequence of 12 colors. +cuda_stream_pool : ``holoscan.resources.CudaStreamPool``, optional + `holoscan.resources.CudaStreamPool` instance to allocate CUDA streams. + Default value is ``None``. +name : str, optional + The name of the operator. +)doc") + +} // namespace ToolTrackingPostprocessorOp +} // namespace holoscan::doc +``` + +We tend to use [NumPy-style docstrings](https://numpydoc.readthedocs.io/en/latest/format.html) for parameters, but also encourage adding a custom section at the top that describes the input and output ports and what type of data is expected on them. This can make it easier for developers to use the operator without having to inspect the source code to determine this information. + +### Configuring with CMake + +We use CMake to configure pybind11 and build the bindings for the C++ operator you wish to wrap. There are two approaches detailed below, one for HoloHub (recommended), one for standalone CMake projects. + +:::{tip} +To have your bindings built, ensure the CMake code below is executed as part of a CMake project which already defines the C++ operator as a CMake target, either built in your project (with `add_library`) or imported (with `find_package` or `find_library`). +::: + +`````{tab-set} +````{tab-item} In HoloHub +We provide a CMake utility function named [pybind11_add_holohub_module](https://github.com/nvidia-holoscan/holohub/blob/main/cmake/pybind11_add_holohub_module.cmake) in HoloHub to facilitate configuring and building your python bindings. + +In our skeleton code below, a top-level CMakeLists.txt which already defined the `tool_tracking_postprocessor` target for the C++ operator would need to do `add_subdirectory(tool_tracking_postprocessor)` to include the following [CMakeLists.txt](https://github.com/nvidia-holoscan/holohub/blob/main/operators/tool_tracking_postprocessor/python/CMakeLists.txt). The `pybind11_add_holohub_module` lists that C++ operator target, the C++ class to wrap, and the path to the C++ binding source code we implemented above. Note how the module name provided as the first argument to PYPBIND11_MODULE needs to match `_` (`_tool_tracking_postprocessor_op` in this case). + +```{code-block} cmake +:caption: tool_tracking_postprocessor/python/CMakeLists.txt + +include(pybind11_add_holohub_module) +pybind11_add_holohub_module( + CPP_CMAKE_TARGET tool_tracking_postprocessor + CLASS_NAME "ToolTrackingPostprocessorOp" + SOURCES tool_tracking_postprocessor.cpp +) +``` + +The key details here are that `CLASS_NAME` should match the name of the C++ class that is being wrapped and is also the name that will be used for the class from Python. `SOURCES` should point to the file where the C++ operator that is being wrapped is defined. The `CPP_CMAKE_TARGET` name will be the name of the holohub package submodule that will contain the operator. + +Note that the python subdirectory where this CMakeLists.txt resides is reachable thanks to the `add_subdirectory(python)` in the [CMakeLists.txt one folder above](https://github.com/nvidia-holoscan/holohub/blob/30d2797e37615f87056075b36ebf1d905b6c770b/operators/tool_tracking_postprocessor/CMakeLists.txt#L33-L35), but that's an arbitrary opinionated location and not a required directory structure. + +```` +````{tab-item} Standalone CMake + +Follow the [pybind11 documentation](https://pybind11.readthedocs.io/en/stable/compiling.html#building-with-cmake) to configure your CMake project to use pybind11. Then, use the [pybind11_add_module](https://pybind11.readthedocs.io/en/stable/compiling.html#pybind11-add-module) function with the cpp files containing the code above, and link against `holoscan::core` and the library that exposes your C++ operator to wrap. + +```{code-block} cmake +:caption: my_op_python/CMakeLists.txt + +pybind11_add_module(my_python_module my_op_pybind.cpp) +target_link_libraries(my_python_module + PRIVATE holoscan::core + PUBLIC my_op +) +``` + +**Example**: in the SDK, this is done [here](https://github.com/nvidia-holoscan/holoscan-sdk/blob/v1.0.3/python/holoscan/CMakeLists.txt). + +```` +````` + +(pybind11-module_name_warning)= +:::{warning} +The name chosen for `CPP_CMAKE_TARGET` **must** also be used (along with a preceding underscore) as the module name passed as the first argument to the [PYBIND11_MODULE macro in the bindings](https://github.com/grlee77/holohub/blob/3adbba16baafb5958950b261a0d6521f7544cfeb/operators/tool_tracking_postprocessor/python/tool_tracking_postprocessor.cpp#L94). + +Note that there is an initial underscore prepended to the name. This is the naming convention used for the shared library and corresponding `__init__.py` file that will be generated by the `pybind11_add_holohub_module` helper function above. + +If the name is specified incorrectly, the build will still complete, but at application run time an `ImportError` such as the following would occur + +```bash +[command] python3 /workspace/holohub/applications/endoscopy_tool_tracking/python/endoscopy_tool_tracking.py --data /workspace/holohub/data/endoscopy +Traceback (most recent call last): + File "/workspace/holohub/applications/endoscopy_tool_tracking/python/endoscopy_tool_tracking.py", line 38, in + from holohub.tool_tracking_postprocessor import ToolTrackingPostprocessorOp + File "/workspace/holohub/build/python/lib/holohub/tool_tracking_postprocessor/__init__.py", line 19, in + from ._tool_tracking_postprocessor import ToolTrackingPostprocessorOp +ImportError: dynamic module does not define module export function (PyInit__tool_tracking_postprocessor) +``` +::: + + +### Importing the class in Python + +`````{tab-set} +````{tab-item} In HoloHub + +When building your project, two files will be generated inside `/python/lib/holohub/` (e.g. `build/python/lib/holohub/tool_tracking_postprocessor/`): +1. the shared library for your bindings (`_tool_tracking_postprocessor_op.cpython---linux-gnu.so`) +2. an `__init__.py` file that makes the necessary imports to expose this in python + +Assuming you have `export PYTHONPATH=/python/lib/`, you should then be able to create an application in Holohub that imports your class via: + +```python +from holohub.tool_tracking_postprocessor_op import ToolTrackingPostProcessorOp +``` +**Example**: `ToolTrackingPostProcessorOp` is imported in the Endoscopy Tool Tracking application on HoloHub [here](https://github.com/nvidia-holoscan/holohub/blob/30d2797e37615f87056075b36ebf1d905b6c770b/applications/endoscopy_tool_tracking/python/endoscopy_tool_tracking.py#L38). + +```` +````{tab-item} Standalone CMake + +When building your project, a shared library file holding the python bindings and named `my_python_module.cpython---linux-gnu.so` will be generated inside `/my_op_python` (configurable with `OUTPUT_NAME` and `LIBRARY_OUTPUT_DIRECTORY` respectively in CMake). + +From there, you can import it in python via: + +```py +import holoscan.core +import holoscan.gxf # if your c++ operator uses gxf extensions + +from .my_op_python import MyOp +``` + +:::{tip} +To imitate HoloHub's behavior, you can also place that file alongside the .so file, name it `__init__.py`, and replace `.` by `.`. It can then be imported as a python module, assuming `` is a module under the `PYTHONPATH` environment variable. +::: +```` +````` + +(pybind11-details)= +## Additional Examples + +In this section we will cover other cases that may occasionally be encountered when writing Python bindings for operators. + +### Optional arguments + +It is also possible to use `std::optional` to handle optional arguments. The `ToolTrackingProcessorOp` example above, for example, has a default argument defined in the spec for `min_prob`. + +```cpp + constexpr float DEFAULT_MIN_PROB = 0.5f; + // ... + + spec.param( + min_prob_, "min_prob", "Minimum probability", "Minimum probability.", DEFAULT_MIN_PROB); +``` + +In the tutorial for `ToolTrackingProcessorOp` above we reproduced this default of 0.5 in both the `PyToolTrackingProcessorOp` constructor function signature as well as the Python bindings defined for it. This carries the risk that the default could change at the C++ operator level without a corresponding change being made for Python. + +An alternative way to define the constructor would have been to use `std::optional` as follows + +```cpp + // Define a constructor that fully initializes the object. + PyToolTrackingPostprocessorOp( + Fragment* fragment, const py::args& args, std::shared_ptr device_allocator, + std::shared_ptr host_allocator, std::optional min_prob = 0.5f, + std::optional>> overlay_img_colors = VIZ_TOOL_DEFAULT_COLORS, + std::shared_ptr cuda_stream_pool = nullptr, + const std::string& name = "tool_tracking_postprocessor") + : ToolTrackingPostprocessorOp(ArgList{Arg{"device_allocator", device_allocator}, + Arg{"host_allocator", host_allocator}, + }) { + if (cuda_stream_pool) { this->add_arg(Arg{"cuda_stream_pool", cuda_stream_pool}); } + if (min_prob.has_value()) { this->add_arg(Arg{"min_prob", min_prob.value() }); } + if (overlay_img_colors.has_value()) { + this->add_arg(Arg{"overlay_img_colors", overlay_img_colors.value() }); + } + add_positional_condition_and_resource_args(this, args); + name_ = name; + fragment_ = fragment; + spec_ = std::make_shared(fragment); + setup(*spec_.get()); + } +``` +where now that `min_prob` and `overlay_img_colors` are optional, they are only conditionally added as an argument to ToolTrackingPostprocessorOp when they have a value. If this approach is used, the Python bindings for the constructor should be updated to use `py::none()` as the default as follows: + +```cpp + .def(py::init, + std::shared_ptr, + float, + std::vector>, + std::shared_ptr, + const std::string&>(), + "fragment"_a, + "device_allocator"_a, + "host_allocator"_a, + "min_prob"_a = py::none(), + "overlay_img_colors"_a = py::none(), + "cuda_stream_pool"_a = py::none(), + "name"_a = "tool_tracking_postprocessor"s, + doc::ToolTrackingPostprocessorOp::doc_ToolTrackingPostprocessorOp_python); +``` + + +### C++ enum parameters as arguments + +Sometimes, operators may use a Parameter with an enum type. It is necessary to wrap the C++ enum to be able to use it as a Python type when providing the argument to the operator. + +The built-in {cpp:class}`holoscan::ops::AJASourceOp` is an example of a C++ operator that takes a [enum Parameter](https://github.com/nvidia-holoscan/holoscan-sdk/blob/v1.0.3/python/holoscan/operators/aja_source/aja_source.cpp#L58) (an `NTV2Channel` enum). + +The enum can easily be wrapped for use from Python via `py::enum_` as shown [here](https://github.com/nvidia-holoscan/holoscan-sdk/blob/v1.0.3/python/holoscan/operators/aja_source/aja_source.cpp#L98-L108). It is recommended in this case to follow Python's convention of using capitalized names in the enum. + +### (Advanced) Custom C++ classes as arguments + +Sometimes it is necessary to accept a custom C++ class type as an argument in the operator's constructor. In this case additional interface code and bindings will likely be necessary to support the type. + +A relatively simple example of this is the {cpp:class}`~holoscan::ops::InferenceProcessorOp::DataVecMap` type used by {cpp:class}`~holoscan::ops::InferenceProcessorOp`. In that case, the type is a structure that holds an internal `std::map>`. The bindings are written to accept a Python dict (`py::dict`) and a [helper function](https://github.com/nvidia-holoscan/holoscan-sdk/blob/v1.0.3/python/holoscan/operators/inference_processor/inference_processor.cpp#L52-L58) is used within the constructor to convert that dictionary to the corresponding C++ `DataVecMap`. + +A more complicated case is the use of a {cpp:class}`~holoscan::ops::HolovizOp::InputSpec` type in the `HolovizOp` bindings. This case involves creating Python bindings for classes {cpp:class}`~holoscan::ops::HolovizOp::InputSpec` and {cpp:class}`~holoscan::ops::HolovizOp::InputSpec::View` as well as a couple of enum types. To avoid the user having to build a `list[holoscan.operators.HolovizOp.InputSpec]` directly to pass as the `tensors` argument, an [additional Python wrapper class](https://github.com/nvidia-holoscan/holoscan-sdk/blob/v1.0.3/python/holoscan/operators/holoviz/__init__.py#L100-L182) was defined in the `__init__.py` to allow passing a simple Python dict for the `tensors` argument and any corresponding InputSpec classes are automatically created in its constructor before calling the underlying Python bindings class. + + +### Customizing the C++ types a Python operator can emit or receive + +In some instances, users may wish to be able to have a Python operator receive and/or emit a custom C++ type. As a first example, suppose we are wrapping an operator that emits a custom C++ type. We need any downstream native Python operators to be able to receive that type. By default the SDK is able to handle the needed C++ types for built in operators like `std::vector`. The SDK provides an `EmitterReceiverRegistry` class that 3rd party projects can use to register `receiver` and `emitter` methods for any custom C++ type that needs to be handled. To handle a new type, users should implement an `emitter_receiver` struct for the desired type as in the example below. We will first cover the general steps necessary to register such a type and then cover where some steps may be omitted in certain simple cases. + +#### Step 1: define emitter_receiver::emit and emitter_receiver::receive methods + +Here is an example for the built-in `std::vector` used by `HolovizOp` to define the input specifications for its received tensors. + +```cpp +#include + +namespace py = pybind11; + +namespace holoscan { + +/* Implements emit and receive capability for the HolovizOp::InputSpec type. + */ +template <> +struct emitter_receiver> { + static void emit(py::object& data, const std::string& name, PyOutputContext& op_output) { + auto input_spec = data.cast>(); + py::gil_scoped_release release; + op_output.emit>(input_spec, name.c_str()); + return; + } + + static py::object receive(std::any result, const std::string& name, PyInputContext& op_input) { + HOLOSCAN_LOG_DEBUG("py_receive: std::vector case"); + // can directly return vector + auto specs = std::any_cast>(result); + py::object py_specs = py::cast(specs); + return py_specs; + } +}; + +} +``` + +This `emitter_receiver` class defines a `receive` method that takes a `std::any` message and casts it to the corresponding Python `list[HolovizOp.InputSpect]` object. Here the `pybind11::cast` call works because we have wrapped the `HolovizOp::InputSpec` class [here](https://github.com/nvidia-holoscan/holoscan-sdk/blob/v2.0.0/python/holoscan/operators/holoviz/holoviz.cpp#L190-L207). + +Similarly, the `emit` method takes a `pybind11::object` (of type `list[HolovizOp.InputSpect]`) and casts it to the corresponding C++ type, `std::vector`. The conversion between `std::vector` and a Python list is one of Pbind11's built-in conversions (available as long as "pybind11/stl.h" has been included). + +The signature of the `emit` and `receive` methods must exactly match the case shown here. + +#### Step 2: Create a register_types method for adding custom types to the EmitterReceiverRegistry. + +The bindings in this operators module, should define a method named `register_types` that takes a reference to an `EmitterReceiverRegistry` as its only argument. Within this function there should be a call to `EmitterReceiverRegistry::add_emitter_receiver` for each type that this operator wished to register. The HolovizOp defines this method using a lambda function + +```cpp + // Import the emitter/receiver registry from holoscan.core and pass it to this function to + // register this new C++ type with the SDK. + m.def("register_types", [](EmitterReceiverRegistry& registry) { + registry.add_emitter_receiver>( + "std::vector"s); + // array camera pose object + registry.add_emitter_receiver>>( + "std::shared_ptr>"s); + // Pose3D camera pose object + registry.add_emitter_receiver>( + "std::shared_ptr"s); + // camera_eye_input, camera_look_at_input, camera_up_input + registry.add_emitter_receiver>("std::array"s); + }); +``` + +Here the following line registers the `std::vector` type +that we wrote an `emitter_receiver` for above. + +```cpp +registry.add_emitter_receiver>( + "std::vector"s); +``` +Internally the registry stores a mapping between the C++ `std::type_index` of the type specified in the template argument and the `emitter_receiver` defined for that type. The second argument is a string that the user can choose which is a label for the type. As we will see later, this label can be used from Python to indicate that we want to emit using the `emitter_receiver::emit` method that was registered for a particular label. + +#### Step 3: In the __init__.py file for the Python module defining the operator call register_types + +To register types with the core SDK, we need to import the `io_type_registry` class (of type `EmitterReceiverRegistry`) from `holoscan.core`. We then pass that class as input to the `register_types` method defined in step 2 to register the 3rd party types with the core SDK. + +```python +from holoscan.core import io_type_registry + +from ._holoviz import register_types as _register_types + +# register methods for receiving or emitting list[HolovizOp.InputSpec] and camera pose types +_register_types(io_type_registry) +``` + +where we chose to import `register_types` with an initial underscore as a common Python convention to indicate it is intended to be "private" to this module. + +#### In some cases steps 1 and 3 as shown above are not necessary. + +When creating Python bindings for an Operator on Holohub, the [pybind11_add_holohub_module.cmake](https://github.com/nvidia-holoscan/holohub/blob/main/cmake/pybind11_add_holohub_module.cmake) utility mentioned above will take care of autogenerating the `__init__.py` as shown in step 3, so it will not be necessary to manually create it in that case. + +For types for which Pybind11's default casting between C++ and Python is adequate, it is not necessary to explicitly define the `emitter_receiver` class as shown in step 1. This is true because there are a couple of [default implementations](https://github.com/nvidia-holoscan/holoscan-sdk/blob/v2.1.0/python/holoscan/core/emitter_receiver_registry.hpp) for `emitter_receiver` and `emitter_receiver>` that already cover common cases. The default emitter_receiver works for the `std::vector` type shown above, which is why the code shown for illustration there is [not found within the operator's bindings](https://github.com/nvidia-holoscan/holoscan-sdk/blob/main/python/holoscan/operators/holoviz/holoviz.cpp). In that case one could immediately implement `register_types` from step 2 without having to explicitly create an `emitter_receiver` class. + +An example where the default `emitter_receiver` would not work is the custom one defined by the SDK for `pybind11::dict`. In this case, to provide convenient emit of multiple tensors via passing a `dict[holoscan::Tensor]` to `op_output.emit` we have special handling of Python dictionaries. The dictionary is inspected and if all keys are strings and all values are tensor-like objects, a single C++ `nvidia::gxf::Entity` containing all of the tensors as an `nvidia::gxf::Tensor` is emitted. If the dictionary is not a tensor map, then it is just emitted as a shared pointer to the Python dict object. The `emitter_receiver` implementations used for the core SDK are defined in [emitter_receivers.hpp](https://github.com/nvidia-holoscan/holoscan-sdk/blob/v2.1.0/python/holoscan/core/emitter_receivers.hpp). These can serve as a reference when creating new ones for additional types. + +#### Runtime behavior of emit and receive + +After registering a new type, receive of that type on any input port will automatically be handled. This is because due to the strong typing of C++, any `op_input.receive` call in an operator's `compute` method can find the registered `receive` method that matches the `std::type_index` of the type and use that to convert to a corresponding Python object. + +Because Python is not strongly typed, on `emit`, the default behavior remains emitting a shared pointer to the Python object itself. If we instead want to `emit` a C++ type, we can pass a 3rd argument to `op_output.emit` to specify the name that we used when registering the types via the `add_emitter_receiver` call as above. + +#### Example of emitting a C++ type +As a concrete example, the SDK already registers `std::string` by default. If we wanted, for instance, to emit a Python string as a C++ `std::string` for use by a downstream operator that is wrapping a C++ operator expecting string input, we would add a 3rd argument to the `op_output.emit` call as follows + +```py +# emit a Python filename string on port "filename_out" using registered type "std::string" +my_string = "filename.raw" +op_output.emit(my_string, "filename_out", "std::string") +``` + +This specifies that the `emit` method that converts to C++ `std::string` should be used instead of the default behavior of emitting the Python string. + +Another example would be to emit a Python `List[float]` as a `std::array` parameter as input to the `camera_eye`, `camera_look_at` or `camera_up` input ports of `HolovizOp`. + +```py +op_output.emit([0.0, 1.0, 0.0], "camera_eye_out", "std::array") +``` + +Only types registered with the SDK can be specified by name in this third argument to `emit`. + +#### Table of types registered by the core SDK + +The list of types that are registered with the SDK's `EmitterReceiverRegistry` are given in the table below. + +C++ Type | name in the EmitterReceiverRegistry +-------------------------------------------------------|------------------------------------------- +holoscan::Tensor | "holoscan::Tensor" +std::shared_ptr<holoscan::GILGuardedPyObject> | "PyObject" +std::string | "std::string" +pybind11::dict | "pybind11::dict" +holoscan::gxf::Entity | "holoscan::gxf::Entity" +holoscan::PyEntity | "holoscan::PyEntity" +nullptr_t | "nullptr_t" +CloudPickleSerializedObject | "CloudPickleSerializedObject" +std::array<float, 3> | "std::array<float, 3>" +std::shared_ptr<std::array<float, 16>> | "std::shared_ptr<std::array<float, 16>>" +std::shared_ptr<nvidia::gxf::Pose3D> | "std::shared_ptr<nvidia::gxf::Pose3D>" +std::vector<holoscan::ops::HolovizOp::InputSpec> | "std::vector<HolovizOp::InputSpec>" + +:::{note} +There is no requirement that the registered name match any particular convention. We generally used +the C++ type as the name to avoid ambiguity, but that is not required. +::: + +The sections above explain how a `register_types` function can be added to bindings to expand this list. It is also possible to get a list of all currently registered types, including those that have been registered by any additional imported 3rd party modules. This can be done via + +```py +from holoscan.core import io_type_registry + +print(io_type_registry.registered_types()) +``` diff --git a/docs/index.md b/docs/index.md index 20d5c893..52339653 100644 --- a/docs/index.md +++ b/docs/index.md @@ -30,6 +30,7 @@ holoscan_packager holoscan_create_operator holoscan_logging holoscan_debugging +holoscan_create_operator_python_bindings ``` ```{toctree} @@ -48,6 +49,7 @@ inference components/schedulers components/conditions components/resources +components/analytics ``` ```{toctree} @@ -83,6 +85,7 @@ gxf/doc/index.md :caption: Performance Tools flow_tracking +gxf_job_statistics latency_tool.rst ``` diff --git a/docs/inference.md b/docs/inference.md index da2155d2..00765f09 100644 --- a/docs/inference.md +++ b/docs/inference.md @@ -88,6 +88,11 @@ Required parameters and related features available with the Holoscan Inference M - Each entry in `device_map` has a unique keyword representing the model (same as used in `model_path_map` and `pre_processor_map`), and GPU identifier as the value. This GPU ID is used to execute the inference for the specified model. - GPUs specified in the `device_map` must have P2P (peer to peer) access and they must be connected to the same PCIE configuration. If P2P access is not possible among GPUs, the host (CPU memory) will be used to transfer the data. - Multi-GPU inferencing is supported for all backends. + - `temporal_map`: Temporal inferencing is enabled if `temporal_map` is populated in the parameter set. + - Each entry in `temporal_map` has a unique keyword representing the model (same as used in `model_path_map` and `pre_processor_map`), and frame delay as the value. Frame delay represents the frame count that are skipped by the operator in doing the inference for that particular model. A model with the value of 1, is inferred per frame. A model with a value of 10 is inferred for every 10th frame coming into the operator, which is the 1st frame, 11th frame, 21st frame and so on. Additionally, the operator will transmit the last inferred result for all the frames that are not inferred. For example, a model with a value of 10 will be inferred at 11th frame and from 12th to 20th frame, the result from 11th frame is transmitted. + - If the `temporal_map` is absent in the parameter set, all models are inferred for all the frames. + - All models are not mandatory in the `temporal_map`. The missing models are inferred per frame. + - Temporal map based inferencing is supported for all backends. - `backend_map`: Multiple backends can be used in the same application with this parameter. - Each entry in `backend_map` has a unique keyword representing the model (same as used in `model_path_map`), and the `backend` as the value. - A sample backend_map is shown below. In the example, model_1 uses the `tensorRT` backend, and model 2 and model 3 uses the `torch` backend for inference. diff --git a/docs/overview.md b/docs/overview.md index 540451dc..db3a1885 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -2,10 +2,6 @@ [NVIDIA Holoscan](https://developer.nvidia.com/holoscan-sdk) is the AI sensor processing platform that combines hardware systems for low-latency sensor and network connectivity, optimized libraries for data processing and AI, and core microservices to run streaming, imaging, and other applications, from embedded to edge to cloud. It can be used to build streaming AI pipelines for a variety of domains, including Medical Devices, High Performance Computing at the Edge, Industrial Inspection and more. -:::{note} -In previous releases, the prefix [`Clara`](https://developer.nvidia.com/industries/healthcare) was used to define Holoscan as a platform designed initially for [medical devices](https://www.nvidia.com/en-us/clara/developer-kits/). As Holoscan has grown, its potential to serve other areas has become apparent. With version 0.4.0, we're proud to announce that the Holoscan SDK is now officially built to be domain-agnostic and can be used to build sensor AI applications in multiple domains. Note that some of the content of the SDK (sample applications) or the documentation might still appear to be healthcare-specific pending additional updates. Going forward, domain specific content will be hosted on the [HoloHub](https://nvidia-holoscan.github.io/holohub) repository. -::: - The Holoscan SDK assists developers by providing: 1. **Various installation strategies** @@ -46,3 +42,7 @@ The Holoscan SDK documentation is composed of: - This user guide, in a [webpage](https://docs.nvidia.com/holoscan/sdk-user-guide/) or [PDF](https://developer.nvidia.com/downloads/holoscan-sdk-user-guide) format - Build and run instructions specific to each {ref}`installation strategy` - [Release notes](https://github.com/nvidia-holoscan/holoscan-sdk/releases) on Github + +:::{note} +In previous releases, the prefix [`Clara`](https://developer.nvidia.com/industries/healthcare) was used to define Holoscan as a platform designed initially for [medical devices](https://www.nvidia.com/en-us/clara/developer-kits/). Starting with version 0.4.0, the Holoscan SDK is built to be domain-agnostic and can be used to build sensor AI applications in multiple domains. Domain specific content will be hosted on the [HoloHub](https://nvidia-holoscan.github.io/holohub) repository. +::: diff --git a/docs/sdk_installation.md b/docs/sdk_installation.md index 7205b595..95ee5d58 100644 --- a/docs/sdk_installation.md +++ b/docs/sdk_installation.md @@ -91,17 +91,18 @@ See details and usage instructions on [NGC][container]. ```` ````{tab-item} Debian package -- **IGX Orin**: Ensure the [compute stack is pre-installed](https://docs.nvidia.com/igx-orin/user-guide/latest/base-os.html#installing-the-compute-stack). -- **Jetson**: Install the latest [CUDA keyring package](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/#network-repo-installation-for-ubuntu) for `ubuntu2204/arm64`. -- **GH200**: Install the latest [CUDA keyring package](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/#network-repo-installation-for-ubuntu) for `ubuntu2204/sbsa`. -- **x86_64**: Install the latest [CUDA keyring package](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/#network-repo-installation-for-ubuntu) for `ubuntu2204/x86_64`. - -Then, install the holoscan SDK: +Try the following to install the holoscan SDK: ```sh sudo apt update sudo apt install holoscan ``` +If `holoscan` is not found, try the following before repeating the steps above: +- **IGX Orin**: Ensure the [compute stack is properly installed](https://docs.nvidia.com/igx-orin/user-guide/latest/base-os.html#installing-the-compute-stack) which should configure the L4T repository source. If you still cannot install the Holoscan SDK, use the [`arm64-sbsa` installer](https://developer.nvidia.com/holoscan-downloads?target_os=Linux&target_arch=arm64-sbsa&Compilation=Native&Distribution=Ubuntu&target_version=22.04&target_type=deb_network) from the CUDA repository. +- **Jetson**: Ensure [JetPack is properly installed](https://developer.nvidia.com/embedded/jetpack) which should configure the L4T repository source. If you still cannot install the Holoscan SDK, use the [`aarch64-jetson` installer](https://developer.nvidia.com/holoscan-downloads?target_os=Linux&target_arch=aarch64-jetson&Compilation=Native&Distribution=Ubuntu&target_version=22.04&target_type=deb_network) from the CUDA repository. +- **GH200**: Use the [`arm64-sbsa` installer](https://developer.nvidia.com/holoscan-downloads?target_os=Linux&target_arch=arm64-sbsa&Compilation=Native&Distribution=Ubuntu&target_version=22.04&target_type=deb_network) from the CUDA repository. +- **x86_64**: Use the [`x86_64` installer](https://developer.nvidia.com/holoscan-downloads?target_os=Linux&target_arch=x86_64&Distribution=Ubuntu&target_version=22.04&target_type=deb_network) from the CUDA repository. + :::{note} To leverage the python module included in the debian package (instead of installing the python wheel), include the path below to your python path. For example: ```sh @@ -166,7 +167,7 @@ For x86_64, ensure that the [CUDA Runtime is installed](https://developer.nvidia [^3]: NPP 12 needed for the FormatConverter and BayerDemosaic operators. Already installed on NVIDIA developer kits with IGX Software and JetPack. [^4]: TensorRT 8.6.1+ and cuDNN needed for the Inference operator. Already installed on NVIDIA developer kits with IGX Software and JetPack. [^5]: Vulkan 1.3.204+ loader needed for the HoloViz operator (+ libegl1 for headless rendering). Already installed on NVIDIA developer kits with IGX Software and JetPack. -[^6]: V4L2 1.22+ needed for the V4L2 operator. Already installed on NVIDIA developer kits with IGX Software and JetPack. +[^6]: V4L2 1.22+ needed for the V4L2 operator. Already installed on NVIDIA developer kits with IGX Software and JetPack. V4L2 also requires libjpeg. [^7]: Torch support requires LibTorch 2.1+, TorchVision 0.16+, OpenBLAS 0.3.20+, OpenMPI (aarch64 only), MKL 2021.1.1 (x86_64 only), libpng and libjpeg. [^8]: To install LibTorch and TorchVision, either build them from source, download our [pre-built packages](https://edge.urm.nvidia.com/artifactory/sw-holoscan-thirdparty-generic-local/), or copy them from the holoscan container (in `/opt`). [^9]: ONNXRuntime 1.15.1+ needed for the Inference operator. Note that ONNX models are also supported through the TensoRT backend of the Inference Operator. @@ -185,4 +186,4 @@ We only recommend building the SDK from source if you need to build it with debu ### Looking for a light runtime container image? -The current Holoscan container on NGC has a large size due to including all the dependencies for each of the built-in operators, but also because of the development tools and libraries that are included. Follow the [instructions on GitHub](https://github.com/nvidia-holoscan/holoscan-sdk#runtime-container) to build a runtime container without these development packages. This page also includes detailed documentation to assist you in only including runtime dependencies your Holoscan application might need. +The current Holoscan container on NGC has a large size due to including all the dependencies for each of the built-in operators, but also because of the development tools and libraries that are included. Follow the [instructions on GitHub](https://github.com/nvidia-holoscan/holoscan-sdk/blob/main/DEVELOP.md#runtime-container) to build a runtime container without these development packages. This page also includes detailed documentation to assist you in only including runtime dependencies your Holoscan application might need. diff --git a/docs/visualization.md b/docs/visualization.md index 12aae63d..18eb5fe8 100644 --- a/docs/visualization.md +++ b/docs/visualization.md @@ -202,14 +202,14 @@ See {enum}`viz::ImageFormat` for supported image formats. Additionally {func}`vi ### Geometry Layers -A geometry layer is used to draw geometric primitives such as points, lines, rectangles, ovals or text. +A geometry layer is used to draw 2d or 3d geometric primitives. 2d primitives are points, lines, line strips, rectangles, ovals or text and are defined with 2d coordinates (x, y). 3d primitives are points, lines, line strips or triangles and are defined with 3d coordinates (x, y, z). -Coordinates start with (0, 0) in the top left and end with (1, 1) in the bottom right. +Coordinates start with (0, 0) in the top left and end with (1, 1) in the bottom right for 2d primitives. `````{tab-set} ````{tab-item} Operator -See [holoviz_geometry.cpp](https://github.com/nvidia-holoscan/holoscan-sdk/blob/main/examples/holoviz/cpp/holoviz_geometry.cpp) and [holoviz_geometry.py](https://github.com/nvidia-holoscan/holoscan-sdk/blob/main/examples/holoviz/python/holoviz_geometry.py). +See [holoviz_geometry.cpp](https://github.com/nvidia-holoscan/holoscan-sdk/blob/main/examples/holoviz/cpp/holoviz_geometry.cpp) and [holoviz_geometry.py](https://github.com/nvidia-holoscan/holoscan-sdk/blob/main/examples/holoviz/python/holoviz_geometry.py) for 2d geometric primitives and and [holoviz_geometry.py](https://github.com/nvidia-holoscan/holoscan-sdk/blob/main/examples/holoviz/python/holoviz_geometry_3d.py) for 3d geometric primitives. ```` ````{tab-item} Module @@ -292,13 +292,6 @@ an 8-bit B component in byte 2, and an 8-bit A component in byte 3 Depth maps are rendered in 3D and support camera movement. -The camera is operated using the mouse. -- Orbit (LMB) -- Pan (LMB + CTRL | MMB) -- Dolly (LMB + SHIFT | RMB | Mouse wheel) -- Look Around (LMB + ALT | LMB + CTRL + SHIFT) -- Zoom (Mouse wheel + SHIFT) - `````{tab-set} ````{tab-item} Operator ```cpp @@ -342,6 +335,27 @@ Use {func}`viz::LayerAddView()` to add a view to a layer. ```` ````` +## Camera + +When rendering 3d geometry using a geometry layer with 3d primitives or using a depth map layer the camera properties can either be set by the application or interactively changed by the user. + +To interactively change the camera, use the mouse: + +- Orbit (LMB) +- Pan (LMB + CTRL | MMB) +- Dolly (LMB + SHIFT | RMB | Mouse wheel) +- Look Around (LMB + ALT | LMB + CTRL + SHIFT) +- Zoom (Mouse wheel + SHIFT) + +`````{tab-set} +````{tab-item} Operator +See [holoviz_camera.cpp](https://github.com/nvidia-holoscan/holoscan-sdk/blob/main/examples/holoviz/cpp/holoviz_camera.cpp). +```` +````{tab-item} Module +Use {func}`viz::SetCamera()` to change the camera. +```` +````` + (holoviz-display-mode)= ## Using a display in exclusive mode @@ -392,9 +406,12 @@ If the name is `nullptr` then the first display is selected. ```` ````` -The name of the display can either be the EDID name as displayed in the NVIDIA Settings, or the output name used by `xrandr`. +The name of the display can either be the EDID name as displayed in the NVIDIA Settings, or the output name provided by `xrandr` or +`hwinfo --monitor`. :::{tip} +`````{tab-set} +````{tab-item} X11 In this example output of `xrandr`, `DP-2` would be an adequate display name to use: ```bash Screen 0: minimum 8 x 8, current 4480 x 1440, maximum 32767 x 32767 @@ -407,6 +424,15 @@ DP-2 connected primary 2560x1440+1920+0 (normal left inverted right x axis y axi 640x480 59.94 USB-C-0 disconnected (normal left inverted right x axis y axis) ``` +```` +````{tab-item} Wayland and X11 +In this example output of `hwinfo`, `MSI MPG343CQR would be an adequate display name to use: +```bash +$ hwinfo --monitor | grep Model + Model: "MSI MPG343CQR" +``` +```` +````` ::: ## CUDA streams diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 2c1f7944..e77b9b86 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -20,7 +20,9 @@ add_subdirectory(cupy_native) add_subdirectory(flow_tracker) add_subdirectory(hello_world) add_subdirectory(holoviz) +add_subdirectory(import_gxf_components) add_subdirectory(multithread) +add_subdirectory(multi_branch_pipeline) add_subdirectory(numpy_native) add_subdirectory(ping_any) add_subdirectory(ping_conditional) diff --git a/examples/README.md b/examples/README.md index 150bdd31..2845b78d 100644 --- a/examples/README.md +++ b/examples/README.md @@ -11,8 +11,8 @@ See [HoloHub](https://nvidia-holoscan.github.io/holohub) to find additional refe ```sh export src_dir="/opt/nvidia/holoscan/examples/" # Add "/cpp" to build a specific example - export build_dir="/opt/nvidia/holoscan/examples/build` # Or the path of your choice - cmake -S $src_dir -B $build_dir + export build_dir="/opt/nvidia/holoscan/examples/build" # Or the path of your choice + cmake -S $src_dir -B $build_dir -D Holoscan_ROOT="/opt/nvidia/holoscan" cmake --build $build_dir -j ``` @@ -60,16 +60,17 @@ The following examples demonstrate the basics of the Holoscan core API, and are The following examples illustrate the use of specific **schedulers** to define when operators are run: -* [**Multithread Scheduler**](multithread): run operators in parallel +* [**Multithread or Event-Based Schedulers**](multithread): run operators in parallel +* [**Multi-Rate Pipeline**](multi_branch_pipeline): Demonstrates how to override default operator port properties to allow parallel downstream branches of a pipeline to operate at different frame rates The following examples illustrate the use of specific **conditions** to modify the behavior of operators: -* [**PeriodicCondition**](conditions/periodic): trigger an operator at a user-defined time interval. +* [**PeriodicCondition**](conditions/periodic): trigger an operator at a user-defined time interval * [**AsynchronousCondition**](conditions/asynchronous): allow operators to run asynchronously (C++ API only) The following examples illustrate the use of specific resource classes that can be passed to operators or schedulers: -* [**Clock**](resources/clock): demonstrates assignment of a user-configured clock to the Holoscan SDK scheduler and how its runtime methods can be accessed from an operator's compute method. +* [**Clock**](resources/clock): demonstrate assignment of a user-configured clock to the Holoscan SDK scheduler and how its runtime methods can be accessed from an operator's compute method. ## Visualization * [**Holoviz**](holoviz): display overlays of various geometric primitives @@ -94,4 +95,5 @@ The following examples demonstrate how sensors can be used as input streams to y ### GXF and Holoscan * [**Tensor interop**](tensor_interop): use the `Entity` message to pass tensors to/from Holoscan operators wrapping GXF codelets in Holoscan applications +* [**Import GXF Components**](import_gxf_components): import the existing GXF Codelets and Components into Holoscan applications * [**Wrap operator as GXF extension**](wrap_operator_as_gxf_extension): wrap Holoscan native operators as GXF codelets to use in GXF applications diff --git a/examples/bring_your_own_model/README.md b/examples/bring_your_own_model/README.md index e60cbe9d..613a741c 100644 --- a/examples/bring_your_own_model/README.md +++ b/examples/bring_your_own_model/README.md @@ -27,11 +27,11 @@ through how to modify the python example code to run the application with an ult ``` * **using deb package install**: ```bash - /opt/nvidia/holoscan/examples/download_example_data - export HOLOSCAN_INPUT_PATH= + sudo /opt/nvidia/holoscan/examples/download_example_data + export HOLOSCAN_INPUT_PATH=/opt/nvidia/holoscan/data export PYTHONPATH=/opt/nvidia/holoscan/python/lib - # Need to enable write permission in the model directory to write the engine file (use with caution) - sudo chmod a+w /opt/nvidia/holoscan/examples/bring_your_own_model/model + # Enable write permission in the sample model directory to write the optimized TensorRT engine file (use with caution) + sudo chown $USER /opt/nvidia/holoscan/examples/bring_your_own_model/model python3 /opt/nvidia/holoscan/examples/bring_your_own_model/python/byom.py ``` * **from NGC container**: diff --git a/examples/bring_your_own_model/python/CMakeLists.min.txt b/examples/bring_your_own_model/python/CMakeLists.min.txt new file mode 100644 index 00000000..51c264dd --- /dev/null +++ b/examples/bring_your_own_model/python/CMakeLists.min.txt @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Testing +if(HOLOSCAN_BUILD_TESTS) + + file(READ ${CMAKE_CURRENT_SOURCE_DIR}/byom.yaml CONFIG_STRING) + string(REPLACE "count: 0" "count: 10" CONFIG_STRING ${CONFIG_STRING}) + set(CONFIG_FILE ${CMAKE_CURRENT_BINARY_DIR}/python_byom_config.yaml) + file(WRITE ${CONFIG_FILE} ${CONFIG_STRING}) + + add_test(NAME EXAMPLE_PYTHON_BYOM_TEST + COMMAND python3 byom.py --config python_byom_config.yaml + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + + set_tests_properties(EXAMPLE_PYTHON_BYOM_TEST PROPERTIES + DEPENDS "byom.py" + PASS_REGULAR_EXPRESSION "Reach end of file or playback count reaches to the limit. Stop ticking." + ) + +endif() diff --git a/examples/bring_your_own_model/python/CMakeLists.txt b/examples/bring_your_own_model/python/CMakeLists.txt index f250b048..1c407a81 100644 --- a/examples/bring_your_own_model/python/CMakeLists.txt +++ b/examples/bring_your_own_model/python/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -39,3 +39,30 @@ install(FILES DESTINATION "${app_relative_dest_path}" COMPONENT "holoscan-examples" ) + +# Install the minimal CMakeLists.txt file +install(FILES CMakeLists.min.txt + RENAME "CMakeLists.txt" + DESTINATION "${app_relative_dest_path}" + COMPONENT holoscan-examples +) + +# Testing +if(HOLOSCAN_BUILD_TESTS) + + file(READ ${CMAKE_CURRENT_SOURCE_DIR}/byom.yaml CONFIG_STRING) + string(REPLACE "count: 0" "count: 10" CONFIG_STRING ${CONFIG_STRING}) + set(CONFIG_FILE ${CMAKE_CURRENT_BINARY_DIR}/python_byom_config.yaml) + file(WRITE ${CONFIG_FILE} ${CONFIG_STRING}) + + add_test(NAME EXAMPLE_PYTHON_BYOM_TEST + COMMAND python3 byom.py --config python_byom_config.yaml + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + + set_tests_properties(EXAMPLE_PYTHON_BYOM_TEST PROPERTIES + DEPENDS "byom.py" + PASS_REGULAR_EXPRESSION "Reach end of file or playback count reaches to the limit. Stop ticking." + ) + +endif() diff --git a/examples/bring_your_own_model/python/byom.py b/examples/bring_your_own_model/python/byom.py index 6240ad90..a00c8321 100644 --- a/examples/bring_your_own_model/python/byom.py +++ b/examples/bring_your_own_model/python/byom.py @@ -58,26 +58,26 @@ def __init__(self, data): raise ValueError(f"Could not find video data: {self.video_dir=}") def compose(self): - host_allocator = UnboundedAllocator(self, name="host_allocator") + allocator = UnboundedAllocator(self, name="allocator") source = VideoStreamReplayerOp( self, name="replayer", directory=self.video_dir, **self.kwargs("replayer") ) preprocessor = FormatConverterOp( - self, name="preprocessor", pool=host_allocator, **self.kwargs("preprocessor") + self, name="preprocessor", pool=allocator, **self.kwargs("preprocessor") ) inference = InferenceOp( self, name="inference", - allocator=host_allocator, + allocator=allocator, model_path_map=self.model_path_map, **self.kwargs("inference"), ) postprocessor = SegmentationPostprocessorOp( - self, name="postprocessor", allocator=host_allocator, **self.kwargs("postprocessor") + self, name="postprocessor", allocator=allocator, **self.kwargs("postprocessor") ) viz = HolovizOp(self, name="viz", **self.kwargs("viz")) @@ -106,7 +106,18 @@ def main(config_file, data): default="none", help=("Set the data path"), ) + parser.add_argument( + "-c", + "--config", + default="none", + help=("Set the configuration file"), + ) args = parser.parse_args() - config_file = os.path.join(os.path.dirname(__file__), "byom.yaml") + + if args.config == "none": + config_file = os.path.join(os.path.dirname(__file__), "byom.yaml") + else: + config_file = args.config + main(config_file=config_file, data=args.data) diff --git a/examples/conditions/asynchronous/CMakeLists.txt b/examples/conditions/asynchronous/CMakeLists.txt index 0dacb07f..35338b09 100644 --- a/examples/conditions/asynchronous/CMakeLists.txt +++ b/examples/conditions/asynchronous/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,7 +14,7 @@ # limitations under the License. add_subdirectory(cpp) -# add_subdirectory(python) +add_subdirectory(python) file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/examples/conditions/asynchronous/README.md b/examples/conditions/asynchronous/README.md index 90415441..481bf7f2 100644 --- a/examples/conditions/asynchronous/README.md +++ b/examples/conditions/asynchronous/README.md @@ -8,10 +8,12 @@ There are two operators involved in this example: 1. a transmitter, set to transmit a sequence of integers from 1-20 to it's 'out' port 2. a receiver that prints the received values to the terminal -The transmit operator will be asynchronous if `async_transmit: true` in `ping_async.yaml`. -The receive operator will be asynchronous if `async_receive: true` in `ping_async.yaml`. +For the C++ application: +- The transmit operator will be asynchronous if `async_transmit: true` in `ping_async.yaml`. +- The receive operator will be asynchronous if `async_receive: true` in `ping_async.yaml`. +- The scheduler to be used can be set via the `scheduler` entry in `ping_async.yaml`. It defaults to `event_based` (an event-based multi-thread scheduler), but can also be set to either `multi_thread` (polling-based) or `greedy` (single thread). -The scheduler to be used can be set via the `scheduler` entry in `ping_async.yaml`. It defaults to `event_based` (an event-based multi-thread scheduler), but can also be set to either `multi_thread` (polling-based) or `greedy` (single thread). +For the Python application, configuration is via command line arguments as described below. *Visit the [SDK User Guide](https://docs.nvidia.com/holoscan/sdk-user-guide/components/conditions.html) to learn more about the Asynchronous Condition.* @@ -42,4 +44,19 @@ sed -i -e 's#^multithreaded:.*#multithreaded: true#' ./examples/ping_async/cpp/p ./examples/ping_async/cpp/ping_async ``` +# Python +```bash +python ./examples/ping_async/python/ping_async.py +``` + +By default, both transmit and receive are asynchronous. To see the available options run the +application using `-h` or `--help`. + +For example, to send 5 messages, waiting 500 ms between messages and use async transmit and +synchronous receive: + +```bash +python ./examples/ping_async/python/ping_async.py --delay=500 --count=5 --sync_rx +``` + > ℹ️ Python apps can run outside those folders if `HOLOSCAN_INPUT_PATH` is set in your environment (automatically done by `./run launch`). diff --git a/examples/conditions/asynchronous/python/CMakeLists.txt b/examples/conditions/asynchronous/python/CMakeLists.txt new file mode 100644 index 00000000..aac597f3 --- /dev/null +++ b/examples/conditions/asynchronous/python/CMakeLists.txt @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Get relative folder path for the app +file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) + +# Copy native operator ping application +add_custom_target(python_ping_async ALL + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/ping_async.py" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "ping_async.py" + BYPRODUCTS "ping_async.py" +) + +# Install the app +install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/ping_async.py" + DESTINATION "${app_relative_dest_path}" + COMPONENT "holoscan-examples" +) + +# Testing +if(HOLOSCAN_BUILD_TESTS) + add_test(NAME EXAMPLE_PYTHON_PING_ASYNC_TEST + COMMAND python3 ping_async.py --delay=100 --count=5 + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_PYTHON_PING_ASYNC_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Rx message value: 5" + PASS_REGULAR_EXPRESSION "waiting for 0.1 s in AsyncPingTxOp.async_send" + PASS_REGULAR_EXPRESSION "waiting for 0.1 s in AsyncPingRxOp.async_receive" + ) +endif() diff --git a/examples/conditions/asynchronous/python/ping_async.py b/examples/conditions/asynchronous/python/ping_async.py new file mode 100644 index 00000000..89682ef5 --- /dev/null +++ b/examples/conditions/asynchronous/python/ping_async.py @@ -0,0 +1,254 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +from argparse import ArgumentParser +from concurrent.futures import Future, ThreadPoolExecutor + +from holoscan.conditions import AsynchronousCondition, AsynchronousEventState, CountCondition +from holoscan.core import Application, Operator, OperatorSpec +from holoscan.operators import PingRxOp, PingTxOp +from holoscan.schedulers import EventBasedScheduler, GreedyScheduler, MultiThreadScheduler + + +class AsyncPingTxOp(Operator): + """Asynchronous transmit operator. + + This operator sends a message asynchronously, where the delay between when the async event will + be set to EVENT_DONE is specified by `delay`. After a specified count, the status will be set + to EVENT_NEVER, and the operator will stop sending messages. + """ + + def __init__(self, fragment, *args, delay=0.2, count=10, **kwargs): + self.index = 0 + + # counter to keep track of number of times compute was called + self.iter = 0 + self.count = count + self.delay = delay + + # add an asynchronous condition + self.async_cond_ = AsynchronousCondition(fragment, name="async_cond") + + # thread pool with 1 worker to run async_send + self.executor_ = ThreadPoolExecutor(max_workers=1) + self.future_ = None # will be set during start() + + # Need to call the base class constructor last + # Note: It is essential that we pass self.async_cond_ to the parent + # class constructor here. + super().__init__(fragment, self.async_cond_, *args, **kwargs) + + def aysnc_send(self): + """Function to be submitted to self.executor by start() + + When the condition's event_state is EVENT_WAITING, set to EVENT_DONE. This function will + only exit once the condition is set to EVENT_NEVER. + """ + + while True: + try: + print(f"waiting for {self.delay} s in AsyncPingTxOp.async_send") + time.sleep(self.delay) + if self.async_cond_.event_state == AsynchronousEventState.EVENT_WAITING: + self.async_cond_.event_state = AsynchronousEventState.EVENT_DONE + elif self.async_cond_.event_state == AsynchronousEventState.EVENT_NEVER: + break + except Exception as e: + self.async_cond_.event_state = AsynchronousEventState.EVENT_NEVER + raise (e) + return + + def setup(self, spec: OperatorSpec): + spec.output("out") + + def start(self): + self.future_ = self.executor_.submit(self.aysnc_send) + assert isinstance(self.future_, Future) + + def compute(self, op_input, op_output, context): + self.iter += 1 + if self.iter < self.count: + self.async_cond_.event_state = AsynchronousEventState.EVENT_WAITING + else: + self.async_cond_.event_state = AsynchronousEventState.EVENT_NEVER + + op_output.emit(self.iter, "out") + + def stop(self): + self.async_cond_.event_state = AsynchronousEventState.EVENT_NEVER + self.future_.result() + + +class AsyncPingRxOp(Operator): + """Asynchronous transmit operator. + + This operator receives a message asynchronously, where the delay between when the async event + will be set to EVENT_DONE is specified by `delay`. + """ + + def __init__(self, fragment, *args, delay=0.2, count=10, **kwargs): + self.index = 0 + + # delay used by async_receive + self.delay = delay + + # add an asynchronous condition + self.async_cond_ = AsynchronousCondition(fragment, name="async_cond") + + # thread pool with 1 worker to run async_send + self.executor_ = ThreadPoolExecutor(max_workers=1) + self.future_ = None # will be set during start() + + # Need to call the base class constructor last + # Note: It is essential that we pass self.async_cond_ to the parent + # class constructor here. + super().__init__(fragment, self.async_cond_, *args, **kwargs) + + def async_receive(self): + """Function to be submitted to self.executor by start() + + When the condition's event_state is EVENT_WAITING, set to EVENT_DONE. This function will + only exit once the condition is set to EVENT_NEVER. + """ + + while True: + try: + print(f"waiting for {self.delay} s in AsyncPingRxOp.async_receive") + time.sleep(self.delay) + if self.async_cond_.event_state == AsynchronousEventState.EVENT_WAITING: + self.async_cond_.event_state = AsynchronousEventState.EVENT_DONE + elif self.async_cond_.event_state == AsynchronousEventState.EVENT_NEVER: + break + except Exception as e: + self.async_cond_.event_state = AsynchronousEventState.EVENT_NEVER + raise (e) + return + + def setup(self, spec: OperatorSpec): + spec.input("in") + + def start(self): + self.future_ = self.executor_.submit(self.async_receive) + assert isinstance(self.future_, Future) + + def compute(self, op_input, op_output, context): + value = op_input.receive("in") + print(f"Rx message value: {value}") + self.async_cond_.event_state = AsynchronousEventState.EVENT_WAITING + + def stop(self): + self.async_cond_.event_state = AsynchronousEventState.EVENT_NEVER + self.future_.result() + + +class MyPingApp(Application): + def __init__(self, *args, delay=10, count=0, async_rx=True, async_tx=True, **kwargs): + self.delay = delay + self.count = count + self.async_rx = async_rx + self.async_tx = async_tx + super().__init__(*args, **kwargs) + + def compose(self): + # Define the tx and rx operators, allowing tx to execute 10 times + if self.async_tx: + tx = AsyncPingTxOp(self, count=self.count, delay=self.delay, name="tx") + else: + tx = PingTxOp(self, CountCondition(self, self.count), name="tx") + if self.async_rx: + rx = AsyncPingRxOp(self, delay=self.delay, name="rx") + else: + rx = PingRxOp(self, name="rx") + + # Define the workflow: tx -> rx + self.add_flow(tx, rx) + + +def main(delay_ms=100, count=10, async_rx=False, async_tx=False, scheduler="event_based"): + app = MyPingApp(delay=delay_ms / 1000.0, count=count, async_rx=async_rx, async_tx=async_tx) + if scheduler == "greedy": + app.scheduler(GreedyScheduler(app)) + elif scheduler == "multi_thread": + app.scheduler(MultiThreadScheduler(app, worker_thread_number=2)) + elif scheduler == "event_based": + app.scheduler(EventBasedScheduler(app, worker_thread_number=2)) + else: + raise ValueError( + f"unrecognized scheduler '{scheduler}', should be one of ('greedy', 'multi_thread', " + "'event_based')" + ) + app.run() + + +if __name__ == "__main__": + parser = ArgumentParser( + description=( + "Asynchronous operator example. By default, both message transmit and receive use an " + "AsynchronousCondition." + ) + ) + parser.add_argument( + "-t", + "--delay", + type=int, + default=100, + help=( + "The delay in ms that the async function will wait before updating from " + "EVENT_WAITING to EVENT_DONE." + ), + ) + parser.add_argument( + "-c", + "--count", + type=int, + default=10, + help=("The number of messages to transmit."), + ) + parser.add_argument( + "--sync_tx", + action="store_true", + help=( + "Sets the application to use the synchronous PingTx transmitter instead of AsyncPingTx." + ), + ) + parser.add_argument( + "--sync_rx", + action="store_true", + help=( + "Sets the application to use the synchronous PingRx receiver instead of AsyncPingRx." + ), + ) + parser.add_argument( + "-s", + "--scheduler", + type=str, + default="event_based", + choices=["event_based", "greedy", "multi_thread"], + help="The scheduler to use for the application.", + ) + args = parser.parse_args() + if args.delay < 0: + raise ValueError(f"delay must be non-negative, got {args.delay}") + if args.count < 0: + raise ValueError(f"count must be positive, got {args.count}") + + main( + delay_ms=args.delay, + count=args.count, + async_rx=not args.sync_rx, + async_tx=not args.sync_tx, + scheduler=args.scheduler, + ) diff --git a/examples/cupy_native/README.md b/examples/cupy_native/README.md index 0e505289..078ccfd5 100644 --- a/examples/cupy_native/README.md +++ b/examples/cupy_native/README.md @@ -8,7 +8,6 @@ This minimal application multiplies two randomly generated matrices on the GPU, ```bash # [Prerequisite] Download example .py file below to `APP_DIR` # [Optional] Start the virtualenv where holoscan is installed - python3 -m pip install cupy-cuda12x python3 /matmul.py ``` * **using deb package install**: diff --git a/examples/holoviz/README.md b/examples/holoviz/README.md index ba35c76d..c3d77644 100644 --- a/examples/holoviz/README.md +++ b/examples/holoviz/README.md @@ -1,9 +1,36 @@ # HolovizOp usage -Includes two examples one showing how to use the [geometry layer and the image laye](#holovizop-geometry-layer-usage) and on showing how to use the [geometry layer with 3d primitives](#holovizop-3d-geometry-layer-usage). +Includes multiple examples showing how to use various features of the Holoviz operator +- use the [camera with 3d primitives](#holovizop-camera-usage) +- use the [geometry layer and the image layer](#holovizop-geometry-layer-usage) +- use the [geometry layer with 3d primitives](#holovizop-3d-geometry-layer-usage) +- use [layer views][#holovizop-views-usage] *Visit the [SDK User Guide](https://docs.nvidia.com/holoscan/sdk-user-guide/visualization.html) to learn more about Visualization in Holoscan.* +## HolovizOp camera usage + +This application demonstrates how to render 3d primitives and view the geometry from different camera positions and also how to retrieve the current camera position from the Holoviz operator. +The `GeometrySourceOp` generates a 3D cube, each side is output as a separate tensor. It also randomly switches between camera positions each second. +The `HolovizOp` renders the 3d primitives, smoothly interpolates between camera positions, allows to use the mouse to change the camera and outputs the current camera position for `CameraPoseRxOp`. +The `CameraPoseRxOp` receives camera pose information and prints to the console (but only once every second). + +### C++ Run instructions + +* **using deb package install or NGC container**: + ```bash + /opt/nvidia/holoscan/examples/holoviz/cpp/holoviz_camera + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + ./examples/holoviz/cpp/holoviz_camera + ``` +* **source (local env)**: + ```bash + ${BUILD_OR_INSTALL_DIR}/examples/holoviz/cpp/holoviz_camera + ``` + ## HolovizOp geometry layer usage As for `example/tensor_interop/python/tensor_interop.py`, this application demonstrates interoperability between a native operator (`ImageProcessingOp`) and two operators (`VideoStreamReplayerOp` and `HolovizOp`) that wrap existing C++-based operators. This application also demonstrates two additional aspects: @@ -15,7 +42,23 @@ As for `example/tensor_interop/python/tensor_interop.py`, this application demon The following dataset is used by this example: [📦️ (NGC) Sample RacerX Video Data](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/clara-holoscan/resources/holoscan_racerx_video/files?version=20231009). -### Run instructions +### C++ Run instructions + +* **using deb package install or NGC container**: + ```bash + /opt/nvidia/holoscan/examples/holoviz/cpp/holoviz_geometry + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + ./examples/holoviz/cpp/holoviz_geometry + ``` +* **source (local env)**: + ```bash + ${BUILD_OR_INSTALL_DIR}/examples/holoviz/cpp/holoviz_geometry + ``` + +### Python Run instructions * **using python wheel**: ```bash @@ -23,16 +66,14 @@ The following dataset is used by this example: export HOLOSCAN_INPUT_PATH= # [Prerequisite] Download example .py file below to `APP_DIR` # [Optional] Start the virtualenv where holoscan is installed - python3 -m pip install numpy - python3 -m pip install cupy-cuda12x + python3 -m pip install "numpy<2.0" python3 /holoviz_geometry.py ``` * **using deb package install**: ```bash - /opt/nvidia/holoscan/examples/download_example_data + sudo /opt/nvidia/holoscan/examples/download_example_data export HOLOSCAN_INPUT_PATH=/opt/nvidia/holoscan/data - python3 -m pip install numpy - python3 -m pip install cupy-cuda12x + python3 -m pip install "numpy<2.0" export PYTHONPATH=/opt/nvidia/holoscan/python/lib python3 /opt/nvidia/holoscan/examples/holoviz/python/holoviz_geometry.py ``` @@ -47,8 +88,7 @@ The following dataset is used by this example: ``` * **source (local env)**: ```bash - python3 -m pip install numpy - python3 -m pip install cupy-cuda12x + python3 -m pip install "numpy<2.0" export PYTHONPATH=${BUILD_OR_INSTALL_DIR}/python/lib export HOLOSCAN_INPUT_PATH=${SRC_DIR}/data python3 ${BUILD_OR_INSTALL_DIR}/examples/holoviz/python/holoviz_geometry.py @@ -66,12 +106,12 @@ As for `example/tensor_interop/python/tensor_interop.py`, this application demon ```bash # [Prerequisite] Download example .py file below to `APP_DIR` # [Optional] Start the virtualenv where holoscan is installed - python3 -m pip install numpy + python3 -m pip install "numpy<2.0" python3 /holoviz_geometry_3d.py ``` * **using deb package install**: ```bash - python3 -m pip install numpy + python3 -m pip install "numpy<2.0" export PYTHONPATH=/opt/nvidia/holoscan/python/lib python3 /opt/nvidia/holoscan/examples/holoviz/python/holoviz_geometry_3d.py ``` @@ -86,7 +126,57 @@ As for `example/tensor_interop/python/tensor_interop.py`, this application demon ``` * **source (local env)**: ```bash - python3 -m pip install numpy + python3 -m pip install "numpy<2.0" export PYTHONPATH=${BUILD_OR_INSTALL_DIR}/python/lib python3 ${BUILD_OR_INSTALL_DIR}/examples/holoviz/python/holoviz_geometry_3d.py ``` + +## HolovizOp views usage + +This example demonstrates how to use views on layers. A layer can be an image or geometry. A view defines how a layer is placed in the output window. A view +defines the 2D offset and size of a layer and also can be placed in 3d space using a 3d transformation matrix. More information can be found [here](https://docs.nvidia.com/holoscan/sdk-user-guide/visualization.html#views). +The `VideoStreamReplayerOp` reads video frames from a file and passes them to the `ImageViewsOp`. +The `ImageViewsOp` takes the frames, defines multiple dynamic and static views and passes the video frames to the `HolovizOp`. The `ImageViewsOp` also generates +the view and data for a rotating frame counter. +The `HolovizOp` renders the video frame views and the frame counter. + +### Data + +The following dataset is used by this example: +[📦️ (NGC) Sample RacerX Video Data](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/clara-holoscan/resources/holoscan_racerx_video/files?version=20231009). + +### Run instructions + +* **using python wheel**: + ```bash + # [Prerequisite] Download NGC dataset above to `DATA_DIR` + export HOLOSCAN_INPUT_PATH= + # [Prerequisite] Download example .py file below to `APP_DIR` + # [Optional] Start the virtualenv where holoscan is installed + python3 -m pip install "numpy<2.0" + python3 /holoviz_views.py + ``` +* **using deb package install**: + ```bash + # [Prerequisite] Download NGC dataset above to `DATA_DIR` + export HOLOSCAN_INPUT_PATH= + python3 -m pip install "numpy<2.0" + export PYTHONPATH=/opt/nvidia/holoscan/python/lib + python3 /opt/nvidia/holoscan/examples/holoviz/python/holoviz_views.py + ``` +* **from NGC container**: + ```bash + python3 /opt/nvidia/holoscan/examples/holoviz/python/holoviz_views.py + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + python3 ./examples/holoviz/python/holoviz_views.py + ``` +* **source (local env)**: + ```bash + python3 -m pip install "numpy<2.0" + export PYTHONPATH=${BUILD_OR_INSTALL_DIR}/python/lib + export HOLOSCAN_INPUT_PATH=${SRC_DIR}/data + python3 ${BUILD_OR_INSTALL_DIR}/examples/holoviz/python/holoviz_views.py + ``` \ No newline at end of file diff --git a/examples/holoviz/cpp/CMakeLists.min.txt b/examples/holoviz/cpp/CMakeLists.min.txt index c49b345f..98505a4c 100644 --- a/examples/holoviz/cpp/CMakeLists.min.txt +++ b/examples/holoviz/cpp/CMakeLists.min.txt @@ -20,6 +20,16 @@ project(holoviz_examples_cpp CXX) find_package(holoscan REQUIRED CONFIG PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install") +add_executable(holoviz_camera + holoviz_camera.cpp +) + +target_link_libraries(holoviz_camera + PRIVATE + holoscan::core + holoscan::ops::holoviz +) + add_executable(holoviz_geometry holoviz_geometry.cpp ) diff --git a/examples/holoviz/cpp/CMakeLists.txt b/examples/holoviz/cpp/CMakeLists.txt index 5af45cfb..6c555483 100644 --- a/examples/holoviz/cpp/CMakeLists.txt +++ b/examples/holoviz/cpp/CMakeLists.txt @@ -13,7 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Create example +# Create examples +add_executable(holoviz_camera + holoviz_camera.cpp +) + +target_link_libraries(holoviz_camera + PRIVATE + holoscan::core + holoscan::ops::holoviz +) + add_executable(holoviz_geometry holoviz_geometry.cpp ) @@ -28,28 +38,29 @@ target_link_libraries(holoviz_geometry # Set the install RPATH based on the location of the Holoscan SDK libraries # The GXF extensions are loaded by the GXF libraries - no need to include here file(RELATIVE_PATH install_lib_relative_path ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_SOURCE_DIR}/${HOLOSCAN_INSTALL_LIB_DIR}) +set_target_properties(holoviz_camera PROPERTIES INSTALL_RPATH "\$ORIGIN/${install_lib_relative_path}") set_target_properties(holoviz_geometry PROPERTIES INSTALL_RPATH "\$ORIGIN/${install_lib_relative_path}") # Get relative folder path for the app file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) if(HOLOSCAN_INSTALL_EXAMPLE_SOURCE) -# Install the source -install(FILES holoviz_geometry.cpp - DESTINATION "${app_relative_dest_path}" - COMPONENT holoscan-examples -) + # Install the source + install(FILES holoviz_camera.cpp holoviz_geometry.cpp + DESTINATION "${app_relative_dest_path}" + COMPONENT holoscan-examples + ) -# Install the minimal CMakeLists.txt file -install(FILES CMakeLists.min.txt - RENAME "CMakeLists.txt" - DESTINATION "${app_relative_dest_path}" - COMPONENT holoscan-examples -) + # Install the minimal CMakeLists.txt file + install(FILES CMakeLists.min.txt + RENAME "CMakeLists.txt" + DESTINATION "${app_relative_dest_path}" + COMPONENT holoscan-examples + ) endif() # Install the app -install(TARGETS holoviz_geometry +install(TARGETS holoviz_camera holoviz_geometry DESTINATION "${app_relative_dest_path}" COMPONENT holoscan-examples ) @@ -64,10 +75,12 @@ if(HOLOSCAN_BUILD_TESTS) file(MAKE_DIRECTORY ${RECORDING_DIR}) # Patch the current example to enable recording the rendering window + set(_patch_file ${CMAKE_SOURCE_DIR}/tests/data/validation_frames/holoviz_geometry/cpp_holoviz_geometry.patch) add_custom_command(OUTPUT holoviz_geometry_test.cpp PRE_LINK COMMAND patch -u -o holoviz_geometry_test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/holoviz_geometry.cpp - ${CMAKE_SOURCE_DIR}/tests/data/validation_frames/holoviz_geometry/cpp_holoviz_geometry.patch + ${_patch_file} + DEPENDS ${_patch_file} ) # Create the test executable @@ -92,13 +105,12 @@ if(HOLOSCAN_BUILD_TESTS) holoscan::ops::format_converter ) - # Add the test and make sure it runs + # Add the geometry test and make sure it runs add_test(NAME EXAMPLE_CPP_HOLOVIZ_GEOMETRY_TEST COMMAND ${CMAKE_CURRENT_BINARY_DIR}/holoviz_geometry_test --count 10 WORKING_DIRECTORY ${CMAKE_BINARY_DIR} ) set_tests_properties(EXAMPLE_CPP_HOLOVIZ_GEOMETRY_TEST PROPERTIES - PASS_REGULAR_EXPRESSION "Received camera pose:" PASS_REGULAR_EXPRESSION "Reach end of file or playback count reaches to the limit. Stop ticking." ) @@ -117,4 +129,15 @@ if(HOLOSCAN_BUILD_TESTS) PASS_REGULAR_EXPRESSION "Valid video output!" ) + # Add the camera example and make sure it runs + add_test(NAME EXAMPLE_CPP_HOLOVIZ_CAMERA_TEST + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/holoviz_camera --count 120 + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_CPP_HOLOVIZ_CAMERA_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Received camera pose:" + PASS_REGULAR_EXPRESSION "Scheduler stopped: Some entities are waiting for execution, but there are no periodic or async entities to get out of the deadlock." + ) + + endif() diff --git a/examples/holoviz/cpp/holoviz_camera.cpp b/examples/holoviz/cpp/holoviz_camera.cpp new file mode 100644 index 00000000..1c9d27f7 --- /dev/null +++ b/examples/holoviz/cpp/holoviz_camera.cpp @@ -0,0 +1,324 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include + +#include +#include +#include +#include + +#include +#include + +#include +#include + +namespace holoscan::ops { +/** + * This operatore receives camera pose information and prints to the console (but only once every + * second). + */ +class CameraPoseRxOp : public Operator { + public: + HOLOSCAN_OPERATOR_FORWARD_ARGS(CameraPoseRxOp) + + CameraPoseRxOp() = default; + + void setup(OperatorSpec& spec) override { spec.input("input"); } + + void start() override { start_time_ = std::chrono::steady_clock::now(); } + + void compute(InputContext& op_input, OutputContext&, ExecutionContext&) override { + auto value = op_input.receive>("input").value(); + + // print once every second + if (std::chrono::steady_clock::now() - start_time_ > std::chrono::seconds(1)) { + HOLOSCAN_LOG_INFO("Received camera pose:\nrotation {}\ntranslation {}", + value->rotation, + value->translation); + + start_time_ = std::chrono::steady_clock::now(); + } + } + + private: + std::chrono::steady_clock::time_point start_time_; +}; + +/** + * The operator generates a 3D cube, each side is output as a separate tensor. It also randomly + * switches between camera positions each second. + */ +class GeometrySourceOp : public Operator { + public: + HOLOSCAN_OPERATOR_FORWARD_ARGS(GeometrySourceOp) + + GeometrySourceOp() = default; + + void initialize() override { + // Create an allocator for the operator + allocator_ = fragment()->make_resource("pool"); + // Add the allocator to the operator so that it is initialized + add_arg(allocator_); + + // Call the base class initialize function + Operator::initialize(); + } + + void setup(OperatorSpec& spec) override { + spec.output("geometry_output"); + spec.output>("camera_eye_output"); + spec.output>("camera_look_at_output"); + spec.output>("camera_up_output"); + } + + void start() override { start_time_ = std::chrono::steady_clock::now(); } + + /** + * Helper function to add a tensor with data to an entity. + */ + template + void add_data(gxf::Entity& entity, const char* name, + const std::array, N>& data, ExecutionContext& context) { + // get Handle to underlying nvidia::gxf::Allocator from std::shared_ptr + auto allocator = nvidia::gxf::Handle::Create(context.context(), + allocator_->gxf_cid()); + // add a tensor + auto tensor = static_cast(entity).add(name).value(); + // reshape the tensor to the size of the data + tensor->reshape( + nvidia::gxf::Shape({N, C}), nvidia::gxf::MemoryStorageType::kHost, allocator.value()); + // copy the data to the tensor + std::memcpy(tensor->pointer(), data.data(), N * C * sizeof(float)); + } + + void compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) override { + auto entity = gxf::Entity::New(&context); + auto specs = std::vector(); + + // Create a colored box + // Each triangle is defined by a set of 3 (x, y, z) coordinate pairs. + add_data<6, 3>(entity, + "back", + {{{-1.f, -1.f, -1.f}, + {1.f, -1.f, -1.f}, + {1.f, 1.f, -1.f}, + {1.f, 1.f, -1.f}, + {-1.f, 1.f, -1.f}, + {-1.f, -1.f, -1.f}}}, + context); + add_data<6, 3>(entity, + "front", + {{{-1.f, -1.f, 1.f}, + {1.f, -1.f, 1.f}, + {1.f, 1.f, 1.f}, + {1.f, 1.f, 1.f}, + {-1.f, 1.f, 1.f}, + {-1.f, -1.f, 1.f}}}, + context); + add_data<6, 3>(entity, + "right", + {{{1.f, -1.f, -1.f}, + {1.f, -1.f, 1.f}, + {1.f, 1.f, 1.f}, + {1.f, 1.f, 1.f}, + {1.f, 1.f, -1.f}, + {1.f, -1.f, -1.f}}}, + context); + add_data<6, 3>(entity, + "left", + {{{-1.f, -1.f, -1.f}, + {-1.f, -1.f, 1.f}, + {-1.f, 1.f, 1.f}, + {-1.f, 1.f, 1.f}, + {-1.f, 1.f, -1.f}, + {-1.f, -1.f, -1.f}}}, + context); + add_data<6, 3>(entity, + "top", + {{{-1.f, 1.f, -1.f}, + {-1.f, 1.f, 1.f}, + {1.f, 1.f, 1.f}, + {1.f, 1.f, 1.f}, + {1.f, 1.f, -1.f}, + {-1.f, 1.f, -1.f}}}, + context); + add_data<6, 3>(entity, + "bottom", + {{{-1.f, -1.f, -1.f}, + {-1.f, -1.f, 1.f}, + {1.f, -1.f, 1.f}, + {1.f, -1.f, 1.f}, + {1.f, -1.f, -1.f}, + {-1.f, -1.f, -1.f}}}, + context); + + // emit the tensors + op_output.emit(entity, "geometry_output"); + + // every second, switch camera + if (std::chrono::steady_clock::now() - start_time_ > std::chrono::seconds(1)) { + const int camera = std::rand() % sizeof(cameras_) / sizeof(cameras_[0]); + camera_eye_ = cameras_[camera][0]; + camera_look_at_ = cameras_[camera][1]; + camera_up_ = cameras_[camera][2]; + + op_output.emit(camera_eye_, "camera_eye_output"); + op_output.emit(camera_look_at_, "camera_look_at_output"); + op_output.emit(camera_up_, "camera_up_output"); + + start_time_ = std::chrono::steady_clock::now(); + } + } + + const std::array& camera_eye() const { return camera_eye_; } + const std::array& camera_look_at() const { return camera_look_at_; } + const std::array& camera_up() const { return camera_up_; } + + private: + std::shared_ptr allocator_; + + std::chrono::steady_clock::time_point start_time_; + + // define some cameras we switch between + static constexpr std::array cameras_[4][3]{ + {{0.f, 0.f, 5.f}, {1.f, 1.f, 0.f}, {0.f, 1.f, 0.f}}, + {{1.f, 1.f, -3.f}, {0.f, 0.f, 0.f}, {0.f, 1.f, 0.f}}, + {{3.f, -4.f, 0.f}, {0.f, 1.f, 1.f}, {1.f, 0.f, 0.f}}, + {{-2.f, 0.f, -3.f}, {-1.f, 0.f, -1.f}, {0.f, 0.f, 1.f}}}; + + std::array camera_eye_ = cameras_[0][0]; + std::array camera_look_at_ = cameras_[0][1]; + std::array camera_up_ = cameras_[0][2]; +}; + +} // namespace holoscan::ops + +/** + * Example of an application that uses the operators defined above. + * + * This application has the following operators: + * + * - GeometrySourceOp + * - HolovizOp + * - CameraPoseRxOp + * + * The GeometrySourceOp creates geometric primitives and camera properties and sends it to the + * HolovizOp. It runs at 60 Hz. + * The HolovizOp displays the geometry and is using the camera properties. + * The CameraPoseRxOp receives camera pose information from the HolovizOp and prints it on the + * console. + */ +class HolovizCameraApp : public holoscan::Application { + public: + /** + * @brief Construct a new HolovizCameraApp object + * + * @param count Limits the number of frames to show before the application ends. + * Set to -1 by default. Any positive integer will limit on the number of frames displayed. + */ + explicit HolovizCameraApp(uint64_t count) : count_(count) {} + + void compose() override { + using namespace holoscan; + + auto source = make_operator( + "source", + // limit frame count + make_condition("frame_limit", count_), + // run at 60 Hz + make_condition("frame_limiter", + Arg("recess_period", std::string("60Hz")))); + + // build the input spec list + std::vector input_spec; + + // Parameters defining the triangle primitives + const std::array spec_names{"back", "front", "left", "right", "top", "bottom"}; + for (int index = 0; index < spec_names.size(); ++index) { + auto& spec = input_spec.emplace_back( + ops::HolovizOp::InputSpec(spec_names[index], ops::HolovizOp::InputType::TRIANGLES_3D)); + spec.color_ = { + float((index + 1) & 1), float(((index + 1) / 2) & 1), float(((index + 1) / 4) & 1), 1.0f}; + } + + auto visualizer = make_operator( + "holoviz", + Arg("width", 1024u), + Arg("height", 1024u), + Arg("tensors", input_spec), + Arg("enable_camera_pose_output", true), + Arg("camera_pose_output_type", std::string("extrinsics_model")), + // pass the initial camera properties to HolovizOp + Arg("camera_eye", source->camera_eye()), + Arg("camera_look_at", source->camera_look_at()), + Arg("camera_up", source->camera_up())); + + auto camera_pose_rx = make_operator("camera_pose_rx"); + + // Define the workflow: source -> holoviz + add_flow(source, visualizer, {{"geometry_output", "receivers"}}); + add_flow(source, visualizer, {{"camera_eye_output", "camera_eye_input"}}); + add_flow(source, visualizer, {{"camera_look_at_output", "camera_look_at_input"}}); + add_flow(source, visualizer, {{"camera_up_output", "camera_up_input"}}); + add_flow(visualizer, camera_pose_rx, {{"camera_pose_output", "input"}}); + } + + private: + uint64_t count_ = -1; +}; + +int main(int argc, char** argv) { + // Parse args + struct option long_options[] = { + {"help", no_argument, 0, 'h'}, {"count", required_argument, 0, 'c'}, {0, 0, 0, 0}}; + uint64_t count = -1; + while (true) { + int option_index = 0; + const int c = getopt_long(argc, argv, "hc:", long_options, &option_index); + + if (c == -1) { break; } + + const std::string argument(optarg ? optarg : ""); + switch (c) { + case 'h': + case '?': + std::cout + << "Usage: " << argv[0] << " [options]" << std::endl + << "Options:" << std::endl + << " -h, --help display this information" << std::endl + << " -c, --count limits the number of frames to show before the application " + "ends. Set to `" + << count + << "` by default. Any positive integer will limit on the number of frames displayed." + << std::endl; + return EXIT_SUCCESS; + + case 'c': + count = std::stoull(argument); + break; + default: + throw std::runtime_error(fmt::format("Unhandled option `{}`", char(c))); + } + } + + auto app = holoscan::make_application(count); + app->run(); + + return 0; +} diff --git a/examples/holoviz/cpp/holoviz_geometry.cpp b/examples/holoviz/cpp/holoviz_geometry.cpp index d527f55a..1eb86c2e 100644 --- a/examples/holoviz/cpp/holoviz_geometry.cpp +++ b/examples/holoviz/cpp/holoviz_geometry.cpp @@ -28,30 +28,6 @@ namespace holoscan::ops { -class CameraPoseRxOp : public Operator { - public: - HOLOSCAN_OPERATOR_FORWARD_ARGS(CameraPoseRxOp) - - CameraPoseRxOp() = default; - - void setup(OperatorSpec& spec) override; - - void compute(InputContext& op_input, OutputContext&, ExecutionContext&) override; - - private: - size_t count_ = 0; -}; - -void CameraPoseRxOp::setup(OperatorSpec& spec) { - spec.input("in"); -} - -void CameraPoseRxOp::compute(InputContext& op_input, OutputContext&, ExecutionContext&) { - auto value = op_input.receive>>("in").value(); - if (count_ == 0) { HOLOSCAN_LOG_INFO("Received camera pose: {}", *value); } - count_++; -} - /** * Example of an operator generating geometric primitives to be displayed by the HolovizOp * @@ -334,18 +310,12 @@ class HolovizGeometryApp : public holoscan::Application { auto visualizer = make_operator("holoviz", Arg("width", 854u), Arg("height", 480u), - Arg("tensors", input_spec), - Arg("enable_camera_pose_output", true)); - - // Set enable_camera_pose_output to true, so can create a receiver for the pose information. - // This example prints the pose information (but only for the first received frame). - auto camera_pose_rx = make_operator("rx"); + Arg("tensors", input_spec)); // Define the workflow: source -> holoviz add_flow(source, visualizer, {{"outputs", "receivers"}}); add_flow(source, visualizer, {{"output_specs", "input_specs"}}); add_flow(replayer, visualizer, {{"output", "receivers"}}); - add_flow(visualizer, camera_pose_rx, {{"camera_pose_output", "in"}}); } private: diff --git a/examples/holoviz/python/holoviz_geometry.py b/examples/holoviz/python/holoviz_geometry.py index b386bdd9..2b00e3dc 100644 --- a/examples/holoviz/python/holoviz_geometry.py +++ b/examples/holoviz/python/holoviz_geometry.py @@ -27,31 +27,6 @@ sample_data_path = os.environ.get("HOLOSCAN_INPUT_PATH", "../data") -class CameraPoseRxOp(Operator): - """Simple receiver operator. - - This operator has a single input port: - input: "in" - - This is an example of a native operator with one input port. - On each tick, it receives an integer from the "in" port. - """ - - def __init__(self, fragment, *args, **kwargs): - self.count = 0 - # Need to call the base class constructor last - super().__init__(fragment, *args, **kwargs) - - def setup(self, spec: OperatorSpec): - spec.input("in") - - def compute(self, op_input, op_output, context): - value = op_input.receive("in") - if self.count == 0: - print(f"Received camera pose: {value}") - self.count += 1 - - # Define custom Operators for use in the demo class GeometryGenerationOp(Operator): """Example creating geometric primitives for overlay on a video. @@ -254,7 +229,6 @@ def compose(self): name="holoviz", width=width, height=height, - enable_camera_pose_output=True, tensors=[ # name="" here to match the output of VideoStreamReplayerOp dict(name="", type="color", opacity=0.5, priority=0), @@ -329,15 +303,9 @@ def compose(self): ), ], ) - # Since we specified `enable_camera_pose_output=True` for the visualizer, we can connect - # this output port to a receiver to print the camera pose. This receiver will just print - # the camera pose the first time one is received. - rx = CameraPoseRxOp(self, name="rx") - self.add_flow(source, visualizer, {("output", "receivers")}) self.add_flow(image_processing, visualizer, {("outputs", "receivers")}) self.add_flow(image_processing, visualizer, {("output_specs", "input_specs")}) - self.add_flow(visualizer, rx, {("camera_pose_output", "in")}) def main(config_count): diff --git a/examples/import_gxf_components/CMakeLists.txt b/examples/import_gxf_components/CMakeLists.txt new file mode 100644 index 00000000..91b8d28e --- /dev/null +++ b/examples/import_gxf_components/CMakeLists.txt @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_subdirectory(cpp) +add_subdirectory(python) + +file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) + +install( + FILES README.md + DESTINATION "${app_relative_dest_path}" + COMPONENT "holoscan-examples" +) diff --git a/examples/import_gxf_components/README.md b/examples/import_gxf_components/README.md new file mode 100644 index 00000000..b7013ae6 --- /dev/null +++ b/examples/import_gxf_components/README.md @@ -0,0 +1,73 @@ +# Importing existing GXF Codelets/Components as Holoscan Operators/Resources + +## Overview + +This application demonstrates how to import existing GXF Codelets and Components as Holoscan Operators/Resources. The example code is adapted from the [Tensor interoperability example](https://github.com/nvidia-holoscan/holoscan-sdk/tree/main/examples/tensor_interop) (`examples/tensor_interop`) and modified to illustrate the use of GXFCodeletOp and GXFComponentResource classes to import GXF Codelet/Component and customize the `setup()` and `initialize()` methods. + +## C++ API + +The main components include a set of custom operators that encapsulate GXF Codelets for sending and receiving tensors, and a resource class for managing block memory pools. + +- **GXFSendTensorOp**: An operator that wraps a GXF Codelet responsible for sending tensors. +- **GXFReceiveTensorOp**: Extends `GXFCodeletOp` to wrap a GXF Codelet that receives tensors, with customizable setup and initialization. +- **MyBlockMemoryPool**: Represents a resource wrapping the `nvidia::gxf::BlockMemoryPool` GXF component, used to manage memory allocation. + +### Run Instructions + +* **using deb package install or NGC container**: + ```bash + /opt/nvidia/holoscan/examples/import_gxf_components/cpp/import_gxf_components + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + ./examples/import_gxf_components/cpp/import_gxf_components + ``` +* **source (local env)**: + ```bash + ${BUILD_OR_INSTALL_DIR}/examples/import_gxf_components/cpp/import_gxf_components + ``` + +## Python API + +Python example shows how to utilize the `GXFCodeletOp` and `GXFComponentResource` in a Holoscan application, focusing on video processing using custom and built-in operators. + +- **ToDeviceMemoryOp**: A derived class from `GXFCodeletOp` for copying tensors to device memory. +- **ImageProcessingOp**: Processes video frames using Gaussian filtering with CuPy. +- **DeviceMemoryPool**: Manages device memory allocations. + +### Run instructions + +* **using python wheel**: + ```bash + # [Prerequisite] Download NGC dataset above to `DATA_DIR` + export HOLOSCAN_INPUT_PATH= + # [Prerequisite] Download example .py and .yaml file below to `APP_DIR` + # [Optional] Start the virtualenv where holoscan is installed + python3 -m pip install cupy-cuda12x + python3 /import_gxf_components.py + ``` +* **using deb package install**: + ```bash + /opt/nvidia/holoscan/examples/download_example_data + export HOLOSCAN_INPUT_PATH=/opt/nvidia/holoscan/data + python3 -m pip install cupy-cuda12x + export PYTHONPATH=/opt/nvidia/holoscan/python/lib + python3 /opt/nvidia/holoscan/examples/import_gxf_components/python/import_gxf_components.py + ``` +* **from NGC container**: + ```bash + python3 /opt/nvidia/holoscan/examples/import_gxf_components/python/import_gxf_components.py + ``` +* **source (dev container)**: + ```bash + ./run launch # optional: append `install` for install tree + python3 ./examples/import_gxf_components/python/import_gxf_components.py + ``` +* **source (local env)**: + ```bash + python3 -m pip install cupy-cuda12x + export PYTHONPATH=${BUILD_OR_INSTALL_DIR}/python/lib + export HOLOSCAN_INPUT_PATH=${SRC_DIR}/data + python3 ${BUILD_OR_INSTALL_DIR}/examples/import_gxf_components/python/import_gxf_components.py + ``` diff --git a/examples/import_gxf_components/cpp/CMakeLists.min.txt b/examples/import_gxf_components/cpp/CMakeLists.min.txt new file mode 100644 index 00000000..dcf15fd0 --- /dev/null +++ b/examples/import_gxf_components/cpp/CMakeLists.min.txt @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the \"License\"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.20) +project(holoscan_tensor_interop CXX) + +# Finds the package holoscan +find_package(holoscan REQUIRED CONFIG + PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install") + +add_executable(import_gxf_components + import_gxf_components.cpp + receive_tensor_gxf.hpp + send_tensor_gxf.hpp +) + +target_link_libraries(import_gxf_components + PRIVATE + holoscan::core + holoscan::ops::gxf_codelet + CUDA::cudart +) + +# Testing +if(BUILD_TESTING) + add_test(NAME EXAMPLE_CPP_IMPORT_GXF_COMPONENTS_TEST + COMMAND import_gxf_components + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_CPP_IMPORT_GXF_COMPONENTS_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "30 30 30 30 30 30" + ) +endif() diff --git a/examples/import_gxf_components/cpp/CMakeLists.txt b/examples/import_gxf_components/cpp/CMakeLists.txt new file mode 100644 index 00000000..9980c3ed --- /dev/null +++ b/examples/import_gxf_components/cpp/CMakeLists.txt @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Create example +add_executable(import_gxf_components + import_gxf_components.cpp + receive_tensor_gxf.hpp + send_tensor_gxf.hpp +) +target_link_libraries(import_gxf_components + PRIVATE + holoscan::core + holoscan::ops::gxf_codelet + CUDA::cudart +) + +# Install example + +# Set the install RPATH based on the location of the Holoscan SDK libraries +# The GXF extensions are loaded by the GXF libraries - no need to include here +file(RELATIVE_PATH install_lib_relative_path ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_SOURCE_DIR}/${HOLOSCAN_INSTALL_LIB_DIR}) +set_target_properties(import_gxf_components PROPERTIES INSTALL_RPATH "\$ORIGIN/${install_lib_relative_path}") + +# Install following the relative folder path +file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) + +# Install the target +install(TARGETS import_gxf_components + DESTINATION "${app_relative_dest_path}" + COMPONENT holoscan-examples +) + +if(HOLOSCAN_INSTALL_EXAMPLE_SOURCE) +# Install the source +install(FILES import_gxf_components.cpp receive_tensor_gxf.hpp send_tensor_gxf.hpp + DESTINATION "${app_relative_dest_path}" + COMPONENT holoscan-examples +) + +# Install the minimal CMakeLists.txt file +install(FILES CMakeLists.min.txt + RENAME "CMakeLists.txt" + DESTINATION "${app_relative_dest_path}" + COMPONENT holoscan-examples +) +endif() + +# Testing +if(HOLOSCAN_BUILD_TESTS) + add_test(NAME EXAMPLE_CPP_IMPORT_GXF_COMPONENTS_TEST + COMMAND import_gxf_components + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_CPP_IMPORT_GXF_COMPONENTS_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "30 30 30 30 30 30" + ) +endif() diff --git a/examples/import_gxf_components/cpp/import_gxf_components.cpp b/examples/import_gxf_components/cpp/import_gxf_components.cpp new file mode 100644 index 00000000..f7e58116 --- /dev/null +++ b/examples/import_gxf_components/cpp/import_gxf_components.cpp @@ -0,0 +1,217 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "./receive_tensor_gxf.hpp" +#include "./send_tensor_gxf.hpp" +#include "holoscan/core/resources/gxf/gxf_component_resource.hpp" +#include "holoscan/operators/gxf_codelet/gxf_codelet.hpp" + +#ifdef CUDA_TRY +#undef CUDA_TRY +#define CUDA_TRY(stmt) \ + { \ + cuda_status = stmt; \ + if (cudaSuccess != cuda_status) { \ + HOLOSCAN_LOG_ERROR("Runtime call {} in line {} of file {} failed with '{}' ({})", \ + #stmt, \ + __LINE__, \ + __FILE__, \ + cudaGetErrorString(cuda_status), \ + cuda_status); \ + } \ + } +#endif + +// Define an operator that wraps the GXF Codelet that sends a tensor +// (`nvidia::gxf::test::SendTensor` class in send_tensor_gxf.hpp) +HOLOSCAN_WRAP_GXF_CODELET_AS_OPERATOR(GXFSendTensorOp, "nvidia::gxf::test::SendTensor") + +// Define an operator that wraps the GXF Codelet that receives a tensor, extends the GXFCodeletOp +// (`nvidia::gxf::test::ReceiveTensor` class in receive_tensor_gxf.hpp). +// If there is no need for custom setup or initialize code, the macro +// `HOLOSCAN_WRAP_GXF_CODELET_AS_OPERATOR` can be used (as shown above) to simplify this process. +class GXFReceiveTensorOp : public ::holoscan::ops::GXFCodeletOp { + public: + HOLOSCAN_OPERATOR_FORWARD_TEMPLATE() + explicit GXFReceiveTensorOp(ArgT&& arg, ArgsT&&... args) + : ::holoscan::ops::GXFCodeletOp("nvidia::gxf::test::ReceiveTensor", std::forward(arg), + std::forward(args)...) {} + GXFReceiveTensorOp() : ::holoscan::ops::GXFCodeletOp("nvidia::gxf::test::ReceiveTensor") {} + + void setup(holoscan::OperatorSpec& spec) override { + using namespace holoscan; + // Ensure the parent class setup() is called before any additional setup code. + ops::GXFCodeletOp::setup(spec); + + // You can add any additional setup code here (if needed). + // You can update conditions of the input/output ports, update the connector types, etc. + // + // Example: + // - `spec.inputs()["signal"]->condition(ConditionType::kNone);` + // to update the condition of the input port to 'kNone'. + // (assuming that the GXF Codelet has a Receiver component named 'signal'.) + } + + void initialize() override { + // You can call any additional initialization code here (if needed). + // + // Example: + // - `register_converter();` to register a converter for a specific type + // - `register_codec("codec_name", bool_overwrite);` to register a codec for a specific type + // - `add_arg(holoscan::Arg("arg_name", arg_value));` + // or `add_arg(holoscan::Arg("arg_name") = arg_value);` to add an argument to the GXF Operator + + // ... + + // The parent class initialize() call should occur after the argument additions specified above. + holoscan::ops::GXFCodeletOp::initialize(); + } +}; + +// Define a resource that wraps the GXF component `nvidia::gxf::BlockMemoryPool` +// (`nvidia::gxf::BlockMemoryPool` class in gxf/std/block_memory_pool.hpp) +// The following class definition can be shortened using +// the `HOLOSCAN_WRAP_GXF_COMPONENT_AS_RESOURCE` macro: +// +// HOLOSCAN_WRAP_GXF_COMPONENT_AS_RESOURCE(MyBlockMemoryPool, "nvidia::gxf::BlockMemoryPool") +// +// Note that this is illustrated here using `BlockMemoryPool` as a concrete example, but in practice +// applications would just import the existing `holoscan::BlockMemoryPool` resource. +// `GXFComponentResource` would be used to wrap some GXF component not already available via +// the resources in the `holoscan` namespace. +class MyBlockMemoryPool : public ::holoscan::GXFComponentResource { + public: + HOLOSCAN_RESOURCE_FORWARD_TEMPLATE() + explicit MyBlockMemoryPool(ArgT&& arg, ArgsT&&... args) + : ::holoscan::GXFComponentResource("nvidia::gxf::BlockMemoryPool", std::forward(arg), + std::forward(args)...) {} + MyBlockMemoryPool() = default; +}; + +class ProcessTensorOp : public holoscan::Operator { + public: + HOLOSCAN_OPERATOR_FORWARD_ARGS(ProcessTensorOp) + + ProcessTensorOp() = default; + + void setup(holoscan::OperatorSpec& spec) override { + spec.input("in"); + spec.output("out"); + } + + void compute(holoscan::InputContext& op_input, holoscan::OutputContext& op_output, + holoscan::ExecutionContext& context) override { + // The type of `in_message` is 'holoscan::TensorMap'. + auto in_message = op_input.receive("in").value(); + // The type of out_message is TensorMap. + holoscan::TensorMap out_message; + + for (auto& [key, tensor] : in_message) { // Process with 'tensor' here. + cudaError_t cuda_status; + size_t data_size = tensor->nbytes(); + std::vector in_data(data_size); + CUDA_TRY(cudaMemcpy(in_data.data(), tensor->data(), data_size, cudaMemcpyDeviceToHost)); + HOLOSCAN_LOG_INFO("ProcessTensorOp Before key: '{}', shape: ({}), data: [{}]", + key, + fmt::join(tensor->shape(), ","), + fmt::join(in_data, ",")); + for (size_t i = 0; i < data_size; i++) { in_data[i] *= 2; } + HOLOSCAN_LOG_INFO("ProcessTensorOp After key: '{}', shape: ({}), data: [{}]", + key, + fmt::join(tensor->shape(), ","), + fmt::join(in_data, ",")); + CUDA_TRY(cudaMemcpy(tensor->data(), in_data.data(), data_size, cudaMemcpyHostToDevice)); + out_message.insert({key, tensor}); + } + // Send the processed message. + op_output.emit(out_message); + }; +}; + +class App : public holoscan::Application { + public: + void register_gxf_codelets() { + gxf_context_t context = executor().context(); + + holoscan::gxf::GXFExtensionRegistrar extension_factory( + context, "TensorSenderReceiver", "Extension for sending and receiving tensors"); + + extension_factory.add_component( + "SendTensor class"); + extension_factory.add_component( + "ReceiveTensor class"); + + if (!extension_factory.register_extension()) { + HOLOSCAN_LOG_ERROR("Failed to register GXF Codelets"); + return; + } + } + + void compose() override { + using namespace holoscan; + + register_gxf_codelets(); + + auto tx = make_operator("tx", + make_condition(15), + Arg("pool") = make_resource( + "pool", + Arg("storage_type") = static_cast(1), + Arg("block_size") = 1024UL, + Arg("num_blocks") = 2UL)); + + // Alternatively, you can use the following code to create the GXFSendTensorOp operator: + // + // auto tx = make_operator("tx", + // "nvidia::gxf::test::SendTensor", + // make_condition(15), + // Arg("pool") = make_resource( + // "pool", + // "nvidia::gxf::BlockMemoryPool", + // Arg("storage_type") = static_cast(1), + // Arg("block_size") = 1024UL, + // Arg("num_blocks") = 2UL)); + + auto mx = make_operator("mx"); + + auto rx = make_operator("rx"); + + add_flow(tx, mx); + add_flow(mx, rx); + } +}; + +int main(int argc, char** argv) { + auto app = holoscan::make_application(); + app->run(); + + return 0; +} diff --git a/examples/import_gxf_components/cpp/receive_tensor_gxf.hpp b/examples/import_gxf_components/cpp/receive_tensor_gxf.hpp new file mode 100644 index 00000000..d8665521 --- /dev/null +++ b/examples/import_gxf_components/cpp/receive_tensor_gxf.hpp @@ -0,0 +1,92 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IMPORT_GXF_COMPONENTS_CPP_RECEIVE_TENSOR_GXF_HPP +#define IMPORT_GXF_COMPONENTS_CPP_RECEIVE_TENSOR_GXF_HPP + +#include + +#include +#include +#include + +#include "gxf/std/allocator.hpp" +#include "gxf/std/codelet.hpp" +#include "gxf/core/parameter_parser_std.hpp" +#include "gxf/std/receiver.hpp" +#include "gxf/std/tensor.hpp" + +#ifndef CUDA_TRY +#define CUDA_TRY(stmt) \ + ({ \ + cudaError_t _holoscan_cuda_err = stmt; \ + if (cudaSuccess != _holoscan_cuda_err) { \ + GXF_LOG_ERROR("CUDA Runtime call %s in line %d of file %s failed with '%s' (%d).\n", \ + #stmt, \ + __LINE__, \ + __FILE__, \ + cudaGetErrorString(_holoscan_cuda_err), \ + _holoscan_cuda_err); \ + } \ + _holoscan_cuda_err; \ + }) +#endif + +namespace nvidia { +namespace gxf { +namespace test { + +class ReceiveTensor : public Codelet { + public: + gxf_result_t registerInterface(Registrar* registrar) override { + gxf::Expected result; + result &= registrar->parameter(signal_, "signal"); + return gxf::ToResultCode(result); + } + gxf_result_t start() override { return GXF_SUCCESS; } + gxf_result_t tick() override { + const auto in_message = signal_->receive(); + if (!in_message || in_message.value().is_null()) { return GXF_CONTRACT_MESSAGE_NOT_AVAILABLE; } + + const auto maybe_in_tensor = in_message.value().get("tensor"); + if (!maybe_in_tensor) { + GXF_LOG_ERROR("Failed to access in tensor with name `tensor`"); + return gxf::ToResultCode(maybe_in_tensor); + } + void* in_data_ptr = maybe_in_tensor.value()->pointer(); + + size_t data_size = maybe_in_tensor->get()->bytes_size(); + std::vector in_data(data_size); + + CUDA_TRY(cudaMemcpy(in_data.data(), in_data_ptr, data_size, cudaMemcpyDeviceToHost)); + + for (size_t i = 0; i < data_size; i++) { std::cout << static_cast(in_data[i]) << " "; } + std::cout << std::endl; + + return GXF_SUCCESS; + } + gxf_result_t stop() override { return GXF_SUCCESS; } + + private: + gxf::Parameter> signal_; +}; + +} // namespace test +} // namespace gxf +} // namespace nvidia + +#endif /* IMPORT_GXF_COMPONENTS_CPP_RECEIVE_TENSOR_GXF_HPP */ diff --git a/examples/import_gxf_components/cpp/send_tensor_gxf.hpp b/examples/import_gxf_components/cpp/send_tensor_gxf.hpp new file mode 100644 index 00000000..2f50d804 --- /dev/null +++ b/examples/import_gxf_components/cpp/send_tensor_gxf.hpp @@ -0,0 +1,106 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IMPORT_GXF_COMPONENTS_CPP_SEND_TENSOR_GXF_HPP +#define IMPORT_GXF_COMPONENTS_CPP_SEND_TENSOR_GXF_HPP + +#include + +#include +#include + +#include "gxf/std/allocator.hpp" +#include "gxf/std/codelet.hpp" +#include "gxf/core/parameter_parser_std.hpp" +#include "gxf/std/tensor.hpp" +#include "gxf/std/transmitter.hpp" + +#ifndef CUDA_TRY +#define CUDA_TRY(stmt) \ + ({ \ + cudaError_t _holoscan_cuda_err = stmt; \ + if (cudaSuccess != _holoscan_cuda_err) { \ + GXF_LOG_ERROR("CUDA Runtime call %s in line %d of file %s failed with '%s' (%d).\n", \ + #stmt, \ + __LINE__, \ + __FILE__, \ + cudaGetErrorString(_holoscan_cuda_err), \ + _holoscan_cuda_err); \ + } \ + _holoscan_cuda_err; \ + }) +#endif + +namespace nvidia { +namespace gxf { +namespace test { + +class SendTensor : public Codelet { + public: + gxf_result_t registerInterface(Registrar* registrar) override { + gxf::Expected result; + result &= registrar->parameter(signal_, "signal"); + result &= registrar->parameter(pool_, "pool", "Pool", "Allocator instance for output tensors."); + return gxf::ToResultCode(result); + } + gxf_result_t start() override { return GXF_SUCCESS; } + gxf_result_t tick() override { + constexpr int rows = 4; + constexpr int cols = 4; + constexpr int out_channels = 3; + constexpr gxf::PrimitiveType element_type = gxf::PrimitiveType::kUnsigned8; + const gxf::Shape tensor_shape{rows, cols, out_channels}; + + gxf::Expected out_message = CreateTensorMap( + context(), + pool_, + {{"tensor", + gxf::MemoryStorageType::kDevice, + tensor_shape, + gxf::PrimitiveType::kUnsigned8, + 0, + gxf::ComputeTrivialStrides(tensor_shape, gxf::PrimitiveTypeSize(element_type))}}); + + const auto maybe_output_tensor = out_message.value().get("tensor"); + + if (!maybe_output_tensor) { + GXF_LOG_ERROR("Failed to access output tensor with name `tensor`"); + return gxf::ToResultCode(maybe_output_tensor); + } + + void* output_data_ptr = maybe_output_tensor.value()->pointer(); + CUDA_TRY(cudaMemset( + output_data_ptr, value_, tensor_shape.size() * gxf::PrimitiveTypeSize(element_type))); + + value_ = (value_ + 1) % 255; + + const auto result = signal_->publish(out_message.value()); + return gxf::ToResultCode(result); + } + gxf_result_t stop() override { return GXF_SUCCESS; } + + private: + gxf::Parameter> signal_; + gxf::Parameter> pool_; + int value_ = 1; +}; + +} // namespace test +} // namespace gxf +} // namespace nvidia + +#endif /* IMPORT_GXF_COMPONENTS_CPP_SEND_TENSOR_GXF_HPP */ diff --git a/examples/import_gxf_components/python/CMakeLists.min.txt b/examples/import_gxf_components/python/CMakeLists.min.txt new file mode 100644 index 00000000..7efb38e9 --- /dev/null +++ b/examples/import_gxf_components/python/CMakeLists.min.txt @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Testing +if(BUILD_TESTING) + + file(READ ${CMAKE_CURRENT_SOURCE_DIR}/import_gxf_components.yaml CONFIG_STRING) + string(REPLACE "count: 0" "count: 10" CONFIG_STRING ${CONFIG_STRING}) + set(CONFIG_FILE ${CMAKE_CURRENT_SOURCE_DIR}/python_import_gxf_components_testing_config.yaml) + file(WRITE ${CONFIG_FILE} ${CONFIG_STRING}) + + add_test(NAME EXAMPLE_PYTHON_IMPORT_GXF_COMPONENTS_TEST + COMMAND python3 import_gxf_components.py --config ${CONFIG_FILE} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + ) + set_tests_properties(EXAMPLE_PYTHON_IMPORT_GXF_COMPONENTS_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Reach end of file or playback count reaches to the limit. Stop ticking." + ) + +endif() diff --git a/examples/import_gxf_components/python/CMakeLists.txt b/examples/import_gxf_components/python/CMakeLists.txt new file mode 100644 index 00000000..0a54a15f --- /dev/null +++ b/examples/import_gxf_components/python/CMakeLists.txt @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Get relative folder path for the app +file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) + +# Copy native operator import_gxf_components application +add_custom_target(python_import_gxf_components ALL + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/import_gxf_components.py" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "import_gxf_components.py" + BYPRODUCTS "import_gxf_components.py" +) + +# Copy the config file +add_custom_target(python_import_gxf_components_yaml + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/import_gxf_components.yaml" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "import_gxf_components.yaml" + BYPRODUCTS "import_gxf_components.yaml" +) +add_dependencies(python_import_gxf_components python_import_gxf_components_yaml racerx_data) + +# Install the app +install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/import_gxf_components.py" + "${CMAKE_CURRENT_SOURCE_DIR}/import_gxf_components.yaml" + DESTINATION "${app_relative_dest_path}" + COMPONENT holoscan-examples +) + +# Install the minimal CMakeLists.txt file +install(FILES CMakeLists.min.txt + RENAME "CMakeLists.txt" + DESTINATION "${app_relative_dest_path}" + COMPONENT holoscan-examples +) + +# Testing +if(HOLOSCAN_BUILD_TESTS) + + file(READ ${CMAKE_CURRENT_SOURCE_DIR}/import_gxf_components.yaml CONFIG_STRING) + string(REPLACE "count: 0" "count: 10" CONFIG_STRING ${CONFIG_STRING}) + set(CONFIG_FILE ${CMAKE_CURRENT_BINARY_DIR}/python_import_gxf_components_testing_config.yaml) + file(WRITE ${CONFIG_FILE} ${CONFIG_STRING}) + + add_test(NAME EXAMPLE_PYTHON_IMPORT_GXF_COMPONENTS_TEST + COMMAND python3 import_gxf_components.py --config ${CONFIG_FILE} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + set_tests_properties(EXAMPLE_PYTHON_IMPORT_GXF_COMPONENTS_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "Reach end of file or playback count reaches to the limit. Stop ticking." + ) + +endif() diff --git a/examples/import_gxf_components/python/import_gxf_components.py b/examples/import_gxf_components/python/import_gxf_components.py new file mode 100644 index 00000000..5194777a --- /dev/null +++ b/examples/import_gxf_components/python/import_gxf_components.py @@ -0,0 +1,246 @@ +""" + SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" # noqa: E501 + +import os + +from holoscan.core import Application, ComponentSpec, Operator, OperatorSpec +from holoscan.operators import GXFCodeletOp, HolovizOp, VideoStreamReplayerOp +from holoscan.resources import GXFComponentResource + +try: + import cupy as cp + import cupyx.scipy.ndimage as ndi +except ImportError: + raise ImportError( + "CuPy must be installed to run this example. See " + "https://docs.cupy.dev/en/stable/install.html" + ) from None + +sample_data_path = os.environ.get("HOLOSCAN_INPUT_PATH", "../data") + + +class DeviceMemoryPool(GXFComponentResource): + """Wrap an existing GXF component for use from Holoscan. + + This is illustrated here using `BlockMemoryPool` as a concrete example, but in practice + applications would just import the existing ``BlockMemoryPool`` class from + ``holoscan.resources``. + ``GXFComponentResource`` would be used to wrap some GXF component not already available via + the ``holoscan.resources`` module. + """ + + def __init__(self, fragment, *args, **kwargs): + # Call the base class constructor with the gxf_typename as the second argument + super().__init__(fragment, "nvidia::gxf::BlockMemoryPool", *args, **kwargs) + + def setup(self, spec: ComponentSpec): + # Ensure the parent class setup() is called before any additional setup code. + super().setup(spec) + + # You can add any additional setup code here (if needed). + + def initialize(self): + # Unlike the C++ API, this initialize() method should not call the parent class's + # initialize() method. + # It is a callback method invoked from the underlying C++ layer. + + # You can call any additional initialization code here (if needed). + pass + + +class ToDeviceMemoryOp(GXFCodeletOp): + def __init__(self, fragment, *args, **kwargs): + # Call the base class constructor with the gxf_typename as the second argument + super().__init__(fragment, "nvidia::gxf::TensorCopier", *args, **kwargs) + + def setup(self, spec: OperatorSpec): + # Ensure the parent class setup() is called before any additional setup code. + super().setup(spec) + + # You can add any additional setup code here (if needed). + # You can update conditions of the input/output ports, update the connector types, etc. + # + # Example: + # - `spec.inputs["receiver"].condition(ConditionType.NONE)` + # to update the condition of the input port to 'NONE'. + # (assuming that the GXF Codelet has a Receiver component named 'receiver'.) + + def initialize(self): + # Unlike the C++ API, this initialize() method should not call the parent class's + # initialize() method. + # It is a callback method invoked from the underlying C++ layer. + + # You can call any additional initialization code here (if needed). + # + # Example: + # ```python + # from holoscan.core import py_object_to_arg + # ... + # # add an argument to the GXF Operator + # self.add_arg(py_object_to_arg(value, name="arg_name")) + # ``` + pass + + +# Define custom Operators for use in the demo +class ImageProcessingOp(Operator): + """Example of an operator processing input video (as a tensor). + + This operator has: + inputs: "input_tensor" + outputs: "output_tensor" + + The data from each input is processed by a CuPy gaussian filter and + the result is sent to the output. + + In this demo, the input and output image (2D RGB) is a 3D array of shape + (height, width, channels). + """ + + def __init__(self, fragment, *args, **kwargs): + self.count = 1 + + # Need to call the base class constructor last + super().__init__(fragment, *args, **kwargs) + + def setup(self, spec: OperatorSpec): + spec.input("input_tensor") + spec.output("output_tensor") + spec.param("sigma") + + def compute(self, op_input, op_output, context): + # in_message is a dict of tensors + in_message = op_input.receive("input_tensor") + + # smooth along first two axes, but not the color channels + sigma = (self.sigma, self.sigma, 0) + + # out_message will be a dict of tensors + out_message = dict() + + for key, value in in_message.items(): + print(f"message received (count: {self.count})") + self.count += 1 + + cp_array = cp.asarray(value) + + # process cp_array + cp_array = ndi.gaussian_filter(cp_array, sigma) + + out_message[key] = cp_array + + op_output.emit(out_message, "output_tensor") + + +# Now define a simple application using the operators defined above +class MyVideoProcessingApp(Application): + """An application that demonstrates video processing using custom and built-in operators. + + This application integrates multiple operators and resources to create a video processing + pipeline: + + - GXFComponentResource: Creates resources from GXF Components + (e.g., nvidia::gxf::BlockMemoryPool). + - DeviceMemoryPool: Derived from GXFComponentResource, manages device memory. + - GXFCodeletOp: Creates operators from GXF Codelets (e.g., nvidia::gxf::TensorCopier). + - ToDeviceMemoryOp: Derived from GXFCodeletOp, handles tensor copying to device memory. + - VideoStreamReplayerOp: Reads video files and outputs video frames. + - ImageProcessingOp: Applies a Gaussian filter to video frames. + - HolovizOp: Visualizes the processed video frames. + + Workflow: + 1. VideoStreamReplayerOp reads a video file and outputs the frames. + 2. TensorCopier (to_system_memory) copies the frames to system memory. + 3. TensorCopier (to_device_memory) copies the frames to device memory. + 4. ImageProcessingOp applies a Gaussian filter to the frames. + 5. HolovizOp displays the processed frames. + """ + + def compose(self): + width = 960 + height = 540 + video_dir = os.path.join(sample_data_path, "racerx") + if not os.path.exists(video_dir): + raise ValueError(f"Could not find video data: {video_dir=}") + source = VideoStreamReplayerOp( + self, + name="replayer", + directory=video_dir, + **self.kwargs("replayer"), + ) + + # For both GXFComponentResource and GXFCodeletOp, the gxf_typename is positional. + system_memory_pool = GXFComponentResource( + self, + "nvidia::gxf::BlockMemoryPool", + name="system_memory_pool", + storage_type=2, + block_size=width * height * 3, + num_blocks=2, + ) + device_memory_pool = DeviceMemoryPool( + self, + name="device_memory_pool", + storage_type=1, + block_size=width * height * 3, + num_blocks=2, + ) + + to_system_memory = GXFCodeletOp( + self, + gxf_typename="nvidia::gxf::TensorCopier", + name="to_system_memory", + allocator=system_memory_pool, + mode=2, # 2 is for copying tensor to system memory + ) + + to_device_memory = ToDeviceMemoryOp( + self, + allocator=device_memory_pool, + mode=0, # 0 is for copying tensor to device memory + name="to_device_memory", + ) + + image_processing = ImageProcessingOp( + self, name="image_processing", **self.kwargs("image_processing") + ) + + visualizer = HolovizOp( + self, + name="holoviz", + width=width, + height=height, + **self.kwargs("holoviz"), + ) + + self.add_flow(source, to_system_memory) + self.add_flow(to_system_memory, to_device_memory) + self.add_flow(to_device_memory, image_processing) + self.add_flow(image_processing, visualizer, {("", "receivers")}) + + +def main(config_file): + app = MyVideoProcessingApp() + # if the --config command line argument was provided, it will override this config_file + app.config(config_file) + app.run() + + +if __name__ == "__main__": + config_file = os.path.join(os.path.dirname(__file__), "import_gxf_components.yaml") + + main(config_file=config_file) diff --git a/examples/import_gxf_components/python/import_gxf_components.yaml b/examples/import_gxf_components/python/import_gxf_components.yaml new file mode 100644 index 00000000..82b3f665 --- /dev/null +++ b/examples/import_gxf_components/python/import_gxf_components.yaml @@ -0,0 +1,33 @@ +%YAML 1.2 +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +replayer: + # directory: "../data/racerx" + basename: "racerx" + frame_rate: 0 # as specified in timestamps + repeat: true # default: false + realtime: true # default: true + count: 0 # default: 0 (no frame count restriction) + +image_processing: + sigma: 5.0 + +holoviz: + tensors: + - name: "" + type: color + opacity: 1.0 + priority: 0 diff --git a/examples/multi_branch_pipeline/CMakeLists.txt b/examples/multi_branch_pipeline/CMakeLists.txt new file mode 100644 index 00000000..91b8d28e --- /dev/null +++ b/examples/multi_branch_pipeline/CMakeLists.txt @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_subdirectory(cpp) +add_subdirectory(python) + +file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) + +install( + FILES README.md + DESTINATION "${app_relative_dest_path}" + COMPONENT "holoscan-examples" +) diff --git a/examples/multi_branch_pipeline/README.md b/examples/multi_branch_pipeline/README.md new file mode 100644 index 00000000..ef0557b4 --- /dev/null +++ b/examples/multi_branch_pipeline/README.md @@ -0,0 +1,66 @@ +# Create an application with multiple processing branches that compute at different rates + +These examples demonstrate how to build an application configured such that a common source operator can act as a source to multiple processing pipelines that execute at different rates. By default, operators have a condition on each input port that requires a message to be available before the `compute` method will be called. Similarly, the default condition on an output port is that there is space any connected operators' input ports to receive a message. This defaults make it relatively easy to connect operators and have things execute in the expected order, but there are some scenarios where we may want to override the defaults. For example, consider the geometry of the application used in this example: + +``` + increment1--rx1 + / + tx + \ + increment2--rx2 +``` + +For the example above, with the default conditions on operator `tx`, it would not execute until both the `increment1` and `increment2` operators are ready to receive a message. This may be okay if both branches should execute at the same rate, but does not support a scenario where one branch runs at a faster rate than another. + +This example shows how the default condition on `tx` can be disabled so that it sends a message regardless of whether there is space in each downstream operator's receiver queue. This also necessitates changing the receiver queue policy on `increment1` and `increment2` so that they can reject the incoming message if the queue is already full. + +*Visit the [Schedulers section of the SDK User Guide](https://docs.nvidia.com/holoscan/sdk-user-guide/components/schedulers.html) to learn more about the schedulers.* + +*See the operator creation guides section on input and output ports ([C++](https://docs.nvidia.com/holoscan/sdk-user-guide/holoscan_create_operator.html#specifying-operator-inputs-and-outputs-c) or [Python](https://docs.nvidia.com/holoscan/sdk-user-guide/holoscan_create_operator.html#specifying-operator-inputs-and-outputs-python)) for how to configure the condition on a port.* + + +## C++ API + +This example shows a simple application using only native operators. There are three types of operators involved (see diagram above): + 1. a transmitter (`tx`), that transmits an integer value on port "out". + 2. increment operators (`increment1` and `increment2`) that increment the received value by a given amount and then transmits that new value + 3. receivers (`rx1` and `rx2`) that print their name and received value + +The user can select the scheduler to be used by the application, but because there is more than one parallel path, it is recommended to use one of the multi-threaded schedulers in this scenario. The number of workers is controlled by the `worker_thread_number` parameter in `multi_branch_pipeline.yaml`. + +The key point of this application is not in the details of the operators involved, but in how their connections are configured so that different branches of the pipeline can execute at different rates. See inline comments in the application code explaining how the output port of `tx` is configured and how the input port of `increment1` and `increment2` are configured. + +### Build instructions + +Built with the SDK, see instructions from the top level README. + +### Run instructions + +First, go in your `build` or `install` directory (automatically done by `./run launch`). + +Set values for `scheduler` and any corresponding parameters such as `worker_thread_number` in `multi_branch_pipeline.yaml`. + +Then, run: +```bash +./examples/multi_branch_pipeline/cpp/multi_branch_pipeline +``` + +For the C++ application, the scheduler to be used can be set via the `scheduler` entry in `multi_branch_pipeline.yaml`. It defaults to `event_based` (an event-based multithread scheduler), but can also be set to either `multi_thread` (polling-based) or `greedy` (single thread). + +## Python API + +- `multi_branch_pipeline.py`: This example is the same as described for the C++ application above. The primary difference is that instead of using a YAML file for the configuration variables, all values are set via the command line. Call the script below with the `--help` option to get a full description of the command line parameters. By default a polling-based multithread scheduler will be used, but if `--event-based` is specified, the event-based multithread scheduler will be used instead. + +### Build instructions + +Built with the SDK, see instructions from the top level README. + +### Run instructions + +First, go in your `build` or `install` directory (automatically done by `./run launch`). + +Then, run the app with the options of your choice. For example + +```bash +python3 ./examples/multi_branch_pipeline/python/multi_branch_pipeline.py --threads 5 +``` diff --git a/examples/multi_branch_pipeline/cpp/CMakeLists.min.txt b/examples/multi_branch_pipeline/cpp/CMakeLists.min.txt new file mode 100644 index 00000000..7e00277a --- /dev/null +++ b/examples/multi_branch_pipeline/cpp/CMakeLists.min.txt @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the \"License\"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an \"AS IS\" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.20) +project(holoscan_multi_branch_pipeline CXX) + +# Finds the package holoscan +find_package(holoscan REQUIRED) + +add_executable(multi_branch_pipeline + multi_branch_pipeline.cpp +) + +target_link_libraries(multi_branch_pipeline + PRIVATE + holoscan::core +) + +# Copy config file to the build tree +add_custom_target(multi_branch_pipeline_yaml + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/multi_branch_pipeline.yaml" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "multi_branch_pipeline.yaml" + BYPRODUCTS "multi_branch_pipeline.yaml" +) +add_dependencies(multi_branch_pipeline multi_branch_pipeline_yaml) + +# Testing +if(BUILD_TESTING) + add_test(NAME EXAMPLE_CPP_MULTI_BRANCH_OPERATOR_TEST + COMMAND multi_branch_pipeline + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + # rx2 should receive close to 100 message (values 0 - 99) + # rx1 will receive only some of these, but it is hard to know exactly which ones so just verify the first + set_tests_properties(EXAMPLE_CPP_MULTI_BRANCH_OPERATOR_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "receiver 'rx2' received value: 90" + PASS_REGULAR_EXPRESSION "receiver 'rx1' received value: 0") +endif() diff --git a/examples/multi_branch_pipeline/cpp/CMakeLists.txt b/examples/multi_branch_pipeline/cpp/CMakeLists.txt new file mode 100644 index 00000000..83dcb055 --- /dev/null +++ b/examples/multi_branch_pipeline/cpp/CMakeLists.txt @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Create examples +add_executable(multi_branch_pipeline + multi_branch_pipeline.cpp +) +target_link_libraries(multi_branch_pipeline + PRIVATE + holoscan::core +) + +# Copy config file to the build tree +add_custom_target(multi_branch_pipeline_yaml + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/multi_branch_pipeline.yaml" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "multi_branch_pipeline.yaml" + BYPRODUCTS "multi_branch_pipeline.yaml" +) +add_dependencies(multi_branch_pipeline multi_branch_pipeline_yaml) + +# Install examples + +# Set the install RPATH based on the location of the Holoscan SDK libraries +# The GXF extensions are loaded by the GXF libraries - no need to include here +file(RELATIVE_PATH install_lib_relative_path ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_SOURCE_DIR}/${HOLOSCAN_INSTALL_LIB_DIR}) +set_target_properties(multi_branch_pipeline PROPERTIES INSTALL_RPATH "\$ORIGIN/${install_lib_relative_path}") + +# Install following the relative folder path +file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) + +# Install the source +install(FILES multi_branch_pipeline.cpp + DESTINATION "${app_relative_dest_path}" + COMPONENT holoscan-examples +) + +# Install the minimal CMakeLists.txt file +install(FILES CMakeLists.min.txt + RENAME "CMakeLists.txt" + DESTINATION "${app_relative_dest_path}" + COMPONENT holoscan-examples +) + +# Install the compiled example +install(TARGETS multi_branch_pipeline + DESTINATION "${app_relative_dest_path}" + COMPONENT holoscan-examples +) + +# Install the configuration file +install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/multi_branch_pipeline.yaml" + DESTINATION "${app_relative_dest_path}" + COMPONENT holoscan-examples +) + +# Testing +if(HOLOSCAN_BUILD_TESTS) + add_test(NAME EXAMPLE_CPP_MULTI_BRANCH_OPERATOR_TEST + COMMAND multi_branch_pipeline + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + + # rx2 should receive close to 100 message (values 0 - 99) + # rx1 will receive only some of these, but it is hard to know exactly which ones so just verify the first + set_tests_properties(EXAMPLE_CPP_MULTI_BRANCH_OPERATOR_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "receiver 'rx2' received value: 90" + PASS_REGULAR_EXPRESSION "receiver 'rx1' received value: 0") +endif() diff --git a/examples/multi_branch_pipeline/cpp/multi_branch_pipeline.cpp b/examples/multi_branch_pipeline/cpp/multi_branch_pipeline.cpp new file mode 100644 index 00000000..7929d5f5 --- /dev/null +++ b/examples/multi_branch_pipeline/cpp/multi_branch_pipeline.cpp @@ -0,0 +1,210 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include +#include +#include + +#include "holoscan/holoscan.hpp" + +namespace holoscan::ops { + +class PingTxOp : public Operator { + public: + HOLOSCAN_OPERATOR_FORWARD_ARGS(PingTxOp) + + PingTxOp() = default; + + void setup(OperatorSpec& spec) override { + // Note: Setting ConditionType::kNone overrides the default of + // ConditionType::kDownstreamMessageAffordable. This means that the operator will be triggered + // regardless of whether any operators connected downstream have space in their queues. + spec.output("out").condition(ConditionType::kNone); + spec.param(initial_value_, + "initial_value", + "Initial value", + "Initial value to emit", + static_cast(0)); + spec.param(increment_, + "increment", + "Increment", + "Integer amount to increment the value by on each subsequent call", + static_cast(1)); + } + + void compute(InputContext&, OutputContext& op_output, ExecutionContext&) override { + auto value = initial_value_.get() + count_ * increment_.get(); + op_output.emit(value, "out"); + count_ += 1; + }; + + private: + Parameter initial_value_; + Parameter increment_; + int64_t count_ = 0; +}; + +class IncrementOp : public Operator { + public: + HOLOSCAN_OPERATOR_FORWARD_ARGS(IncrementOp) + + IncrementOp() = default; + + /* Setup the input and output ports with custom settings on the input port. + * + * Notes + * ===== + * For policy: + * - 0 = pop the oldest value in favor of the new one when the queue is full + * - 1 = reject the new value when the queue is full + * - 2 = fault if queue is full (default) + * + * For capacity: + * When capacity > 1, even once messages stop arriving, this entity will continue to + * call ``compute`` for each remaining item in the queue. + * + * The ``condition`` method call here is the same as the default setting, and is shown + * only for completeness. `min_size` = 1 means that this operator will not call compute + * unless there is at least one message in the queue. + */ + void setup(OperatorSpec& spec) override { + spec.input("in") + .connector(IOSpec::ConnectorType::kDoubleBuffer, + Arg("capacity", static_cast(1)), + Arg("policy", static_cast(1)) // 1 = reject + ) + .condition( // arguments to condition here are the same as the defaults + ConditionType::kMessageAvailable, + Arg("min_size", static_cast(1)), + Arg("front_stage_max_size", static_cast(1))); + spec.output("out"); + + spec.param(increment_, + "increment", + "Increment", + "Integer amount to increment the input value by", + static_cast(0)); + } + + void compute(InputContext& op_input, OutputContext& op_output, ExecutionContext&) override { + int64_t value = op_input.receive("in").value(); + // increment value by the specified increment + int64_t new_value = value + increment_.get(); + op_output.emit(new_value, "out"); + }; + + private: + Parameter increment_; +}; + +class PingRxOp : public Operator { + public: + HOLOSCAN_OPERATOR_FORWARD_ARGS(PingRxOp) + + PingRxOp() = default; + + void setup(OperatorSpec& spec) override { spec.input("in"); } + + void compute(InputContext& op_input, OutputContext&, ExecutionContext&) override { + auto value = op_input.receive("in").value(); + HOLOSCAN_LOG_INFO("receiver '{}' received value: {}", name(), value); + }; +}; + +} // namespace holoscan::ops + +/* This application has a single transmitter connected to two parallel branches + * + * The geometry of the application is as shown below: + * + * increment1--rx1 + * / + * tx + * \ + * increment2--rx2 + * + * The top branch is forced via a PeriodicCondition to run at a slower rate than + * the source. It is currently configured to discard any extra messages that arrive + * at increment1 before it is ready to execute again, but different behavior could be + * achieved via other settings to policy and/or queue sizes. + */ +class MultiRateApp : public holoscan::Application { + public: + void compose() override { + using namespace holoscan; + + // Configure the operators. Here we use CountCondition to terminate + // execution after a specific number of messages have been sent and a + // PeriodicCondition to control how often messages are sent. + int64_t source_rate_hz = 60; // messages sent per second + int64_t period_source_ns = 1'000'000'000 / source_rate_hz; // period in nanoseconds + auto tx = make_operator( + "tx", + make_condition("count", 100), + make_condition("tx-period", period_source_ns)); + + // first branch will have a periodic condition so it can't run faster than 5 Hz + int64_t branch1_hz = 5; + int64_t period_ns1 = 1'000'000'000 / branch1_hz; + auto increment1 = make_operator( + "increment1", make_condition("increment1-period", period_ns1)); + auto rx1 = make_operator("rx1"); + add_flow(tx, increment1); + add_flow(increment1, rx1); + + // second branch is the same, but no periodic condition so will tick on every received message + auto increment2 = make_operator("increment2"); + auto rx2 = make_operator("rx2"); + add_flow(tx, increment2); + add_flow(increment2, rx2); + } +}; + +int main(int argc, char** argv) { + auto app = holoscan::make_application(); + + // Get the configuration + auto config_path = std::filesystem::canonical(argv[0]).parent_path(); + config_path += "/multi_branch_pipeline.yaml"; + app->config(config_path); + + std::string scheduler = app->from_config("scheduler").as(); + if (scheduler == "multi_thread") { + // use MultiThreadScheduler instead of the default GreedyScheduler + app->scheduler(app->make_scheduler( + "multithread-scheduler", app->from_config("multi_thread_scheduler"))); + } else if (scheduler == "event_based") { + // use EventBasedScheduler instead of the default GreedyScheduler + app->scheduler(app->make_scheduler( + "event-based-scheduler", app->from_config("event_based_scheduler"))); + } else if (scheduler == "greedy") { + app->scheduler(app->make_scheduler( + "greedy-scheduler", app->from_config("greedy_scheduler"))); + } else if (scheduler != "default") { + throw std::runtime_error(fmt::format( + "unrecognized scheduler option '{}', should be one of {'multi_thread', 'event_based', " + "'greedy', 'default'}", + scheduler)); + } + + app->run(); + + return 0; +} diff --git a/examples/multi_branch_pipeline/cpp/multi_branch_pipeline.yaml b/examples/multi_branch_pipeline/cpp/multi_branch_pipeline.yaml new file mode 100644 index 00000000..497b5781 --- /dev/null +++ b/examples/multi_branch_pipeline/cpp/multi_branch_pipeline.yaml @@ -0,0 +1,31 @@ +%YAML 1.2 +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +scheduler: multi_thread # event_based, multi_thread or greedy + +greedy_scheduler: + stop_on_deadlock: true + stop_on_deadlock_timeout: 500 + +multi_thread_scheduler: + worker_thread_number: 5 + stop_on_deadlock: true + stop_on_deadlock_timeout: 500 + +event_based_scheduler: + worker_thread_number: 5 + stop_on_deadlock: true + stop_on_deadlock_timeout: 500 diff --git a/examples/multi_branch_pipeline/python/CMakeLists.txt b/examples/multi_branch_pipeline/python/CMakeLists.txt new file mode 100644 index 00000000..3534a68d --- /dev/null +++ b/examples/multi_branch_pipeline/python/CMakeLists.txt @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Get relative folder path for the app +file(RELATIVE_PATH app_relative_dest_path ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) + +# Copy native operator multi_branch_pipeline application +add_custom_target(python_multi_branch_pipeline ALL + COMMAND ${CMAKE_COMMAND} -E copy "${CMAKE_CURRENT_SOURCE_DIR}/multi_branch_pipeline.py" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "multi_branch_pipeline.py" + BYPRODUCTS "multi_branch_pipeline.py" +) + +# Install the app +install(FILES + "${CMAKE_CURRENT_SOURCE_DIR}/multi_branch_pipeline.py" + DESTINATION "${app_relative_dest_path}" + COMPONENT "holoscan-examples" +) + +# Testing +if(HOLOSCAN_BUILD_TESTS) + add_test(NAME EXAMPLE_PYTHON_MULTI_BRANCH_TEST + COMMAND python3 multi_branch_pipeline.py + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + # rx2 should receive close to 100 message (values 0 - 99) + # rx1 will receive only some of these, but it is hard to know exactly which ones so just verify the first + set_tests_properties(EXAMPLE_PYTHON_MULTI_BRANCH_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "receiver 'rx2' received value: 90" + PASS_REGULAR_EXPRESSION "receiver 'rx1' received value: 0") + + add_test(NAME EXAMPLE_PYTHON_MULTI_BRANCH_EVENT_BASED_TEST + COMMAND python3 multi_branch_pipeline.py --event_based + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + # rx2 should receive close to 100 message (values 0 - 99) + # rx1 will receive only some of these, but it is hard to know exactly which ones so just verify the first + set_tests_properties(EXAMPLE_PYTHON_MULTI_BRANCH_TEST PROPERTIES + PASS_REGULAR_EXPRESSION "receiver 'rx2' received value: 90" + PASS_REGULAR_EXPRESSION "receiver 'rx1' received value: 0") +endif() diff --git a/examples/multi_branch_pipeline/python/multi_branch_pipeline.py b/examples/multi_branch_pipeline/python/multi_branch_pipeline.py new file mode 100644 index 00000000..7e6d3a18 --- /dev/null +++ b/examples/multi_branch_pipeline/python/multi_branch_pipeline.py @@ -0,0 +1,227 @@ +""" + SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" # noqa: E501 + +import time +from argparse import ArgumentParser + +from holoscan.conditions import CountCondition, PeriodicCondition +from holoscan.core import Application, ConditionType, IOSpec, Operator, OperatorSpec +from holoscan.schedulers import EventBasedScheduler, GreedyScheduler, MultiThreadScheduler + + +class PingTxOp(Operator): + """Simple transmitter operator. + + This operator has: + outputs: "out" + + On each tick, it transmits an integer on the "out" port. The value on the first call to + compute is equal to `start` and it increments by `increment` on each subsequent call. + """ + + def __init__(self, fragment, *args, initial_value=0, increment=0, **kwargs): + self.count = 0 + self.initial_value = initial_value + self.increment = increment + + # Need to call the base class constructor last + super().__init__(fragment, *args, **kwargs) + + def setup(self, spec: OperatorSpec): + # Note: Setting ConditionType.NONE overrides the default of + # ConditionType.DOWNSTREAM_MESSAGE_AFFORDABLE. This means that the operator will be + # triggered regardless of whether any operators connected downstream have space in their + # queues. + spec.output("out").condition(ConditionType.NONE) + + def compute(self, op_input, op_output, context): + value = self.initial_value + self.count * self.increment + op_output.emit(value, "out") + self.count += 1 + + +class IncrementOp(Operator): + """Add a fixed value to the input and transmit the result.""" + + def __init__(self, fragment, *args, increment=0, **kwargs): + self.increment = increment + + # Need to call the base class constructor last + super().__init__(fragment, *args, **kwargs) + + def setup(self, spec: OperatorSpec): + """Setup the input and output ports with custom settings on the input port. + + Notes + ===== + For policy: + + - 0 = pop the oldest value in favor of the new one when the queue is full + - 1 = reject the new value when the queue is full + - 2 = fault if queue is full (default) + + For capacity: + When capacity > 1, even once messages stop arriving, this entity will continue to + call ``compute`` for each remaining item in the queue. + + The ``condition`` method call here is the same as the default setting, and is shown + only for completeness. `min_size` = 1 means that this operator will not call compute + unless there is at least one message in the queue. + """ + spec.input("in").connector( + IOSpec.ConnectorType.DOUBLE_BUFFER, + capacity=1, + policy=1, # 1 = reject + ).condition(ConditionType.MESSAGE_AVAILABLE, min_size=1, front_stage_max_size=1) + + spec.output("out") + + def compute(self, op_input, op_output, context): + value = op_input.receive("in") + new_value = value + self.increment + op_output.emit(new_value, "out") + + +class PingRxOp(Operator): + """Simple (multi)-receiver operator. + + This is an example of a native operator with one input port. + On each tick, it receives an integer from the "in" port. + """ + + def __init__(self, fragment, *args, **kwargs): + # Need to call the base class constructor last + super().__init__(fragment, *args, **kwargs) + + def setup(self, spec: OperatorSpec): + spec.input("in") + + def compute(self, op_input, op_output, context): + # In this case, nothing will be printed until all messages have + # been received. + value = op_input.receive("in") + print(f"receiver '{self.name}' received value: {value}") + + +# Now define a simple application using the operators defined above +class MultiRateApp(Application): + """This application has a single transmitter connected to two parallel branches + + The geometry of the application is as shown below: + + increment1--rx1 + / + tx + \ + increment2--rx2 + + The top branch is forced via a PeriodicCondition to run at a slower rate than + the source. It is currently configured to discard any extra messages that arrive + at increment1 before it is ready to execute again, but different behavior could be + achieved via other settings to policy and/or queue sizes. + + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def compose(self): + # Configure the operators. Here we use CountCondition to terminate + # execution after a specific number of messages have been sent and a + # PeriodicCondition to control how often messages are sent. + source_rate_hz = 60 # messages sent per second + period_source_ns = int(1e9 / source_rate_hz) # period in nanoseconds + tx = PingTxOp( + self, + CountCondition(self, 100), + PeriodicCondition(self, recess_period=period_source_ns), + start=1, + increment=1, + name="tx", + ) + + # first branch will have a periodic condition so it can't run faster than 5 Hz + branch1_hz = 5 + period_ns1 = int(1e9 / branch1_hz) + increment1 = IncrementOp( + self, + PeriodicCondition(self, recess_period=period_ns1), + name="increment1", + ) + rx1 = PingRxOp(self, name="rx1") + self.add_flow(tx, increment1) + self.add_flow(increment1, rx1) + + # second branch does not have a periodic condition so will tick on every message sent by tx + increment2 = IncrementOp( + self, + name="increment2", + ) + rx2 = PingRxOp(self, name="rx2") + self.add_flow(tx, increment2) + self.add_flow(increment2, rx2) + + +def main(threads, event_based): + app = MultiRateApp() + if threads == 0: + # Explicitly setting GreedyScheduler is not strictly required as it is the default. + scheduler = GreedyScheduler(app, name="greedy_scheduler") + else: + scheduler_class = EventBasedScheduler if event_based else MultiThreadScheduler + scheduler = scheduler_class( + app, + worker_thread_number=threads, + stop_on_deadlock=True, + stop_on_deadlock_timeout=500, + name="multithread_scheduler", + ) + app.scheduler(scheduler) + tstart = time.time() + app.run() + duration = time.time() - tstart + print(f"Total app runtime = {duration:0.3f} s") + + +if __name__ == "__main__": + # Parse args + parser = ArgumentParser(description="Multi-rate pipeline example") + parser.add_argument( + "-t", + "--threads", + type=int, + default=5, + help=( + "The number of threads to use for multi-threaded schedulers. Set this to 0 to use " + "the default greedy scheduler instead. To use the event-based scheduler instead of " + "the default multi-thread scheduler, please specify --event_based." + ), + ) + parser.add_argument( + "--event_based", + action="store_true", + help=( + "Sets the application to use the event-based scheduler instead of the default " + "multi-thread scheduler when threads > 0." + ), + ) + + args = parser.parse_args() + if args.threads < 0: + raise ValueError("threads must be non-negative") + + main(threads=args.threads, event_based=args.event_based) diff --git a/examples/multithread/README.md b/examples/multithread/README.md index 88bd252d..364a3d98 100644 --- a/examples/multithread/README.md +++ b/examples/multithread/README.md @@ -15,8 +15,7 @@ The user can configure the number of delay operators via the `num_delay_op` para The number of workers used by the multi-threaded scheduler is controlled by the `worker_thread_number` parameter in `app_config.yaml`. -Data Flow Tracking is also optionally enabled by changing the `tracking` field in the YAML file. -It is set to `false` by default. +Data Flow Tracking is also optionally enabled by changing the `tracking` field in the YAML file. It is set to `false` by default. Other options in the YAML include a `silent` option to suppress verbose output from the operators (`false` by default) and a `count` option that can be used to control how many messages are sent from the transmitter (1 by default). ### Build instructions diff --git a/examples/multithread/cpp/multithread.cpp b/examples/multithread/cpp/multithread.cpp index 3abae8c9..802f89fa 100644 --- a/examples/multithread/cpp/multithread.cpp +++ b/examples/multithread/cpp/multithread.cpp @@ -35,7 +35,7 @@ class PingTxOp : public Operator { void setup(OperatorSpec& spec) override { spec.output>("out"); } void compute(InputContext&, OutputContext& op_output, ExecutionContext&) override { - auto value = std::make_shared(0); + int value = 0; op_output.emit(value, "out"); }; }; @@ -47,30 +47,33 @@ class DelayOp : public Operator { DelayOp() = default; void setup(OperatorSpec& spec) override { - spec.input>("in"); - spec.output>("out_val"); + spec.input("in"); + spec.output("out_val"); spec.output("out_name"); spec.param( delay_, "delay", "Delay", "Amount of delay before incrementing the input value", 0.5); spec.param( increment_, "increment", "Increment", "Integer amount to increment the input value by", 1); + spec.param(silent_, "silent", "Silent mode?", "Whether to log info on receive", false); } void compute(InputContext& op_input, OutputContext& op_output, ExecutionContext&) override { - auto value = op_input.receive>("in").value(); - - HOLOSCAN_LOG_INFO("{}: now waiting {} s", name(), delay_.get()); - - // sleep for the specified time (rounded down to the nearest microsecond) - int delay_us = static_cast(delay_ * 1000000); - usleep(delay_us); - HOLOSCAN_LOG_INFO("{}: finished waiting", name()); + auto value = op_input.receive("in").value(); // increment value by the specified increment - auto new_value = std::make_shared(*value + increment_.get()); - auto nm = std::string(name()); - - HOLOSCAN_LOG_INFO("{}: sending new value ({})", name(), *new_value); + int new_value = value + increment_.get(); + auto nm = std::string(name_); + + double delay = delay_.get(); + bool silent = silent_.get(); + if (delay > 0) { + if (!silent) { HOLOSCAN_LOG_INFO("{}: now waiting {} s", name(), delay); } + // sleep for the specified time (rounded down to the nearest microsecond) + int delay_us = static_cast(delay * 1000000); + usleep(delay_us); + if (!silent) { HOLOSCAN_LOG_INFO("{}: finished waiting", name()); } + } + if (!silent) { HOLOSCAN_LOG_INFO("{}: sending new value ({})", name(), new_value); } op_output.emit(new_value, "out_val"); op_output.emit(nm, "out_name"); }; @@ -78,13 +81,14 @@ class DelayOp : public Operator { private: Parameter delay_; Parameter increment_; + Parameter silent_; }; class PingRxOp : public Operator { public: HOLOSCAN_OPERATOR_FORWARD_ARGS(PingRxOp) - PingRxOp() = default; + explicit PingRxOp(bool silent) : silent_(silent) {} void setup(OperatorSpec& spec) override { spec.param( @@ -94,19 +98,23 @@ class PingRxOp : public Operator { } void compute(InputContext& op_input, OutputContext&, ExecutionContext&) override { - auto value_vector = op_input.receive>>("values").value(); - auto name_vector = op_input.receive>("names").value(); - - HOLOSCAN_LOG_INFO("number of received names: {}", name_vector.size()); - HOLOSCAN_LOG_INFO("number of received values: {}", value_vector.size()); + std::vector value_vector; + std::vector name_vector; + value_vector = op_input.receive>("values").value(); + name_vector = op_input.receive>("names").value(); + if (!silent_) { + HOLOSCAN_LOG_INFO("number of received names: {}", name_vector.size()); + HOLOSCAN_LOG_INFO("number of received values: {}", value_vector.size()); + } int total = 0; - for (auto vp : value_vector) { total += *vp; } - HOLOSCAN_LOG_INFO("sum of received values: {}", total); + for (auto vp : value_vector) { total += vp; } + if (!silent_) { HOLOSCAN_LOG_INFO("sum of received values: {}", total); } }; private: Parameter> names_; Parameter> values_; + bool silent_ = false; }; } // namespace holoscan::ops @@ -114,18 +122,22 @@ class PingRxOp : public Operator { class App : public holoscan::Application { public: void set_num_delays(int num_delays) { num_delays_ = num_delays; } + void set_count(int64_t count) { count_ = count; } void set_delay(double delay) { delay_ = delay; } void set_delay_step(double delay_step) { delay_step_ = delay_step; } + void set_silent(bool silent) { silent_ = silent; } void compose() override { using namespace holoscan; - auto tx = make_operator("tx", make_condition(1)); - auto rx = make_operator("rx"); + auto tx = make_operator("tx", make_condition(count_)); + auto rx = make_operator("rx", silent_); for (int i = 0; i < num_delays_; ++i) { std::string delay_name = fmt::format("mx{}", i); - auto del_op = make_operator( - delay_name, Arg{"delay", delay_ + delay_step_ * i}, Arg{"increment", i}); + auto del_op = make_operator(delay_name, + Arg{"delay", delay_ + delay_step_ * i}, + Arg{"increment", i}, + Arg{"silent", silent_}); add_flow(tx, del_op); add_flow(del_op, rx, {{"out_val", "values"}, {"out_name", "names"}}); } @@ -133,8 +145,10 @@ class App : public holoscan::Application { private: int num_delays_ = 32; + int64_t count_ = 1; double delay_ = 0.2; double delay_step_ = 0.05; + bool silent_ = false; }; int main(int argc, char** argv) { @@ -154,9 +168,13 @@ int main(int argc, char** argv) { int num_delay_ops = app->from_config("num_delay_ops").as(); double delay = app->from_config("delay").as(); double delay_step = app->from_config("delay_step").as(); + int count = app->from_config("count").as(); + bool silent = app->from_config("silent").as(); app->set_num_delays(num_delay_ops); app->set_delay(delay); app->set_delay_step(delay_step); + app->set_count(count); + app->set_silent(silent); std::string scheduler = app->from_config("scheduler").as(); if (scheduler == "multi_thread") { @@ -172,8 +190,8 @@ int main(int argc, char** argv) { "greedy-scheduler", app->from_config("greedy_scheduler"))); } else if (scheduler != "default") { throw std::runtime_error(fmt::format( - "unrecognized scheduler option '{}', should be one of {'multi_thread', 'event_based', " - "'greedy', 'default'}", + "unrecognized scheduler option '{}', should be one of ('multi_thread', 'event_based', " + "'greedy', 'default')", scheduler)); } diff --git a/examples/multithread/cpp/multithread.yaml b/examples/multithread/cpp/multithread.yaml index 76c6472c..a76e6c82 100644 --- a/examples/multithread/cpp/multithread.yaml +++ b/examples/multithread/cpp/multithread.yaml @@ -14,13 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. --- -extensions: - - libgxf_std.so scheduler: event_based # event_based, multi_thread or greedy num_delay_ops: 32 delay: 0.1 delay_step: 0.01 +count: 1 +silent: false +tracking: false greedy_scheduler: stop_on_deadlock: true @@ -35,5 +36,3 @@ event_based_scheduler: worker_thread_number: 8 stop_on_deadlock: true stop_on_deadlock_timeout: 500 - -tracking: false diff --git a/examples/multithread/python/CMakeLists.txt b/examples/multithread/python/CMakeLists.txt index 1d18fc5d..92a62290 100644 --- a/examples/multithread/python/CMakeLists.txt +++ b/examples/multithread/python/CMakeLists.txt @@ -39,7 +39,7 @@ if(HOLOSCAN_BUILD_TESTS) set_tests_properties(EXAMPLE_PYTHON_MULTITHREAD_TEST PROPERTIES PASS_REGULAR_EXPRESSION "sum of received values: 496") - add_test(NAME EXAMPLE_PYTHON_EVENT_BASED_TEST + add_test(NAME EXAMPLE_PYTHON_MULTITHREAD_EVENT_BASED_TEST COMMAND python3 multithread.py --event_based WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) diff --git a/examples/multithread/python/multithread.py b/examples/multithread/python/multithread.py index 4f6aaaf8..15fe49fa 100644 --- a/examples/multithread/python/multithread.py +++ b/examples/multithread/python/multithread.py @@ -20,7 +20,7 @@ from argparse import ArgumentParser from holoscan.conditions import CountCondition -from holoscan.core import Application, Operator, OperatorSpec +from holoscan.core import Application, Operator, OperatorSpec, Tracker from holoscan.schedulers import EventBasedScheduler, GreedyScheduler, MultiThreadScheduler @@ -48,9 +48,10 @@ class DelayOp(Operator): value by a user-specified integer increment. """ - def __init__(self, fragment, *args, delay=0.25, increment=1, **kwargs): + def __init__(self, fragment, *args, delay=0.25, increment=1, silent=False, **kwargs): self.delay = delay self.increment = increment + self.silent = silent # Need to call the base class constructor last super().__init__(fragment, *args, **kwargs) @@ -61,12 +62,17 @@ def setup(self, spec: OperatorSpec): spec.output("out_val") def compute(self, op_input, op_output, context): - print(f"{self.name}: now waiting {self.delay:0.3f} s") - time.sleep(self.delay) - print(f"{self.name}: finished waiting") new_value = op_input.receive("in") + self.increment + + if self.delay > 0: + if not self.silent: + print(f"{self.name}: now waiting {self.delay:0.3f} s") + time.sleep(self.delay) + if not self.silent: + print(f"{self.name}: finished waiting") op_output.emit(self.name, "out_name") - print(f"{self.name}: sending new value ({new_value})") + if not self.silent: + print(f"{self.name}: sending new value ({new_value})") op_output.emit(new_value, "out_val") @@ -77,7 +83,9 @@ class PingRxOp(Operator): number of inputs connected to is "receivers" port. """ - def __init__(self, fragment, *args, **kwargs): + def __init__(self, fragment, *args, silent=False, **kwargs): + self.silent = silent + # Need to call the base class constructor last super().__init__(fragment, *args, **kwargs) @@ -90,37 +98,50 @@ def compute(self, op_input, op_output, context): # been received. names = op_input.receive("names") values = op_input.receive("values") - print(f"number of received names: {len(names)}") - print(f"number of received values: {len(values)}") - print(f"sum of received values: {sum(values)}") + if not self.silent: + print(f"number of received names: {len(names)}") + print(f"number of received values: {len(values)}") + print(f"sum of received values: {sum(values)}") # Now define a simple application using the operators defined above class ParallelPingApp(Application): - def __init__(self, *args, num_delays=8, delay=0.5, delay_step=0.1, **kwargs): + def __init__( + self, *args, num_delays=8, delay=0.5, delay_step=0.1, count=1, silent=False, **kwargs + ): self.num_delays = num_delays self.delay = delay self.delay_step = delay_step + self.silent = silent + self.count = count super().__init__(*args, **kwargs) def compose(self): # Configure the operators. Here we use CountCondition to terminate # execution after a specific number of messages have been sent. - tx = PingTxOp(self, CountCondition(self, 1), name="tx") + tx = PingTxOp(self, CountCondition(self, self.count), name="tx") delay_ops = [ - DelayOp(self, delay=self.delay + self.delay_step * n, increment=n, name=f"delay{n:02d}") + DelayOp( + self, + delay=self.delay + self.delay_step * n, + increment=n, + silent=self.silent, + name=f"delay{n:02d}", + ) for n in range(self.num_delays) ] - rx = PingRxOp(self, name="rx") + rx = PingRxOp(self, silent=self.silent, name="rx") for d in delay_ops: self.add_flow(tx, d) self.add_flow(d, rx, {("out_val", "values"), ("out_name", "names")}) -def main(threads, num_delays, delay, delay_step, event_based): - app = ParallelPingApp(num_delays=num_delays, delay=delay, delay_step=delay_step) +def main(threads, num_delays, delay, delay_step, event_based, count, silent, track): + app = ParallelPingApp( + num_delays=num_delays, delay=delay, delay_step=delay_step, count=count, silent=silent + ) if threads == 0: # Explicitly setting GreedyScheduler is not strictly required as it is the default. scheduler = GreedyScheduler(app, name="greedy_scheduler") @@ -135,7 +156,15 @@ def main(threads, num_delays, delay, delay_step, event_based): ) app.scheduler(scheduler) tstart = time.time() - app.run() + if track: + with Tracker( + app, filename="logger.log", num_start_messages_to_skip=2, num_last_messages_to_discard=2 + ) as tracker: + app.run() + tracker.print() + else: + app.run() + duration = time.time() - tstart print(f"Total app runtime = {duration:0.3f} s") @@ -191,6 +220,23 @@ def main(threads, num_delays, delay, delay_step, event_based): "multi-thread scheduler when threads > 0." ), ) + parser.add_argument( + "-c", + "--count", + type=int, + default=1, + help="The number of messages to transmit.", + ) + parser.add_argument( + "--silent", + action="store_true", + help="Disable info logging during operator compute.", + ) + parser.add_argument( + "--track", + action="store_true", + help="enable data flow tracking", + ) args = parser.parse_args() if args.delay < 0: @@ -201,6 +247,8 @@ def main(threads, num_delays, delay, delay_step, event_based): raise ValueError("num_delay_ops must be >= 1") if args.threads < -1: raise ValueError("threads must be non-negative or -1 (for all threads)") + if args.count < 1: + raise ValueError("count must be a positive integer") elif args.threads == -1: # use up to maximum number of available threads args.threads = min(args.num_delay_ops, multiprocessing.cpu_count()) @@ -211,4 +259,7 @@ def main(threads, num_delays, delay, delay_step, event_based): delay=args.delay, delay_step=args.delay_step, event_based=args.event_based, + count=args.count, + silent=args.silent, + track=args.track, ) diff --git a/examples/tensor_interop/README.md b/examples/tensor_interop/README.md index a6a707e7..9fcf22d5 100644 --- a/examples/tensor_interop/README.md +++ b/examples/tensor_interop/README.md @@ -12,6 +12,8 @@ This application demonstrates interoperability between a native operator (`Proce Notably, the two GXF codelets have not been wrapped as Holoscan operators, but are instead registered at runtime in the `compose` method of the application. +Note that this C++ example shows how to explicitly wrap GXF codelets (`SendTensor` and `ReceiveTensor`) for use from Holoscan by creating a class that inherits from `holoscan::ops::GXFOperator`. More recently there is a `holoscan::ops::GXFCodelet` class which simplifies this process and allows the codelets to directly be used via `make_operator`. This second approach to using an existing GXF codelet is demonstrated in `examples/import_gxf_components`. + ### Run instructions * **using deb package install or NGC container**: @@ -50,12 +52,11 @@ The following dataset is used by this example: export HOLOSCAN_INPUT_PATH= # [Prerequisite] Download example .py and .yaml file below to `APP_DIR` # [Optional] Start the virtualenv where holoscan is installed - python3 -m pip install cupy-cuda12x python3 /tensor_interop.py ``` * **using deb package install**: ```bash - /opt/nvidia/holoscan/examples/download_example_data + sudo /opt/nvidia/holoscan/examples/download_example_data export HOLOSCAN_INPUT_PATH=/opt/nvidia/holoscan/data python3 -m pip install cupy-cuda12x export PYTHONPATH=/opt/nvidia/holoscan/python/lib diff --git a/examples/tensor_interop/cpp/tensor_interop.cpp b/examples/tensor_interop/cpp/tensor_interop.cpp index 7bddc9af..150446b0 100644 --- a/examples/tensor_interop/cpp/tensor_interop.cpp +++ b/examples/tensor_interop/cpp/tensor_interop.cpp @@ -87,7 +87,7 @@ class ProcessTensorOp : public Operator { ExecutionContext& context) override { // The type of `in_message` is 'holoscan::TensorMap'. auto in_message = op_input.receive("in").value(); - // the type of out_message is TensorMap + // The type of out_message is TensorMap TensorMap out_message; for (auto& [key, tensor] : in_message) { // Process with 'tensor' here. diff --git a/examples/tensor_interop/python/tensor_interop.py b/examples/tensor_interop/python/tensor_interop.py index c2309f63..0a6b666b 100644 --- a/examples/tensor_interop/python/tensor_interop.py +++ b/examples/tensor_interop/python/tensor_interop.py @@ -59,13 +59,13 @@ def setup(self, spec: OperatorSpec): spec.param("sigma") def compute(self, op_input, op_output, context): - # in_message is of dict + # in_message is a dict of tensors in_message = op_input.receive("input_tensor") # smooth along first two axes, but not the color channels sigma = (self.sigma, self.sigma, 0) - # out_message is of dict + # out_message will be a dict of tensors out_message = dict() for key, value in in_message.items(): diff --git a/examples/v4l2_camera/README.md b/examples/v4l2_camera/README.md index d5d04f54..2468de5d 100644 --- a/examples/v4l2_camera/README.md +++ b/examples/v4l2_camera/README.md @@ -5,7 +5,7 @@ This app captures video streams using [Video4Linux](https://www.kernel.org/doc/h #### Notes on the V4L2 operator * The V4L2 operator can read a range of pixel formats, though it will always output RGBA32 at this time. -* If pixel format is not specified in the yaml configuration file, it will be automatically selected if `AB24` or `YUYV` is supported by the device. For other formats, you will need to specify the `pixel_format` parameter in the yaml file which will then be used. However, note that the operator expects that this format can be encoded as RGBA32. If not, the behavior is undefined. +* If the pixel format is not specified in the YAML configuration file, it will automatically select either `AB24`, `YUYV`, or `MJPG` if supported by the device. The first supported format in the order provided will be used. For other formats, you will need to specify the `pixel_format` parameter in the yaml file which will then be used but note that the operator expects that these formats can be encoded as RGBA32. If not, the behavior is undefined. * The V4L2 operator outputs data on host. In order to move data from host to GPU device, use `holoscan::ops::FormatConverterOp`. ## Requirements @@ -46,7 +46,7 @@ There are a few parameters that can be specified: * Default: `"/dev/video0"` * List available options with `v4l2-ctl --list-devices` * `pixel_format`: The [V4L2 pixel format](https://docs.kernel.org/userspace-api/media/v4l/pixfmt-intro.html) of the device, as FourCC code - * Default: auto selects `AB24` or `YUYV` based on device support + * Default: auto selects `AB24`, `YUYV`, or `MJPG` based on device support * List available options with `v4l2-ctl -d /dev/ --list-formats` * `width` and `height`: The frame dimensions * Default: device default diff --git a/examples/video_replayer/README.md b/examples/video_replayer/README.md index 3becbb15..e933baab 100644 --- a/examples/video_replayer/README.md +++ b/examples/video_replayer/README.md @@ -17,7 +17,7 @@ The following dataset is used by this example: * **using deb package install**: ```bash - /opt/nvidia/holoscan/examples/download_example_data + sudo /opt/nvidia/holoscan/examples/download_example_data export HOLOSCAN_INPUT_PATH=/opt/nvidia/holoscan/data ./examples/video_replayer/cpp/video_replayer ``` @@ -48,7 +48,7 @@ The following dataset is used by this example: ``` * **using deb package install**: ```bash - /opt/nvidia/holoscan/examples/download_example_data + sudo /opt/nvidia/holoscan/examples/download_example_data export HOLOSCAN_INPUT_PATH=/opt/nvidia/holoscan/data export PYTHONPATH=/opt/nvidia/holoscan/python/lib python3 /opt/nvidia/holoscan/examples/video_replayer/python/video_replayer.py diff --git a/examples/video_replayer_distributed/README.md b/examples/video_replayer_distributed/README.md index 58fe2830..3f19ab25 100644 --- a/examples/video_replayer_distributed/README.md +++ b/examples/video_replayer_distributed/README.md @@ -22,7 +22,7 @@ Please refer to the [user guide](https://docs.nvidia.com/holoscan/sdk-user-guide * **using deb package install**: ```bash - /opt/nvidia/holoscan/examples/download_example_data + sudo /opt/nvidia/holoscan/examples/download_example_data export HOLOSCAN_INPUT_PATH=/opt/nvidia/holoscan/data # Set the application folder @@ -31,8 +31,6 @@ Please refer to the [user guide](https://docs.nvidia.com/holoscan/sdk-user-guide * **from NGC container**: ```bash - # HOLOSCAN_INPUT_PATH is set to /opt/nvidia/data by default - # Set the application folder APP_DIR=/opt/nvidia/holoscan/examples/video_replayer_distributed/cpp ``` @@ -83,7 +81,7 @@ Please refer to the [user guide](https://docs.nvidia.com/holoscan/sdk-user-guide ``` * **using deb package install**: ```bash - /opt/nvidia/holoscan/examples/download_example_data + sudo /opt/nvidia/holoscan/examples/download_example_data export HOLOSCAN_INPUT_PATH=/opt/nvidia/holoscan/data export PYTHONPATH=/opt/nvidia/holoscan/python/lib @@ -92,8 +90,6 @@ Please refer to the [user guide](https://docs.nvidia.com/holoscan/sdk-user-guide ``` * **from NGC container**: ```bash - # HOLOSCAN_INPUT_PATH is set to /opt/nvidia/data by default - # Set the application folder APP_DIR=/opt/nvidia/holoscan/examples/video_replayer_distributed/python ``` diff --git a/include/holoscan/core/analytics/csv_data_exporter.hpp b/include/holoscan/core/analytics/csv_data_exporter.hpp new file mode 100644 index 00000000..f520b734 --- /dev/null +++ b/include/holoscan/core/analytics/csv_data_exporter.hpp @@ -0,0 +1,127 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HOLOSCAN_CORE_ANALYTICS_CSVDATA_EXPORTER_HPP +#define HOLOSCAN_CORE_ANALYTICS_CSVDATA_EXPORTER_HPP + +#include +#include +#include + +#include "./data_exporter.hpp" + +namespace holoscan { + +// The default output file name for analytics data. +constexpr const char* kAnalyticsOutputFileName = "data.csv"; + +/** + * @brief A class to support exporting Holoscan application data in CSV format for Holoscan + * Federated Analytics. + * + * The directory will be created with the app name in the data root directory if it is not present + * already. Inside the application directory, a directory with the current timestamp will be + * created. + * + * The output file name can be specified using the environment variable + * `HOLOSCAN_ANALYTICS_DATA_FILE_NAME`. If not specified, the output file named `data.csv` will + * be created inside the timestamp directory. The column names are added to the output file as a + * first row. + * + * Using this class mainly involves two steps: + * - Create `CsvDataExporter` object specifying app name and columns. + * - Call `export_data()` method to add a single row to the output file. + * + * Example: + * + * ```cpp + * #include "holoscan/core/analytics/csv_data_exporter.hpp" + * + * void export_data() { + * const std::string app_name = "sample_app"; + * const std::vector columns = {"column1", "column2", "column3"}; + * CsvDataExporter data_exporter(app_name, columns); + * + * const std::vector data = {"1", "2", "3"}; + * data_exporter.export_data(data); + * ... + * } + * ``` + */ +class CsvDataExporter : public DataExporter { + public: + /** + * @brief The constructor creates required directories and CSV file with the specified names. + * + * @param app_name The application name. + * + * @param columns The column names list which will be added to the CSV file as a first row. + * + */ + CsvDataExporter(const std::string& app_name, const std::vector& columns); + + ~CsvDataExporter(); + + /** + * @brief Exports given data to a CSV file. + * + * Each call to the function will add one more row to the csv file. + * + * @param data The data to be written to the CSV file. + */ + void export_data(const std::vector& data) override; + + /** + * @brief Get the value of analytics output file name environment variable + * `HOLOSCAN_ANALYTICS_DATA_FILE_NAME`. + * + * @return A string if the environment variable is set else it returns + * error code. + */ + static expected get_analytics_data_file_name_env(); + + /** + * @brief Returns output file name. + * + */ + const std::string& output_file_name() const { return file_name_; } + + /** + * @brief Returns the column names. + * + */ + const std::vector& columns() const { return columns_; } + + private: + /** + * @brief Write one row to a CSV file. + * + * Each call to the function will just add one more row to the csv file. + * + * @param data The data to be written to the CSV file. + * The number of strings passed should be same as the number of columns in CSV file. + */ + void write_row(const std::vector& data); + + std::string file_name_; + std::vector columns_; + std::ofstream file_; +}; + +} // namespace holoscan + +#endif /* HOLOSCAN_CORE_ANALYTICS_CSVDATA_EXPORTER_HPP */ diff --git a/include/holoscan/core/analytics/data_exporter.hpp b/include/holoscan/core/analytics/data_exporter.hpp new file mode 100644 index 00000000..e8405133 --- /dev/null +++ b/include/holoscan/core/analytics/data_exporter.hpp @@ -0,0 +1,95 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HOLOSCAN_CORE_ANALYTICS_DATA_EXPORTER_HPP +#define HOLOSCAN_CORE_ANALYTICS_DATA_EXPORTER_HPP + +#include +#include + +#include "holoscan/core/errors.hpp" +#include "holoscan/core/expected.hpp" + +namespace holoscan { + +/** + * @brief A base class to support exporting Holoscan application data for + * Federated Analytics. + * + * This class will create a directory with the application name passed to the constructor. It will + * also create a subdirectory based on the current timestamp within the application directory. + * + * The root directory for the application data can be specified by using + * environment variable `HOLOSCAN_ANALYTICS_DATA_DIRECTORY`. If not specified, + * it will default to current application directory. + * + */ +class DataExporter { + public: + explicit DataExporter(const std::string& app_name); + virtual ~DataExporter() = default; + + /** + * @brief Get the value of analytics data directory environment variable + * `HOLOSCAN_ANALYTICS_DATA_DIRECTORY`. + * + * @return A string if the environment variable is set else it returns + * error code. + */ + static expected get_analytics_data_directory_env(); + + /** + * @brief A pure virtual function that needs to be implemented by subclasses + * to export the data in required format. + * + * @param Data The data to be written to the CSV file. + */ + virtual void export_data(const std::vector& data) = 0; + + /** + * @brief Return the application name. + * + */ + const std::string& app_name() const { return app_name_; } + + /** + * @brief Returns a data directory name. + * + */ + const std::string& data_directory() const { return directory_name_; } + + /** + * @brief Remove the data directory and its contents. + * + */ + void cleanup_data_directory(); + + protected: + std::string app_name_; + std::string directory_name_; + + private: + /** + * @brief Create a data directory with the current timestamp inside the + * application directory. + */ + void create_data_directory_with_timestamp(); +}; + +} // namespace holoscan + +#endif /* HOLOSCAN_CORE_ANALYTICS_DATA_EXPORTER_HPP */ diff --git a/include/holoscan/core/arg.hpp b/include/holoscan/core/arg.hpp index 3adbe602..ae3698eb 100644 --- a/include/holoscan/core/arg.hpp +++ b/include/holoscan/core/arg.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -34,7 +34,7 @@ #include #include "./type_traits.hpp" -#include "./common.hpp" +#include "holoscan/logger/logger.hpp" // #include "gxf/std/complex.hpp" // nvidia::gxf::complex64, complex128 @@ -314,6 +314,13 @@ class Arg { */ YAML::Node to_yaml_node() const; + /** + * @brief Get a YAML representation of the argument value. + * + * @ return YAML node including the value of the argument. + */ + YAML::Node value_to_yaml_node() const; + /** * @brief Get a description of the argument. * diff --git a/include/holoscan/core/codecs.hpp b/include/holoscan/core/codecs.hpp index 8e26ac88..5bccecd2 100644 --- a/include/holoscan/core/codecs.hpp +++ b/include/holoscan/core/codecs.hpp @@ -168,6 +168,26 @@ struct codec { } }; +// will hold Python cloudpickle strings in this container to differentiate from std::string +struct CloudPickleSerializedObject { + std::string serialized; +}; + +// codec for CloudPickleSerializedObject +template <> +struct codec { + static expected serialize(const CloudPickleSerializedObject& value, + Endpoint* endpoint) { + return serialize_binary_blob(value.serialized, endpoint); + } + static expected deserialize(Endpoint* endpoint) { + auto maybe_string = deserialize_binary_blob(endpoint); + if (!maybe_string) { return forward_error(maybe_string); } + CloudPickleSerializedObject cloudpickle_obj{std::move(maybe_string.value())}; + return cloudpickle_obj; + } +}; + ////////////////////////////////////////////////////////////////////////////////////////////////// // Codec type 4: serialization of std::vector only // diff --git a/include/holoscan/core/common.hpp b/include/holoscan/core/common.hpp index 2c9811e9..8b598aaf 100644 --- a/include/holoscan/core/common.hpp +++ b/include/holoscan/core/common.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,6 +21,13 @@ #include "./errors.hpp" #include "./expected.hpp" #include "./forward_def.hpp" +// clang-format off + +// Include parameter.hpp before logger.hpp for supporting holoscan::Parameter +// with fmt::format. +#include "./parameter.hpp" #include "holoscan/logger/logger.hpp" +// clang-format on + #endif /* HOLOSCAN_CORE_COMMON_HPP */ diff --git a/include/holoscan/core/component.hpp b/include/holoscan/core/component.hpp index 4e41e932..2ff26121 100644 --- a/include/holoscan/core/component.hpp +++ b/include/holoscan/core/component.hpp @@ -32,12 +32,13 @@ #include "./arg.hpp" #include "./forward_def.hpp" -#define HOLOSCAN_COMPONENT_FORWARD_TEMPLATE() \ - template > && \ - (std::is_same_v> || \ - std::is_same_v>)>> +#define HOLOSCAN_COMPONENT_FORWARD_TEMPLATE() \ + template > && \ + (std::is_same_v<::holoscan::Arg, std::decay_t> || \ + std::is_same_v<::holoscan::ArgList, std::decay_t>)>> #define HOLOSCAN_COMPONENT_FORWARD_ARGS(class_name) \ HOLOSCAN_COMPONENT_FORWARD_TEMPLATE() \ class_name(ArgT&& arg, ArgsT&&... args) \ diff --git a/include/holoscan/core/condition.hpp b/include/holoscan/core/condition.hpp index dd9f513d..a9567843 100644 --- a/include/holoscan/core/condition.hpp +++ b/include/holoscan/core/condition.hpp @@ -35,12 +35,13 @@ #include "./gxf/gxf_component.hpp" #include "./gxf/gxf_utils.hpp" -#define HOLOSCAN_CONDITION_FORWARD_TEMPLATE() \ - template > && \ - (std::is_same_v> || \ - std::is_same_v>)>> +#define HOLOSCAN_CONDITION_FORWARD_TEMPLATE() \ + template > && \ + (std::is_same_v<::holoscan::Arg, std::decay_t> || \ + std::is_same_v<::holoscan::ArgList, std::decay_t>)>> /** * @brief Forward the arguments to the super class. @@ -98,6 +99,7 @@ namespace holoscan { // Forward declarations class Operator; +// Note: Update `IOSpec::to_yaml_node()` if you add new condition types enum class ConditionType { kNone, ///< No condition kMessageAvailable, ///< Default for input port (nvidia::gxf::MessageAvailableSchedulingTerm) diff --git a/include/holoscan/core/conditions/gxf/asynchronous.hpp b/include/holoscan/core/conditions/gxf/asynchronous.hpp index 8f133168..a1bd17e1 100644 --- a/include/holoscan/core/conditions/gxf/asynchronous.hpp +++ b/include/holoscan/core/conditions/gxf/asynchronous.hpp @@ -31,6 +31,9 @@ using nvidia::gxf::AsynchronousEventState; /** * @brief Condition class to support asynchronous execution of operators. * + * This condition waits on an asynchronous event which can happen outside of the regular compute + * function of an operator. + * * The method `event_state()` method is used to get or set the asynchronous condition's state. * The possible states are: * - AsynchronousEventState::READY ///< Initial state, first compute call is pending @@ -43,9 +46,9 @@ using nvidia::gxf::AsynchronousEventState; * - AsynchronousEventState::EVENT_NEVER ///< Entity will not call compute again, end of * execution * - * TODO: expand documentation - * - * This class wraps GXF SchedulingTerm(`nvidia::gxf::AsynchronousSchedulingTerm`). + * This class wraps GXF SchedulingTerm(`nvidia::gxf::AsynchronousSchedulingTerm`). The event used + * corresponds to `gxf_event_t` enum value `GXF_EVENT_EXTERNAL` which is supported by all + * schedulers. */ class AsynchronousCondition : public gxf::GXFCondition { public: diff --git a/include/holoscan/core/domain/tensor.hpp b/include/holoscan/core/domain/tensor.hpp index 81261612..7abe0d9f 100644 --- a/include/holoscan/core/domain/tensor.hpp +++ b/include/holoscan/core/domain/tensor.hpp @@ -109,6 +109,13 @@ class Tensor { */ std::vector strides() const; + /** + * @brief Check if the tensor a has contiguous, row-major memory layout. + * + * @return True if the tensor is contiguous, False otherwise. + */ + bool is_contiguous() const; + /** * @brief Get the size (number of elements) in the Tensor. * diff --git a/include/holoscan/core/executors/gxf/gxf_parameter_adaptor.hpp b/include/holoscan/core/executors/gxf/gxf_parameter_adaptor.hpp index cedab034..5259946d 100644 --- a/include/holoscan/core/executors/gxf/gxf_parameter_adaptor.hpp +++ b/include/holoscan/core/executors/gxf/gxf_parameter_adaptor.hpp @@ -76,6 +76,19 @@ class GXFParameterAdaptor { return func(context, uid, key, param_wrap.arg_type(), param_wrap.value()); } + static gxf_result_t set_param(gxf_context_t context, gxf_uid_t uid, const char* key, + const ArgType& arg_type, std::any& any_value) { + auto& instance = get_instance(); + const auto index = std::type_index(any_value.type()); + const AdaptFunc& func = instance.get_arg_param_handler(index); + if (&func == &none_param_handler) { + HOLOSCAN_LOG_ERROR("Unable to handle parameter: {}", key); + return GXF_FAILURE; + } + + return func(context, uid, key, arg_type, any_value); + } + template static void ensure_type() { auto& instance = get_instance(); @@ -91,13 +104,24 @@ class GXFParameterAdaptor { return handler; } + AdaptFunc& get_arg_param_handler(std::type_index index) { + if (arg_function_map_.find(index) == arg_function_map_.end()) { + HOLOSCAN_LOG_WARN("No parameter handler for type '{}' exists", index.name()); + return GXFParameterAdaptor::none_param_handler; + } + auto& handler = arg_function_map_[index]; + return handler; + } + template void add_param_handler(AdaptFunc func) { function_map_.try_emplace(std::type_index(typeid(typeT)), func); + arg_function_map_.try_emplace(std::type_index(typeid(typeT)), func); } void add_param_handler(std::type_index index, AdaptFunc func) { function_map_.try_emplace(index, func); + arg_function_map_.try_emplace(index, func); } template @@ -124,337 +148,371 @@ class GXFParameterAdaptor { } } - auto& value = param.get(); - switch (arg_type.container_type()) { - case ArgContainerType::kNative: { - switch (arg_type.element_type()) { - case ArgElementType::kBoolean: { - if constexpr (std::is_same_v) { - return GxfParameterSetBool(context, uid, key, value); - } - break; - } - case ArgElementType::kInt8: { - if constexpr (std::is_same_v) { - return GxfParameterSetInt8(context, uid, key, value); - } - break; - } - case ArgElementType::kUnsigned8: { - if constexpr (std::is_same_v) { - return GxfParameterSetUInt8(context, uid, key, value); - } - break; - } - case ArgElementType::kInt16: { - if constexpr (std::is_same_v) { - return GxfParameterSetInt16(context, uid, key, value); - } - break; - } - case ArgElementType::kUnsigned16: { - if constexpr (std::is_same_v) { - return GxfParameterSetUInt16(context, uid, key, value); - } - break; - } - case ArgElementType::kInt32: { - if constexpr (std::is_same_v) { - return GxfParameterSetInt32(context, uid, key, value); - } - break; - } - case ArgElementType::kUnsigned32: { - if constexpr (std::is_same_v) { - return GxfParameterSetUInt32(context, uid, key, value); - } - break; - } - case ArgElementType::kInt64: { - if constexpr (std::is_same_v) { - return GxfParameterSetInt64(context, uid, key, value); - } - break; - } - case ArgElementType::kUnsigned64: { - if constexpr (std::is_same_v) { - return GxfParameterSetUInt64(context, uid, key, value); - } - break; - } - case ArgElementType::kFloat32: { - if constexpr (std::is_same_v) { - return GxfParameterSetFloat32(context, uid, key, value); - } - break; - } - case ArgElementType::kFloat64: { - if constexpr (std::is_same_v) { - return GxfParameterSetFloat64(context, uid, key, value); - } - break; - } - case ArgElementType::kComplex64: { - // GXF Doesn't have parameter setter for complex or complex - if constexpr (std::is_same_v>) { - YAML::Node yaml_node; - yaml_node.push_back(value); - YAML::Node value_node = yaml_node[0]; - return GxfParameterSetFromYamlNode(context, uid, key, &value_node, ""); - } - break; - } - case ArgElementType::kComplex128: { - // GXF Doesn't have parameter setter for complex or complex - if constexpr (std::is_same_v>) { - YAML::Node yaml_node; - yaml_node.push_back(value); - YAML::Node value_node = yaml_node[0]; - return GxfParameterSetFromYamlNode(context, uid, key, &value_node, ""); - } - break; - } - case ArgElementType::kString: { - if constexpr (std::is_same_v) { - return GxfParameterSetStr(context, uid, key, value.c_str()); - } - break; - } - case ArgElementType::kHandle: { - HOLOSCAN_LOG_ERROR("Unable to set handle parameter for key '{}'", key); - return GXF_FAILURE; + typeT& value = param.get(); + + gxf_result_t result = set_gxf_parameter_value(context, uid, key, arg_type, value); + return result; + } catch (const std::bad_any_cast& e) { + HOLOSCAN_LOG_ERROR("Bad any cast exception: {}", e.what()); + } + + return GXF_FAILURE; + }; + + const AdaptFunc& arg_func = [](gxf_context_t context, + gxf_uid_t uid, + const char* key, + const ArgType& arg_type, + const std::any& any_value) { + (void)context; // avoid `-Werror=unused-but-set-parameter` due to `constexpr` + (void)uid; // avoid `-Werror=unused-but-set-parameter` due to `constexpr` + try { + typeT value = std::any_cast(any_value); + gxf_result_t result = set_gxf_parameter_value(context, uid, key, arg_type, value); + return result; + } catch (const std::bad_any_cast& e) { + HOLOSCAN_LOG_ERROR("Bad any cast exception: {}", e.what()); + } + + return GXF_FAILURE; + }; + + function_map_.try_emplace(std::type_index(typeid(typeT)), func); + arg_function_map_.try_emplace(std::type_index(typeid(typeT)), arg_func); + } + + template + static gxf_result_t set_gxf_parameter_value(gxf_context_t context, gxf_uid_t uid, const char* key, + const ArgType& arg_type, typeT& value) { + switch (arg_type.container_type()) { + case ArgContainerType::kNative: { + switch (arg_type.element_type()) { + case ArgElementType::kBoolean: { + if constexpr (std::is_same_v) { + return GxfParameterSetBool(context, uid, key, value); + } + break; + } + case ArgElementType::kInt8: { + if constexpr (std::is_same_v) { + return GxfParameterSetInt8(context, uid, key, value); + } + break; + } + case ArgElementType::kUnsigned8: { + if constexpr (std::is_same_v) { + return GxfParameterSetUInt8(context, uid, key, value); + } + break; + } + case ArgElementType::kInt16: { + if constexpr (std::is_same_v) { + return GxfParameterSetInt16(context, uid, key, value); + } + break; + } + case ArgElementType::kUnsigned16: { + if constexpr (std::is_same_v) { + return GxfParameterSetUInt16(context, uid, key, value); + } + break; + } + case ArgElementType::kInt32: { + if constexpr (std::is_same_v) { + return GxfParameterSetInt32(context, uid, key, value); + } + break; + } + case ArgElementType::kUnsigned32: { + if constexpr (std::is_same_v) { + return GxfParameterSetUInt32(context, uid, key, value); + } + break; + } + case ArgElementType::kInt64: { + if constexpr (std::is_same_v) { + return GxfParameterSetInt64(context, uid, key, value); + } + break; + } + case ArgElementType::kUnsigned64: { + if constexpr (std::is_same_v) { + return GxfParameterSetUInt64(context, uid, key, value); + } + break; + } + case ArgElementType::kFloat32: { + if constexpr (std::is_same_v) { + return GxfParameterSetFloat32(context, uid, key, value); + } + break; + } + case ArgElementType::kFloat64: { + if constexpr (std::is_same_v) { + return GxfParameterSetFloat64(context, uid, key, value); + } + break; + } + case ArgElementType::kComplex64: { + // GXF Doesn't have parameter setter for complex or complex + if constexpr (std::is_same_v>) { + YAML::Node yaml_node; + yaml_node.push_back(value); + YAML::Node value_node = yaml_node[0]; + return GxfParameterSetFromYamlNode(context, uid, key, &value_node, ""); + } + break; + } + case ArgElementType::kComplex128: { + // GXF Doesn't have parameter setter for complex or complex + if constexpr (std::is_same_v>) { + YAML::Node yaml_node; + yaml_node.push_back(value); + YAML::Node value_node = yaml_node[0]; + return GxfParameterSetFromYamlNode(context, uid, key, &value_node, ""); + } + break; + } + case ArgElementType::kString: { + if constexpr (std::is_same_v) { + return GxfParameterSetStr(context, uid, key, value.c_str()); + } + break; + } + case ArgElementType::kHandle: { + HOLOSCAN_LOG_ERROR("Unable to set handle parameter for key '{}'", key); + return GXF_FAILURE; + } + case ArgElementType::kYAMLNode: { + if constexpr (std::is_same_v) { + return GxfParameterSetFromYamlNode(context, uid, key, &value, ""); + } else { + HOLOSCAN_LOG_ERROR("Unable to handle ArgElementType::kYAMLNode for key '{}'", key); + return GXF_FAILURE; + } + } + case ArgElementType::kIOSpec: { + if constexpr (std::is_same_v) { + if (value) { + auto gxf_resource = std::dynamic_pointer_cast(value->connector()); + gxf_uid_t cid = gxf_resource->gxf_cid(); + + return GxfParameterSetHandle(context, uid, key, cid); + } else { + // If the IOSpec is null, do not set the parameter. + return GXF_SUCCESS; } - case ArgElementType::kYAMLNode: { - if constexpr (std::is_same_v) { - return GxfParameterSetFromYamlNode(context, uid, key, &value, ""); - } else { - HOLOSCAN_LOG_ERROR("Unable to handle ArgElementType::kYAMLNode for key '{}'", - key); - return GXF_FAILURE; + } + break; + } + case ArgElementType::kResource: { + if constexpr (std::is_same_v::element_type, + std::shared_ptr> && + holoscan::type_info::dimension == 0) { + auto gxf_resource = std::dynamic_pointer_cast(value); + if (gxf_resource) { + // Initialize GXF component if it is not already initialized. + if (gxf_resource->gxf_context() == nullptr) { + gxf_resource->gxf_eid( + gxf::get_component_eid(context, uid)); // set Entity ID of the component + + gxf_resource->initialize(); } + return GxfParameterSetHandle(context, uid, key, gxf_resource->gxf_cid()); + } else { + HOLOSCAN_LOG_TRACE("Resource is null for key '{}'. Not setting parameter.", key); + return GXF_SUCCESS; } - case ArgElementType::kIOSpec: { - if constexpr (std::is_same_v) { - if (value) { - auto gxf_resource = std::dynamic_pointer_cast(value->connector()); - gxf_uid_t cid = gxf_resource->gxf_cid(); - - return GxfParameterSetHandle(context, uid, key, cid); - } else { - // If the IOSpec is null, do not set the parameter. - return GXF_SUCCESS; - } + } + HOLOSCAN_LOG_ERROR("Unable to handle ArgElementType::kResource for key '{}'", key); + break; + } + case ArgElementType::kCondition: { + if constexpr (std::is_same_v::element_type, + std::shared_ptr> && + holoscan::type_info::dimension == 0) { + auto gxf_condition = std::dynamic_pointer_cast(value); + if (gxf_condition) { + // Initialize GXF component if it is not already initialized. + if (gxf_condition->gxf_context() == nullptr) { + gxf_condition->gxf_eid( + gxf::get_component_eid(context, uid)); // set Entity ID of the component + + gxf_condition->initialize(); } - break; + return GxfParameterSetHandle(context, uid, key, gxf_condition->gxf_cid()); } - case ArgElementType::kResource: { - if constexpr (std::is_same_v::element_type, - std::shared_ptr> && - holoscan::type_info::dimension == 0) { - // Set the handle parameter only if the resource is valid. - if (value) { - auto gxf_resource = std::dynamic_pointer_cast(value); - // Initialize GXF component if it is not already initialized. - if (gxf_resource->gxf_context() == nullptr) { - gxf_resource->gxf_eid( - gxf::get_component_eid(context, uid)); // set Entity ID of the component - - gxf_resource->initialize(); - } - return GxfParameterSetHandle(context, uid, key, gxf_resource->gxf_cid()); - } else { - HOLOSCAN_LOG_TRACE("Resource is null for key '{}'. Not setting parameter.", - key); - return GXF_SUCCESS; - } + HOLOSCAN_LOG_ERROR("Unable to handle ArgElementType::kCondition for key '{}'", key); + } + break; + } + case ArgElementType::kCustom: { + HOLOSCAN_LOG_ERROR("Unable to handle ArgElementType::kCustom for key '{}'", key); + return GXF_FAILURE; + } + } + break; + } + case ArgContainerType::kVector: { + switch (arg_type.element_type()) { + case ArgElementType::kBoolean: + case ArgElementType::kInt8: + case ArgElementType::kUnsigned8: + case ArgElementType::kInt16: + case ArgElementType::kUnsigned16: + case ArgElementType::kInt32: + case ArgElementType::kUnsigned32: + case ArgElementType::kInt64: + case ArgElementType::kUnsigned64: + case ArgElementType::kFloat32: + case ArgElementType::kFloat64: + case ArgElementType::kComplex64: + case ArgElementType::kComplex128: + case ArgElementType::kString: { + // GXF Doesn't support std::vector or std::vector> parameter + // types so use a workaround with GxfParameterSetFromYamlNode. + if constexpr (holoscan::is_one_of_v::element_type, + bool, + int8_t, + uint8_t, + int16_t, + uint16_t, + int32_t, + uint32_t, + int64_t, + uint64_t, + float, + double, + std::complex, + std::complex, + std::string>) { + if constexpr (holoscan::dimension_of_v == 1) { + // Create vector of Handles + YAML::Node yaml_node = YAML::Load("[]"); // Create an empty sequence + for (typename holoscan::type_info::element_type item : value) { + yaml_node.push_back(item); } - HOLOSCAN_LOG_ERROR("Unable to handle ArgElementType::kResource for key '{}'", key); - break; - } - case ArgElementType::kCondition: { - if constexpr (std::is_same_v::element_type, - std::shared_ptr> && - holoscan::type_info::dimension == 0) { - auto gxf_condition = std::dynamic_pointer_cast(value); - if (value) { - // Initialize GXF component if it is not already initialized. - if (gxf_condition->gxf_context() == nullptr) { - gxf_condition->gxf_eid( - gxf::get_component_eid(context, uid)); // set Entity ID of the component - - gxf_condition->initialize(); - } - return GxfParameterSetHandle(context, uid, key, gxf_condition->gxf_cid()); + return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); + } else if constexpr (holoscan::dimension_of_v == 2) { + YAML::Node yaml_node = YAML::Load("[]"); // Create an empty sequence + for (std::vector::element_type>& vec : value) { + YAML::Node inner_yaml_node = YAML::Load("[]"); // Create an empty sequence + for (typename holoscan::type_info::element_type item : vec) { + inner_yaml_node.push_back(item); } - HOLOSCAN_LOG_ERROR("Unable to handle ArgElementType::kCondition for key '{}'", - key); + if (inner_yaml_node.size() > 0) { yaml_node.push_back(inner_yaml_node); } } - break; - } - case ArgElementType::kCustom: { - HOLOSCAN_LOG_ERROR("Unable to handle ArgElementType::kCustom for key '{}'", key); - return GXF_FAILURE; + return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); } } break; } - case ArgContainerType::kVector: { - switch (arg_type.element_type()) { - case ArgElementType::kBoolean: - case ArgElementType::kInt8: - case ArgElementType::kUnsigned8: - case ArgElementType::kInt16: - case ArgElementType::kUnsigned16: - case ArgElementType::kInt32: - case ArgElementType::kUnsigned32: - case ArgElementType::kInt64: - case ArgElementType::kUnsigned64: - case ArgElementType::kFloat32: - case ArgElementType::kFloat64: - case ArgElementType::kComplex64: - case ArgElementType::kComplex128: - case ArgElementType::kString: { - // GXF Doesn't support std::vector or std::vector> parameter - // types so use a workaround with GxfParameterSetFromYamlNode. - if constexpr (holoscan::is_one_of_v< - typename holoscan::type_info::element_type, - bool, - int8_t, - uint8_t, - int16_t, - uint16_t, - int32_t, - uint32_t, - int64_t, - uint64_t, - float, - double, - std::complex, - std::complex, - std::string>) { - if constexpr (holoscan::dimension_of_v == 1) { - // Create vector of Handles - YAML::Node yaml_node = YAML::Load("[]"); // Create an empty sequence - for (typename holoscan::type_info::element_type item : value) { - yaml_node.push_back(item); - } - return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); - } else if constexpr (holoscan::dimension_of_v == 2) { - YAML::Node yaml_node = YAML::Load("[]"); // Create an empty sequence - for (std::vector::element_type>& vec : - value) { - YAML::Node inner_yaml_node = YAML::Load("[]"); // Create an empty sequence - for (typename holoscan::type_info::element_type item : vec) { - inner_yaml_node.push_back(item); - } - if (inner_yaml_node.size() > 0) { yaml_node.push_back(inner_yaml_node); } - } - return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); - } - } - break; - } - case ArgElementType::kHandle: { - HOLOSCAN_LOG_ERROR( - "Unable to handle vector of ArgElementType::kHandle for key '{}'", key); - return GXF_FAILURE; - } - case ArgElementType::kYAMLNode: { - HOLOSCAN_LOG_ERROR( - "Unable to handle vector of ArgElementType::kYAMLNode for key '{}'", key); - return GXF_FAILURE; - } - case ArgElementType::kIOSpec: { - if constexpr (std::is_same_v>) { - // Create vector of Handles - YAML::Node yaml_node = YAML::Load("[]"); // Create an empty sequence - for (auto& io_spec : value) { - if (io_spec) { // Only consider non-null IOSpecs - auto gxf_resource = - std::dynamic_pointer_cast(io_spec->connector()); - yaml_node.push_back(gxf_resource->gxf_cname()); - } - } - return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); + case ArgElementType::kHandle: { + HOLOSCAN_LOG_ERROR("Unable to handle vector of ArgElementType::kHandle for key '{}'", + key); + return GXF_FAILURE; + } + case ArgElementType::kYAMLNode: { + HOLOSCAN_LOG_ERROR("Unable to handle vector of ArgElementType::kYAMLNode for key '{}'", + key); + return GXF_FAILURE; + } + case ArgElementType::kIOSpec: { + if constexpr (std::is_same_v>) { + // Create vector of Handles + YAML::Node yaml_node = YAML::Load("[]"); // Create an empty sequence + for (auto& io_spec : value) { + if (io_spec) { // Only consider non-null IOSpecs + auto gxf_resource = std::dynamic_pointer_cast(io_spec->connector()); + yaml_node.push_back(gxf_resource->gxf_cname()); } - HOLOSCAN_LOG_ERROR( - "Unable to handle vector of std::vector> for key: " - "'{}'", - key); - break; } - case ArgElementType::kResource: { - if constexpr (std::is_same_v::element_type, - std::shared_ptr> && - holoscan::type_info::dimension == 1) { - // Create vector of Handles - YAML::Node yaml_node; - for (auto& resource : value) { - auto gxf_resource = std::dynamic_pointer_cast(resource); - // Push back the resource's gxf_cname only if it is not null. - if (gxf_resource) { - gxf_uid_t resource_cid = gxf_resource->gxf_cid(); - std::string full_resource_name = - gxf::get_full_component_name(context, resource_cid); - yaml_node.push_back(full_resource_name.c_str()); - } else { - HOLOSCAN_LOG_TRACE( - "Resource item in the vector is null. Skipping it for key '{}'", key); - } + return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); + } + HOLOSCAN_LOG_ERROR( + "Unable to handle vector of std::vector> for key: " + "'{}'", + key); + break; + } + case ArgElementType::kResource: { + if constexpr (std::is_same_v::element_type, + std::shared_ptr> && + holoscan::type_info::dimension == 1) { + // Create vector of Handles + YAML::Node yaml_node; + for (auto& resource : value) { + auto gxf_resource = std::dynamic_pointer_cast(resource); + // Push back the resource's gxf_cname only if it is not null. + if (gxf_resource) { + // Initialize GXF component if it is not already initialized. + if (gxf_resource->gxf_context() == nullptr) { + gxf_resource->gxf_eid( + gxf::get_component_eid(context, uid)); // set Entity ID of the component + + gxf_resource->initialize(); } - return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); + gxf_uid_t resource_cid = gxf_resource->gxf_cid(); + std::string full_resource_name = + gxf::get_full_component_name(context, resource_cid); + yaml_node.push_back(full_resource_name.c_str()); + } else { + HOLOSCAN_LOG_TRACE( + "Resource item in the vector is null. Skipping it for key '{}'", key); } - HOLOSCAN_LOG_ERROR( - "Unable to handle vector of ArgElementType::kResource for key '{}'", key); - break; } - case ArgElementType::kCondition: { - if constexpr (std::is_same_v::element_type, - std::shared_ptr> && - holoscan::type_info::dimension == 1) { - // Create vector of Handles - YAML::Node yaml_node; - for (auto& condition : value) { - auto gxf_condition = std::dynamic_pointer_cast(condition); - // Initialize GXF component if it is not already initialized. - if (gxf_condition->gxf_context() == nullptr) { - gxf_condition->gxf_eid( - gxf::get_component_eid(context, uid)); // set Entity ID of the component - - gxf_condition->initialize(); - } - gxf_uid_t condition_cid = gxf_condition->gxf_cid(); - std::string full_condition_name = - gxf::get_full_component_name(context, condition_cid); - yaml_node.push_back(full_condition_name.c_str()); + return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); + } + HOLOSCAN_LOG_ERROR("Unable to handle vector of ArgElementType::kResource for key '{}'", + key); + break; + } + case ArgElementType::kCondition: { + if constexpr (std::is_same_v::element_type, + std::shared_ptr> && + holoscan::type_info::dimension == 1) { + // Create vector of Handles + YAML::Node yaml_node; + for (auto& condition : value) { + auto gxf_condition = std::dynamic_pointer_cast(condition); + // Push back the condition's gxf_cname only if it is not null. + if (gxf_condition) { + // Initialize GXF component if it is not already initialized. + if (gxf_condition->gxf_context() == nullptr) { + gxf_condition->gxf_eid( + gxf::get_component_eid(context, uid)); // set Entity ID of the component + + gxf_condition->initialize(); } - return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); + gxf_uid_t condition_cid = gxf_condition->gxf_cid(); + std::string full_condition_name = + gxf::get_full_component_name(context, condition_cid); + yaml_node.push_back(full_condition_name.c_str()); + } else { + HOLOSCAN_LOG_TRACE( + "Condition item in the vector is null. Skipping it for key '{}'", key); } - HOLOSCAN_LOG_ERROR( - "Unable to handle vector of ArgElementType::kCondition for key '{}'", key); - break; - } - case ArgElementType::kCustom: { - HOLOSCAN_LOG_ERROR( - "Unable to handle vector of ArgElementType::kCustom type for key '{}'", key); - return GXF_FAILURE; } + return GxfParameterSetFromYamlNode(context, uid, key, &yaml_node, ""); } + HOLOSCAN_LOG_ERROR("Unable to handle vector of ArgElementType::kCondition for key '{}'", + key); break; } - case ArgContainerType::kArray: { - HOLOSCAN_LOG_ERROR("Unable to handle ArgContainerType::kArray type for key '{}'", key); - break; + case ArgElementType::kCustom: { + HOLOSCAN_LOG_ERROR( + "Unable to handle vector of ArgElementType::kCustom type for key '{}'", key); + return GXF_FAILURE; } } - } catch (const std::bad_any_cast& e) { - HOLOSCAN_LOG_ERROR("Bad any cast exception: {}", e.what()); + break; } - - return GXF_FAILURE; - }; - - function_map_.try_emplace(std::type_index(typeid(typeT)), func); + case ArgContainerType::kArray: { + HOLOSCAN_LOG_ERROR("Unable to handle ArgContainerType::kArray type for key '{}'", key); + break; + } + } + return GXF_SUCCESS; } private: @@ -508,6 +566,7 @@ class GXFParameterAdaptor { } std::unordered_map function_map_; + std::unordered_map arg_function_map_; }; } // namespace holoscan::gxf diff --git a/include/holoscan/core/gxf/gxf_component_info.hpp b/include/holoscan/core/gxf/gxf_component_info.hpp new file mode 100644 index 00000000..12f77cd4 --- /dev/null +++ b/include/holoscan/core/gxf/gxf_component_info.hpp @@ -0,0 +1,143 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HOLOSCAN_CORE_GXF_GXF_COMPONENT_INFO_HPP +#define HOLOSCAN_CORE_GXF_GXF_COMPONENT_INFO_HPP + +#include + +#include +#include +#include + +#include "holoscan/core/arg.hpp" + +namespace holoscan::gxf { + +/** + * @brief A class that encapsulates the information about a GXF component. + * + * This class provides methods to access various properties of a GXF component, + * such as its receiver and transmitter TIDs, parameter keys, parameter infos, etc. + */ +class ComponentInfo { + public: + /// Maximum number of parameters a component can have. + static constexpr int MAX_PARAM_COUNT = 512; + + /** + * @brief Construct a new component info object. + * + * @param context The GXF context. + * @param tid The TID of the component. + */ + ComponentInfo(gxf_context_t context, gxf_tid_t tid); + + /** + * @brief Destroy the component info object. + */ + ~ComponentInfo(); + + /** + * @brief Get the arg type object + * + * Returns the Holoscan argument type for the given GXF parameter info. + * + * @param param_info The GXF parameter info. + * @return The argument type of the parameter. + */ + static ArgType get_arg_type(const gxf_parameter_info_t& param_info); + + /** + * @brief Get the receiver TID of the component. + * + * @return The receiver TID. + */ + gxf_tid_t receiver_tid() const; + + /** + * @brief Get the transmitter TID of the component. + * + * @return The transmitter TID. + */ + gxf_tid_t transmitter_tid() const; + + /** + * @brief Get the component info. + * + * @return The component info. + */ + const gxf_component_info_t& component_info() const; + + /** + * @brief Get the parameter keys of the component. + * + * @return The parameter keys. + */ + const std::vector& parameter_keys() const; + + /** + * @brief Get the parameter infos of the component. + * + * @return The parameter infos. + */ + const std::vector& parameter_infos() const; + + /** + * @brief Get the parameter info map of the component. + * + * @return The parameter info map. + */ + const std::unordered_map& parameter_info_map() const; + + /** + * @brief Get the receiver parameters of the component. + * + * @return The receiver parameters. + */ + const std::vector& receiver_parameters() const; + + /** + * @brief Get the transmitter parameters of the component. + * + * @return The transmitter parameters. + */ + const std::vector& transmitter_parameters() const; + + /** + * @brief Get the normal parameters of the component. + * + * @return The normal parameters. + */ + const std::vector& normal_parameters() const; + + private: + gxf_context_t gxf_context_ = nullptr; ///< The GXF context. + gxf_tid_t component_tid_ = GxfTidNull(); ///< The TID of the component. + gxf_component_info_t component_info_{}; ///< The component info. + std::vector parameter_keys_; ///< The parameter keys. + std::vector parameter_infos_; ///< The parameter infos. + /// The parameter info map. + std::unordered_map parameter_info_map_; + std::vector receiver_parameters_; ///< The receiver parameters. + std::vector transmitter_parameters_; ///< The transmitter parameters. + std::vector normal_parameters_; ///< The normal parameters. +}; + +} // namespace holoscan::gxf + +#endif /* HOLOSCAN_CORE_GXF_GXF_COMPONENT_INFO_HPP */ diff --git a/include/holoscan/core/gxf/gxf_io_context.hpp b/include/holoscan/core/gxf/gxf_io_context.hpp index c65271d7..3963f188 100644 --- a/include/holoscan/core/gxf/gxf_io_context.hpp +++ b/include/holoscan/core/gxf/gxf_io_context.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,7 +26,7 @@ namespace holoscan::gxf { -nvidia::gxf::Receiver* get_gxf_receiver(const std::unique_ptr& input_spec); +nvidia::gxf::Receiver* get_gxf_receiver(const std::shared_ptr& input_spec); /** * @brief Class to hold the input context for a GXF Operator. @@ -51,7 +51,7 @@ class GXFInputContext : public InputContext { * @param inputs inputs The references to the map of the input specs. */ GXFInputContext(ExecutionContext* execution_context, Operator* op, - std::unordered_map>& inputs); + std::unordered_map>& inputs); /** * @brief Get a pointer to the GXF execution runtime. @@ -87,7 +87,7 @@ class GXFOutputContext : public OutputContext { * @param outputs outputs The references to the map of the output specs. */ GXFOutputContext(ExecutionContext* execution_context, Operator* op, - std::unordered_map>& outputs); + std::unordered_map>& outputs); /** * @brief Get pointer to the GXF execution runtime. diff --git a/include/holoscan/core/gxf/gxf_operator.hpp b/include/holoscan/core/gxf/gxf_operator.hpp index 320a0224..f953845b 100644 --- a/include/holoscan/core/gxf/gxf_operator.hpp +++ b/include/holoscan/core/gxf/gxf_operator.hpp @@ -21,6 +21,7 @@ #include #include +#include #include #include "../executors/gxf/gxf_parameter_adaptor.hpp" @@ -31,6 +32,11 @@ namespace holoscan::ops { class GXFOperator : public holoscan::Operator { public: + /** + * @brief Construct a new GXFOperator object. + * + * @param args The arguments to be passed to the operator. + */ HOLOSCAN_OPERATOR_FORWARD_TEMPLATE() explicit GXFOperator(ArgT&& arg, ArgsT&&... args) : Operator(std::forward(arg), std::forward(args)...) { @@ -172,8 +178,22 @@ class GXFOperator : public holoscan::Operator { } protected: + /** + * This method is invoked by 'GXFExecutor::initialize_operator(Operator* op)' during + * the initialization of the operator. + * By overriding this method, additional setup tasks are performed for the operator, including: + * - Initializing the `spec_` object with the codelet's parameters. + * + * @return The codelet component id corresponding to GXF codelet. + */ gxf_uid_t add_codelet_to_graph_entity() override; + /** + * This method is invoked at the end of 'GXFExecutor::initialize_operator(Operator* op)' during + * the initialization of the operator. + * By overriding this method, we can modify how GXF Codelet's parameters are set from the + * arguments. + */ void set_parameters() override; /** @@ -226,9 +246,12 @@ class GXFOperator : public holoscan::Operator { }); } - gxf_context_t gxf_context_ = nullptr; ///< The GXF context. - gxf_uid_t gxf_eid_ = 0; ///< GXF entity ID - gxf_uid_t gxf_cid_ = 0; ///< The GXF component ID. + gxf_context_t gxf_context_ = nullptr; ///< The GXF context. + gxf_uid_t gxf_eid_ = 0; ///< GXF entity ID + gxf_uid_t gxf_cid_ = 0; ///< The GXF component ID. + nvidia::gxf::Handle codelet_handle_; ///< The codelet handle. + /// The GXF type name (used for GXFCodeletOp) + std::string gxf_typename_ = "unknown_gxf_typename"; }; } // namespace holoscan::ops diff --git a/include/holoscan/core/gxf/gxf_resource.hpp b/include/holoscan/core/gxf/gxf_resource.hpp index 34261067..be710ba5 100644 --- a/include/holoscan/core/gxf/gxf_resource.hpp +++ b/include/holoscan/core/gxf/gxf_resource.hpp @@ -37,7 +37,23 @@ class GXFResource : public holoscan::Resource, public gxf::GXFComponent { void initialize() override; - void add_to_graph_entity(Operator* op); + protected: + // Make GXFExecutor a friend class so it can call protected initialization methods + friend class holoscan::gxf::GXFExecutor; + // Operator::initialize_resources() needs to call add_to_graph_entity() + friend class holoscan::Operator; + + virtual void add_to_graph_entity(Operator* op); + + /** + * This method is invoked by `GXFResource::initialize()`. + * By overriding this method, we can modify how GXF Codelet's parameters are set from the + * arguments. + */ + void set_parameters() override; + bool handle_dev_id(std::optional& dev_id_value); + /// The GXF type name (used for GXFComponentResource) + std::string gxf_typename_ = "unknown_gxf_typename"; }; } // namespace holoscan::gxf diff --git a/include/holoscan/core/io_context.hpp b/include/holoscan/core/io_context.hpp index 4d7226d1..030d7004 100644 --- a/include/holoscan/core/io_context.hpp +++ b/include/holoscan/core/io_context.hpp @@ -27,6 +27,7 @@ #include #include +#include #include "./common.hpp" #include "./domain/tensor_map.hpp" #include "./errors.hpp" @@ -39,7 +40,7 @@ namespace holoscan { static inline std::string get_well_formed_name( - const char* name, const std::unordered_map>& io_list) { + const char* name, const std::unordered_map>& io_list) { std::string well_formed_name; if (name == nullptr || name[0] == '\0') { if (io_list.size() == 1) { @@ -68,7 +69,7 @@ class InputContext { * @param inputs The references to the map of the input specs. */ InputContext(ExecutionContext* execution_context, Operator* op, - std::unordered_map>& inputs) + std::unordered_map>& inputs) : execution_context_(execution_context), op_(op), inputs_(inputs) {} /** @@ -99,7 +100,7 @@ class InputContext { * @brief Return the reference to the map of the input specs. * @return The reference to the map of the input specs. */ - std::unordered_map>& inputs() const { return inputs_; } + std::unordered_map>& inputs() const { return inputs_; } /** * @brief Return whether the input port has any data. @@ -368,8 +369,11 @@ class InputContext { return tensor_map; } auto error_message = fmt::format( - "Unable to cast the received data to the specified type (DataT) for input {}: {}", + "Unable to cast the received data to the specified type ({}) for input {} of type {}: " + "{}", + nvidia::TypenameAsString(), name, + value.type().name(), e.what()); HOLOSCAN_LOG_DEBUG(error_message); return make_unexpected( @@ -409,7 +413,7 @@ class InputContext { ExecutionContext* execution_context_ = nullptr; ///< The execution context that is associated with. Operator* op_ = nullptr; ///< The operator that this context is associated with. - std::unordered_map>& inputs_; ///< The inputs. + std::unordered_map>& inputs_; ///< The inputs. }; /** @@ -438,7 +442,7 @@ class OutputContext { * @param outputs The references to the map of the output specs. */ OutputContext(ExecutionContext* execution_context, Operator* op, - std::unordered_map>& outputs) + std::unordered_map>& outputs) : execution_context_(execution_context), op_(op), outputs_(outputs) {} /** @@ -458,7 +462,7 @@ class OutputContext { * @brief Return the reference to the map of the output specs. * @return The reference to the map of the output specs. */ - std::unordered_map>& outputs() const { return outputs_; } + std::unordered_map>& outputs() const { return outputs_; } /** * @brief The output data type. @@ -651,7 +655,7 @@ class OutputContext { ExecutionContext* execution_context_ = nullptr; ///< The execution context that is associated with. Operator* op_ = nullptr; ///< The operator that this context is associated with. - std::unordered_map>& outputs_; ///< The outputs. + std::unordered_map>& outputs_; ///< The outputs. }; } // namespace holoscan diff --git a/include/holoscan/core/network_context.hpp b/include/holoscan/core/network_context.hpp index 28aac9c5..71661800 100644 --- a/include/holoscan/core/network_context.hpp +++ b/include/holoscan/core/network_context.hpp @@ -34,12 +34,13 @@ #include "./forward_def.hpp" #include "./resource.hpp" -#define HOLOSCAN_NETWORK_CONTEXT_FORWARD_TEMPLATE() \ - template > && \ - (std::is_same_v> || \ - std::is_same_v>)>> +#define HOLOSCAN_NETWORK_CONTEXT_FORWARD_TEMPLATE() \ + template > && \ + (std::is_same_v<::holoscan::Arg, std::decay_t> || \ + std::is_same_v<::holoscan::ArgList, std::decay_t>)>> /** * @brief Forward the arguments to the super class. diff --git a/include/holoscan/core/operator.hpp b/include/holoscan/core/operator.hpp index 6daf524d..a7557c08 100644 --- a/include/holoscan/core/operator.hpp +++ b/include/holoscan/core/operator.hpp @@ -594,10 +594,10 @@ class Operator : public ComponentBase { */ virtual gxf_uid_t add_codelet_to_graph_entity(); - /// Initialize conditions and add GXF conditions to graph_entity_; + /// Initialize conditions and add GXF conditions to graph_entity_ void initialize_conditions(); - /// Initialize resources and add GXF resources to graph_entity_; + /// Initialize resources and add GXF resources to graph_entity_ void initialize_resources(); using ComponentBase::update_params_from_args; diff --git a/include/holoscan/core/operator_spec.hpp b/include/holoscan/core/operator_spec.hpp index 0af9a617..ccad639a 100644 --- a/include/holoscan/core/operator_spec.hpp +++ b/include/holoscan/core/operator_spec.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -48,7 +48,7 @@ class OperatorSpec : public ComponentSpec { * * @return The reference to the input specifications of this operator. */ - std::unordered_map>& inputs() { return inputs_; } + std::unordered_map>& inputs() { return inputs_; } /** * @brief Define an input specification for this operator. @@ -70,7 +70,7 @@ class OperatorSpec : public ComponentSpec { */ template IOSpec& input(std::string name) { - auto spec = std::make_unique(this, name, IOSpec::IOType::kInput, &typeid(DataT)); + auto spec = std::make_shared(this, name, IOSpec::IOType::kInput, &typeid(DataT)); auto [iter, is_exist] = inputs_.insert_or_assign(name, std::move(spec)); if (!is_exist) { HOLOSCAN_LOG_ERROR("Input port '{}' already exists", name); } return *(iter->second.get()); @@ -81,7 +81,7 @@ class OperatorSpec : public ComponentSpec { * * @return The reference to the output specifications of this operator. */ - std::unordered_map>& outputs() { return outputs_; } + std::unordered_map>& outputs() { return outputs_; } /** * @brief Define an output specification for this operator. @@ -103,7 +103,7 @@ class OperatorSpec : public ComponentSpec { */ template IOSpec& output(std::string name) { - auto spec = std::make_unique(this, name, IOSpec::IOType::kOutput, &typeid(DataT)); + auto spec = std::make_shared(this, name, IOSpec::IOType::kOutput, &typeid(DataT)); auto [iter, is_exist] = outputs_.insert_or_assign(name, std::move(spec)); if (!is_exist) { HOLOSCAN_LOG_ERROR("Output port '{}' already exists", name); } return *(iter->second.get()); @@ -262,8 +262,8 @@ class OperatorSpec : public ComponentSpec { YAML::Node to_yaml_node() const override; protected: - std::unordered_map> inputs_; ///< Input specs - std::unordered_map> outputs_; ///< Outputs specs + std::unordered_map> inputs_; ///< Input specs + std::unordered_map> outputs_; ///< Outputs specs }; } // namespace holoscan diff --git a/include/holoscan/core/parameter.hpp b/include/holoscan/core/parameter.hpp index c1ca7289..0216dc5e 100644 --- a/include/holoscan/core/parameter.hpp +++ b/include/holoscan/core/parameter.hpp @@ -18,6 +18,10 @@ #ifndef HOLOSCAN_CORE_PARAMETER_HPP #define HOLOSCAN_CORE_PARAMETER_HPP +// Include fmt library for specialized formatting +#include +#include // allows fmt to format std::array, std::vector, etc. + #include #include #include @@ -26,8 +30,7 @@ #include #include -#include "./arg.hpp" -#include "./common.hpp" +#include "./type_traits.hpp" namespace holoscan { @@ -46,73 +49,6 @@ enum class ParameterFlag { kDynamic = 2, }; -/** - * @brief Class to wrap a parameter with std::any. - */ -class ParameterWrapper { - public: - ParameterWrapper() = default; - - /** - * @brief Construct a new ParameterWrapper object. - * - * @tparam typeT The type of the parameter. - * @param param The parameter to wrap. - */ - template - explicit ParameterWrapper(Parameter& param) - : type_(&typeid(typeT)), - arg_type_(ArgType::create()), - value_(¶m), - storage_ptr_(static_cast(¶m)) {} - - /** - * @brief Construct a new ParameterWrapper object. - * - * @param value The parameter to wrap. - * @param type The type of the parameter. - * @param arg_type The type of the parameter as an ArgType. - */ - ParameterWrapper(std::any value, const std::type_info* type, const ArgType& arg_type) - : type_(type), arg_type_(arg_type), value_(std::move(value)) {} - - /** - * @brief Get the type of the parameter. - * - * @return The type info of the parameter. - */ - const std::type_info& type() const { - if (type_) { return *type_; } - return typeid(void); - } - /** - * @brief Get the type of the parameter as an ArgType. - * - * @return The type of the parameter as an ArgType. - */ - const ArgType& arg_type() const { return arg_type_; } - - /** - * @brief Get the value of the parameter. - * - * @return The reference to the value of the parameter. - */ - std::any& value() { return value_; } - - /** - * @brief Get the pointer to the parameter storage. - * - * @return The pointer to the parameter storage. - */ - void* storage_ptr() const { return storage_ptr_; } - - private: - const std::type_info* type_ = nullptr; ///< The element type of Parameter - ArgType arg_type_; ///< The type of the argument - std::any value_; ///< The value of the parameter - void* storage_ptr_ = nullptr; ///< The pointer to the parameter storage -}; - /** * @brief Class to define a parameter. */ @@ -133,6 +69,18 @@ class MetaParameter { * @param value The value of the parameter. */ explicit MetaParameter(ValueT&& value) : value_(std::move(value)) {} + /** + * @brief Construct a new MetaParameter object + * + * @param value The value of the parameter. + * @param key The key (name) of the parameter. + * @param headline The headline of the parameter. + * @param description The description of the parameter. + * @param flag The flag of the parameter (default: ParameterFlag::kNone). + */ + MetaParameter(const ValueT& value, const char* key, const char* headline, const char* description, + ParameterFlag flag) + : key_(key), headline_(headline), description_(description), flag_(flag), value_(value) {} /** * @brief Define the assignment operator. @@ -293,4 +241,107 @@ class MetaParameter { } // namespace holoscan +// ------------------------------------------------------------------------------------------------ +// holoscan::Parameter format support for fmt::format +// +// After defining the holoscan::Parameter class, we need to specialize the fmt::formatter +// struct for the holoscan::Parameter type to use it with fmt::format. Here, we specialize the +// fmt::formatter struct for the holoscan::Parameter type before including the +// holoscan/logger/logger.hpp file. +// ------------------------------------------------------------------------------------------------ + +namespace fmt { + +template +struct formatter> : formatter { + template + auto format(const holoscan::Parameter& v, FormatContext& ctx) const { + return formatter::format(const_cast&>(v).get(), ctx); + } +}; + +} // namespace fmt + +// Include the logger.hpp after the fmt::formatter specialization +#include "holoscan/logger/logger.hpp" + +// ------------------------------------------------------------------------------------------------ + +// Define ParameterWrapper class + +#include "./arg.hpp" + +namespace holoscan { + +/** + * @brief Class to wrap a parameter with std::any. + */ +class ParameterWrapper { + public: + ParameterWrapper() = default; + + /** + * @brief Construct a new ParameterWrapper object. + * + * @tparam typeT The type of the parameter. + * @param param The parameter to wrap. + */ + template + explicit ParameterWrapper(Parameter& param) + : type_(&typeid(typeT)), + arg_type_(ArgType::create()), + value_(¶m), + storage_ptr_(static_cast(¶m)) {} + + /** + * @brief Construct a new ParameterWrapper object. + * + * @param value The parameter to wrap. + * @param type The type of the parameter. + * @param arg_type The type of the parameter as an ArgType. + * @param storage_ptr + */ + ParameterWrapper(std::any value, const std::type_info* type, const ArgType& arg_type, + void* storage_ptr = nullptr) + : type_(type), arg_type_(arg_type), value_(std::move(value)), storage_ptr_(storage_ptr) {} + + /** + * @brief Get the type of the parameter. + * + * @return The type info of the parameter. + */ + const std::type_info& type() const { + if (type_) { return *type_; } + return typeid(void); + } + /** + * @brief Get the type of the parameter as an ArgType. + * + * @return The type of the parameter as an ArgType. + */ + const ArgType& arg_type() const { return arg_type_; } + + /** + * @brief Get the value of the parameter. + * + * @return The reference to the value of the parameter. + */ + std::any& value() { return value_; } + + /** + * @brief Get the pointer to the parameter storage. + * + * @return The pointer to the parameter storage. + */ + void* storage_ptr() const { return storage_ptr_; } + + private: + const std::type_info* type_ = nullptr; ///< The element type of Parameter + ArgType arg_type_; ///< The type of the argument + std::any value_; ///< The value of the parameter + void* storage_ptr_ = nullptr; ///< The pointer to the parameter storage +}; + +} // namespace holoscan + #endif /* HOLOSCAN_CORE_PARAMETER_HPP */ diff --git a/include/holoscan/core/resource.hpp b/include/holoscan/core/resource.hpp index 5c121ad3..bc5f8c55 100644 --- a/include/holoscan/core/resource.hpp +++ b/include/holoscan/core/resource.hpp @@ -27,12 +27,13 @@ #include "./gxf/gxf_component.hpp" #include "./gxf/gxf_utils.hpp" -#define HOLOSCAN_RESOURCE_FORWARD_TEMPLATE() \ - template > && \ - (std::is_same_v> || \ - std::is_same_v>)>> +#define HOLOSCAN_RESOURCE_FORWARD_TEMPLATE() \ + template > && \ + (std::is_same_v<::holoscan::Arg, std::decay_t> || \ + std::is_same_v<::holoscan::ArgList, std::decay_t>)>> /** * @brief Forward the arguments to the super class. @@ -214,6 +215,14 @@ class Resource : public Component { using Component::reset_graph_entities; + using ComponentBase::update_params_from_args; + + /// Update parameters based on the specified arguments + void update_params_from_args(); + + /// Set the parameters based on defaults (sets GXF parameters for GXF components) + virtual void set_parameters(); + ResourceType resource_type_ = ResourceType::kNative; ///< The type of the resource. bool is_initialized_ = false; ///< Whether the resource is initialized. }; diff --git a/include/holoscan/core/resources/gxf/gxf_component_resource.hpp b/include/holoscan/core/resources/gxf/gxf_component_resource.hpp new file mode 100644 index 00000000..a545584a --- /dev/null +++ b/include/holoscan/core/resources/gxf/gxf_component_resource.hpp @@ -0,0 +1,111 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HOLOSCAN_CORE_RESOURCES_GXF_GXF_COMPONENT_RESOURCE_HPP +#define HOLOSCAN_CORE_RESOURCES_GXF_GXF_COMPONENT_RESOURCE_HPP + +#include "../../gxf/gxf_resource.hpp" + +#include +#include +#include + +#include "holoscan/core/gxf/gxf_component_info.hpp" + +namespace holoscan { + +/** + * @brief Wrap a GXF Component as a Holoscan Resource. + * + * This macro is designed to simplify the creation of Holoscan resources that encapsulate a GXF + * Component. It defines a class derived from `holoscan::GXFResource` and sets up the + * constructor to forward arguments to the base class while automatically setting the GXF type name. + * + * The resulting class is intended to act as a bridge, allowing GXF Components to be used directly + * within the Holoscan framework as resources, facilitating seamless integration and usage. + * + * Example Usage: + * + * ```cpp + * // Define a Holoscan resource that wraps a GXF Component within a Holoscan application + * class App : public holoscan::Application { + * ... + * HOLOSCAN_WRAP_GXF_CODELET_AS_OPERATOR(MyTensorOp, "nvidia::gxf::test::SendTensor") + * HOLOSCAN_WRAP_GXF_COMPONENT_AS_RESOURCE(MyBlockMemoryPool, "nvidia::gxf::BlockMemoryPool") + * + * void compose() override { + * using namespace holoscan; + * ... + * auto tx = make_operator( + * "tx", + * make_condition(15), + * Arg("pool") = make_resource( + * "pool", + * Arg("storage_type") = static_cast(1), + * Arg("block_size") = 1024UL, + * Arg("num_blocks") = 2UL)); + * ... + * } + * ... + * }; + * ``` + * + * @param class_name The name of the new Holoscan resource class. + * @param gxf_typename The GXF type name that identifies the specific GXF Component being wrapped. + */ +#define HOLOSCAN_WRAP_GXF_COMPONENT_AS_RESOURCE(class_name, gxf_typename) \ + class class_name : public ::holoscan::GXFComponentResource { \ + public: \ + HOLOSCAN_RESOURCE_FORWARD_TEMPLATE() \ + explicit class_name(ArgT&& arg, ArgsT&&... args) \ + : ::holoscan::GXFComponentResource(gxf_typename, std::forward(arg), \ + std::forward(args)...) {} \ + class_name() = default; \ + }; + +/** + * @brief Class that wraps a GXF Component as a Holoscan Resource. + */ +class GXFComponentResource : public gxf::GXFResource { + public: + // Constructor + template + explicit GXFComponentResource(const char* gxf_typename, ArgsT&&... args) + : GXFResource(std::forward(args)...) { + gxf_typename_ = gxf_typename; + } + + // Default constructor + GXFComponentResource() = default; + + // Returns the type name of the GXF component + const char* gxf_typename() const override; + + // Sets up the component spec + void setup(ComponentSpec& spec) override; + + protected: + // Sets the parameters of the component + void set_parameters() override; + + std::shared_ptr gxf_component_info_; ///< The GXF component info. + std::list> parameters_; ///< The fake parameters for the description. +}; + +} // namespace holoscan + +#endif /* HOLOSCAN_CORE_RESOURCES_GXF_GXF_COMPONENT_RESOURCE_HPP */ diff --git a/include/holoscan/core/scheduler.hpp b/include/holoscan/core/scheduler.hpp index f4c5db1e..a6baa865 100644 --- a/include/holoscan/core/scheduler.hpp +++ b/include/holoscan/core/scheduler.hpp @@ -34,12 +34,13 @@ #include "./component.hpp" #include "./resource.hpp" -#define HOLOSCAN_SCHEDULER_FORWARD_TEMPLATE() \ - template > && \ - (std::is_same_v> || \ - std::is_same_v>)>> +#define HOLOSCAN_SCHEDULER_FORWARD_TEMPLATE() \ + template > && \ + (std::is_same_v<::holoscan::Arg, std::decay_t> || \ + std::is_same_v<::holoscan::ArgList, std::decay_t>)>> /** * @brief Forward the arguments to the super class. diff --git a/include/holoscan/core/type_traits.hpp b/include/holoscan/core/type_traits.hpp index 942a9c35..7c65707c 100644 --- a/include/holoscan/core/type_traits.hpp +++ b/include/holoscan/core/type_traits.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,7 +24,7 @@ #include #include -#include "./common.hpp" +#include "./forward_def.hpp" namespace holoscan { diff --git a/include/holoscan/logger/logger.hpp b/include/holoscan/logger/logger.hpp index aad1f9a2..0bddb6e3 100644 --- a/include/holoscan/logger/logger.hpp +++ b/include/holoscan/logger/logger.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef HOLOSCAN_CORE_LOGGER_HPP -#define HOLOSCAN_CORE_LOGGER_HPP +#ifndef HOLOSCAN_LOGGER_LOGGER_HPP +#define HOLOSCAN_LOGGER_LOGGER_HPP #include #include // allows fmt to format std::array, std::vector, etc. @@ -167,12 +167,13 @@ class Logger { function_name, level, format, - fmt::make_args_checked(format, args...)); + fmt::make_args_checked(format, std::forward(args)...)); } template static void log(LogLevel level, const FormatT& format, ArgsT&&... args) { - log_message(level, format, fmt::make_args_checked(format, args...)); + log_message( + level, format, fmt::make_args_checked(format, std::forward(args)...)); } /** @@ -326,4 +327,4 @@ inline void log_message(const char* file, int line, const char* function_name, L } // namespace holoscan -#endif /* HOLOSCAN_CORE_LOGGER_HPP */ +#endif /* HOLOSCAN_LOGGER_LOGGER_HPP */ diff --git a/include/holoscan/operators/README.md b/include/holoscan/operators/README.md index 2b7f2e03..c5468288 100644 --- a/include/holoscan/operators/README.md +++ b/include/holoscan/operators/README.md @@ -5,6 +5,7 @@ These are the operators included as part of the Holoscan SDK: - **aja_source**: support AJA capture card as source - **bayer_demosaic**: perform color filter array (CFA) interpolation for 1-channel inputs of 8 or 16-bit unsigned integer and outputs an RGB or RGBA image - **format_converter**: provides common video or tensor operations in inference pipelines to change datatypes, resize images, reorder channels, and normalize and scale values. +- **gxf_codelet**: Provides a generic import interface for a GXF Codelet. - **holoviz**: handles compositing, blending, and visualization of RGB or RGBA images, masks, geometric primitives, text and depth maps - **inference**: performs AI inference using APIs from `HoloInfer` module. - **inference_processor**: performs processing of data using APIs from `HoloInfer` module. In the current release, a limited set of operations are supported on CPU. diff --git a/include/holoscan/operators/bayer_demosaic/bayer_demosaic.hpp b/include/holoscan/operators/bayer_demosaic/bayer_demosaic.hpp index 6746b2f0..c969d33f 100644 --- a/include/holoscan/operators/bayer_demosaic/bayer_demosaic.hpp +++ b/include/holoscan/operators/bayer_demosaic/bayer_demosaic.hpp @@ -38,11 +38,11 @@ namespace holoscan::ops { * * - **receiver** : `nvidia::gxf::Tensor` or `nvidia::gxf::VideoBuffer` * - The input video frame to process. If the input is a VideoBuffer it must be an 8-bit - * unsigned grayscale video (`nvidia::gxf::VideoFormat::GXF_VIDEO_FORMAT_GRAY`). The video - * buffer may be in either host or device memory (a host->device copy is performed if - * needed). If a video buffer is not found, the input port message is searched for a tensor - * with the name specified by `in_tensor_name`. This must be a device tensor in either - * 8-bit or 16-bit unsigned integer format. + * unsigned grayscale video (`nvidia::gxf::VideoFormat::GXF_VIDEO_FORMAT_GRAY`). If a video + * buffer is not found, the input port message is searched for a device + * tensor with the name specified by `in_tensor_name`. The tensor must have either 8-bit or + * 16-bit unsigned integer format. The tensor or video buffer may be in either host or device + * memory (a host->device copy is performed if needed). * * ==Named Outputs== * @@ -50,7 +50,8 @@ namespace holoscan::ops { * - The output video frame after demosaicing. This will be a 3-channel RGB image if * `alpha_value` is true, otherwise it will be a 4-channel RGBA image. The data type * will be either 8-bit or 16-bit unsigned integer (matching the bit depth of the input). - * The name of the tensor that is output is controlled by `out_tensor_name`. + * The name of the tensor that is output is controlled by `out_tensor_name`. The output will + * be in device memory. * * ==Parameters== * @@ -86,6 +87,15 @@ namespace holoscan::ops { * - **generate_alpha**: Generate alpha channel. Optional (default: `false`). * - **alpha_value**: Alpha value to be generated if `generate_alpha` is set to `true`. Optional * (default: `255`). + * + * ==Device Memory Requirements== + * + * When using this operator with a `BlockMemoryPool`, the minimum `block_size` is + * `(rows * columns * output_channels * element_size_bytes)` where `output_channels` is 4 when + * `generate_alpha` is true and 3 otherwise. If the input tensor or video buffer is already on the + * device, only a single memory block is needed. However, if the input is on the host, a + * second memory block will also be needed in order to make an internal copy of the input to the + * device. The memory buffer must be on device (`storage_type` = 1). */ class BayerDemosaicOp : public holoscan::Operator { public: diff --git a/include/holoscan/operators/format_converter/format_converter.hpp b/include/holoscan/operators/format_converter/format_converter.hpp index d23f4458..8a2d04e0 100644 --- a/include/holoscan/operators/format_converter/format_converter.hpp +++ b/include/holoscan/operators/format_converter/format_converter.hpp @@ -55,12 +55,11 @@ enum class FormatConversionType { * * - **source_video** : `nvidia::gxf::Tensor` or `nvidia::gxf::VideoBuffer` * - The input video frame to process. If the input is a VideoBuffer it must be in format - * GXF_VIDEO_FORMAT_RGBA, GXF_VIDEO_FORMAT_RGB or GXF_VIDEO_FORMAT_NV12. This video - * buffer may be in either host or device memory (a host->device copy is performed if - * needed). If a video buffer is not found, the input port message is searched for a tensor - * with the name specified by `in_tensor_name`. This must be a device tensor in one of - * several supported formats (unsigned 8-bit int or float32 graycale, unsigned 8-bit int - * RGB or RGBA YUV420 or NV12). + * GXF_VIDEO_FORMAT_RGBA, GXF_VIDEO_FORMAT_RGB or GXF_VIDEO_FORMAT_NV12. If a video buffer is + * not found, the input port message is searched for a tensor with the name specified by + * `in_tensor_name`. This must be a tensor in one of several supported formats (unsigned 8-bit + * int or float32 graycale, unsigned 8-bit int RGB or RGBA YUV420 or NV12). The tensor or video + * buffer may be in either host or device memory (a host->device copy is performed if needed). * * ==Named Outputs== * @@ -119,6 +118,24 @@ enum class FormatConversionType { * Optional (default: `[0, 1, 2]` for 3-channel images and `[0, 1, 2, 3]` for 4-channel images). * - **cuda_stream_pool**: `holoscan::CudaStreamPool` instance to allocate CUDA streams. * Optional (default: `nullptr`). + * + * ==Device Memory Requirements== + * + * When using this operator with a `BlockMemoryPool`, between 1 and 3 device memory blocks + * (`storage_type` = 1) will be required based on the input tensors and parameters: + * - 1.) In all cases there is a memory block needed for the output tensor. The size of this + * block will be `out_height * out_width * out_channels * out_element_size_bytes` where + * (out_height, out_width) will either be (in_height, in_width) (or + * (resize_height, resize_width) a resize was specified). `out_element_size` is the element + * size in bytes (e.g. 1 for RGB888 or 4 for Float32). + * - 2.) If a resize is being done, another memory block is required for this. This block will + * have size `resize_height * resize_width * in_channels * in_element_size_bytes`. + * - 3.) If the input tensor will be in host memory, a memory block is needed to copy the input + * to the device. This block will have size + * `in_height * in_width * in_channels * in_element_size_bytes`. + * + * Thus when declaring the memory pool, `num_blocks` should be between 1-3 and `block_size` should + * be set to the maximum of the individual blocks sizes described above. */ class FormatConverterOp : public holoscan::Operator { public: diff --git a/include/holoscan/operators/gxf_codelet/gxf_codelet.hpp b/include/holoscan/operators/gxf_codelet/gxf_codelet.hpp new file mode 100644 index 00000000..342e8fab --- /dev/null +++ b/include/holoscan/operators/gxf_codelet/gxf_codelet.hpp @@ -0,0 +1,101 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HOLOSCAN_OPERATORS_GXF_CODELET_GXF_CODELET_HPP +#define HOLOSCAN_OPERATORS_GXF_CODELET_GXF_CODELET_HPP + +#include "holoscan/core/gxf/gxf_operator.hpp" + +#include +#include +#include +#include +#include +#include + +#include "holoscan/core/gxf/gxf_component_info.hpp" + +/** + * @brief Wrap a GXF Codelet as a Holoscan Operator. + * + * This macro is designed to simplify the creation of Holoscan operators that encapsulate GXF + * Codelets. It defines a class derived from `holoscan::ops::GXFCodeletOp` and sets up the + * constructor to forward arguments to the base class while automatically setting the GXF type name. + * + * The resulting class is intended to act as a bridge, allowing GXF Codelets to be used directly + * within the Holoscan framework as operators, facilitating seamless integration and usage. + * + * Example Usage: + * + * ```cpp + * // Define a Holoscan operator that wraps a GXF Codelet within a Holoscan application + * class App : public holoscan::Application { + * ... + * HOLOSCAN_WRAP_GXF_CODELET_AS_OPERATOR(MyTensorOp, "nvidia::gxf::test::SendTensor") + * + * void compose() override { + * using namespace holoscan; + * ... + * auto tx = make_operator( + * "tx", + * make_condition(15), + * Arg("pool") = make_resource("pool")); + * ... + * } + * ... + * }; + * ``` + * + * @param class_name The name of the new Holoscan operator class. + * @param gxf_typename The GXF type name that identifies the specific GXF Codelet being wrapped. + */ +#define HOLOSCAN_WRAP_GXF_CODELET_AS_OPERATOR(class_name, gxf_typename) \ + class class_name : public ::holoscan::ops::GXFCodeletOp { \ + public: \ + HOLOSCAN_OPERATOR_FORWARD_TEMPLATE() \ + explicit class_name(ArgT&& arg, ArgsT&&... args) \ + : ::holoscan::ops::GXFCodeletOp(gxf_typename, std::forward(arg), \ + std::forward(args)...) {} \ + class_name() : ::holoscan::ops::GXFCodeletOp(gxf_typename) {} \ + }; + +namespace holoscan::ops { + +class GXFCodeletOp : public holoscan::ops::GXFOperator { + public: + template + explicit GXFCodeletOp(const char* gxf_typename, ArgsT&&... args) + : holoscan::ops::GXFOperator(std::forward(args)...) { + gxf_typename_ = gxf_typename; + } + + GXFCodeletOp() = default; + + const char* gxf_typename() const override; + + void setup(OperatorSpec& spec) override; + + protected: + void set_parameters() override; + + std::shared_ptr gxf_component_info_; ///< The GXF component info. + std::list> parameters_; ///< The fake parameters for the description. +}; + +} // namespace holoscan::ops + +#endif /* HOLOSCAN_OPERATORS_GXF_CODELET_GXF_CODELET_HPP */ diff --git a/include/holoscan/operators/holoviz/buffer_info.hpp b/include/holoscan/operators/holoviz/buffer_info.hpp index 03304309..a8f80d36 100644 --- a/include/holoscan/operators/holoviz/buffer_info.hpp +++ b/include/holoscan/operators/holoviz/buffer_info.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -31,14 +31,14 @@ struct BufferInfo { /** * Initialize with tensor * - * @returns error code + * @return error code */ gxf_result_t init(const nvidia::gxf::Handle& tensor); /** * Initialize with video buffer * - * @returns error code + * @return error code */ gxf_result_t init(const nvidia::gxf::Handle& video); diff --git a/include/holoscan/operators/holoviz/holoviz.hpp b/include/holoscan/operators/holoviz/holoviz.hpp index 4543b1c7..91ae2dea 100644 --- a/include/holoscan/operators/holoviz/holoviz.hpp +++ b/include/holoscan/operators/holoviz/holoviz.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef HOLOSCAN_OPERATORS_HOLOVIZ_HOLOVIZ_HPP -#define HOLOSCAN_OPERATORS_HOLOVIZ_HOLOVIZ_HPP +#ifndef INCLUDE_HOLOSCAN_OPERATORS_HOLOVIZ_HOLOVIZ_HPP +#define INCLUDE_HOLOSCAN_OPERATORS_HOLOVIZ_HOLOVIZ_HPP #include #include @@ -48,9 +48,20 @@ struct BufferInfo; * * This high-speed viewer handles compositing, blending, and visualization of RGB or RGBA images, * masks, geometric primitives, text and depth maps. The operator can auto detect the format of the - * input tensors when only the `receivers` parameter list is specified. Else the input specification - * can be set at creation time using the `tensors` parameter or at runtime when passing input - * specifications to the `input_specs` port. + * input tensors acquired at the `receivers` port. Else the input specification can be set at + * creation time using the `tensors` parameter or at runtime when passing input specifications to + * the `input_specs` port. + * + * Depth maps and 3D geometry are rendered in 3D and support camera movement. The camera is + * controlled using the mouse: + * - Orbit (LMB) + * - Pan (LMB + CTRL | MMB) + * - Dolly (LMB + SHIFT | RMB | Mouse wheel) + * - Look Around (LMB + ALT | LMB + CTRL + SHIFT) + * - Zoom (Mouse wheel + SHIFT) + * Or by providing new values at the `camera_eye_input`, `camera_look_at_input` or `camera_up_input` + * input ports. The camera pose can be output at the `camera_pose_output` port when + * `enable_camera_pose_output` is set to `true`. * * ==Named Inputs== * @@ -75,6 +86,12 @@ struct BufferInfo; * GXF_VIDEO_FORMAT_RGBA and be in device memory. This input port only exists if * `enable_render_buffer_input` was set to true, in which case `compute` will only be * called when a message arrives on this input. + * - **camera_eye_input** : `std::array` (optional) + * - Camera eye position. The camera is animated to reach the new position. + * - **camera_look_at_input** : `std::array` (optional) + * - Camera look at position. The camera is animated to reach the new position. + * - **camera_up_input** : : `std::array` (optional) + * - Camera up vector. The camera is animated to reach the new vector. * * ==Named Outputs== * @@ -84,10 +101,11 @@ struct BufferInfo; * GXF_VIDEO_FORMAT_RGBA and will be in device memory. This output is useful for offline * rendering or headless mode. This output port only exists if `enable_render_buffer_output` * was set to true. - * - **camera_pose_output** : `std::array` (optional) - * - The camera pose. The parameters returned represent the values of a 4x4 row major - * projection matrix. This output port only exists if `enable_camera_pose_output` was set to - * true. + * - **camera_pose_output** : `std::array` or `nvidia::gxf::Pose3D` (optional) + * - Output the camera pose. Depending on the value of `camera_pose_output_type` this outputs a + * 4x4 row major projection matrix (type `std::array`) or the camera extrinsics + * model (type `nvidia::gxf::Pose3D`). This output port only exists if + * `enable_camera_pose_output` was set to `True`. * * ==Parameters== * @@ -96,18 +114,10 @@ struct BufferInfo; * - type: `std::vector>` * - **enable_render_buffer_input**: Enable `render_buffer_input` (default: `false`) * - type: `bool` - * - **render_buffer_input**: Input for an empty render buffer, type `gxf::VideoBuffer`. - * - type: `gxf::Handle` * - **enable_render_buffer_output**: Enable `render_buffer_output` (default: `false`) * - type: `bool` - * - **render_buffer_output**: Output for a filled render buffer. If an input render buffer is - * specified at `render_buffer_input` it uses that one, otherwise it allocates a new buffer. - * - type: `gxf::Handle` * - **enable_camera_pose_output**: Enable `camera_pose_output` (default: `false`) * - type: `bool` - * - **camera_pose_output**: Output the camera pose. The camera parameters are returned in a - * 4x4 row major projection matrix. - * - type: `std::array` * - **tensors**: List of input tensor specifications (default: `[]`) * - type: `std::vector` * - **name**: name of the tensor containing the input data to display @@ -166,8 +176,8 @@ struct BufferInfo; * - type: `std::vector>` * - **window_title**: Title on window canvas (default: `"Holoviz"`) * - type: `std::string` - * - **display_name**: In exclusive mode, name of display to use as shown with xrandr (default: - * `DP-0`) + * - **display_name**: In exclusive mode, name of display to use as shown with `xrandr` + * or `hwinfo --monitor` (default: `DP-0`) * - type: `std::string` * - **width**: Window width or display resolution width if in exclusive or fullscreen mode * (default: `1920`) @@ -193,6 +203,36 @@ struct BufferInfo; * - type: `std::string` * - **cuda_stream_pool**: Instance of gxf::CudaStreamPool * - type: `gxf::Handle` + * - **camera_pose_output_type**: Type of data output at `camera_pose_output`. Supported values are + * `projection_matrix` and `extrinsics_model`. Default value is `projection_matrix`. + * - type: `std::string` + * - **camera_eye**: Initial camera eye position. + * - type: `std::array` + * - **camera_look_at**: Initial camera look at position. + * - type: `std::array` + * - **camera_up**: Initial camera up vector. + * - type: `std::array` + * + * ==Device Memory Requirements== + * + * If `render_buffer_input` is enabled, the provided buffer is used and no memory block will be + * allocated. Otherwise, when using this operator with a `BlockMemoryPool`, a single device memory + * block is needed (`storage_type` = 1). The size of this memory block can be determined by + * rounding the width and height up to the nearest even size and then padding the rows as needed so + * that the row stride is a multiple of 256 bytes. C++ code to calculate the block size is as + * follows: + * + * ```cpp + * #include + * + * int64_t get_block_size(int32_t height, int32_t width) { + * int32_t height_even = height + (height & 1); + * int32_t width_even = width + (width & 1); + * int64_t row_bytes = width_even * 4; // 4 bytes per pixel for 8-bit RGBA + * int64_t row_stride = (row_bytes % 256 == 0) ? row_bytes : ((row_bytes / 256 + 1) * 256); + * return height_even * row_stride; + * } + * ``` * * ==Notes== * @@ -266,13 +306,7 @@ struct BufferInfo; * * The type of geometry drawn can be selected by setting `depth_map_render_mode`. * - * Depth maps are rendered in 3D and support camera movement. The camera is controlled using the - * mouse: - * - Orbit (LMB) - * - Pan (LMB + CTRL | MMB) - * - Dolly (LMB + SHIFT | RMB | Mouse wheel) - * - Look Around (LMB + ALT | LMB + CTRL + SHIFT) - * - Zoom (Mouse wheel + SHIFT) + * Depth maps are rendered in 3D and support camera movement. * * 4. Output * @@ -359,17 +393,17 @@ class HolovizOp : public Operator { InputSpec(const std::string& tensor_name, const std::string& type_str); /** - * @returns an InputSpec from the YAML form output by description() + * @return an InputSpec from the YAML form output by description() */ explicit InputSpec(const std::string& yaml_description); /** - * @returns true if the input spec is valid + * @return true if the input spec is valid */ explicit operator bool() const noexcept { return !tensor_name_.empty(); } /** - * @returns a YAML string representation of the InputSpec + * @return a YAML string representation of the InputSpec */ std::string description() const; @@ -425,6 +459,9 @@ class HolovizOp : public Operator { void set_input_spec_geometry(const InputSpec& input_spec); void read_frame_buffer(InputContext& op_input, OutputContext& op_output, ExecutionContext& context); + void render_color_image(const InputSpec& input_spec, BufferInfo& buffer_info); + void render_geometry(const ExecutionContext& context, const InputSpec& input_spec, + BufferInfo& buffer_info); void render_depth_map(InputSpec* const input_spec_depth_map, const BufferInfo& buffer_info_depth_map, InputSpec* const input_spec_depth_map_color, @@ -451,6 +488,10 @@ class HolovizOp : public Operator { Parameter> window_close_scheduling_term_; Parameter> allocator_; Parameter font_path_; + Parameter camera_pose_output_type_; + Parameter> camera_eye_; + Parameter> camera_look_at_; + Parameter> camera_up_; // internal state viz::InstanceHandle instance_ = nullptr; @@ -461,7 +502,11 @@ class HolovizOp : public Operator { bool render_buffer_output_enabled_; bool camera_pose_output_enabled_; bool is_first_tick_ = true; + + std::array camera_eye_cur_; //< current camera eye position + std::array camera_look_at_cur_; //< current camera look at position + std::array camera_up_cur_; //< current camera up vector }; } // namespace holoscan::ops -#endif /* HOLOSCAN_OPERATORS_HOLOVIZ_HOLOVIZ_HPP */ +#endif /* INCLUDE_HOLOSCAN_OPERATORS_HOLOVIZ_HOLOVIZ_HPP */ diff --git a/include/holoscan/operators/inference/inference.hpp b/include/holoscan/operators/inference/inference.hpp index 65ad4258..c1c0246b 100644 --- a/include/holoscan/operators/inference/inference.hpp +++ b/include/holoscan/operators/inference/inference.hpp @@ -69,6 +69,7 @@ namespace holoscan::ops { * - **pre_processor_map**: Pre processed data to model map. * - **device_map**: Mapping of model (`DataMap`) to GPU ID for inference. Optional. * - **backend_map**: Mapping of model (`DataMap`) to backend type for inference. + * - **temporal_map**: Mapping of model (`DataMap`) to a frame delay for model inference. Optional. * Backend options: `"trt"` or `"torch"`. Optional. * - **in_tensor_names**: Input tensors (`std::vector`). Optional. * - **out_tensor_names**: Output tensors (`std::vector`). Optional. @@ -83,6 +84,14 @@ namespace holoscan::ops { * (default: `false`). * - **cuda_stream_pool**: `holoscan::CudaStreamPool` instance to allocate CUDA streams. Optional * (default: `nullptr`). + * + * ==Device Memory Requirements== + * + * When using this operator with a `BlockMemoryPool`, `num_blocks` must be greater than or equal to + * the number of output tensors that will be produced. The `block_size` in bytes must be greater + * than or equal to the largest output tensor (in bytes). If `output_on_cuda` is true, the blocks + * should be in device memory (`storage_type`=1), otherwise they should be CUDA pinned host memory + * (`storage_type`=0). */ class InferenceOp : public holoscan::Operator { public: @@ -139,6 +148,9 @@ class InferenceOp : public holoscan::Operator { /// @brief Map with key as model name and value as GPU ID for inference Parameter device_map_; + /// @brief Map with key as model name and value as frame delay for model inference + Parameter temporal_map_; + /// @brief Input tensor names Parameter> in_tensor_names_; diff --git a/include/holoscan/operators/inference_processor/inference_processor.hpp b/include/holoscan/operators/inference_processor/inference_processor.hpp index cce1fa6c..78198a9f 100644 --- a/include/holoscan/operators/inference_processor/inference_processor.hpp +++ b/include/holoscan/operators/inference_processor/inference_processor.hpp @@ -71,6 +71,14 @@ namespace holoscan::ops { * - **config_path**: File path to the config file. Optional (default: `""`). * - **disable_transmitter**: If `true`, disable the transmitter output port of the operator. * Optional (default: `false`). + * + * ==Device Memory Requirements== + * + * When using this operator with a `BlockMemoryPool`, `num_blocks` must be greater than or equal to + * the number of output tensors that will be produced. The `block_size` in bytes must be greater + * than or equal to the largest output tensor (in bytes). If `output_on_cuda` is true, the blocks + * should be in device memory (`storage_type`=1), otherwise they should be CUDA pinned host memory + * (`storage_type`=0). */ class InferenceProcessorOp : public holoscan::Operator { public: diff --git a/include/holoscan/operators/segmentation_postprocessor/segmentation_postprocessor.hpp b/include/holoscan/operators/segmentation_postprocessor/segmentation_postprocessor.hpp index af133744..6ed17971 100644 --- a/include/holoscan/operators/segmentation_postprocessor/segmentation_postprocessor.hpp +++ b/include/holoscan/operators/segmentation_postprocessor/segmentation_postprocessor.hpp @@ -40,14 +40,14 @@ namespace holoscan::ops { * ==Named Inputs== * * - **in_tensor** : `nvidia::gxf::Tensor` - * - Expects a message containing a 32-bit floating point tensor with name + * - Expects a message containing a 32-bit floating point device tensor with name * `in_tensor_name`. The expected data layout of this tensor is HWC, NCHW or NHWC format as - * specified via `data_format`. + * specified via `data_format`. If batch dimension, N, is present it should be size 1. * * ==Named Outputs== * * - **out_tensor** : `nvidia::gxf::Tensor` - * - Emits a message containing a tensor named "out_tensor" that contains the segmentation + * - Emits a message containing a device tensor named "out_tensor" that contains the segmentation * labels. This tensor will have unsigned 8-bit integer data type and shape (H, W, 1). * * ==Parameters== @@ -58,6 +58,11 @@ namespace holoscan::ops { * - **data_format**: Data format of network output. Optional (default: `"hwc"`). * - **cuda_stream_pool**: `holoscan::CudaStreamPool` instance to allocate CUDA streams. * Optional (default: `nullptr`). + * + * ==Device Memory Requirements== + * + * When used with a `BlockMemoryPool`, this operator requires only a single device memory + * block (`storage_type` = 1) of size `height * width` bytes. */ class SegmentationPostprocessorOp : public Operator { public: diff --git a/include/holoscan/operators/v4l2_video_capture/v4l2_video_capture.hpp b/include/holoscan/operators/v4l2_video_capture/v4l2_video_capture.hpp index d07c7a79..0df55093 100644 --- a/include/holoscan/operators/v4l2_video_capture/v4l2_video_capture.hpp +++ b/include/holoscan/operators/v4l2_video_capture/v4l2_video_capture.hpp @@ -33,7 +33,7 @@ namespace holoscan::ops { * * Inputs a video stream from a V4L2 node, including USB cameras and HDMI IN. * - Input stream is on host. If no pixel format is specified in the yaml configuration file, the - * pixel format will be automatically selected. However, only `AB24` and `YUYV` are then + * pixel format will be automatically selected. However, only `AB24`, `YUYV`, and MJPG are then * supported. * If a pixel format is specified in the yaml file, then this format will be used. However, note * that the operator then expects that this format can be encoded as RGBA32. If not, the behavior @@ -71,6 +71,24 @@ namespace holoscan::ops { * - When not set by the user, V4L2_CID_AUTOGAIN is set to false (if supported). * - When set by the user, V4L2_CID_AUTOGAIN is set to true (if supported). The provided value is * then used to set V4L2_CID_GAIN. + * + * ==Device Memory Requirements== + * + * When using this operator with a `BlockMemoryPool`, a single device memory block is needed + * (`storage_type` = 1). The size of this memory block can be determined by rounding the width and + * height up to the nearest even size and then padding the rows as needed so that the row stride is + * a multiple of 256 bytes. C++ code to calculate the block size is as follows: + * + * ```cpp + * #include + * + * int64_t get_block_size(int32_t height, int32_t width) { + * int32_t height_even = height + (height & 1); + * int32_t width_even = width + (width & 1); + * int64_t row_bytes = width_even * 4; // 4 bytes per pixel for 8-bit RGBA + * int64_t row_stride = (row_bytes % 256 == 0) ? row_bytes : ((row_bytes / 256 + 1) * 256); + * return height_even * row_stride; + * } */ class V4L2VideoCaptureOp : public Operator { public: @@ -109,6 +127,7 @@ class V4L2VideoCaptureOp : public Operator { void v4l2_read_buffer(v4l2_buffer& buf); void YUYVToRGBA(const void* yuyv, void* rgba, size_t width, size_t height); + void MJPEGToRGBA(const void* mjpg, void* rgba, size_t width, size_t height); struct Buffer { void* ptr; diff --git a/include/holoscan/utils/holoinfer_utils.hpp b/include/holoscan/utils/holoinfer_utils.hpp index 2a9ec3d3..dfce5c24 100644 --- a/include/holoscan/utils/holoinfer_utils.hpp +++ b/include/holoscan/utils/holoinfer_utils.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -43,7 +43,7 @@ namespace holoscan::utils { * @param module Module that called for data extraction * @param context GXF execution context * @param cuda_stream_handler Cuda steam handler - * @returns GXF result code + * @return GXF result code */ gxf_result_t get_data_per_model(InputContext& op_input, const std::vector& in_tensors, HoloInfer::DataMap& data_per_input_tensor, @@ -67,7 +67,7 @@ gxf_result_t get_data_per_model(InputContext& op_input, const std::vector PUBLIC + $ $ $ $ diff --git a/modules/holoinfer/src/include/holoinfer.hpp b/modules/holoinfer/src/include/holoinfer.hpp index ac1e3835..8a0be6dd 100644 --- a/modules/holoinfer/src/include/holoinfer.hpp +++ b/modules/holoinfer/src/include/holoinfer.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -40,7 +40,7 @@ class _HOLOSCAN_EXTERNAL_API_ InferContext { * * @param inference_specs Pointer to inference specifications * - * @returns InferStatus with appropriate holoinfer_code and message. + * @return InferStatus with appropriate holoinfer_code and message. */ InferStatus set_inference_params(std::shared_ptr& inference_specs); @@ -51,14 +51,14 @@ class _HOLOSCAN_EXTERNAL_API_ InferContext { * @param preprocess_data_map Map of model names as key mapped to the preprocessed input data * @param output_data_map Map of tensor names as key mapped to the inferred data * - * @returns InferStatus with appropriate holoinfer_code and message. + * @return InferStatus with appropriate holoinfer_code and message. */ InferStatus execute_inference(DataMap& preprocess_data_map, DataMap& output_data_map); /** * Gets output dimension per model * - * @returns Map of model as key mapped to the output dimension (of inferred data) + * @return Map of model as key mapped to the output dimension (of inferred data) */ DimType get_output_dimensions() const; @@ -78,7 +78,7 @@ class _HOLOSCAN_EXTERNAL_API_ ProcessorContext { * @param process_operations Map of tensor name as key, mapped to list of operations to be * applied in sequence on the tensor * - * @returns InferStatus with appropriate holoinfer_code and message. + * @return InferStatus with appropriate holoinfer_code and message. */ InferStatus initialize(const MultiMappings& process_operations, const std::string config_path); @@ -95,7 +95,7 @@ class _HOLOSCAN_EXTERNAL_API_ ProcessorContext { * @param dimension_map Map is updated with model name as key mapped to dimension of processed * data as a vector * - * @returns InferStatus with appropriate holoinfer_code and message. + * @return InferStatus with appropriate holoinfer_code and message. */ InferStatus process(const MultiMappings& tensor_oper_map, const MultiMappings& in_out_tensor_map, DataMap& processed_result_map, @@ -105,7 +105,7 @@ class _HOLOSCAN_EXTERNAL_API_ ProcessorContext { * Get output data per Tensor * Toolkit supports one output per Tensor, in float32 type * - * @returns Map of tensor name as key mapped to the output float32 type data as a vector + * @return Map of tensor name as key mapped to the output float32 type data as a vector */ DataMap get_processed_data() const; @@ -113,7 +113,7 @@ class _HOLOSCAN_EXTERNAL_API_ ProcessorContext { * Get output dimension per model * Toolkit supports one output per model * - * @returns Map of model as key mapped to the output dimension (of processed data) as a vector + * @return Map of model as key mapped to the output dimension (of processed data) as a vector */ DimType get_processed_data_dims() const; }; diff --git a/modules/holoinfer/src/include/holoinfer_buffer.hpp b/modules/holoinfer/src/include/holoinfer_buffer.hpp index 4da75d9b..3909aeb5 100644 --- a/modules/holoinfer/src/include/holoinfer_buffer.hpp +++ b/modules/holoinfer/src/include/holoinfer_buffer.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -44,7 +44,7 @@ namespace inference { * * @param element_type Input data type. Float32 is the only supported element type. * - * @returns Bytes used in storing element type + * @return Bytes used in storing element type */ uint32_t get_element_size(holoinfer_datatype t) noexcept; @@ -87,21 +87,21 @@ class DeviceBuffer { /** * @brief Get the data buffer * - * @returns Void pointer to the buffer + * @return Void pointer to the buffer */ void* data(); /** * @brief Get the size of the allocated buffer * - * @returns size + * @return size */ size_t size() const; /** * @brief Get the bytes allocated * - * @returns allocated bytes + * @return allocated bytes */ size_t get_bytes() const; @@ -206,6 +206,7 @@ struct InferenceSpecs { * @param pre_processor_map Map with model name as key, input tensor names in vector form as value * @param inference_map Map with model name as key, output tensor names in vector form as value * @param device_map Map with model name as key, GPU ID for inference as value + * @param temporal_map Map with model name as key, frame number to skip for inference as value * @param is_engine_path Input path to model is trt engine * @param oncpu Perform inference on CPU * @param parallel_proc Perform parallel inference of multiple models @@ -216,14 +217,15 @@ struct InferenceSpecs { InferenceSpecs(const std::string& backend, const Mappings& backend_map, const Mappings& model_path_map, const MultiMappings& pre_processor_map, const MultiMappings& inference_map, const Mappings& device_map, - bool is_engine_path, bool oncpu, bool parallel_proc, bool use_fp16, - bool cuda_buffer_in, bool cuda_buffer_out) + const Mappings& temporal_map, bool is_engine_path, bool oncpu, bool parallel_proc, + bool use_fp16, bool cuda_buffer_in, bool cuda_buffer_out) : backend_type_(backend), backend_map_(backend_map), model_path_map_(model_path_map), pre_processor_map_(pre_processor_map), inference_map_(inference_map), device_map_(device_map), + temporal_map_(temporal_map), is_engine_path_(is_engine_path), oncuda_(!oncpu), parallel_processing_(parallel_proc), @@ -249,6 +251,12 @@ struct InferenceSpecs { */ Mappings get_device_map() const { return device_map_; } + /** + * @brief Get the Temporal map + * @return Mappings data + */ + Mappings get_temporal_map() const { return temporal_map_; } + /// @brief Backend type (for all models) std::string backend_type_{""}; @@ -267,6 +275,9 @@ struct InferenceSpecs { /// @brief Map with key as model name and value as GPU ID for inference Mappings device_map_; + /// @brief Map with key as model name and frame number to skip for inference as value + Mappings temporal_map_; + /// @brief Flag showing if input model path is path to engine files bool is_engine_path_ = false; @@ -302,7 +313,7 @@ struct InferenceSpecs { * @param keyname Storage name in the map against the created DataBuffer * @param allocate_cuda flag to allocate cuda buffer * @param device_id GPU ID to allocate buffers on - * @returns InferStatus with appropriate code and message + * @return InferStatus with appropriate code and message */ InferStatus allocate_buffers(DataMap& buffers, std::vector& dims, holoinfer_datatype datatype, const std::string& keyname, diff --git a/modules/holoinfer/src/include/holoinfer_utils.hpp b/modules/holoinfer/src/include/holoinfer_utils.hpp index c8419081..2b38a99f 100644 --- a/modules/holoinfer/src/include/holoinfer_utils.hpp +++ b/modules/holoinfer/src/include/holoinfer_utils.hpp @@ -58,7 +58,7 @@ cudaError_t check_cuda(cudaError_t result); * @param module Module of error occurrence * @param submodule Submodule/Function of error occurrence with the error message (as string) * - * @returns GXF Error code: GXF_FAILURE + * @return GXF Error code: GXF_FAILURE */ gxf_result_t _HOLOSCAN_EXTERNAL_API_ report_error(const std::string& module, const std::string& submodule); diff --git a/modules/holoinfer/src/infer/torch/core.cpp b/modules/holoinfer/src/infer/torch/core.cpp index 585c0c66..083f4c53 100644 --- a/modules/holoinfer/src/infer/torch/core.cpp +++ b/modules/holoinfer/src/infer/torch/core.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -284,7 +284,6 @@ InferStatus TorchInferImpl::transfer_to_output( default: return InferStatus(holoinfer_code::H_ERROR, "Unsupported datatype for transfer."); } - return InferStatus(); } void TorchInfer::print_model_details() { @@ -499,7 +498,7 @@ InferStatus TorchInfer::do_inference(const std::vectoroutput_names_[a]).toTensor(); - auto status = impl_->transfer_to_output(output_buffer, current_tensor, a); + auto status = impl_->transfer_to_output(output_buffer, std::move(current_tensor), a); if (status.get_code() != holoinfer_code::H_SUCCESS) { HOLOSCAN_LOG_ERROR("Transfer of Tensor {} failed in inferece core.", impl_->output_names_[a]); diff --git a/modules/holoinfer/src/infer/trt/utils.hpp b/modules/holoinfer/src/infer/trt/utils.hpp index 046dd8cd..430348a1 100644 --- a/modules/holoinfer/src/infer/trt/utils.hpp +++ b/modules/holoinfer/src/infer/trt/utils.hpp @@ -41,7 +41,11 @@ namespace inference { */ class Logger : public nvinfer1::ILogger { void log(Severity severity, const char* msg) noexcept override { - if (severity <= Severity::kWARNING) { HOLOSCAN_LOG_INFO(msg); } + if (severity <= Severity::kWARNING) { + try { // ignore potential fmt::format_error exception + HOLOSCAN_LOG_INFO(msg); + } catch (std::exception& e) {} + } }; }; diff --git a/modules/holoinfer/src/manager/infer_manager.cpp b/modules/holoinfer/src/manager/infer_manager.cpp index 1b65013a..273014bd 100644 --- a/modules/holoinfer/src/manager/infer_manager.cpp +++ b/modules/holoinfer/src/manager/infer_manager.cpp @@ -35,6 +35,7 @@ InferStatus ManagerInfer::set_inference_params(std::shared_ptr& auto multi_model_map = inference_specs->get_path_map(); auto device_map = inference_specs->get_device_map(); + auto temporal_map = inference_specs->get_temporal_map(); auto backend_type = inference_specs->backend_type_; auto backend_map = inference_specs->get_backend_map(); cuda_buffer_in_ = inference_specs->cuda_buffer_in_; @@ -74,27 +75,37 @@ InferStatus ManagerInfer::set_inference_params(std::shared_ptr& } // set up gpu-dt - if (device_map.find("gpu-dt") != device_map.end()) { - auto dev_id = std::stoi(device_map.at("gpu-dt")); - device_gpu_dt = dev_id; - HOLOSCAN_LOG_INFO("ID of data transfer GPU updated to: {}", device_gpu_dt); - } - std::set unique_gpu_ids; - unique_gpu_ids.insert(device_gpu_dt); - - for (auto const& [_, gpu_id] : device_map) { - auto dev_id = std::stoi(gpu_id); - cudaDeviceProp device_prop; - auto cstatus = cudaGetDeviceProperties(&device_prop, dev_id); - if (cstatus != cudaSuccess) { - HOLOSCAN_LOG_ERROR("Error in getting device properties for gpu id: {}.", dev_id); - HOLOSCAN_LOG_INFO("Use integer id's displayed after the GPU after executing: nvidia-smi -L"); - status.set_message("Incorrect gpu id in the configuration."); - return status; + + try { + if (device_map.find("gpu-dt") != device_map.end()) { + auto dev_id = std::stoi(device_map.at("gpu-dt")); + device_gpu_dt = dev_id; + HOLOSCAN_LOG_INFO("ID of data transfer GPU updated to: {}", device_gpu_dt); } - unique_gpu_ids.insert(dev_id); - } + + unique_gpu_ids.insert(device_gpu_dt); + + for (auto const& [_, gpu_id] : device_map) { + auto dev_id = std::stoi(gpu_id); + cudaDeviceProp device_prop; + auto cstatus = cudaGetDeviceProperties(&device_prop, dev_id); + if (cstatus != cudaSuccess) { + HOLOSCAN_LOG_ERROR("Error in getting device properties for gpu id: {}.", dev_id); + HOLOSCAN_LOG_INFO( + "Use integer id's displayed after the GPU after executing: nvidia-smi -L"); + status.set_message("Incorrect gpu id in the configuration."); + return status; + } + unique_gpu_ids.insert(dev_id); + } + } catch (std::invalid_argument const& ex) { + HOLOSCAN_LOG_ERROR("Invalid argument in Device map: {}", ex.what()); + raise_error("Inference Manager", "Error in Device map."); + } catch (std::out_of_range const& ex) { + HOLOSCAN_LOG_ERROR("Invalid range in Device map: {}", ex.what()); + raise_error("Inference Manager", "Error in Device map."); + } catch (...) { raise_error("Inference Manager", "Error in Device map."); } auto vec_unique_gpu_ids = std::vector(unique_gpu_ids.begin(), unique_gpu_ids.end()); @@ -176,6 +187,21 @@ InferStatus ManagerInfer::set_inference_params(std::shared_ptr& infer_param_.at(model_name)->set_device_id(device_id); + unsigned int temporal_id = 1; + if (temporal_map.find(model_name) != temporal_map.end()) { + try { + temporal_id = std::stoul(temporal_map.at(model_name)); + HOLOSCAN_LOG_INFO("Temporal id: {} for Model: {}", temporal_id, model_name); + } catch (std::invalid_argument const& ex) { + HOLOSCAN_LOG_ERROR("Invalid argument in Temporal map: {}", ex.what()); + throw; + } catch (std::out_of_range const& ex) { + HOLOSCAN_LOG_ERROR("Invalid range in Temporal map: {}", ex.what()); + throw; + } + } + infer_param_.at(model_name)->set_temporal_id(temporal_id); + // Get input and output tensor maps of the model from inference_specs auto out_tensor_names = inference_specs->inference_map_.at(model_name); auto in_tensor_names = inference_specs->pre_processor_map_.at(model_name); @@ -259,6 +285,7 @@ InferStatus ManagerInfer::set_inference_params(std::shared_ptr& if (!new_ort_infer) { HOLOSCAN_LOG_ERROR(dlerror()); status.set_message("ONNX Runtime context setup failure."); + dlclose(handle); return status; } dlclose(handle); @@ -658,6 +685,8 @@ InferStatus ManagerInfer::execute_inference(DataMap& permodel_preprocess_data, DataMap& permodel_output_data) { InferStatus status = InferStatus(); + if (frame_counter_++ == UINT_MAX - 1) { frame_counter_ = 0; } + if (infer_param_.size() == 0) { status.set_code(holoinfer_code::H_ERROR); status.set_message( @@ -672,24 +701,27 @@ InferStatus ManagerInfer::execute_inference(DataMap& permodel_preprocess_data, std::map> inference_futures; s_time = std::chrono::steady_clock::now(); for (const auto& [model_instance, _] : infer_param_) { - if (!parallel_processing_) { - InferStatus infer_status = - run_core_inference(model_instance, permodel_preprocess_data, permodel_output_data); - if (infer_status.get_code() != holoinfer_code::H_SUCCESS) { - status.set_code(holoinfer_code::H_ERROR); - infer_status.display_message(); - status.set_message("Inference manager, Inference failed in execution for " + - model_instance); - return status; + auto temporal_id = infer_param_.at(model_instance)->get_temporal_id(); + if (frame_counter_ % temporal_id == 0) { + if (!parallel_processing_) { + InferStatus infer_status = + run_core_inference(model_instance, permodel_preprocess_data, permodel_output_data); + if (infer_status.get_code() != holoinfer_code::H_SUCCESS) { + status.set_code(holoinfer_code::H_ERROR); + infer_status.display_message(); + status.set_message("Inference manager, Inference failed in execution for " + + model_instance); + return status; + } + } else { + inference_futures.insert({model_instance, + std::async(std::launch::async, + std::bind(&ManagerInfer::run_core_inference, + this, + model_instance, + permodel_preprocess_data, + permodel_output_data))}); } - } else { - inference_futures.insert({model_instance, - std::async(std::launch::async, - std::bind(&ManagerInfer::run_core_inference, - this, - model_instance, - permodel_preprocess_data, - permodel_output_data))}); } } diff --git a/modules/holoinfer/src/manager/infer_manager.hpp b/modules/holoinfer/src/manager/infer_manager.hpp index e4e7adb3..a78944bd 100644 --- a/modules/holoinfer/src/manager/infer_manager.hpp +++ b/modules/holoinfer/src/manager/infer_manager.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -64,7 +64,7 @@ class ManagerInfer { * * @param inference_specs specifications for inference * - * @returns InferStatus with appropriate code and message + * @return InferStatus with appropriate code and message */ InferStatus set_inference_params(std::shared_ptr& inference_specs); @@ -74,7 +74,7 @@ class ManagerInfer { * @param preprocess_data_map Input DataMap with model name as key and DataBuffer as value * @param output_data_map Output DataMap with tensor name as key and DataBuffer as value * - * @returns InferStatus with appropriate code and message + * @return InferStatus with appropriate code and message */ InferStatus execute_inference(DataMap& preprocess_data_map, DataMap& output_data_map); @@ -85,7 +85,7 @@ class ManagerInfer { * @param permodel_preprocess_data Input DataMap with model name as key and DataBuffer as value * @param permodel_output_data Output DataMap with tensor name as key and DataBuffer as value * - * @returns InferStatus with appropriate code and message + * @return InferStatus with appropriate code and message */ InferStatus run_core_inference(const std::string& model_name, DataMap& permodel_preprocess_data, DataMap& permodel_output_data); @@ -98,14 +98,14 @@ class ManagerInfer { /** * @brief Get input dimension per model * - * @returns Map with model name as key and dimension as value + * @return Map with model name as key and dimension as value */ DimType get_input_dimensions() const; /** * @brief Get output dimension per tensor * - * @returns Map with tensor name as key and dimension as value + * @return Map with tensor name as key and dimension as value */ DimType get_output_dimensions() const; @@ -153,6 +153,9 @@ class ManagerInfer { /// Input buffer for multi-gpu inference std::map mgpu_input_buffer_; + /// Frame counter into the inference engine + unsigned int frame_counter_ = 0; + /// Data transfer GPU. Default: 0. Not configurable in this release. int device_gpu_dt = 0; diff --git a/modules/holoinfer/src/manager/process_manager.cpp b/modules/holoinfer/src/manager/process_manager.cpp index e0d2463c..c8e4acb5 100644 --- a/modules/holoinfer/src/manager/process_manager.cpp +++ b/modules/holoinfer/src/manager/process_manager.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -124,8 +124,9 @@ InferStatus ManagerProcessor::process( std::vector processed_dims; std::vector out_tensor_names, custom_strings; - // if operation is print, then no need to allocate output memory - if (operation.find("print") != std::string::npos) { + // if operation is print or export, then no need to allocate output memory + if (operation.find("print") != std::string::npos || + operation.find("export") != std::string::npos) { if (operation.compare("print") == 0 || operation.compare("print_int32") == 0) { std::cout << "Printing results from " << tensor_name << " -> "; } else { @@ -143,6 +144,20 @@ InferStatus ManagerProcessor::process( } operation = custom_strings[0]; custom_strings.erase(custom_strings.begin()); + } else if (operation.find("export_binary_classification_to_csv") != std::string::npos) { + std::istringstream cstrings(operation); + + std::string custom_string; + while (std::getline(cstrings, custom_string, ',')) { + custom_strings.push_back(custom_string); + } + if (custom_strings.size() != 5) { + return InferStatus( + holoinfer_code::H_ERROR, + "Process manager, Export output to CSV operation must generate 5 strings"); + } + operation = custom_strings[0]; + custom_strings.erase(custom_strings.begin()); } } } else { diff --git a/modules/holoinfer/src/manager/process_manager.hpp b/modules/holoinfer/src/manager/process_manager.hpp index f9a9c26c..009641f4 100644 --- a/modules/holoinfer/src/manager/process_manager.hpp +++ b/modules/holoinfer/src/manager/process_manager.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -51,7 +51,7 @@ class ManagerProcessor { * the tensor as vector of strings. Each value in the vector of strings is the supported * operation. * - * @returns InferStatus with appropriate code and message + * @return InferStatus with appropriate code and message */ InferStatus initialize(const MultiMappings& process_operations, const std::string config_path); @@ -65,7 +65,7 @@ class ManagerProcessor { * @param inferred_result_map Map with output tensor name as key, and related DataBuffer as * value * @param dimension_map Map with tensor name as key and related output dimension as value. - * @returns InferStatus with appropriate code and message + * @return InferStatus with appropriate code and message */ InferStatus process(const MultiMappings& tensor_oper_map, const MultiMappings& in_out_tensor_map, DataMap& inferred_result_map, @@ -80,7 +80,7 @@ class ManagerProcessor { * @param inferred_result_map Map Contains output tensor name as key, and related DataBuffer as * value * @param dimension_map Map with tensor name as key and related output dimension as value. - * @returns InferStatus with appropriate code and message + * @return InferStatus with appropriate code and message */ InferStatus process_multi_tensor_operation( const std::string tensor_name, const std::vector& tensor_oper_map, @@ -88,14 +88,14 @@ class ManagerProcessor { /* * @brief Get processed data * - * @returns DataMap with tensor name as key and related DataBuffer as value + * @return DataMap with tensor name as key and related DataBuffer as value */ DataMap get_processed_data() const; /* * @brief Get processed data dimensions * - * @returns DataMap with tensor name as key and related dimension as value + * @return DataMap with tensor name as key and related dimension as value */ DimType get_processed_data_dims() const; diff --git a/modules/holoinfer/src/params/infer_param.cpp b/modules/holoinfer/src/params/infer_param.cpp index a807c0dc..0197c7bc 100644 --- a/modules/holoinfer/src/params/infer_param.cpp +++ b/modules/holoinfer/src/params/infer_param.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,13 +22,18 @@ namespace holoscan { namespace inference { -Params::Params(const std::string& _model_path, const std::string& _infer_name, bool _usecuda) - : model_file_path_(_model_path), instance_name_(_infer_name), use_cuda_(_usecuda) {} +Params::Params(const std::string& _model_path, const std::string& _infer_name, bool _usecuda, + int _device_id) + : model_file_path_(_model_path), + instance_name_(_infer_name), + use_cuda_(_usecuda), + device_id_(_device_id) {} Params::Params() { model_file_path_ = ""; instance_name_ = ""; use_cuda_ = false; + device_id_ = 0; } bool Params::get_cuda_flag() const { @@ -63,6 +68,14 @@ int Params::get_device_id() const { return device_id_; } +void Params::set_temporal_id(unsigned int& temporal_id) { + temporal_id_ = temporal_id; +} + +unsigned int Params::get_temporal_id() const { + return temporal_id_; +} + void Params::set_tensor_names(const std::vector& _tensor_names, bool type) { if (type) { in_tensor_names_.assign(_tensor_names.begin(), _tensor_names.end()); diff --git a/modules/holoinfer/src/params/infer_param.hpp b/modules/holoinfer/src/params/infer_param.hpp index b5c53973..ecfb4df1 100644 --- a/modules/holoinfer/src/params/infer_param.hpp +++ b/modules/holoinfer/src/params/infer_param.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,15 +28,17 @@ namespace inference { class Params { public: Params(); - Params(const std::string&, const std::string&, bool); + Params(const std::string&, const std::string&, bool, int device_id_ = 0); const std::string get_model_path() const; const std::string get_instance_name() const; const std::vector get_input_tensor_names() const; const std::vector get_output_tensor_names() const; bool get_cuda_flag() const; int get_device_id() const; + unsigned int get_temporal_id() const; void set_model_path(const std::string&); void set_device_id(int); + void set_temporal_id(unsigned int&); void set_instance_name(const std::string&); void set_cuda_flag(bool); void set_tensor_names(const std::vector&, bool); @@ -46,6 +48,7 @@ class Params { std::string model_file_path_; std::string instance_name_; int device_id_; + unsigned int temporal_id_; std::vector in_tensor_names_; std::vector out_tensor_names_; }; diff --git a/modules/holoinfer/src/process/data_processor.cpp b/modules/holoinfer/src/process/data_processor.cpp index beab0efc..1996a8fe 100644 --- a/modules/holoinfer/src/process/data_processor.cpp +++ b/modules/holoinfer/src/process/data_processor.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,9 +19,12 @@ #include #include #include +#include #include #include +#include + namespace holoscan { namespace inference { @@ -44,35 +47,35 @@ InferStatus DataProcessor::initialize(const MultiMappings& process_operations, } for (const auto& _op : _operations) { - if (_op.find("print") == std::string::npos) { - if (supported_transforms_.find(_op) != supported_transforms_.end()) { - // In future releases, this will be generalized with addition of more transforms. - if (_op.compare("generate_boxes") == 0) { - // unique key is created as a combination of input tensors and operation - auto key = fmt::format("{}-{}", p_op.first, _op); - HOLOSCAN_LOG_INFO("Transform map updated with key: {}", key); - transforms_.insert({key, std::make_unique(config_path_)}); - - std::vector tensor_tokens; - string_split(p_op.first, tensor_tokens, ':'); - - auto status = transforms_.at(key)->initialize(tensor_tokens); - if (status.get_code() != holoinfer_code::H_SUCCESS) { - status.display_message(); - return status; - } + std::vector operation_strings; + string_split(_op, operation_strings, ','); + const std::string operation = operation_strings[0]; + if (supported_transforms_.find(operation) != supported_transforms_.end()) { + // In future releases, this will be generalized with addition of more transforms. + if (operation == "generate_boxes") { + // unique key is created as a combination of input tensors and operation + auto key = fmt::format("{}-{}", p_op.first, operation); + HOLOSCAN_LOG_INFO("Transform map updated with key: {}", key); + transforms_.insert({key, std::make_unique(config_path_)}); + + std::vector tensor_tokens; + string_split(p_op.first, tensor_tokens, ':'); + + auto status = transforms_.at(key)->initialize(tensor_tokens); + if (status.get_code() != holoinfer_code::H_SUCCESS) { + status.display_message(); + return status; } - } else if (supported_compute_operations_.find(_op) == supported_compute_operations_.end()) { - return InferStatus(holoinfer_code::H_ERROR, - "Data processor, Operation " + _op + " not supported."); - } - } else { - if (supported_print_operations_.find(_op) == supported_print_operations_.end() && - _op.find("print_custom_binary_classification") == std::string::npos) { - return InferStatus(holoinfer_code::H_ERROR, - "Data processor, Print operation: " + _op + " not supported."); + continue; } } + + if (supported_compute_operations_.find(operation) == supported_compute_operations_.end() && + supported_print_operations_.find(operation) == supported_print_operations_.end() && + supported_export_operations_.find(operation) == supported_export_operations_.end()) { + return InferStatus(holoinfer_code::H_ERROR, + "Data processor, Operation " + _op + " not supported."); + } } } @@ -147,6 +150,55 @@ InferStatus DataProcessor::print_custom_binary_classification( return InferStatus(); } +InferStatus DataProcessor::export_binary_classification_to_csv( + const std::vector& dimensions, const void* indata, + const std::vector& custom_strings) { + size_t dsize = accumulate(dimensions.begin(), dimensions.end(), 1, std::multiplies()); + if (dsize < 1) { + HOLOSCAN_LOG_ERROR("Input data size must be at least 1."); + return InferStatus(holoinfer_code::H_ERROR, "Data processor, Incorrect input data size"); + } + auto indata_float = static_cast(indata); + + auto strings_count = custom_strings.size(); + if (strings_count != 4) { + HOLOSCAN_LOG_INFO("The number of custom strings passed : {}", strings_count); + HOLOSCAN_LOG_INFO( + "This is export binary classification results to CSV file operation, size must be 4."); + return InferStatus(); + } + + if (dsize != 2) { + HOLOSCAN_LOG_INFO("Input data size: {}", dsize); + HOLOSCAN_LOG_INFO( + "This is export binary classification results to CSV file operation, size must be 2."); + return InferStatus(); + } + + if (!data_exporter_) { + const std::string app_name = custom_strings[0]; + const std::vector columns = { + custom_strings[1], custom_strings[2], custom_strings[3]}; + data_exporter_ = std::make_unique(app_name, columns); + } + + auto first_value = 1.0 / (1 + exp(-indata_float[0])); + auto second_value = 1.0 / (1 + exp(-indata_float[1])); + std::vector data; + if (first_value > second_value) { + std::ostringstream confidence_score_ss; + confidence_score_ss << first_value; + data = {"1", "0", confidence_score_ss.str()}; + } else { + std::ostringstream confidence_score_ss; + confidence_score_ss << second_value; + data = {"0", "1", confidence_score_ss.str()}; + } + data_exporter_->export_data(data); + + return InferStatus(); +} + InferStatus DataProcessor::scale_intensity_cpu(const std::vector& dimensions, const void* indata, std::vector& processed_dims, @@ -329,10 +381,6 @@ InferStatus DataProcessor::process_operation(const std::string& operation, return InferStatus(holoinfer_code::H_ERROR, "Data processor, Exception in running " + operation); } - if (operation.find("print") == std::string::npos && processed_data_map.size() == 0) { - return InferStatus(holoinfer_code::H_ERROR, "Data processor, Processed data map empty"); - } - return InferStatus(); } } // namespace inference diff --git a/modules/holoinfer/src/process/data_processor.hpp b/modules/holoinfer/src/process/data_processor.hpp index 644be182..be1e4417 100644 --- a/modules/holoinfer/src/process/data_processor.hpp +++ b/modules/holoinfer/src/process/data_processor.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -31,6 +31,7 @@ #include #include +#include #include namespace holoscan { @@ -67,7 +68,7 @@ class DataProcessor { * operation. * @param config_path Path to the processing configuration settings * - * @returns InferStatus with appropriate code and message + * @return InferStatus with appropriate code and message */ InferStatus initialize(const MultiMappings& process_operations, const std::string config_path); @@ -81,7 +82,7 @@ class DataProcessor { * @param processed_data_map Output data map, that will be populated * @param output_tensors Tensor names to be populated in the out_data_map * @param custom_strings Strings to display for custom print operations - * @returns InferStatus with appropriate code and message + * @return InferStatus with appropriate code and message */ InferStatus process_operation(const std::string& operation, const std::vector& in_dims, const void* in_data, std::vector& processed_dims, @@ -98,7 +99,7 @@ class DataProcessor { * @param indims Map with key as tensor name and value as dimension of the input tensor * @param processed_data Output data map, that will be populated * @param processed_dims Dimension of the output tensor, is populated during the processing - * @returns InferStatus with appropriate code and message + * @return InferStatus with appropriate code and message */ InferStatus process_transform(const std::string& transform, const std::string& key, const std::map& indata, @@ -161,6 +162,20 @@ class DataProcessor { const void* in_data, const std::vector& custom_strings); + /** + * @brief Export binary classification results in the input buffer to CSV file using Data Exporter + * API. + * + * @param in_dims Dimension of the input tensor + * @param in_data Input data buffer + * @param custom_strings The comma separated list of strings containing information for + * the output CSV file. It should include application name as a first string + * required for the Data Exporter API and column names. + */ + InferStatus export_binary_classification_to_csv(const std::vector& in_dims, + const void* in_data, + const std::vector& custom_strings); + private: /// Map defining supported operations by DataProcessor Class. /// Keyword in this map must be used exactly by the user in configuration. @@ -187,6 +202,11 @@ class DataProcessor { {"print_int32", holoinfer_data_processor::h_HOST}, {"print_custom_binary_classification", holoinfer_data_processor::h_HOST}}; + /// Map defining supported formats by DataProcessor Class to export results using Data Exporter + /// API. + inline static const std::map supported_export_operations_{ + {"export_binary_classification_to_csv", holoinfer_data_processor::h_HOST}}; + /// Mapped function call for the function pointer of max_per_channel_scaled processor_FP max_per_channel_scaled_fp_ = [this](auto& in_dims, const void* in_data, std::vector& out_dims, DataMap& out_data, @@ -216,6 +236,14 @@ class DataProcessor { return print_custom_binary_classification(in_dims, in_data, custom_strings); }; + /// Mapped function call for the function pointer of exporting binary classification + /// results to the CSV file using the Data Exporter API. + processor_FP export_binary_classification_to_csv_fp_ = + [this](auto& in_dims, const void* in_data, std::vector& out_dims, DataMap& out_data, + auto& output_tensors, auto& custom_strings) { + return export_binary_classification_to_csv(in_dims, in_data, custom_strings); + }; + /// Mapped function call for the function pointer of print int32 processor_FP print_results_i32_fp_ = [this](auto& in_dims, const void* in_data, std::vector& out_dims, DataMap& out_data, @@ -229,7 +257,8 @@ class DataProcessor { {"scale_intensity_cpu", scale_intensity_cpu_fp_}, {"print", print_results_fp_}, {"print_int32", print_results_i32_fp_}, - {"print_custom_binary_classification", print_custom_binary_classification_fp_}}; + {"print_custom_binary_classification", print_custom_binary_classification_fp_}, + {"export_binary_classification_to_csv", export_binary_classification_to_csv_fp_}}; /// Mapped function call for the function pointer of generate_boxes transforms_FP generate_boxes_fp_ = [this](const std::string& key, @@ -245,6 +274,9 @@ class DataProcessor { /// Configuration path std::string config_path_ = {}; + + /// Data exporter + std::unique_ptr data_exporter_ = nullptr; }; } // namespace inference } // namespace holoscan diff --git a/modules/holoinfer/src/process/transform.hpp b/modules/holoinfer/src/process/transform.hpp index 23b46bae..24d8f52d 100644 --- a/modules/holoinfer/src/process/transform.hpp +++ b/modules/holoinfer/src/process/transform.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -32,6 +32,8 @@ namespace inference { */ class TransformBase { public: + virtual ~TransformBase() = default; + /** * @brief Does the transform execution * @param indata Map with key as tensor name and value as raw data buffer diff --git a/modules/holoinfer/src/process/transforms/generate_boxes.hpp b/modules/holoinfer/src/process/transforms/generate_boxes.hpp index 6e3298ee..a9764e63 100644 --- a/modules/holoinfer/src/process/transforms/generate_boxes.hpp +++ b/modules/holoinfer/src/process/transforms/generate_boxes.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -42,7 +42,17 @@ class GenerateBoxes : public TransformBase { * @brief Default Constructor */ GenerateBoxes() {} + + /** + * @brief Explicit Constructor + */ explicit GenerateBoxes(const std::string& config_path) : config_path_(config_path) {} + + /** + * @brief Default Destructor + */ + ~GenerateBoxes() override = default; + /** * @brief Initializer. Parses the config file and populates all required variables to be used in * the execution process diff --git a/modules/holoinfer/src/utils/infer_buffer.cpp b/modules/holoinfer/src/utils/infer_buffer.cpp index cdb1dfe1..adb925b6 100644 --- a/modules/holoinfer/src/utils/infer_buffer.cpp +++ b/modules/holoinfer/src/utils/infer_buffer.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -31,7 +31,7 @@ namespace inference { * * @param element_type Input data type. Float32 is the only supported element type. * - * @returns Bytes used in storing element type + * @return Bytes used in storing element type */ uint32_t get_element_size(holoinfer_datatype element_type) noexcept { switch (element_type) { @@ -52,10 +52,13 @@ InferStatus allocate_buffers(DataMap& buffers, std::vector& dims, bool allocate_cuda, int device_id) { size_t buffer_size = accumulate(dims.begin(), dims.end(), 1, std::multiplies()); - auto data_buffer = std::make_shared(datatype, device_id); - if (!data_buffer) { + std::shared_ptr data_buffer; + try { + data_buffer = std::make_shared(datatype, device_id); + } catch (std::exception& e) { InferStatus status = InferStatus(holoinfer_code::H_ERROR); - status.set_message("Data buffer creation failed for " + keyname); + status.set_message( + fmt::format("Data buffer creation failed for {} with error {}", keyname, e.what())); return status; } data_buffer->host_buffer.resize(buffer_size); @@ -74,9 +77,11 @@ void DeviceFree::operator()(void* ptr) const { DataBuffer::DataBuffer(holoinfer_datatype data_type, int device_id) : type_(data_type), device_id_(device_id) { - device_buffer = std::make_shared(type_); - if (!device_buffer) { - throw std::runtime_error("Device buffer creation failed in DataBuffer constructor"); + try { + device_buffer = std::make_shared(type_); + } catch (std::exception& e) { + throw std::runtime_error( + fmt::format("Device buffer creation failed in DataBuffer constructor with {}", e.what())); } host_buffer.set_type(type_); } diff --git a/modules/holoviz/examples/demo/Main.cpp b/modules/holoviz/examples/demo/Main.cpp index 621477ae..825f802a 100644 --- a/modules/holoviz/examples/demo/Main.cpp +++ b/modules/holoviz/examples/demo/Main.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -477,7 +477,7 @@ int main(int argc, char** argv) { << " -b, --bench benchmark mode" << std::endl << " -l, --headless headless mode" << std::endl << " -d, --display name of the display to use in exclusive mode" - " (either EDID or xrandr name)" + " (either EDID, `xrandr` or `hwinfo --monitor` name)" << std::endl; return EXIT_SUCCESS; diff --git a/modules/holoviz/src/CMakeLists.txt b/modules/holoviz/src/CMakeLists.txt index fe4cdff9..adb43de3 100644 --- a/modules/holoviz/src/CMakeLists.txt +++ b/modules/holoviz/src/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,7 +19,6 @@ include(GNUInstallDirs) include(GenHeaderFromBinaryFile) find_package(CUDAToolkit REQUIRED) -find_package(X11 REQUIRED) find_package(Vulkan REQUIRED) add_library(${PROJECT_NAME} SHARED) @@ -58,6 +57,7 @@ target_sources(${PROJECT_NAME} glfw_window.cpp headless_window.cpp holoviz.cpp + window.cpp cuda/convert.cu cuda/cuda_service.cpp diff --git a/modules/holoviz/src/context.hpp b/modules/holoviz/src/context.hpp index 705e6e4b..e1c46917 100644 --- a/modules/holoviz/src/context.hpp +++ b/modules/holoviz/src/context.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -54,12 +54,12 @@ class Context : public NonCopyable { ~Context(); /** - * @returns the context instance current to this thread, nullptr if none is current + * @return the context instance current to this thread, nullptr if none is current */ static Context* get_current(); /** - * @returns the context instance, create one if none is current + * @return the context instance, create one if none is current */ static Context& get(); @@ -89,7 +89,8 @@ class Context : public NonCopyable { * Initialize the context and create a exclusive window with the given properties. * * @param display_name name of the display, this can either be the EDID name as displayed - * in the NVIDIA Settings, or the output name used by xrandr, + * in the NVIDIA Settings, or the output name provided by `xrandr` or + * `hwinfo --monitor`. * if nullptr then the first display is selected. * @param width desired width, ignored if 0 * @param height desired height, ignored if 0 @@ -119,7 +120,7 @@ class Context : public NonCopyable { void set_cuda_stream(CUstream stream); /** - * @returns the currently active cuda stream + * @return the currently active cuda stream */ CUstream get_cuda_stream() const; @@ -180,17 +181,17 @@ class Context : public NonCopyable { CUdeviceptr device_ptr, size_t row_pitch = 0); /** - * @returns the active layer + * @return the active layer */ Layer* get_active_layer() const; /** - * @returns the active image layer + * @return the active image layer */ ImageLayer* get_active_image_layer() const; /** - * @returns the active geometry layer + * @return the active geometry layer */ GeometryLayer* get_active_geometry_layer() const; diff --git a/modules/holoviz/src/cuda/cuda_service.hpp b/modules/holoviz/src/cuda/cuda_service.hpp index b9d35e3e..a9818820 100644 --- a/modules/holoviz/src/cuda/cuda_service.hpp +++ b/modules/holoviz/src/cuda/cuda_service.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -119,7 +119,7 @@ class CudaService { ~CudaService(); /** - * @returns true if running on a multi GPU system + * @return true if running on a multi GPU system */ bool IsMultiGPU() const; @@ -128,7 +128,7 @@ class CudaService { * * @param device_ptr CUDA device memory to check * - * @returns true if the memory is on the same device as the CudaService + * @return true if the memory is on the same device as the CudaService */ bool IsMemOnDevice(CUdeviceptr device_ptr) const; diff --git a/modules/holoviz/src/exclusive_window.cpp b/modules/holoviz/src/exclusive_window.cpp index 45e0b974..ad8c7dde 100644 --- a/modules/holoviz/src/exclusive_window.cpp +++ b/modules/holoviz/src/exclusive_window.cpp @@ -17,9 +17,7 @@ #include "exclusive_window.hpp" -#include #include -#include #include #include @@ -44,13 +42,9 @@ struct ExclusiveWindow::Impl { uint32_t width_ = 0; uint32_t height_ = 0; uint32_t refresh_rate_ = 0; - - Display* dpy_ = nullptr; }; -ExclusiveWindow::~ExclusiveWindow() { - if (impl_->dpy_) { XCloseDisplay(impl_->dpy_); } -} +ExclusiveWindow::~ExclusiveWindow() {} ExclusiveWindow::ExclusiveWindow(const char* display_name, uint32_t width, uint32_t height, uint32_t refresh_rate, InitFlags flags) @@ -66,10 +60,11 @@ void ExclusiveWindow::init_im_gui() {} void ExclusiveWindow::setup_callbacks( std::function frame_buffer_size_cb) {} +void ExclusiveWindow::restore_callbacks() {} + const char** ExclusiveWindow::get_required_instance_extensions(uint32_t* count) { static char const* extensions[]{VK_KHR_SURFACE_EXTENSION_NAME, VK_KHR_DISPLAY_EXTENSION_NAME, - VK_EXT_ACQUIRE_XLIB_DISPLAY_EXTENSION_NAME, VK_EXT_DIRECT_MODE_DISPLAY_EXTENSION_NAME}; *count = sizeof(extensions) / sizeof(extensions[0]); @@ -136,23 +131,6 @@ vk::SurfaceKHR ExclusiveWindow::create_surface(vk::PhysicalDevice physical_devic const vk::DisplayKHR display = selected_display.display; - // If the X11 server is running, acquire permission from the X-Server to directly - // access the display in Vulkan - impl_->dpy_ = XOpenDisplay(NULL); - if (impl_->dpy_) { - const PFN_vkAcquireXlibDisplayEXT vkAcquireXlibDisplayEXT = - PFN_vkAcquireXlibDisplayEXT(vkGetInstanceProcAddr(instance, "vkAcquireXlibDisplayEXT")); - if (!vkAcquireXlibDisplayEXT) { - throw std::runtime_error("Could not get proc address of vkAcquireXlibDisplayEXT"); - } - HOLOSCAN_LOG_INFO("X server is running, trying to acquire display"); - VkResult result = vkAcquireXlibDisplayEXT(physical_device, impl_->dpy_, display); - if (result < 0) { - nvvk::checkResult(result); - throw std::runtime_error("Failed to acquire display from X-Server."); - } - } - // pick highest available resolution const std::vector modes = physical_device.getDisplayModePropertiesKHR(display); @@ -262,7 +240,10 @@ void ExclusiveWindow::im_gui_new_frame() { void ExclusiveWindow::begin() {} -void ExclusiveWindow::end() {} +void ExclusiveWindow::end() { + // call the base class + Window::end(); +} float ExclusiveWindow::get_aspect_ratio() { return float(impl_->width_) / float(impl_->height_); diff --git a/modules/holoviz/src/exclusive_window.hpp b/modules/holoviz/src/exclusive_window.hpp index af8a5f41..471b163f 100644 --- a/modules/holoviz/src/exclusive_window.hpp +++ b/modules/holoviz/src/exclusive_window.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -37,7 +37,8 @@ class ExclusiveWindow : public Window { * Construct a new exclusive window. * * @param display_name name of the display, this can either be the EDID name as displayed - * in the NVIDIA Settings, or the output name used by xrandr, + * in the NVIDIA Settings, or the output name provided by `xrandr` or + * `hwinfo --monitor`. * if nullptr then the first display is selected. * @param width desired width, ignored if 0 * @param height desired height, ignored if 0 @@ -62,6 +63,7 @@ class ExclusiveWindow : public Window { ///@{ void init_im_gui() override; void setup_callbacks(std::function frame_buffer_size_cb) override; + void restore_callbacks() override; const char** get_required_instance_extensions(uint32_t* count) override; const char** get_required_device_extensions(uint32_t* count) override; diff --git a/modules/holoviz/src/export.map b/modules/holoviz/src/export.map index 9c6fe2fd..b97f91c3 100644 --- a/modules/holoviz/src/export.map +++ b/modules/holoviz/src/export.map @@ -64,7 +64,9 @@ "holoscan::viz::ReadFramebuffer(holoscan::viz::ImageFormat, unsigned int, unsigned int, unsigned long, unsigned long long, unsigned long)"; + "holoscan::viz::SetCamera(float, float, float, float, float, float, float, float, float, bool)"; "holoscan::viz::GetCameraPose(unsigned long, float*)"; + "holoscan::viz::GetCameraPose(float (&) [9], float (&) [3])"; }; local: *; diff --git a/modules/holoviz/src/glfw_window.cpp b/modules/holoviz/src/glfw_window.cpp index 71320f73..a2eaec0c 100644 --- a/modules/holoviz/src/glfw_window.cpp +++ b/modules/holoviz/src/glfw_window.cpp @@ -111,10 +111,6 @@ GLFWWindow::GLFWWindow(GLFWwindow* window) : impl_(new Impl) { int width, height; glfwGetWindowSize(window, &width, &height); impl_->frame_buffer_size_cb(window, width, height); - - // setup camera - CameraManip.setLookat( - nvmath::vec3f(0.f, 0.f, 1.f), nvmath::vec3f(0.f, 0.f, 0.f), nvmath::vec3f(0.f, 1.f, 0.f)); } GLFWWindow::GLFWWindow(uint32_t width, uint32_t height, const char* title, InitFlags flags) @@ -131,10 +127,6 @@ GLFWWindow::GLFWWindow(uint32_t width, uint32_t height, const char* title, InitF // set framebuffer size with initial window size impl_->frame_buffer_size_cb(impl_->window_, width, height); - - // setup camera - CameraManip.setLookat( - nvmath::vec3f(0.f, 0.f, 1.f), nvmath::vec3f(0.f, 0.f, 0.f), nvmath::vec3f(0.f, 1.f, 0.f)); } GLFWWindow::~GLFWWindow() {} @@ -222,6 +214,20 @@ void GLFWWindow::setup_callbacks(std::function fram impl_->prev_key_cb_ = glfwSetKeyCallback(impl_->window_, &GLFWWindow::Impl::key_cb); } +void GLFWWindow::restore_callbacks() { + glfwSetFramebufferSizeCallback(impl_->window_, impl_->prev_frame_buffer_size_cb_); + glfwSetMouseButtonCallback(impl_->window_, impl_->prev_mouse_button_cb_); + glfwSetScrollCallback(impl_->window_, impl_->prev_scroll_cb_); + glfwSetCursorPosCallback(impl_->window_, impl_->prev_cursor_pos_cb_); + glfwSetKeyCallback(impl_->window_, impl_->prev_key_cb_); + + impl_->frame_buffer_size_cb_ = nullptr; + impl_->prev_key_cb_ = nullptr; + impl_->prev_cursor_pos_cb_ = nullptr; + impl_->prev_mouse_button_cb_ = nullptr; + impl_->prev_scroll_cb_ = nullptr; +} + const char** GLFWWindow::get_required_instance_extensions(uint32_t* count) { return glfwGetRequiredInstanceExtensions(count); } @@ -278,11 +284,9 @@ void GLFWWindow::begin() { glfwPollEvents(); } -void GLFWWindow::end() {} - -void GLFWWindow::get_view_matrix(nvmath::mat4f* view_matrix) { - *view_matrix = nvmath::perspectiveVK(CameraManip.getFov(), 1.f /*aspectRatio*/, 0.1f, 1000.0f) * - CameraManip.getMatrix(); +void GLFWWindow::end() { + // call the base class + Window::end(); } float GLFWWindow::get_aspect_ratio() { diff --git a/modules/holoviz/src/glfw_window.hpp b/modules/holoviz/src/glfw_window.hpp index 7bb15bcd..e7c630df 100644 --- a/modules/holoviz/src/glfw_window.hpp +++ b/modules/holoviz/src/glfw_window.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef HOLOSCAN_VIZ_GLFW_WINDOW_HPP -#define HOLOSCAN_VIZ_GLFW_WINDOW_HPP +#ifndef MODULES_HOLOVIZ_SRC_GLFW_WINDOW_HPP +#define MODULES_HOLOVIZ_SRC_GLFW_WINDOW_HPP #include #include @@ -64,6 +64,7 @@ class GLFWWindow : public Window { ///@{ void init_im_gui() override; void setup_callbacks(std::function frame_buffer_size_cb) override; + void restore_callbacks() override; const char** get_required_instance_extensions(uint32_t* count) override; const char** get_required_device_extensions(uint32_t* count) override; @@ -81,8 +82,6 @@ class GLFWWindow : public Window { void begin() override; void end() override; - void get_view_matrix(nvmath::mat4f* view_matrix) override; - float get_aspect_ratio() override; ///@} @@ -93,4 +92,4 @@ class GLFWWindow : public Window { } // namespace holoscan::viz -#endif /* HOLOSCAN_VIZ_GLFW_WINDOW_HPP */ +#endif /* MODULES_HOLOVIZ_SRC_GLFW_WINDOW_HPP */ diff --git a/modules/holoviz/src/headless_window.cpp b/modules/holoviz/src/headless_window.cpp index fef8a565..af21b8f6 100644 --- a/modules/holoviz/src/headless_window.cpp +++ b/modules/holoviz/src/headless_window.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -43,6 +43,8 @@ void HeadlessWindow::init_im_gui() {} void HeadlessWindow::setup_callbacks( std::function frame_buffer_size_cb) {} +void HeadlessWindow::restore_callbacks() {} + const char** HeadlessWindow::get_required_instance_extensions(uint32_t* count) { static char const* extensions[]{}; @@ -89,7 +91,10 @@ void HeadlessWindow::im_gui_new_frame() { void HeadlessWindow::begin() {} -void HeadlessWindow::end() {} +void HeadlessWindow::end() { + // call the base class + Window::end(); +} float HeadlessWindow::get_aspect_ratio() { return float(impl_->width_) / float(impl_->height_); diff --git a/modules/holoviz/src/headless_window.hpp b/modules/holoviz/src/headless_window.hpp index 6928e51c..74a388d9 100644 --- a/modules/holoviz/src/headless_window.hpp +++ b/modules/holoviz/src/headless_window.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -55,6 +55,7 @@ class HeadlessWindow : public Window { ///@{ void init_im_gui() override; void setup_callbacks(std::function frame_buffer_size_cb) override; + void restore_callbacks() override; const char** get_required_instance_extensions(uint32_t* count) override; const char** get_required_device_extensions(uint32_t* count) override; diff --git a/modules/holoviz/src/holoviz.cpp b/modules/holoviz/src/holoviz.cpp index 57c5fff5..34c45b85 100644 --- a/modules/holoviz/src/holoviz.cpp +++ b/modules/holoviz/src/holoviz.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -195,6 +195,14 @@ void ReadFramebuffer(ImageFormat fmt, uint32_t width, uint32_t height, size_t bu Context::get().read_framebuffer(fmt, width, height, buffer_size, device_ptr, row_pitch); } +void SetCamera(float eye_x, float eye_y, float eye_z, float look_at_x, float look_at_y, + float look_at_z, float up_x, float up_y, float up_z, bool anim) { + nvmath::vec3f eye(eye_x, eye_y, eye_z); + nvmath::vec3f look_at(look_at_x, look_at_y, look_at_z); + nvmath::vec3f up(up_x, up_y, up_z); + Context::get().get_window()->set_camera(eye, look_at, up, anim); +} + void GetCameraPose(size_t size, float* matrix) { if (size != 16) { throw std::invalid_argument("Size of the matrix array should be 16"); } if (matrix == nullptr) { throw std::invalid_argument("Pointer to matrix should not be nullptr"); } @@ -208,4 +216,18 @@ void GetCameraPose(size_t size, float* matrix) { } } +void GetCameraPose(float (&rotation)[9], float (&translation)[3]) { + nvmath::mat4f camera_matrix; + Context::get().get_window()->get_camera_matrix(&camera_matrix); + + // nvmath::mat4f is column major, the outgoing matrix is row major, transpose while copying + for (uint32_t row = 0; row < 3; ++row) { + for (uint32_t col = 0; col < 3; ++col) { rotation[row * 3 + col] = camera_matrix(row, col); } + } + + translation[0] = camera_matrix(0, 3); + translation[1] = camera_matrix(1, 3); + translation[2] = camera_matrix(2, 3); +} + } // namespace holoscan::viz diff --git a/modules/holoviz/src/holoviz/holoviz.hpp b/modules/holoviz/src/holoviz/holoviz.hpp index 7fc2c0c3..119fa8a5 100644 --- a/modules/holoviz/src/holoviz/holoviz.hpp +++ b/modules/holoviz/src/holoviz/holoviz.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -117,7 +117,7 @@ InstanceHandle Create(); void SetCurrent(InstanceHandle instance); /** - * @returns the current instance for the active thread. + * @return the current instance for the active thread. */ InstanceHandle GetCurrent(); @@ -153,7 +153,8 @@ void Init(uint32_t width, uint32_t height, const char* title, InitFlags flags = * SSH into the machine, stop the X server with `sudo systemctl stop display-manager`. * * @param displayName name of the display, this can either be the EDID name as displayed in the - * NVIDIA Settings, or the output name used by xrandr, + * NVIDIA Settings, or the output name provided by `xrandr` or + * `hwinfo --monitor`. * if nullptr then the first display is selected. * @param width desired width, ignored if 0 * @param height desired height, ignored if 0 @@ -188,14 +189,14 @@ void SetCudaStream(CUstream stream); /** * Check if the window close button had been pressed * - * @returns true if the window close button had been pressed + * @return true if the window close button had been pressed */ bool WindowShouldClose(); /** * Check if the window is minimized. This can be used to skip rendering on minimized windows. * - * @returns true if the window is minimized + * @return true if the window is minimized */ bool WindowIsMinimized(); @@ -479,6 +480,17 @@ void EndLayer(); void ReadFramebuffer(ImageFormat fmt, uint32_t width, uint32_t height, size_t buffer_size, CUdeviceptr device_ptr, size_t row_pitch = 0); +/** + * Set the camera eye, look at and up vectors. + * + * @param eye_x, eye_y, eye_z eye position + * @param look_at_x, look_at_y, look_at_z look at position + * @param up_x, up_y, up_z up vector + * @param anim animate transition + */ +void SetCamera(float eye_x, float eye_y, float eye_z, float look_at_x, float look_at_y, + float look_at_z, float up_x, float up_y, float up_z, bool anim = false); + /** * Get the camera pose. * @@ -496,6 +508,27 @@ void ReadFramebuffer(ImageFormat fmt, uint32_t width, uint32_t height, size_t bu */ void GetCameraPose(size_t size, float* matrix); +/** + * Get the camera pose. + * + * The camera parameters are returned as camera extrinsics parameters. + * The extrinsics camera matrix is defined as follow: + * @code + * T = [R | t] + * @endcode + * + * The camera is operated using the mouse. + * - Orbit (LMB) + * - Pan (LMB + CTRL | MMB) + * - Dolly (LMB + SHIFT | RMB | Mouse wheel) + * - Look Around (LMB + ALT | LMB + CTRL + SHIFT) + * - Zoom (Mouse wheel + SHIFT) + * + * @param rotation row major rotation matrix + * @param translation translation vector + */ +void GetCameraPose(float (&rotation)[9], float (&translation)[3]); + } // namespace holoscan::viz #endif /* HOLOVIZ_SRC_HOLOVIZ_HOLOVIZ_HPP */ diff --git a/modules/holoviz/src/layers/geometry_layer.cpp b/modules/holoviz/src/layers/geometry_layer.cpp index 89287d98..381cb2d8 100644 --- a/modules/holoviz/src/layers/geometry_layer.cpp +++ b/modules/holoviz/src/layers/geometry_layer.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -69,7 +69,7 @@ class Primitive { Primitive() = delete; /** - * @returns true for 3D primitives + * @return true for 3D primitives */ bool three_dimensional() const { switch (topology_) { diff --git a/modules/holoviz/src/layers/image_layer.cpp b/modules/holoviz/src/layers/image_layer.cpp index fb36a019..22bdebbe 100644 --- a/modules/holoviz/src/layers/image_layer.cpp +++ b/modules/holoviz/src/layers/image_layer.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,7 +29,7 @@ namespace holoscan::viz { namespace { -/// @returns true if fmt is a depth format +/// @return true if fmt is a depth format bool is_depth_format(ImageFormat fmt) { return ((fmt == ImageFormat::D16_UNORM) || (fmt == ImageFormat::X8_D24_UNORM) || (fmt == ImageFormat::D32_SFLOAT)); diff --git a/modules/holoviz/src/layers/layer.hpp b/modules/holoviz/src/layers/layer.hpp index 3ad69b68..e99794b8 100644 --- a/modules/holoviz/src/layers/layer.hpp +++ b/modules/holoviz/src/layers/layer.hpp @@ -57,12 +57,12 @@ class Layer { virtual ~Layer(); /** - * @returns the layer type + * @return the layer type */ Type get_type() const; /** - * @returns the layer priority + * @return the layer priority */ int32_t get_priority() const; @@ -74,7 +74,7 @@ class Layer { void set_priority(int32_t priority); /** - * @returns the layer opacity + * @return the layer opacity */ float get_opacity() const; @@ -99,7 +99,7 @@ class Layer { }; /** - * @returns the layer views + * @return the layer views */ const std::vector& get_views() const; diff --git a/modules/holoviz/src/util/unique_value.hpp b/modules/holoviz/src/util/unique_value.hpp index 26858f6f..dec5fd7f 100644 --- a/modules/holoviz/src/util/unique_value.hpp +++ b/modules/holoviz/src/util/unique_value.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -61,7 +61,7 @@ class UniqueValue : public NonCopyable { /** * Release the value * - * @returns value + * @return value */ T release() noexcept { T value = value_; @@ -101,27 +101,27 @@ class UniqueValue : public NonCopyable { T get() const noexcept { return value_; } /** - * @returns true if the value is set + * @return true if the value is set */ explicit operator bool() const noexcept { return (value_ != T()); } /** - * @returns reference to value + * @return reference to value */ T& operator*() const { return value_; } /** - * @returns value + * @return value */ T operator->() const noexcept { return value_; } /** - * @returns true if equal + * @return true if equal */ bool operator==(const UniqueValue& other) const { return (value_ == other.value_); } /** - * @returns true if not equal + * @return true if not equal */ bool operator!=(const UniqueValue& other) const { return !(operator==(other)); } diff --git a/modules/holoviz/src/vulkan/framebuffer_sequence.hpp b/modules/holoviz/src/vulkan/framebuffer_sequence.hpp index 4d1b8902..6b122a4c 100644 --- a/modules/holoviz/src/vulkan/framebuffer_sequence.hpp +++ b/modules/holoviz/src/vulkan/framebuffer_sequence.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -70,32 +70,32 @@ class FramebufferSequence { void acquire(); /** - * @returns the framebuffer color format/ + * @return the framebuffer color format/ */ vk::Format get_color_format() const { return color_format_; } /** - * @returns the framebuffer depth format/ + * @return the framebuffer depth format/ */ vk::Format get_depth_format() const { return depth_format_; } /** - * @returns the image view of a color buffer + * @return the image view of a color buffer */ vk::ImageView get_color_image_view(uint32_t i) const; /** - * @returns the image view of a depth buffer + * @return the image view of a depth buffer */ vk::ImageView get_depth_image_view(uint32_t i) const; /** - * @returns the color buffer count + * @return the color buffer count */ uint32_t get_image_count() const { return image_count_; } /** - * @returns the index of the current active color buffer + * @return the index of the current active color buffer */ uint32_t get_active_image_index() const; @@ -112,28 +112,28 @@ class FramebufferSequence { * semaphore of the previous frame, else it's the semaphore used to acquire the swap queue * image. * - * @returns active read semaphore + * @return active read semaphore */ vk::Semaphore get_active_read_semaphore() const; /** * Get the active write semaphore. * - * @returns active write semaphore + * @return active write semaphore */ vk::Semaphore get_active_written_semaphore() const; /** * Get the active color image. * - * @returns active color image + * @return active color image */ vk::Image get_active_color_image() const; /** * Get the active depth image. * - * @returns active depth image + * @return active depth image */ vk::Image get_active_depth_image() const; diff --git a/modules/holoviz/src/vulkan/vulkan_app.cpp b/modules/holoviz/src/vulkan/vulkan_app.cpp index 34201e6f..b398cf97 100644 --- a/modules/holoviz/src/vulkan/vulkan_app.cpp +++ b/modules/holoviz/src/vulkan/vulkan_app.cpp @@ -751,6 +751,8 @@ class Vulkan::Impl { Vulkan::Impl::~Impl() { try { if (device_) { + window_->restore_callbacks(); + device_.waitIdle(); cleanup_transfer_jobs(); diff --git a/modules/holoviz/src/window.cpp b/modules/holoviz/src/window.cpp new file mode 100644 index 00000000..d45de701 --- /dev/null +++ b/modules/holoviz/src/window.cpp @@ -0,0 +1,49 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "window.hpp" + +#include + +namespace holoscan::viz { + +Window::Window() { + // setup camera + CameraManip.setLookat( + nvmath::vec3f(0.f, 0.f, 1.f), nvmath::vec3f(0.f, 0.f, 0.f), nvmath::vec3f(0.f, 1.f, 0.f)); +} + +void Window::end() { + // update the camera + CameraManip.updateAnim(); +} + +void Window::set_camera(const nvmath::vec3f& eye, const nvmath::vec3f& look_at, + const nvmath::vec3f& up, bool anim) { + CameraManip.setLookat(eye, look_at, up, !anim); +} + +void Window::get_view_matrix(nvmath::mat4f* view_matrix) { + *view_matrix = nvmath::perspectiveVK(CameraManip.getFov(), 1.f /*aspectRatio*/, 0.1f, 1000.0f) * + CameraManip.getMatrix(); +} + +void Window::get_camera_matrix(nvmath::mat4f* camera_matrix) { + *camera_matrix = CameraManip.getMatrix(); +} + +} // namespace holoscan::viz diff --git a/modules/holoviz/src/window.hpp b/modules/holoviz/src/window.hpp index 8e01f090..e7e22e06 100644 --- a/modules/holoviz/src/window.hpp +++ b/modules/holoviz/src/window.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -36,7 +36,7 @@ class Window { /** * Construct a new Window object. */ - Window() {} + Window(); /** * Destroy the Window object. @@ -55,6 +55,11 @@ class Window { */ virtual void setup_callbacks(std::function frame_buffer_size_cb) = 0; + /** + * Restore call backs. + */ + virtual void restore_callbacks() = 0; + /** * Get the required instance extensions for vulkan. * @@ -99,12 +104,12 @@ class Window { vk::Instance instance) = 0; /** - * @returns true if the window should be closed + * @return true if the window should be closed */ virtual bool should_close() = 0; /** - * @returns true if the window is minimized + * @return true if the window is minimized */ virtual bool is_minimized() = 0; @@ -123,15 +128,33 @@ class Window { */ virtual void end() = 0; + /** + * Set the camera eye, look at and up vectors. + * + * @param eye_x, eye_y, eye_z eye position + * @param look_at_x, look_at_y, look_at_z look at position + * @param up_x, up_y, up_z up vector + * @param anim animate transition + */ + void set_camera(const nvmath::vec3f& eye, const nvmath::vec3f& look_at, const nvmath::vec3f& up, + bool anim); + + /** + * Get the view matrix + * + * @param view_matrix + */ + void get_view_matrix(nvmath::mat4f* view_matrix); + /** * Get the view matrix * * @param view_matrix */ - virtual void get_view_matrix(nvmath::mat4f* view_matrix) { *view_matrix = nvmath::mat4f(1); } + void get_camera_matrix(nvmath::mat4f* camera_matrix); /** - * @returns the horizontal aspect ratio + * @return the horizontal aspect ratio */ virtual float get_aspect_ratio() = 0; }; diff --git a/modules/holoviz/tests/functional/CMakeLists.txt b/modules/holoviz/tests/functional/CMakeLists.txt index b47ac4b0..468eeb27 100644 --- a/modules/holoviz/tests/functional/CMakeLists.txt +++ b/modules/holoviz/tests/functional/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -37,7 +37,7 @@ target_sources(${PROJECT_NAME} PRIVATE camera_pose_test.cpp geometry_layer_test.cpp - headless_fixture.cpp + test_fixture.cpp im_gui_layer_test.cpp image_layer_test.cpp init_test.cpp @@ -64,7 +64,6 @@ target_link_libraries(${PROJECT_NAME} PRIVATE glfw Vulkan::Vulkan - X11::X11 holoscan::viz holoscan::viz::imgui GTest::gtest_main diff --git a/modules/holoviz/tests/functional/camera_pose_test.cpp b/modules/holoviz/tests/functional/camera_pose_test.cpp index 45319efa..33b05c59 100644 --- a/modules/holoviz/tests/functional/camera_pose_test.cpp +++ b/modules/holoviz/tests/functional/camera_pose_test.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -17,26 +17,112 @@ #include +#include + #include -#include "headless_fixture.hpp" +#include "test_fixture.hpp" namespace viz = holoscan::viz; class CameraPose : public TestHeadless {}; -TEST_F(CameraPose, Get) { - std::array pose; +TEST_F(CameraPose, Set) { + EXPECT_NO_THROW(viz::SetCamera(1.f, 2.f, 3.f, 4.f, 5.f, 6.f, 0.f, 1.f, 0.f)); + + float rotation[9]; + float translation[3]; + EXPECT_NO_THROW(viz::GetCameraPose(rotation, translation)); + + // There are test errors on some systems when using EXPECT_FLOAT_EQ() (includes a error margin of + // 4 ULP, see https://google.github.io/googletest/reference/assertions.html#floating-point). + // Use EXPECT_NEAR() with a higher epsilon. + constexpr float epsilon = 1e-6f; + EXPECT_NEAR(rotation[0], -0.707106769f, epsilon); + EXPECT_NEAR(rotation[1], 0.f, epsilon); + EXPECT_NEAR(rotation[2], 0.707106769f, epsilon); + EXPECT_NEAR(rotation[3], -0.408248335f, epsilon); + EXPECT_NEAR(rotation[4], 0.81649667f, epsilon); + EXPECT_NEAR(rotation[5], -0.408248335f, epsilon); + EXPECT_NEAR(rotation[6], -0.577350259f, epsilon); + EXPECT_NEAR(rotation[7], -0.577350259f, epsilon); + EXPECT_NEAR(rotation[8], -0.577350259f, epsilon); + EXPECT_NEAR(translation[0], -1.41421342f, epsilon); + EXPECT_NEAR(translation[1], 0.f, epsilon); + EXPECT_NEAR(translation[2], 3.46410155f, epsilon); +} + +TEST_F(CameraPose, GetDefault) { + float rotation[9]; + float translation[3]; + + EXPECT_NO_THROW(viz::GetCameraPose(rotation, translation)); + for (uint32_t row = 0; row < 3; ++row) { + for (uint32_t col = 0; col < 3; ++col) { + EXPECT_FLOAT_EQ(rotation[row * 3 + col], ((row == col) ? 1.f : 0.f)); + } + } + EXPECT_FLOAT_EQ(translation[0], 0.f); + EXPECT_FLOAT_EQ(translation[1], 0.f); + EXPECT_FLOAT_EQ(translation[2], -1.f); + std::array pose; // it's an error to specify a size less than 16 EXPECT_THROW(viz::GetCameraPose(15, pose.data()), std::invalid_argument); // it's an error to specify a null pointer for matrix EXPECT_THROW(viz::GetCameraPose(16, nullptr), std::invalid_argument); - // in headless mode the camera matrix is the identity matrix + // this is the default setup for the matrix, see Window class constructor + std::array expected_pose{1.73205066f, + 0.f, + 0.f, + 0.f, + 0.f, + -1.73205066f, + 0.f, + 0.f, + 0.f, + 0.f, + -1.00010002f, + 0.900090039f, + 0.f, + 0.f, + -1.f, + 1.f}; EXPECT_NO_THROW(viz::GetCameraPose(pose.size(), pose.data())); - for (uint32_t row = 0; row < 4; ++row) { - for (uint32_t col = 0; col < 4; ++col) { - EXPECT_TRUE(pose[row * 4 + col] == ((row == col) ? 1.f : 0.f)); + for (int i = 0; i < 16; ++i) { EXPECT_FLOAT_EQ(pose[i], expected_pose[i]); } +} + +TEST_F(CameraPose, Anim) { + // move the camera in x direction + EXPECT_NO_THROW(viz::SetCamera(100.f, 0.f, 1.f, 100.f, 0.f, 0.f, 0.f, 1.f, 0.f, true)); + + // start animation (duration default is 500 ms) + EXPECT_NO_THROW(viz::Begin()); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + EXPECT_NO_THROW(viz::End()); + + float rotation[9]; + float translation[3]; + + EXPECT_NO_THROW(viz::GetCameraPose(rotation, translation)); + // translation has changed + EXPECT_NE(translation[0], 0.f); + EXPECT_FLOAT_EQ(translation[1], 0.f); + EXPECT_NE(translation[2], -1.f); + + // wait for the end + EXPECT_NO_THROW(viz::Begin()); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + EXPECT_NO_THROW(viz::End()); + + EXPECT_NO_THROW(viz::GetCameraPose(rotation, translation)); + for (uint32_t row = 0; row < 3; ++row) { + for (uint32_t col = 0; col < 3; ++col) { + EXPECT_FLOAT_EQ(rotation[row * 3 + col], ((row == col) ? 1.f : 0.f)); } } + + EXPECT_FLOAT_EQ(translation[0], -100.f); + EXPECT_FLOAT_EQ(translation[1], 0.f); + EXPECT_FLOAT_EQ(translation[2], -1.f); } diff --git a/modules/holoviz/tests/functional/geometry_layer_test.cpp b/modules/holoviz/tests/functional/geometry_layer_test.cpp index d768b1b3..444fdb54 100644 --- a/modules/holoviz/tests/functional/geometry_layer_test.cpp +++ b/modules/holoviz/tests/functional/geometry_layer_test.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +22,7 @@ #include #include -#include "headless_fixture.hpp" +#include "test_fixture.hpp" namespace viz = holoscan::viz; @@ -87,12 +87,12 @@ TEST_P(PrimitiveTopology, Primitive) { data.push_back(0.2f); data.push_back(0.4f); color_crc = { - 0xe96c7246, // RTX 6000, RTX A5000, RTX A6000 - 0x5f7bf4d3 // T4 + 0xe96c7246, // Quadro + 0x5f7bf4d3 // non-Quadro }; depth_crc = { - 0x802dbbb0, // RTX 6000, RTX A5000, RTX A6000 - 0xbd6bedea // T4 + 0x802dbbb0, // Quadro + 0xbd6bedea // non-Quadro }; break; case viz::PrimitiveTopology::LINE_STRIP: @@ -105,12 +105,12 @@ TEST_P(PrimitiveTopology, Primitive) { data.push_back(0.3f); data.push_back(0.2f); color_crc = { - 0x162496c0, // RTX 6000, RTX A5000, RTX A6000 - 0x9118f5cb // T4 + 0x162496c0, // Quadro + 0x9118f5cb // non-Quadro }; depth_crc = { - 0xfae233b9, // RTX 6000, RTX A5000, RTX A6000 - 0x92c04b5 // T4 + 0xfae233b9, // Quadro + 0x92c04b5 // non-Quadro }; break; case viz::PrimitiveTopology::TRIANGLE_LIST: @@ -141,12 +141,12 @@ TEST_P(PrimitiveTopology, Primitive) { data.push_back(0.3f); data.push_back(0.01f); color_crc = { - 0xb507fa88, // RTX 6000, RTX A5000, RTX A6000 - 0xf298654 // T4 + 0xb507fa88, // Quadro + 0xf298654 // non-Quadro }; depth_crc = { - 0x44098c3f, // RTX 6000, RTX A5000, RTX A6000 - 0x6fe44aee // T4 + 0x44098c3f, // Quadro + 0x6fe44aee // non-Quadro }; break; case viz::PrimitiveTopology::RECTANGLE_LIST: @@ -161,12 +161,12 @@ TEST_P(PrimitiveTopology, Primitive) { data.push_back(0.5f); data.push_back(0.3f); color_crc = { - 0x19a05481, // RTX 6000, RTX A5000, RTX A6000 - 0xf1f8f1b3 // T4 + 0x19a05481, // Quadro + 0xf1f8f1b3 // non-Quadro }; depth_crc = { - 0xf67bacdc, // RTX 6000, RTX A5000, RTX A6000 - 0x41396ef5 // T4 + 0xf67bacdc, // Quadro + 0x41396ef5 // non-Quadro }; break; case viz::PrimitiveTopology::OVAL_LIST: @@ -181,12 +181,12 @@ TEST_P(PrimitiveTopology, Primitive) { data.push_back(0.05f); data.push_back(0.07f); color_crc = { - 0x2341eef6, // RTX 6000, RTX A5000, RTX A6000 - 0xae3f0636 // T4 + 0x2341eef6, // Quadro + 0xae3f0636 // non-Quadro }; depth_crc = { - 0x41d7da93, // RTX 6000, RTX A5000, RTX A6000 - 0x7e44520d // T4 + 0x41d7da93, // Quadro + 0x7e44520d // non-Quadro }; break; case viz::PrimitiveTopology::POINT_LIST_3D: @@ -194,8 +194,8 @@ TEST_P(PrimitiveTopology, Primitive) { data.push_back(-0.5f); data.push_back(0.5f); data.push_back(0.8f); - color_crc = {0x83063d37}; - depth_crc = {0x1273ab78}; + color_crc = {0xd8f49994}; + depth_crc = {0x4e371ba0}; break; case viz::PrimitiveTopology::LINE_LIST_3D: primitive_count = 2; @@ -213,12 +213,12 @@ TEST_P(PrimitiveTopology, Primitive) { data.push_back(0.4f); data.push_back(0.5f); color_crc = { - 0x30cd7e29, // RTX 6000, RTX A5000, RTX A6000 - 0x697c858d // T4 + 0xc7762cc5, // Quadro + 0xe9f3dbc3 // non-Quadro }; depth_crc = { - 0xa31c2460, // RTX 6000, RTX A5000, RTX A6000 - 0x2eb67c7e // T4 + 0x782f15cf, // Quadro + 0xed2056f8 // non-Quadro }; break; case viz::PrimitiveTopology::LINE_STRIP_3D: @@ -234,12 +234,12 @@ TEST_P(PrimitiveTopology, Primitive) { data.push_back(-0.2f); data.push_back(0.2f); color_crc = { - 0x6c8cfdee, // RTX 6000, RTX A5000, RTX A6000 - 0xf8e2cd1f // T4 + 0x135ba8af, // Quadro + 0x322d3fdd // non-Quadro }; depth_crc = { - 0xc2d73af1, // RTX 6000, RTX A5000, RTX A6000 - 0x48ecd6f9 // T4 + 0x38dcc175, // Quadro + 0xa2292265 // non-Quadro }; break; case viz::PrimitiveTopology::TRIANGLE_LIST_3D: @@ -263,8 +263,8 @@ TEST_P(PrimitiveTopology, Primitive) { data.push_back(0.25f); data.push_back(0.6f); data.push_back(0.5f); - color_crc = {0x9d0d88}; - depth_crc = {0xb45187a8}; + color_crc = {0xf372dff7}; + depth_crc = {0x90e4e07d}; break; default: EXPECT_TRUE(false) << "Unhandled primitive topology"; @@ -399,16 +399,16 @@ TEST_P(DepthMapRenderMode, DepthMap) { std::vector crc; switch (depth_map_render_mode) { case viz::DepthMapRenderMode::POINTS: - crc = {0xcd990f6d}; + crc = {0x46e021cb}; break; case viz::DepthMapRenderMode::LINES: crc = { - 0x92a330ea, // RTX 6000, RTX A5000, RTX A6000 - 0xfd6f60f0 // T4 + 0x6b63061e, // Quadro + 0x69207440 // non-Quadro }; break; case viz::DepthMapRenderMode::TRIANGLES: - crc = {0x97856df3}; + crc = {0x9cb8d951}; break; } EXPECT_NO_THROW(viz::Begin()); diff --git a/modules/holoviz/tests/functional/im_gui_layer_test.cpp b/modules/holoviz/tests/functional/im_gui_layer_test.cpp index 4eb23f56..4a0c1fc8 100644 --- a/modules/holoviz/tests/functional/im_gui_layer_test.cpp +++ b/modules/holoviz/tests/functional/im_gui_layer_test.cpp @@ -22,7 +22,7 @@ #include #include -#include "headless_fixture.hpp" +#include "test_fixture.hpp" namespace viz = holoscan::viz; diff --git a/modules/holoviz/tests/functional/image_layer_test.cpp b/modules/holoviz/tests/functional/image_layer_test.cpp index 935421b8..d4a53b36 100644 --- a/modules/holoviz/tests/functional/image_layer_test.cpp +++ b/modules/holoviz/tests/functional/image_layer_test.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -27,7 +27,7 @@ #include #include -#include "headless_fixture.hpp" +#include "test_fixture.hpp" namespace viz = holoscan::viz; diff --git a/modules/holoviz/tests/functional/init_test.cpp b/modules/holoviz/tests/functional/init_test.cpp index a5843e01..f4cc39c4 100644 --- a/modules/holoviz/tests/functional/init_test.cpp +++ b/modules/holoviz/tests/functional/init_test.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,7 +19,6 @@ #define GLFW_INCLUDE_NONE #define GLFW_INCLUDE_VULKAN #include -#include #include #include @@ -27,25 +26,41 @@ namespace viz = holoscan::viz; TEST(Init, GLFWWindow) { - Display* display = XOpenDisplay(NULL); - if (!display) { - GTEST_SKIP() << "X11 server is not running or DISPLAY variable is not set, skipping test."; + if (glfwInit() == GLFW_FALSE) { + const char* description; + int code = glfwGetError(&description); + ASSERT_EQ(code, GLFW_PLATFORM_UNAVAILABLE) + << "Expected `GLFW_PLATFORM_UNAVAILABLE` but got `" << code << "`: `" << description << "`"; + GTEST_SKIP() << "No display server available, skipping test." << description; + } + + if (glfwGetPlatform() == GLFW_PLATFORM_WAYLAND) { + // No longer works after statically linking with glfw after + // https://gitlab-master.nvidia.com/holoscan/holoscan-sdk/-/merge_requests/2143. + // Error: + // Reason: GLFW maintains a global variable with state, when statically linking the GLFW library + // binaries (such as the Holoviz shared lib and this test binary) there are different + // global variables per binary. + GTEST_SKIP() << "With Wayland and statically linked GLFW creating the GLFW window outside of " + "Holoviz is not supported."; } - EXPECT_EQ(glfwInit(), GLFW_TRUE); glfwWindowHint(GLFW_CLIENT_API, GLFW_NO_API); GLFWwindow* const window = glfwCreateWindow(128, 64, "Holoviz test", NULL, NULL); - EXPECT_NO_THROW(viz::Init(window)); + ASSERT_NO_THROW(viz::Init(window)); EXPECT_FALSE(viz::WindowShouldClose()); EXPECT_FALSE(viz::WindowIsMinimized()); EXPECT_NO_THROW(viz::Shutdown()); } TEST(Init, CreateWindow) { - Display* display = XOpenDisplay(NULL); - if (!display) { - GTEST_SKIP() << "X11 server is not running or DISPLAY variable is not set, skipping test."; + if (glfwInit() == GLFW_FALSE) { + const char* description; + int code = glfwGetError(&description); + ASSERT_EQ(code, GLFW_PLATFORM_UNAVAILABLE) + << "Expected `GLFW_PLATFORM_UNAVAILABLE` but got `" << code << "`: `" << description << "`"; + GTEST_SKIP() << "No display server available, skipping test." << description; } EXPECT_NO_THROW(viz::Init(128, 64, "Holoviz test")); @@ -55,9 +70,12 @@ TEST(Init, CreateWindow) { } TEST(Init, Fullscreen) { - Display* display = XOpenDisplay(NULL); - if (!display) { - GTEST_SKIP() << "X11 server is not running or DISPLAY variable is not set, skipping test."; + if (glfwInit() == GLFW_FALSE) { + const char* description; + int code = glfwGetError(&description); + ASSERT_EQ(code, GLFW_PLATFORM_UNAVAILABLE) + << "Expected `GLFW_PLATFORM_UNAVAILABLE` but got `" << code << "`: `" << description << "`"; + GTEST_SKIP() << "No display server available, skipping test." << description; } // There is an issue when setting a mode with lower resolution than the current mode, in diff --git a/modules/holoviz/tests/functional/layer_test.cpp b/modules/holoviz/tests/functional/layer_test.cpp index fddbba25..311a0db4 100644 --- a/modules/holoviz/tests/functional/layer_test.cpp +++ b/modules/holoviz/tests/functional/layer_test.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,7 +21,7 @@ #include #include -#include "headless_fixture.hpp" +#include "test_fixture.hpp" namespace viz = holoscan::viz; diff --git a/modules/holoviz/tests/functional/read_framebuffer_test.cpp b/modules/holoviz/tests/functional/read_framebuffer_test.cpp index 78d7a222..f782324a 100644 --- a/modules/holoviz/tests/functional/read_framebuffer_test.cpp +++ b/modules/holoviz/tests/functional/read_framebuffer_test.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +22,7 @@ #include #include -#include "headless_fixture.hpp" +#include "test_fixture.hpp" namespace viz = holoscan::viz; diff --git a/modules/holoviz/tests/functional/headless_fixture.cpp b/modules/holoviz/tests/functional/test_fixture.cpp similarity index 91% rename from modules/holoviz/tests/functional/headless_fixture.cpp rename to modules/holoviz/tests/functional/test_fixture.cpp index 4bf6494a..55df2582 100644 --- a/modules/holoviz/tests/functional/headless_fixture.cpp +++ b/modules/holoviz/tests/functional/test_fixture.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,11 +15,15 @@ * limitations under the License. */ -#include "headless_fixture.hpp" +#include "test_fixture.hpp" #define STB_IMAGE_WRITE_IMPLEMENTATION #include +#define GLFW_INCLUDE_NONE +#define GLFW_INCLUDE_VULKAN +#include + #include #include #include @@ -51,19 +55,33 @@ void Fill(void* data, size_t elements, float min, float max) { } } -void TestHeadless::SetUp() { - ASSERT_NO_THROW(viz::Init(width_, height_, "Holoviz test", viz::InitFlags::HEADLESS)); +void TestBase::SetUp() { + if (~(init_flags_ & viz::InitFlags::HEADLESS)) { + if (glfwInit() == GLFW_FALSE) { + const char* description; + int code = glfwGetError(&description); + ASSERT_EQ(code, GLFW_PLATFORM_UNAVAILABLE) << "Expected `GLFW_PLATFORM_UNAVAILABLE` but got `" + << code << "`: `" << description << "`"; + GTEST_SKIP() << "No display server available, skipping test." << description; + } + } + + ASSERT_NO_THROW(viz::Init(width_, height_, "Holoviz test", init_flags_)); + initialized_ = true; } -void TestHeadless::TearDown() { - ASSERT_NO_THROW(viz::Shutdown()); +void TestBase::TearDown() { + if (initialized_) { + ASSERT_NO_THROW(viz::Shutdown()); + initialized_ = false; + } } -void TestHeadless::SetCUDADevice(uint32_t device_ordinal) { +void TestBase::SetCUDADevice(uint32_t device_ordinal) { device_ordinal_ = device_ordinal; } -void TestHeadless::SetupData(viz::ImageFormat format, uint32_t rand_seed) { +void TestBase::SetupData(viz::ImageFormat format, uint32_t rand_seed) { std::srand(rand_seed); uint32_t channels; @@ -183,7 +201,7 @@ void TestHeadless::SetupData(viz::ImageFormat format, uint32_t rand_seed) { } } -void TestHeadless::ReadColorData(std::vector& color_data) { +void TestBase::ReadColorData(std::vector& color_data) { const size_t data_size = width_ * height_ * sizeof(uint8_t) * 4; viz::CudaService cuda_service(device_ordinal_); @@ -206,7 +224,7 @@ void TestHeadless::ReadColorData(std::vector& color_data) { ASSERT_EQ(cuMemcpyDtoH(color_data.data(), device_ptr.get(), color_data.size()), CUDA_SUCCESS); } -void TestHeadless::ReadDepthData(std::vector& depth_data) { +void TestBase::ReadDepthData(std::vector& depth_data) { const size_t data_size = width_ * height_ * sizeof(float); viz::CudaService cuda_service(device_ordinal_); @@ -246,7 +264,7 @@ static std::string BuildFileName(const std::string& end) { return file_name; } -bool TestHeadless::CompareColorResult() { +bool TestBase::CompareColorResult() { const uint32_t components = color_data_.size() / (width_ * height_); if ((components != 1) && (components != 3) && (components != 4)) { EXPECT_TRUE(false) << "Can compare R8_UNORM, R8G8B8_UNORM or R8G8B8A8_UNORM data only"; @@ -277,7 +295,7 @@ bool TestHeadless::CompareColorResult() { return true; } -bool TestHeadless::CompareDepthResult() { +bool TestBase::CompareDepthResult() { if (depth_data_.size() != width_ * height_ * sizeof(float)) { EXPECT_TRUE(false) << "Can compare D32_SFLOAT data only"; return false; @@ -317,7 +335,7 @@ Compare these images with the `_fail` images. Update or add the CRC values of th accordingly. )"; -bool TestHeadless::CompareColorResultCRC32(const std::vector crc32) { +bool TestBase::CompareColorResultCRC32(const std::vector crc32) { std::vector read_data; ReadColorData(read_data); @@ -352,7 +370,7 @@ bool TestHeadless::CompareColorResultCRC32(const std::vector crc32) { return passed; } -bool TestHeadless::CompareDepthResultCRC32(const std::vector crc32) { +bool TestBase::CompareDepthResultCRC32(const std::vector crc32) { std::vector read_data; ReadDepthData(read_data); diff --git a/modules/holoviz/tests/functional/headless_fixture.hpp b/modules/holoviz/tests/functional/test_fixture.hpp similarity index 60% rename from modules/holoviz/tests/functional/headless_fixture.hpp rename to modules/holoviz/tests/functional/test_fixture.hpp index 562e8f81..f28ee111 100644 --- a/modules/holoviz/tests/functional/headless_fixture.hpp +++ b/modules/holoviz/tests/functional/test_fixture.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef HOLOVIZ_TESTS_FUNCTIONAL_HEADLESS_FIXTURE_HPP -#define HOLOVIZ_TESTS_FUNCTIONAL_HEADLESS_FIXTURE_HPP +#ifndef MODULES_HOLOVIZ_TESTS_FUNCTIONAL_TEST_FIXTURE_HPP +#define MODULES_HOLOVIZ_TESTS_FUNCTIONAL_TEST_FIXTURE_HPP #include @@ -25,23 +25,28 @@ #include /** - * Fixture that initializes Holoviz in headless mode and support functions to setup, read back + * Fixture that initializes Holoviz and has support functions to setup, read back * and compare data. */ -class TestHeadless : public ::testing::Test { +class TestBase : public ::testing::Test { protected: /** - * Construct a new TestHeadless object + * Construct a new TestBase object + * + * @param init_flags init flags */ - TestHeadless() = default; + explicit TestBase(holoscan::viz::InitFlags init_flags) : init_flags_(init_flags) {} + TestBase() = delete; /** - * Construct a new TestHeadless object with a given window size + * Construct a new TestBase object with a given window size * * @param width window width * @param height window height + * @param init_flags init flags */ - TestHeadless(uint32_t width, uint32_t height) : width_(width), height_(height) {} + TestBase(uint32_t width, uint32_t height, holoscan::viz::InitFlags init_flags) + : width_(width), height_(height), init_flags_(init_flags) {} /// ::testing::Test virtual members ///@{ @@ -81,14 +86,14 @@ class TestHeadless : public ::testing::Test { /** * Read back color data and compare with the data generated with SetupData(). * - * @returns false if read back and generated data do not match + * @return false if read back and generated data do not match */ bool CompareColorResult(); /** * Read back depth data and compare with the data generated with SetupData(). * - * @returns false if read back and generated data do not match + * @return false if read back and generated data do not match */ bool CompareDepthResult(); @@ -97,7 +102,7 @@ class TestHeadless : public ::testing::Test { * * @param crc32 vector of expected CRC32's * - * @returns false if CRC32 of read back does not match provided CRC32 + * @return false if CRC32 of read back does not match provided CRC32 */ bool CompareColorResultCRC32(const std::vector crc32); @@ -106,7 +111,7 @@ class TestHeadless : public ::testing::Test { * * @param crc32 vector of expected CRC32's * - * @returns false if CRC32 of read back does not match provided CRC32 + * @return false if CRC32 of read back does not match provided CRC32 */ bool CompareDepthResultCRC32(const std::vector crc32); @@ -119,6 +124,37 @@ class TestHeadless : public ::testing::Test { std::vector color_data_; std::vector depth_data_; + + protected: + holoscan::viz::InitFlags init_flags_ = holoscan::viz::InitFlags::NONE; + + private: + bool initialized_ = false; +}; + +/** + * Fixture that initializes Holoviz in headless mode. + */ +class TestHeadless : public TestBase { + public: + TestHeadless() : TestBase(holoscan::viz::InitFlags::HEADLESS) {} + + /** + * Construct a new TestHeadless object with a given window size + * + * @param width window width + * @param height window height + */ + TestHeadless(uint32_t width, uint32_t height) + : TestBase(width, height, holoscan::viz::InitFlags::HEADLESS) {} +}; + +/** + * Fixture that initializes Holoviz in headless mode. + */ +class TestWindow : public TestBase { + public: + TestWindow() : TestBase(holoscan::viz::InitFlags::NONE) {} }; -#endif /* HOLOVIZ_TESTS_FUNCTIONAL_HEADLESS_FIXTURE_HPP */ +#endif /* MODULES_HOLOVIZ_TESTS_FUNCTIONAL_TEST_FIXTURE_HPP */ diff --git a/modules/holoviz/thirdparty/nvpro_core/nvh/cameramanipulator.cpp b/modules/holoviz/thirdparty/nvpro_core/nvh/cameramanipulator.cpp index d01b0005..becee614 100644 --- a/modules/holoviz/thirdparty/nvpro_core/nvh/cameramanipulator.cpp +++ b/modules/holoviz/thirdparty/nvpro_core/nvh/cameramanipulator.cpp @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. * - * SPDX-FileCopyrightText: Copyright (c) 2018-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2018-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ //-------------------------------------------------------------------- @@ -101,6 +101,7 @@ void CameraManipulator::updateAnim() { m_current = m_goal; m_anim_done = true; + update(); return; } diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index a8d7edfd..12404ffa 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -70,19 +70,27 @@ if(HOLOSCAN_BUILD_TESTS) set_tests_properties(python-api-system-distributed-ebs-tests PROPERTIES FAIL_REGULAR_EXPRESSION "Fatal Python error") - # set environment variables used by distributed applications in the tests - # - UCX_TCP_CM_REUSEADDR=y : Reuse port for UCX - # (see https://github.com/openucx/ucx/issues/8585 and https://github.com/rapidsai/ucxx#c-1) - # - HOLOSCAN_STOP_ON_DEADLOCK_TIMEOUT=2500 : Set deadlock timeout for distributed app - # - HOLOSCAN_MAX_DURATION_MS=2500 : Set max duration for distributed app +# set environment variables used by distributed applications in the tests +# - UCX_TCP_CM_REUSEADDR=y : Reuse port for UCX +# (see https://github.com/openucx/ucx/issues/8585 and https://github.com/rapidsai/ucxx#c-1) +# - HOLOSCAN_STOP_ON_DEADLOCK_TIMEOUT=2500 : Set deadlock timeout for distributed app +# - HOLOSCAN_MAX_DURATION_MS=2500 : Set max duration for distributed app +if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "aarch64") set(CMAKE_DISTRIBUTED_TEST_FLAGS "\ -UCX_TCP_CM_REUSEADDR=y;\ -UCX_PROTO_ENABLE=y;\ -HOLOSCAN_STOP_ON_DEADLOCK_TIMEOUT=2500;\ -HOLOSCAN_MAX_DURATION_MS=2500\ +HOLOSCAN_STOP_ON_DEADLOCK_TIMEOUT=6000;\ +HOLOSCAN_MAX_DURATION_MS=6000\ " ) +else() + set(CMAKE_DISTRIBUTED_TEST_FLAGS +"\ +HOLOSCAN_STOP_ON_DEADLOCK_TIMEOUT=3000;\ +HOLOSCAN_MAX_DURATION_MS=3000\ +" + ) +endif() + set_tests_properties(python-api-system-distributed-tests PROPERTIES ENVIRONMENT "${CMAKE_DISTRIBUTED_TEST_FLAGS} HOLOSCAN_DISTRIBUTED_APP_SCHEDULER=multi_thread" ) diff --git a/python/holoscan/cli/common/artifact_sources.py b/python/holoscan/cli/common/artifact_sources.py index 0646568d..1ee53986 100644 --- a/python/holoscan/cli/common/artifact_sources.py +++ b/python/holoscan/cli/common/artifact_sources.py @@ -39,7 +39,7 @@ class ArtifactSources: def __init__(self) -> None: self._logger = logging.getLogger("common") - self._supported_holoscan_versions = ["2.0.0"] + self._supported_holoscan_versions = ["2.0.0", "2.1.0"] @property def holoscan_versions(self) -> List[str]: diff --git a/python/holoscan/cli/common/dockerutils.py b/python/holoscan/cli/common/dockerutils.py index cc2e4009..d1036ab1 100644 --- a/python/holoscan/cli/common/dockerutils.py +++ b/python/holoscan/cli/common/dockerutils.py @@ -199,7 +199,7 @@ def docker_run( """ volumes = [] environment_variables = { - "NVIDIA_DRIVER_CAPABILITIES": "graphics,video,compute,utility,display", + "NVIDIA_DRIVER_CAPABILITIES": "all", "HOLOSCAN_HOSTING_SERVICE": "HOLOSCAN_RUN", "UCX_CM_USE_ALL_DEVICES": "y" if use_all_nics else "n", } @@ -216,6 +216,16 @@ def docker_run( display = os.environ.get("DISPLAY", None) if display is not None: environment_variables["DISPLAY"] = display + xdg_session_type = os.environ.get("XDG_SESSION_TYPE", None) + if xdg_session_type is not None: + environment_variables["XDG_SESSION_TYPE"] = xdg_session_type + xdg_runtime_dir = os.environ.get("XDG_RUNTIME_DIR", None) + if xdg_runtime_dir is not None: + volumes.append((xdg_runtime_dir, xdg_runtime_dir)) + environment_variables["XDG_RUNTIME_DIR"] = xdg_runtime_dir + wayland_display = os.environ.get("WAYLAND_DISPLAY", None) + if wayland_display is not None: + environment_variables["WAYLAND_DISPLAY"] = wayland_display # Use user-specified --gpu values if gpu_enum is not None: diff --git a/python/holoscan/cli/packager/templates/tools.sh b/python/holoscan/cli/packager/templates/tools.sh index 671b25ba..eca83dc5 100755 --- a/python/holoscan/cli/packager/templates/tools.sh +++ b/python/holoscan/cli/packager/templates/tools.sh @@ -1,6 +1,6 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -394,7 +394,7 @@ main() { c_echo nocolor "docker run --runtime nvidia \\" c_echo nocolor " --gpus all \\" c_echo nocolor " -it \\" - c_echo nocolor " -e NVIDIA_DRIVER_CAPABILITIES=graphics,video,compute,utility,display \\" + c_echo nocolor " -e NVIDIA_DRIVER_CAPABILITIES=all \\" c_echo nocolor " -e DISPLAY=\$DISPLAY \\" c_echo nocolor " -v /tmp/.X11-unix:/tmp/.X11-unix \\" c_echo nocolor " -v \${MY-INPUT-DATA}:/var/holoscan/input \\" diff --git a/python/holoscan/conditions/CMakeLists.txt b/python/holoscan/conditions/CMakeLists.txt index 856d6217..f0b6dbd3 100644 --- a/python/holoscan/conditions/CMakeLists.txt +++ b/python/holoscan/conditions/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,6 +14,7 @@ # limitations under the License. holoscan_pybind11_module(conditions + asynchronous.cpp boolean.cpp conditions.cpp count.cpp diff --git a/python/holoscan/conditions/__init__.py b/python/holoscan/conditions/__init__.py index 314f0c2b..e974bab2 100644 --- a/python/holoscan/conditions/__init__.py +++ b/python/holoscan/conditions/__init__.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,6 +16,7 @@ .. autosummary:: + holoscan.conditions.AsynchronousCondition holoscan.conditions.BooleanCondition holoscan.conditions.CountCondition holoscan.conditions.DownstreamMessageAffordableCondition @@ -24,6 +25,8 @@ """ from ._conditions import ( + AsynchronousCondition, + AsynchronousEventState, BooleanCondition, CountCondition, DownstreamMessageAffordableCondition, @@ -32,6 +35,8 @@ ) __all__ = [ + "AsynchronousCondition", + "AsynchronousEventState", "BooleanCondition", "CountCondition", "DownstreamMessageAffordableCondition", diff --git a/python/holoscan/conditions/asynchronous.cpp b/python/holoscan/conditions/asynchronous.cpp new file mode 100644 index 00000000..681ca6a7 --- /dev/null +++ b/python/holoscan/conditions/asynchronous.cpp @@ -0,0 +1,93 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include + +#include "./asynchronous_pydoc.hpp" +#include "holoscan/core/component_spec.hpp" +#include "holoscan/core/conditions/gxf/count.hpp" +#include "holoscan/core/fragment.hpp" +#include "holoscan/core/gxf/gxf_resource.hpp" + +using std::string_literals::operator""s; +using pybind11::literals::operator""_a; + +#define STRINGIFY(x) #x +#define MACRO_STRINGIFY(x) STRINGIFY(x) + +namespace py = pybind11; + +namespace holoscan { + +/* Trampoline classes for handling Python kwargs + * + * These add a constructor that takes a Fragment for which to initialize the condition. + * The explicit parameter list and default arguments take care of providing a Pythonic + * kwarg-based interface with appropriate default values matching the condition's + * default parameters in the C++ API `setup` method. + * + * The sequence of events in this constructor is based on Fragment::make_condition + */ + +class PyAsynchronousCondition : public AsynchronousCondition { + public: + /* Inherit the constructors */ + using AsynchronousCondition::AsynchronousCondition; + + // Define a constructor that fully initializes the object. + explicit PyAsynchronousCondition(Fragment* fragment, + const std::string& name = "noname_async_condition") + : AsynchronousCondition() { + name_ = name; + fragment_ = fragment; + spec_ = std::make_shared(fragment); + setup(*spec_.get()); + } +}; + +void init_asynchronous(py::module_& m) { + py::enum_(m, "AsynchronousEventState") + .value("READY", holoscan::AsynchronousEventState::READY) + .value("WAIT", holoscan::AsynchronousEventState::WAIT) + .value("EVENT_WAITING", holoscan::AsynchronousEventState::EVENT_WAITING) + .value("EVENT_DONE", holoscan::AsynchronousEventState::EVENT_DONE) + .value("EVENT_NEVER", holoscan::AsynchronousEventState::EVENT_NEVER); + + py::class_>( + m, "AsynchronousCondition", doc::AsynchronousCondition::doc_AsynchronousCondition) + .def(py::init(), + "fragment"_a, + "name"_a = "noname_async_condition"s, + doc::AsynchronousCondition::doc_AsynchronousCondition_python) + .def_property_readonly("gxf_typename", + &AsynchronousCondition::gxf_typename, + doc::AsynchronousCondition::doc_gxf_typename) + .def_property( + "event_state", + py::overload_cast<>(&AsynchronousCondition::event_state, py::const_), + py::overload_cast(&AsynchronousCondition::event_state), + doc::AsynchronousCondition::doc_event_state) + .def("setup", &AsynchronousCondition::setup, "spec"_a, doc::AsynchronousCondition::doc_setup); +} +} // namespace holoscan diff --git a/python/holoscan/conditions/asynchronous_pydoc.hpp b/python/holoscan/conditions/asynchronous_pydoc.hpp new file mode 100644 index 00000000..09858ae4 --- /dev/null +++ b/python/holoscan/conditions/asynchronous_pydoc.hpp @@ -0,0 +1,79 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PYHOLOSCAN_CONDITIONS_ASYNCHRONOUS_PYDOC_HPP +#define PYHOLOSCAN_CONDITIONS_ASYNCHRONOUS_PYDOC_HPP + +#include + +#include "../macros.hpp" + +namespace holoscan::doc { + +namespace AsynchronousCondition { + +PYDOC(AsynchronousCondition, R"doc( +Asynchronous condition class. + +Used to control whether an entity is executed. +)doc") + +// PyAsynchronousCondition Constructor +PYDOC(AsynchronousCondition_python, R"doc( +Asynchronous condition. + +Parameters +---------- +fragment : holoscan.core.Fragment + The fragment the condition will be associated with +name : str, optional + The name of the condition. +)doc") + +PYDOC(gxf_typename, R"doc( +The GXF type name of the condition. + +Returns +------- +str + The GXF type name of the condition +)doc") + +PYDOC(event_state, R"doc( +Event state property + +- AsynchronousEventState.READY +- AsynchronousEventState.WAIT +- AsynchronousEventState.EVENT_WAITING +- AsynchronousEventState.EVENT_DONE +- AsynchronousEventState.EVENT_NEVER +)doc") + +PYDOC(setup, R"doc( +Define the component specification. + +Parameters +---------- +spec : holoscan.core.ComponentSpec + Component specification associated with the condition. +)doc") + +} // namespace AsynchronousCondition + +} // namespace holoscan::doc + +#endif // PYHOLOSCAN_CONDITIONS_ASYNCHRONOUS_PYDOC_HPP diff --git a/python/holoscan/conditions/boolean.cpp b/python/holoscan/conditions/boolean.cpp index 53f00017..bccd4499 100644 --- a/python/holoscan/conditions/boolean.cpp +++ b/python/holoscan/conditions/boolean.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -69,7 +69,7 @@ void init_boolean(py::module_& m) { "fragment"_a, "enable_tick"_a = true, "name"_a = "noname_boolean_condition"s, - doc::BooleanCondition::doc_BooleanCondition_python) + doc::BooleanCondition::doc_BooleanCondition) .def_property_readonly( "gxf_typename", &BooleanCondition::gxf_typename, doc::BooleanCondition::doc_gxf_typename) .def("enable_tick", &BooleanCondition::enable_tick, doc::BooleanCondition::doc_enable_tick) diff --git a/python/holoscan/conditions/boolean_pydoc.hpp b/python/holoscan/conditions/boolean_pydoc.hpp index ad93aa3e..6c01c5ab 100644 --- a/python/holoscan/conditions/boolean_pydoc.hpp +++ b/python/holoscan/conditions/boolean_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -27,13 +27,6 @@ namespace holoscan::doc { namespace BooleanCondition { PYDOC(BooleanCondition, R"doc( -Boolean condition class. - -Used to control whether an entity is executed. -)doc") - -// PyBooleanCondition Constructor -PYDOC(BooleanCondition_python, R"doc( Boolean condition. Parameters diff --git a/python/holoscan/conditions/conditions.cpp b/python/holoscan/conditions/conditions.cpp index b2a74ef9..4f911798 100644 --- a/python/holoscan/conditions/conditions.cpp +++ b/python/holoscan/conditions/conditions.cpp @@ -24,6 +24,7 @@ namespace py = pybind11; namespace holoscan { +void init_asynchronous(py::module_&); void init_boolean(py::module_&); void init_count(py::module_&); void init_periodic(py::module_&); @@ -32,21 +33,12 @@ void init_message_available(py::module_&); PYBIND11_MODULE(_conditions, m) { m.doc() = R"pbdoc( - Holoscan SDK Python Bindings + Holoscan SDK Conditions Python Bindings --------------------------------------- .. currentmodule:: _conditions - .. autosummary:: - :toctree: _generate - add - subtract )pbdoc"; -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif - + init_asynchronous(m); init_boolean(m); init_count(m); init_periodic(m); diff --git a/python/holoscan/conditions/count.cpp b/python/holoscan/conditions/count.cpp index 4d58efe8..900fbdad 100644 --- a/python/holoscan/conditions/count.cpp +++ b/python/holoscan/conditions/count.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -70,7 +70,7 @@ void init_count(py::module_& m) { "fragment"_a, "count"_a = 1L, "name"_a = "noname_count_condition"s, - doc::CountCondition::doc_CountCondition_python) + doc::CountCondition::doc_CountCondition) .def_property_readonly( "gxf_typename", &CountCondition::gxf_typename, doc::CountCondition::doc_gxf_typename) .def("setup", &CountCondition::setup, doc::CountCondition::doc_setup) diff --git a/python/holoscan/conditions/count_pydoc.hpp b/python/holoscan/conditions/count_pydoc.hpp index 92cb2526..b0ab4abf 100644 --- a/python/holoscan/conditions/count_pydoc.hpp +++ b/python/holoscan/conditions/count_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -27,11 +27,6 @@ namespace holoscan::doc { namespace CountCondition { PYDOC(CountCondition, R"doc( -Count condition class. -)doc") - -// PyCountCondition Constructor -PYDOC(CountCondition_python, R"doc( Count condition. Parameters diff --git a/python/holoscan/conditions/downstream_message_affordable.cpp b/python/holoscan/conditions/downstream_message_affordable.cpp index 60310fa5..e4a705be 100644 --- a/python/holoscan/conditions/downstream_message_affordable.cpp +++ b/python/holoscan/conditions/downstream_message_affordable.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -79,8 +79,7 @@ void init_downstream_message_affordable(py::module_& m) { "fragment"_a, "min_size"_a = 1L, "name"_a = "noname_downstream_affordable_condition"s, - doc::DownstreamMessageAffordableCondition:: - doc_DownstreamMessageAffordableCondition_python) + doc::DownstreamMessageAffordableCondition::doc_DownstreamMessageAffordableCondition) .def_property_readonly("gxf_typename", &DownstreamMessageAffordableCondition::gxf_typename, doc::DownstreamMessageAffordableCondition::doc_gxf_typename) diff --git a/python/holoscan/conditions/downstream_message_affordable_pydoc.hpp b/python/holoscan/conditions/downstream_message_affordable_pydoc.hpp index e89f1647..89f96401 100644 --- a/python/holoscan/conditions/downstream_message_affordable_pydoc.hpp +++ b/python/holoscan/conditions/downstream_message_affordable_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,11 +28,11 @@ namespace DownstreamMessageAffordableCondition { PYDOC(DownstreamMessageAffordableCondition, R"doc( Condition that permits execution when the downstream operator can accept new messages. -)doc") -// PyDownstreamMessageAffordableCondition Constructor -PYDOC(DownstreamMessageAffordableCondition_python, R"doc( -Condition that permits execution when the downstream operator can accept new messages. +Satisfied when the receiver queue of any connected downstream operators has at least a certain +number of elements free. The minimum number of messages that permits the execution of +the entity is specified by `min_size`. It can be used for operators to prevent operators from +sending a message when the downstream operator is not ready to receive it. Parameters ---------- diff --git a/python/holoscan/conditions/message_available.cpp b/python/holoscan/conditions/message_available.cpp index fa65979b..0c5ea890 100644 --- a/python/holoscan/conditions/message_available.cpp +++ b/python/holoscan/conditions/message_available.cpp @@ -78,7 +78,7 @@ void init_message_available(py::module_& m) { "min_size"_a = 1UL, "front_stage_max_size"_a = 1UL, "name"_a = "noname_message_available_condition"s, - doc::MessageAvailableCondition::doc_MessageAvailableCondition_python) + doc::MessageAvailableCondition::doc_MessageAvailableCondition) .def_property_readonly("gxf_typename", &MessageAvailableCondition::gxf_typename, doc::MessageAvailableCondition::doc_gxf_typename) diff --git a/python/holoscan/conditions/message_available_pydoc.hpp b/python/holoscan/conditions/message_available_pydoc.hpp index b9d271c6..2f031f70 100644 --- a/python/holoscan/conditions/message_available_pydoc.hpp +++ b/python/holoscan/conditions/message_available_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,7 +29,7 @@ namespace MessageAvailableCondition { PYDOC(MessageAvailableCondition, R"doc( Condition that permits execution when an upstream message is available. -Executed when the associated receiver queue has at least a certain number of +Satisfied when the associated receiver queue has at least a certain number of elements. The receiver is specified using the receiver parameter of the scheduling term. The minimum number of messages that permits the execution of the entity is specified by `min_size`. An optional parameter for this @@ -37,11 +37,6 @@ scheduling term is `front_stage_max_size`, the maximum front stage message count. If this parameter is set, the scheduling term will only allow execution if the number of messages in the queue does not exceed this count. It can be used for operators which do not consume all messages from the queue. -)doc") - -// PyMessageAvailableCondition Constructor -PYDOC(MessageAvailableCondition_python, R"doc( -Condition that permits execution when an upstream message is available. Parameters ---------- @@ -101,4 +96,4 @@ time, and uses a light-weight initialization. } // namespace holoscan::doc -#endif // PYHOLOSCAN_CONDITIONS_MESSAGE_AVAILABLE_PYDOC_HPP +#endif /* PYHOLOSCAN_CONDITIONS_MESSAGE_AVAILABLE_PYDOC_HPP */ diff --git a/python/holoscan/conditions/periodic.cpp b/python/holoscan/conditions/periodic.cpp index ae2148e5..2bb6f50e 100644 --- a/python/holoscan/conditions/periodic.cpp +++ b/python/holoscan/conditions/periodic.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -91,7 +91,7 @@ void init_periodic(py::module_& m) { "fragment"_a, "recess_period"_a, "name"_a = "noname_periodic_condition"s, - doc::PeriodicCondition::doc_PeriodicCondition_python) + doc::PeriodicCondition::doc_PeriodicCondition) .def_property_readonly("gxf_typename", &PeriodicCondition::gxf_typename, doc::PeriodicCondition::doc_gxf_typename) diff --git a/python/holoscan/conditions/periodic_pydoc.hpp b/python/holoscan/conditions/periodic_pydoc.hpp index 7ebb5252..3936fd5c 100644 --- a/python/holoscan/conditions/periodic_pydoc.hpp +++ b/python/holoscan/conditions/periodic_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -43,11 +43,6 @@ For example: datetime.timedelta(minutes=1), datetime.timedelta(seconds=1), datetime.timedelta(milliseconds=1) and datetime.timedelta(microseconds=1). Supported argument names are: weeks| days | hours | minutes | seconds | millisecons | microseconds This requires `import datetime`. -)doc") - -// PyPeriodicCondition Constructor -PYDOC(PeriodicCondition_python, R"doc( -Condition class to support periodic execution of operators. Parameters ---------- diff --git a/python/holoscan/core/CMakeLists.txt b/python/holoscan/core/CMakeLists.txt index e6e09fab..05450b32 100644 --- a/python/holoscan/core/CMakeLists.txt +++ b/python/holoscan/core/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,6 +23,7 @@ holoscan_pybind11_module( core.cpp dataflow_tracker.cpp dl_converter.cpp + emitter_receiver_registry.cpp execution_context.cpp executor.cpp fragment.cpp @@ -36,3 +37,15 @@ holoscan_pybind11_module( tensor.cpp ../gxf/entity.cpp ) + +# Copy headers +install( + DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + DESTINATION include/holoscan/python + FILE_PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ + DIRECTORY_PERMISSIONS OWNER_READ OWNER_EXECUTE OWNER_WRITE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE + COMPONENT "holoscan-python_libs" + FILES_MATCHING + PATTERN "*.hpp" + PATTERN "*pydoc.hpp" EXCLUDE +) diff --git a/python/holoscan/core/__init__.py b/python/holoscan/core/__init__.py index 4702f58b..8c1e3e9b 100644 --- a/python/holoscan/core/__init__.py +++ b/python/holoscan/core/__init__.py @@ -72,7 +72,6 @@ ArgType, CLIOptions, Component, - ComponentSpec, Condition, ConditionType, Config, @@ -87,6 +86,8 @@ from ._core import InputContext, IOSpec, Message, NetworkContext from ._core import Operator as _Operator from ._core import OutputContext, ParameterFlag +from ._core import PyComponentSpec as ComponentSpec +from ._core import PyRegistryContext as _RegistryContext from ._core import PyOperatorSpec as OperatorSpec from ._core import PyTensor as Tensor from ._core import ( @@ -97,6 +98,7 @@ kwargs_to_arglist, py_object_to_arg, ) +from ._core import register_types as _register_types Graph = OperatorGraph # define alias for backward compatibility @@ -137,6 +139,7 @@ "Tracker", "arg_to_py_object", "arglist_to_kwargs", + "io_type_registry", "kwargs_to_arglist", "py_object_to_arg", ] @@ -356,3 +359,9 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, exc_tb): if self.enable_logging: self.tracker.end_logging() + + +_registry_context = _RegistryContext() +io_type_registry = _registry_context.registry() + +_register_types(io_type_registry) diff --git a/python/holoscan/core/application.hpp b/python/holoscan/core/application.hpp index 4f2c50c9..5dc3e0c0 100644 --- a/python/holoscan/core/application.hpp +++ b/python/holoscan/core/application.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef PYBIND11_CORE_APPLICATION_HPP -#define PYBIND11_CORE_APPLICATION_HPP +#ifndef PYHOLOSCAN_CORE_APPLICATION_HPP +#define PYHOLOSCAN_CORE_APPLICATION_HPP #include #include @@ -103,4 +103,4 @@ class PyApplication : public Application { } // namespace holoscan -#endif /* PYBIND11_CORE_APPLICATION_HPP */ +#endif /* PYHOLOSCAN_CORE_APPLICATION_HPP */ diff --git a/python/holoscan/core/arg.hpp b/python/holoscan/core/arg.hpp index bb2654d8..1933bfcb 100644 --- a/python/holoscan/core/arg.hpp +++ b/python/holoscan/core/arg.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef PYBIND11_CORE_ARG_HPP -#define PYBIND11_CORE_ARG_HPP +#ifndef PYHOLOSCAN_CORE_ARG_HPP +#define PYHOLOSCAN_CORE_ARG_HPP #include @@ -32,4 +32,4 @@ void init_arg(py::module_&); } // namespace holoscan -#endif /* PYBIND11_CORE_ARG_HPP */ +#endif /* PYHOLOSCAN_CORE_ARG_HPP */ diff --git a/python/holoscan/core/component.cpp b/python/holoscan/core/component.cpp index 6aa59a21..c05b28f7 100644 --- a/python/holoscan/core/component.cpp +++ b/python/holoscan/core/component.cpp @@ -20,6 +20,7 @@ #include #include +#include #include "component_pydoc.hpp" #include "holoscan/core/arg.hpp" @@ -32,6 +33,55 @@ namespace py = pybind11; namespace holoscan { +class PyComponentSpec : public ComponentSpec { + public: + /* Inherit the constructors */ + using ComponentSpec::ComponentSpec; + + // Override the constructor to get the py::object for the Python class + explicit PyComponentSpec(Fragment* fragment = nullptr, py::object op = py::none()) + : ComponentSpec(fragment), py_op_(op) {} + + // TOIMPROVE: Should we parse headline and description from kwargs or just + // add them to the function signature? + void py_param(const std::string& name, const py::object& default_value, const ParameterFlag& flag, + const py::kwargs& kwargs) { + using std::string_literals::operator""s; + + bool is_receivers = false; + std::string headline{""s}; + std::string description{""s}; + for (const auto& [name, value] : kwargs) { + std::string param_name = name.cast(); + if (param_name == "headline") { + headline = value.cast(); + } else if (param_name == "description") { + description = value.cast(); + } else { + throw std::runtime_error("unsupported kwarg: "s + param_name); + } + } + + // Create parameter object + py_params_.emplace_back(py_op()); + + // Register parameter + auto& parameter = py_params_.back(); + param(parameter, name.c_str(), headline.c_str(), description.c_str(), default_value, flag); + } + + py::object py_op() const { return py_op_; } + + std::list>& py_params() { return py_params_; } + + private: + py::object py_op_ = py::none(); + // NOTE: we use std::list instead of std::vector because we register the address of Parameter + // object to the GXF framework. The address of a std::vector element may change when the vector is + // resized. + std::list> py_params_; +}; + class PyComponentBase : public ComponentBase { public: /* Inherit the constructors */ @@ -72,6 +122,25 @@ void init_component(py::module_& m) { [](const ComponentSpec& spec) { return spec.description(); }, R"doc(Return repr(self).)doc"); + py::enum_(m, "ParameterFlag", doc::ParameterFlag::doc_ParameterFlag) + .value("NONE", ParameterFlag::kNone) + .value("OPTIONAL", ParameterFlag::kOptional) + .value("DYNAMIC", ParameterFlag::kDynamic); + + py::class_>( + m, "PyComponentSpec", R"doc(Operator specification class.)doc") + .def(py::init(), + "fragment"_a, + "op"_a = py::none(), + doc::ComponentSpec::doc_ComponentSpec) + .def("param", + &PyComponentSpec::py_param, + "Register parameter", + "name"_a, + "default_value"_a = py::none(), + "flag"_a = ParameterFlag::kNone, + doc::ComponentSpec::doc_param); + py::class_>( m, "ComponentBase", doc::Component::doc_Component) .def(py::init<>(), doc::Component::doc_Component) diff --git a/python/holoscan/core/component_pydoc.hpp b/python/holoscan/core/component_pydoc.hpp index 551210d3..a4c04018 100644 --- a/python/holoscan/core/component_pydoc.hpp +++ b/python/holoscan/core/component_pydoc.hpp @@ -24,6 +24,20 @@ namespace holoscan::doc { +namespace ParameterFlag { + +// Constructor +PYDOC(ParameterFlag, R"doc( +Enum class for parameter flags. + +The following flags are supported: +- `NONE`: The parameter is mendatory and static. It cannot be changed at runtime. +- `OPTIONAL`: The parameter is optional and might not be available at runtime. +- `DYNAMIC`: The parameter is dynamic and might change at runtime. +)doc") + +} // namespace ParameterFlag + namespace ComponentSpec { // Constructor @@ -52,6 +66,49 @@ PYDOC(description, R"doc( YAML formatted string describing the component spec. )doc") +PYDOC(param, R"doc( +Add a parameter to the specification. + +Parameters +---------- +param : name + The name of the parameter. +default_value : object + The default value for the parameter. + +Additional Parameters +--------------------- +headline : str, optional + If provided, this is a brief "headline" description for the parameter. +description : str, optional + If provided, this is a description for the parameter (typically more verbose than the brief + description provided via `headline`). +kind : str, optional + In most cases, this keyword should not be specified. If specified, the only valid option is + currently ``kind="receivers"``, which can be used to create a parameter holding a vector of + receivers. This effectively creates a multi-receiver input port to which any number of + operators can be connected. +flag: holoscan.core.ParameterFlag, optional + If provided, this is a flag that can be used to control the behavior of the parameter. + By default, `ParameterFlag.NONE` is used. + + The following flags are supported: + - `ParameterFlag.NONE`: The parameter is mendatory and static. It cannot be changed at runtime. + - `ParameterFlag.OPTIONAL`: The parameter is optional and might not be available at runtime. + - `ParameterFlag.DYNAMIC`: The parameter is dynamic and might change at runtime. + +Notes +----- +This method is intended to be called within the `setup` method of an Operator. + +In general, for native Python operators, it is not necessary to call `param` to register a +parameter with the class. Instead, one can just directly add parameters to the Python operator +class (e.g. For example, directly assigning ``self.param_name = value`` in __init__.py). + +The one case which cannot be implemented without a call to `param` is adding a multi-receiver port +to an operator via a parameter with ``kind="receivers"`` set. +)doc") + } // namespace ComponentSpec namespace Component { diff --git a/python/holoscan/core/condition_pydoc.hpp b/python/holoscan/core/condition_pydoc.hpp index 71d27de5..4ab986e7 100644 --- a/python/holoscan/core/condition_pydoc.hpp +++ b/python/holoscan/core/condition_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -46,7 +46,7 @@ Class representing a condition. Can be initialized with any number of Python positional and keyword arguments. If a `name` keyword argument is provided, it must be a `str` and will be -used to set the name of the Operator. +used to set the name of the condition. If a `fragment` keyword argument is provided, it must be of type `holoscan.core.Fragment` (or diff --git a/python/holoscan/core/core.cpp b/python/holoscan/core/core.cpp index ac8632aa..7e569cf9 100644 --- a/python/holoscan/core/core.cpp +++ b/python/holoscan/core/core.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -32,20 +32,11 @@ namespace holoscan { PYBIND11_MODULE(_core, m) { m.doc() = R"pbdoc( - Holoscan SDK Python Bindings - --------------------------------------- + Holoscan SDK Core Python Bindings + --------------------------------- .. currentmodule:: _core - .. autosummary:: - :toctree: _generate - add - subtract )pbdoc"; -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif init_arg(m); init_kwarg_handling(m); init_component(m); diff --git a/python/holoscan/core/core.hpp b/python/holoscan/core/core.hpp index 28e15691..a15d8e52 100644 --- a/python/holoscan/core/core.hpp +++ b/python/holoscan/core/core.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef PYBIND11_CORE_CORE_HPP -#define PYBIND11_CORE_CORE_HPP +#ifndef PYHOLOSCAN_CORE_CORE_HPP +#define PYHOLOSCAN_CORE_CORE_HPP #include @@ -47,4 +47,4 @@ void init_cli(py::module_&); } // namespace holoscan -#endif /* PYBIND11_CORE_CORE_HPP */ +#endif /* PYHOLOSCAN_CORE_CORE_HPP */ diff --git a/python/holoscan/core/dl_converter.hpp b/python/holoscan/core/dl_converter.hpp index d8c756d9..abfa9960 100644 --- a/python/holoscan/core/dl_converter.hpp +++ b/python/holoscan/core/dl_converter.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef PYBIND11_CORE_DL_CONVERTER_HPP -#define PYBIND11_CORE_DL_CONVERTER_HPP +#ifndef PYHOLOSCAN_CORE_DL_CONVERTER_HPP +#define PYHOLOSCAN_CORE_DL_CONVERTER_HPP #include #include @@ -131,4 +131,4 @@ pybind11::tuple array2pytuple(const T* arr, size_t length) { } // namespace holoscan -#endif /* PYBIND11_CORE_DL_CONVERTER_HPP */ +#endif /* PYHOLOSCAN_CORE_DL_CONVERTER_HPP */ diff --git a/python/holoscan/core/emitter_receiver_registry.cpp b/python/holoscan/core/emitter_receiver_registry.cpp new file mode 100644 index 00000000..e627a020 --- /dev/null +++ b/python/holoscan/core/emitter_receiver_registry.cpp @@ -0,0 +1,129 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "./emitter_receiver_registry.hpp" + +#include +#include + +namespace holoscan { + +EmitterReceiverRegistry& EmitterReceiverRegistry::get_instance() { + static EmitterReceiverRegistry instance; + return instance; +} + +const EmitterReceiverRegistry::EmitterReceiver& EmitterReceiverRegistry::get_emitter_receiver( + const std::type_index& index) const { + auto maybe_name = index_to_name(index); + if (!maybe_name) { + HOLOSCAN_LOG_WARN("No emitter_receiver for type '{}' exists", index.name()); + return EmitterReceiverRegistry::none_emitter_receiver; + } + auto& emitter_receiver = emitter_receiver_map_.at(maybe_name.value()); + return emitter_receiver; +} + +bool EmitterReceiverRegistry::has_emitter_receiver(const std::type_index& index) const { + auto maybe_name = index_to_name(index); + if (maybe_name) { return emitter_receiver_map_.count(maybe_name.value()) > 0 ? true : false; } + return false; +} + +const EmitterReceiverRegistry::EmitterReceiver& EmitterReceiverRegistry::get_emitter_receiver( + const std::string& name) const { + auto loc = emitter_receiver_map_.find(name); + if (loc == emitter_receiver_map_.end()) { + HOLOSCAN_LOG_WARN("No emitter_receiver for name '{}' exists", name); + return EmitterReceiverRegistry::none_emitter_receiver; + } + auto& emitter_receiver = loc->second; + return emitter_receiver; +} + +const EmitterReceiverRegistry::EmitFunc& EmitterReceiverRegistry::get_emitter( + const std::string& name) const { + auto loc = emitter_receiver_map_.find(name); + if (loc == emitter_receiver_map_.end()) { + HOLOSCAN_LOG_WARN("No emitter for name '{}' exists", name); + return EmitterReceiverRegistry::none_emit; + } + auto& emitter_receiver = loc->second; + return emitter_receiver.first; +} + +const EmitterReceiverRegistry::EmitFunc& EmitterReceiverRegistry::get_emitter( + const std::type_index& index) const { + auto maybe_name = index_to_name(index); + if (!maybe_name) { + HOLOSCAN_LOG_WARN("No emitter for type '{}' exists", index.name()); + return EmitterReceiverRegistry::none_emit; + } + auto& emitter = emitter_receiver_map_.at(maybe_name.value()).first; + return emitter; +} + +const EmitterReceiverRegistry::ReceiveFunc& EmitterReceiverRegistry::get_receiver( + const std::string& name) const { + auto loc = emitter_receiver_map_.find(name); + if (loc == emitter_receiver_map_.end()) { + HOLOSCAN_LOG_WARN("No receiver for name '{}' exists", name); + return EmitterReceiverRegistry::none_receive; + } + auto& emitter_receiver = loc->second; + return emitter_receiver.second; +} + +const EmitterReceiverRegistry::ReceiveFunc& EmitterReceiverRegistry::get_receiver( + const std::type_index& index) const { + auto maybe_name = index_to_name(index); + if (!maybe_name) { + HOLOSCAN_LOG_WARN("No receiver for type '{}' exists", index.name()); + return EmitterReceiverRegistry::none_receive; + } + auto& receiver = emitter_receiver_map_.at(maybe_name.value()).second; + return receiver; +} + +expected EmitterReceiverRegistry::name_to_index( + const std::string& name) const { + auto loc = name_to_index_map_.find(name); + if (loc == name_to_index_map_.end()) { + auto err_msg = fmt::format("No emitter_receiver for name '{}' exists", name); + return make_unexpected(RuntimeError(ErrorCode::kFailure, err_msg)); + } + return loc->second; +} + +expected EmitterReceiverRegistry::index_to_name( + const std::type_index& index) const { + auto loc = index_to_name_map_.find(index); + if (loc == index_to_name_map_.end()) { + auto err_msg = fmt::format("No emitter_receiver for type '{}' exists", index.name()); + return make_unexpected(RuntimeError(ErrorCode::kFailure, err_msg)); + } + return loc->second; +} + +std::vector EmitterReceiverRegistry::registered_types() const { + std::vector names; + names.reserve(emitter_receiver_map_.size()); + for (auto& [key, _] : emitter_receiver_map_) { names.emplace_back(key); } + return names; +} + +} // namespace holoscan diff --git a/python/holoscan/core/emitter_receiver_registry.hpp b/python/holoscan/core/emitter_receiver_registry.hpp new file mode 100644 index 00000000..51b0a29b --- /dev/null +++ b/python/holoscan/core/emitter_receiver_registry.hpp @@ -0,0 +1,294 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HOLOSCAN_CORE_EMITTER_RECEIVER_REGISTRY_HPP +#define HOLOSCAN_CORE_EMITTER_RECEIVER_REGISTRY_HPP + +#include +#include // needed for py::cast to work with STL container types + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "holoscan/core/errors.hpp" +#include "holoscan/core/expected.hpp" +#include "holoscan/logger/logger.hpp" +#include "io_context.hpp" + +using std::string_literals::operator""s; + +namespace py = pybind11; + +namespace holoscan { + +/* Emit and receive of any type T that pybind11 can cast via pybind11::object.cast() and + * py::cast(). + * + * For example emitter_receiver could be used to convert between C++ and Python + * strings. + */ +template +struct emitter_receiver { + static void emit(py::object& data, const std::string& name, PyOutputContext& op_output) { + auto cpp_type = data.cast(); + py::gil_scoped_release release; + op_output.emit(cpp_type, name.c_str()); + return; + } + + static py::object receive(std::any result, const std::string& name, PyInputContext& op_input) { + auto cpp_obj = std::any_cast(result); + return py::cast(cpp_obj); + } +}; + +/* Implements a receiver for the array camera pose type accepted by HolovizOp. + */ +template +struct emitter_receiver> { + static void emit(py::object& data, const std::string& name, PyOutputContext& op_output) { + auto cpp_obj = std::make_shared(data.cast()); + py::gil_scoped_release release; + op_output.emit>(cpp_obj, name.c_str()); + return; + } + static py::object receive(std::any result, const std::string& name, PyInputContext& op_input) { + auto camera_pose = std::any_cast>(result); + py::object py_camera_pose = py::cast(*camera_pose); + return py_camera_pose; + } +}; + +/** + * @brief Class to set emitter/receivers for data types. + * + * This class is used to set emitter/receivers (emitter + receiver) for data types. + */ +class EmitterReceiverRegistry { + public: + /** + * @brief Function type for emitting a data type + */ + using EmitFunc = std::function; + + /** + * @brief Function type for receiving a data type + */ + using ReceiveFunc = + std::function; + + /** + * @brief Function tuple type for emitting and receiving a data type + */ + using EmitterReceiver = std::pair; + + inline static EmitFunc none_emit = []([[maybe_unused]] py::object& data, + [[maybe_unused]] const std::string& name, + [[maybe_unused]] PyOutputContext& op_output) -> void { + HOLOSCAN_LOG_ERROR( + "Unable to emit message (op: '{}', port: '{}')", op_output.op()->name(), name); + return; + }; + + inline static ReceiveFunc none_receive = + []([[maybe_unused]] std::any result, [[maybe_unused]] const std::string& name, + [[maybe_unused]] PyInputContext& op_input) -> py::object { + HOLOSCAN_LOG_ERROR( + "Unable to receive message (op: '{}', port: '{}')", op_input.op()->name(), name); + return py::none(); + }; + + /** + * @brief Default @ref EmitterReceiver for Arg. + */ + inline static EmitterReceiver none_emitter_receiver = std::make_pair(none_emit, none_receive); + + /** + * @brief Get the instance object. + * + * @return The reference to the EmitterReceiverRegistry instance. + */ + static EmitterReceiverRegistry& get_instance(); + + /** + * @brief Emit the message object. + * + * @tparam typeT The data type within the message. + * @param data The Python object corresponding to the data of typeT. + * @param name The name of the entity emitted. + * @param op_output The PyOutputContext used to emit the data. + */ + template + static void emit(py::object& data, const std::string& name, PyOutputContext& op_output) { + auto& instance = get_instance(); + const std::type_index index = std::type_index(typeid(typeT)); + const EmitFunc& func = instance.get_emitter(index); + return func(data, name, op_output); + } + + /** + * @brief Receive the message object. + * + * @tparam typeT The data type within the message. + * @param message The message to serialize. + * @param endpoint The serialization endpoint (buffer). + */ + template + static py::object receive(std::any result, const std::string& name, PyInputContext& op_input) { + auto& instance = get_instance(); + const std::type_index index = std::type_index(typeid(typeT)); + const ReceiveFunc& func = instance.get_receiver(index); + return func(result, name, op_input); + } + + /** + * @brief Get the emitter/receiver function tuple. + * + * @param index The type index of the parameter. + * @return The reference to the EmitterReceiver object. + */ + const EmitterReceiver& get_emitter_receiver(const std::type_index& index) const; + + /** + * @brief Check if a given emitter/receiver exists based on the type index. + * + * @param index The type index of the parameter. + * @return boolean indicating if an emitter/receiver exists for the given index. + */ + bool has_emitter_receiver(const std::type_index& index) const; + + /** + * @brief Get the emitter/receiver function tuple. + * + * @param name The name of the emitter/receiver. + * @return The reference to the EmitterReceiver object. + */ + const EmitterReceiver& get_emitter_receiver(const std::string& name) const; + + /** + * @brief Get the emitter function. + * + * @param name The name of the emitter/receiver. + * @return The reference to the emitter function. + */ + const EmitFunc& get_emitter(const std::string& name) const; + + /** + * @brief Get the emitter function. + * + * @param index The type index of the parameter. + * @return The reference to the emitter function. + */ + const EmitFunc& get_emitter(const std::type_index& index) const; + + /** + * @brief Get the receiver function. + * + * @param name The name of the emitter/receiver. + * @return The reference to the receiver function. + */ + const ReceiveFunc& get_receiver(const std::string& name) const; + + /** + * @brief Get the receiver function. + * + * @param index The type index of the parameter. + * @return The reference to the receiver function. + */ + const ReceiveFunc& get_receiver(const std::type_index& index) const; + + /** + * @brief Get the std::type_index corresponding to a emitter/receiver name + * + * @param name The name of the emitter/receiver. + * @return The std::type_index corresponding to the name. + */ + expected name_to_index(const std::string& name) const; + + /** + * @brief Get the name corresponding to a std::type_index + * + * @param index The std::type_index corresponding to the parameter. + * @return The name of the emitter/receiver. + */ + expected index_to_name(const std::type_index& index) const; + + /** + * @brief Add a emitter/receiver for the type. + * + * @tparam typeT the type for which a emitter/receiver is being added + * @param name The name of the emitter/receiver to add. + * @param overwrite if true, any existing emitter/receiver with matching name will be overwritten. + */ + template + void add_emitter_receiver(const std::string& name, bool overwrite = false) { + auto name_search = name_to_index_map_.find(name); + auto index = std::type_index(typeid(typeT)); + if (name_search != name_to_index_map_.end()) { + if (!overwrite) { + HOLOSCAN_LOG_WARN( + "Existing emitter_receiver for name '{}' found, keeping the previous one.", name); + return; + } + if (index != name_search->second) { + HOLOSCAN_LOG_ERROR( + "Existing emitter_receiver for name '{}' found, but with non-matching type_index. ", + "If you did not intend to replace the existing emitter_receiver, please choose a " + "different name.", + name); + } + HOLOSCAN_LOG_DEBUG("Replacing existing emitter_receiver with name '{}'.", name); + emitter_receiver_map_.erase(name); + } + HOLOSCAN_LOG_DEBUG("Added emitter/receiver for type named: {}", name); + name_to_index_map_.try_emplace(name, index); + index_to_name_map_.try_emplace(index, name); + emitter_receiver_map_.emplace( + name, std::make_pair(emitter_receiver::emit, emitter_receiver::receive)); + } + + /** + * @brief List the names of the types with an emitter and/or receiver registered. + * + * @return A vector of the names of the types with an emitter and/or receiver registered. + */ + std::vector registered_types() const; + + private: + // private constructor (retrieve static instance via get_instance) + EmitterReceiverRegistry() {} + + // define maps to and from type_index and string (since type_index may vary across platforms) + std::unordered_map + index_to_name_map_; ///< Mapping from type_index to name + std::unordered_map + name_to_index_map_; ///< Mapping from name to type_index + + std::unordered_map + emitter_receiver_map_; ///< Map of emitter/receiver name to function tuple +}; + +} // namespace holoscan + +#endif /* HOLOSCAN_CORE_EMITTER_RECEIVER_REGISTRY_HPP */ diff --git a/python/holoscan/core/emitter_receivers.hpp b/python/holoscan/core/emitter_receivers.hpp new file mode 100644 index 00000000..84324e74 --- /dev/null +++ b/python/holoscan/core/emitter_receivers.hpp @@ -0,0 +1,370 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PYHOLOSCAN_CORE_EMITTER_RECEIVERS_HPP +#define PYHOLOSCAN_CORE_EMITTER_RECEIVERS_HPP + +#include +#include // needed for py::cast to work with STL container types + +#include +#include +#include +#include +#include +#include + +#include "../gxf/entity.hpp" + +#include "emitter_receiver_registry.hpp" +#include "gil_guarded_pyobject.hpp" +#include "holoscan/core/domain/tensor.hpp" +#include "holoscan/core/gxf/entity.hpp" +#include "holoscan/core/io_context.hpp" +#include "holoscan/operators/holoviz/holoviz.hpp" +#include "io_context.hpp" +#include "tensor.hpp" // for PyTensor + +using std::string_literals::operator""s; + +namespace py = pybind11; + +namespace holoscan { + +py::tuple vector2pytuple(const std::vector>& vec) { + py::tuple result(vec.size()); + int counter = 0; + for (auto& arg_value : vec) { + // Increase the ref count of the Python object because we are going to store (copy) it in a + // tuple. + arg_value->obj().inc_ref(); + // We should not release arg_value->obj() because the Python object can be referenced by the + // input context of other operators + PyTuple_SET_ITEM(result.ptr(), counter++, arg_value->obj().ptr()); + } + return result; +} + +py::object gxf_entity_to_py_object(holoscan::gxf::Entity in_entity) { + // Create a shared Entity (increase ref count) + holoscan::PyEntity entity_wrapper(in_entity); + + try { + auto components_expected = entity_wrapper.findAll(); + auto components = components_expected.value(); + auto n_components = components.size(); + + HOLOSCAN_LOG_DEBUG("py_receive: Entity Case"); + if ((n_components == 1) && (components[0]->name()[0] == '#')) { + // special case for single non-TensorMap tensor + // (will have been serialized with a name starting with #numpy, #cupy or #holoscan) + HOLOSCAN_LOG_DEBUG("py_receive: SINGLE COMPONENT WITH # NAME"); + auto component = components[0]; + std::string component_name = component->name(); + py::object holoscan_pytensor = entity_wrapper.py_get(component_name.c_str()); + + if (component_name.find("#numpy") != std::string::npos) { + HOLOSCAN_LOG_DEBUG("py_receive: name starting with #numpy"); + // cast the holoscan::Tensor to a NumPy array + py::module_ numpy; + try { + numpy = py::module_::import("numpy"); + } catch (const pybind11::error_already_set& e) { + if (e.matches(PyExc_ImportError)) { + throw pybind11::import_error( + fmt::format("Failed to import numpy to deserialize array with " + "__array_interface__ attribute: {}", + e.what())); + } else { + throw; + } + } + // py::object holoscan_pytensor_obj = py::cast(holoscan_pytensor); + py::object numpy_array = numpy.attr("asarray")(holoscan_pytensor); + return numpy_array; + } else if (component_name.find("#cupy") != std::string::npos) { + HOLOSCAN_LOG_DEBUG("py_receive: name starting with #cupy"); + // cast the holoscan::Tensor to a CuPy array + py::module_ cupy; + try { + cupy = py::module_::import("cupy"); + } catch (const pybind11::error_already_set& e) { + if (e.matches(PyExc_ImportError)) { + throw pybind11::import_error( + fmt::format("Failed to import cupy to deserialize array with " + "__cuda_array_interface__ attribute: {}", + e.what())); + } else { + throw; + } + } + py::object cupy_array = cupy.attr("asarray")(holoscan_pytensor); + return cupy_array; + } else if (component_name.find("#holoscan") != std::string::npos) { + HOLOSCAN_LOG_DEBUG("py_receive: name starting with #holoscan"); + return holoscan_pytensor; + } else { + throw std::runtime_error( + fmt::format("Invalid tensor name (if # is the first character in the name, the " + "name must start with #numpy, #cupy or #holoscan). Found: {}", + component_name)); + } + } else { + HOLOSCAN_LOG_DEBUG("py_receive: TensorMap case"); + py::dict dict_tensor; + for (size_t i = 0; i < n_components; i++) { + auto component = components[i]; + auto component_name = component->name(); + if (std::string(component_name).compare("message_label") == 0) { + // Skip checking for Tensor as it's the message label for flow tracking + continue; + } + if (std::string(component_name).compare("cuda_stream_id_") == 0) { + // Skip checking for Tensor as it's a stream ID from CudaStreamHandler + continue; + } + auto holoscan_pytensor = entity_wrapper.py_get(component_name); + if (holoscan_pytensor) { dict_tensor[component_name] = holoscan_pytensor; } + } + return dict_tensor; + } + } catch (const std::bad_any_cast& e) { + throw std::runtime_error( + fmt::format("Unable to cast the received data to the specified type (holoscan::gxf::" + "Entity): {}", + e.what())); + } +} + +/* Emitter for case where user explicitly creates a GXF holoscan.gxf.Entity from Python + * + * Using holoscan.gxf.Entity was the primary way to transmit multiple tensors before TensorMap + * support was added. After tensormap, it is more common to transmit a dict of tensor-like + * objects instead. + */ +template <> +struct emitter_receiver { + static void emit(py::object& data, const std::string& name, PyOutputContext& op_output) { + py::gil_scoped_release release; + auto entity = gxf::Entity(static_cast(data.cast())); + op_output.emit(entity, name.c_str()); + return; + } + static py::object receive(std::any result, const std::string& name, PyInputContext& op_input) { + // unused (receive type is always holoscan::gxf:Entity, not holoscan::PyEntity) + return py::none(); + } +}; + +/* Receiver for holoscan::gxf::Entity. + * + * This receiver currently only extracts Holoscan, NumPy or CuPy tensors from the entity. It + * explicitly ignores any "cuda_stream_pool" component that may have been added by + * CudaStreamHandler. It also explicitly ignores any component named "message_label" that may + * have been added by the data flow tracking feature. + */ +template <> +struct emitter_receiver { + static void emit(py::object& data, const std::string& name, PyOutputContext& op_output) { + // unused (emit type is holoscan::PyEntity, not holoscan::gxf:Entity) + return; + } + + static py::object receive(std::any result, const std::string& name, PyInputContext& op_input) { + auto in_entity = std::any_cast(result); + return gxf_entity_to_py_object(in_entity); + } +}; + +/* Emitter for holoscan.core.Tensor with special handling for 3rd party tensor interoperability. + * + * Names any holoscan::Tensor supporting ``__cuda_array_interface__`` as "#cupy: tensor". + * This will cause ``emitter_receiver::receive`` to convert this tensor + * to a CuPy tensor on receive by a downstream Python operator. + * + * Names any holoscan::Tensor supporting ``__array_interface__`` as "#numpy: tensor". + * This will cause ``emitter_receiver::receive`` to convert this tensor + * to a NumPy tensor on receive by a downstream Python operator. + * + * If the tensor-like object, ``data``, doesn't support either ``__array_interface__`` or + * ``__cuda_array_interface__``, but only DLPack then it will be received by a downstream + * Python operator as a ``holoscan::Tensor``. + * + * This special handling is done to allow equivalent behavior when sending NumPy and CuPy + * tensors between operators in both single fragment and distributed applications. + */ +template <> +struct emitter_receiver { + static void emit(py::object& data, const std::string& name, PyOutputContext& op_output) { + HOLOSCAN_LOG_DEBUG("py_emit: tensor-like over UCX connector"); + // For tensor-like data, we should create an entity and transmit using the holoscan::Tensor + // serializer. cloudpickle fails to serialize PyTensor and we want to avoid using it anyways + // as it would be inefficient to serialize the tensor to a string. + py::gil_scoped_release release; + auto entity = nvidia::gxf::Entity::New(op_output.gxf_context()); + if (!entity) { throw std::runtime_error("Failed to create entity"); } + auto py_entity = static_cast(entity.value()); + + py::gil_scoped_acquire acquire; + auto py_tensor_obj = PyTensor::as_tensor(data); + if (py::hasattr(data, "__cuda_array_interface__")) { + // checking with __cuda_array_interface__ instead of + // if (py::isinstance(value, cupy.attr("ndarray"))) + + // This way we don't have to add try/except logic around importing the CuPy module. + // One consequence of this is that Non-CuPy arrays having __cuda_array_interface__ will be + // cast to CuPy arrays on deserialization. + py_entity.py_add(py_tensor_obj, "#cupy: tensor"); + } else if (py::hasattr(data, "__array_interface__")) { + // objects with __array_interface__ defined will be cast to NumPy array on + // deserialization. + py_entity.py_add(py_tensor_obj, "#numpy: tensor"); + } else { + py_entity.py_add(py_tensor_obj, "#holoscan: tensor"); + } + py::gil_scoped_release release2; + op_output.emit(py_entity, name.c_str()); + return; + } + static py::object receive(std::any result, const std::string& name, PyInputContext& op_input) { + return py::none(); + } +}; + +/* Emit a Python dict as a TensorMap entity when possible, otherwise emit the Python object. + * + * If every entry in the dict is a tensor-like object then create a holoscan::gxf::Entity + * containing a TensorMap corresponding to the tensors (no data copying is done). This allows + * interfacing with any wrapped C++ operators taking tensors as input. + * + * Otherwise, emit the dictionary as-is via a std::shared_ptr. + */ +template <> +struct emitter_receiver { + static void emit(py::object& data, const std::string& name, PyOutputContext& op_output) { + bool is_tensormap = true; + auto dict_obj = data.cast(); + + // Check if all items in the dict are tensor-like + for (auto& item : dict_obj) { + auto& value = item.second; + // Check if item is tensor-like + if (!is_tensor_like(py::reinterpret_borrow(value))) { + is_tensormap = false; + break; + } + } + if (is_tensormap) { + // Create an Entity containing the TensorMap + auto dict_obj = data.cast(); + py::gil_scoped_release release; + auto entity = nvidia::gxf::Entity::New(op_output.gxf_context()); + if (!entity) { throw std::runtime_error("Failed to create entity"); } + auto py_entity = static_cast(entity.value()); + + py::gil_scoped_acquire acquire; + for (auto& item : dict_obj) { + std::string key = item.first.cast(); + auto& value = item.second; + auto value_obj = py::reinterpret_borrow(value); + auto py_tensor_obj = PyTensor::as_tensor(value_obj); + py_entity.py_add(py_tensor_obj, key.c_str()); + } + py::gil_scoped_release release2; + op_output.emit(py_entity, name.c_str()); + return; + } else { + // If the dict is not a TensorMap, pass it as a Python object + HOLOSCAN_LOG_DEBUG("py_emit: dict, but not a tensormap"); + auto data_ptr = std::make_shared(data); + py::gil_scoped_release release; + op_output.emit>(data_ptr, name.c_str()); + return; + } + } + // pybind11::dict is never received directly, but only as a std::shared_ptr + static py::object receive(std::any result, const std::string& name, PyInputContext& op_input) { + return py::none(); + } +}; + +/* Emit and receive generic Python objects + * + * This is the typical emitter/receiver used for within-fragment communication between native + * Python operators. + */ +template <> +struct emitter_receiver> { + static void emit(py::object& data, const std::string& name, PyOutputContext& op_output) { + // Emit everything else as a Python object. + auto data_ptr = std::make_shared(data); + py::gil_scoped_release release; + op_output.emit>(data_ptr, name.c_str()); + return; + } + + static py::object receive(std::any result, const std::string& name, PyInputContext& op_input) { + HOLOSCAN_LOG_DEBUG("py_receive: Python object case"); + auto in_message = std::any_cast>(result); + return in_message->obj(); + } +}; + +/* Emit and receive Python objects that have been serialized to a string via ``cloudpickle``. + * + * This is used to send native Python objects between fragments of a distributed application. + */ +template <> +struct emitter_receiver { + static void emit(py::object& data, const std::string& name, PyOutputContext& op_output) { + // use cloudpickle to serialize as a string + py::module_ cloudpickle = py::module_::import("cloudpickle"); + py::bytes serialized = cloudpickle.attr("dumps")(data); + py::gil_scoped_release release; + auto serialized_str = serialized.cast(); + CloudPickleSerializedObject serialized_obj{serialized_str}; + op_output.emit(serialized_obj, name.c_str()); + return; + } + + static py::object receive(std::any result, const std::string& name, PyInputContext& op_input) { + HOLOSCAN_LOG_DEBUG("py_receive: cloudpickle string case"); + auto serialized_obj = std::any_cast(result); + py::module_ cloudpickle = py::module_::import("cloudpickle"); + py::object deserialized = cloudpickle.attr("loads")(py::bytes(serialized_obj.serialized)); + return deserialized; + } +}; + +/* Emit or receive a nullptr. + * + * A Python operator receiving a C++ nullptr will convert it to Python's None. + */ +template <> +struct emitter_receiver { + static void emit(py::object& data, const std::string& name, PyOutputContext& op_output) { + op_output.emit(nullptr, name.c_str()); + return; + } + static py::object receive(std::any result, const std::string& name, PyInputContext& op_input) { + return py::none(); + } +}; + +} // namespace holoscan + +#endif /* PYHOLOSCAN_CORE_EMITTER_RECEIVERS_HPP */ diff --git a/python/holoscan/core/execution_context.hpp b/python/holoscan/core/execution_context.hpp index 974c8cca..f46659d4 100644 --- a/python/holoscan/core/execution_context.hpp +++ b/python/holoscan/core/execution_context.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef PYBIND11_CORE_EXECUTION_CONTEXT_HPP -#define PYBIND11_CORE_EXECUTION_CONTEXT_HPP +#ifndef PYHOLOSCAN_CORE_EXECUTION_CONTEXT_HPP +#define PYHOLOSCAN_CORE_EXECUTION_CONTEXT_HPP #include @@ -52,4 +52,4 @@ class PyExecutionContext : public gxf::GXFExecutionContext { } // namespace holoscan -#endif /* PYBIND11_CORE_EXECUTION_CONTEXT_HPP */ +#endif /* PYHOLOSCAN_CORE_EXECUTION_CONTEXT_HPP */ diff --git a/python/holoscan/core/fragment.hpp b/python/holoscan/core/fragment.hpp index 90f89b44..bcbe5a77 100644 --- a/python/holoscan/core/fragment.hpp +++ b/python/holoscan/core/fragment.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef PYBIND11_CORE_FRAGMENT_HPP -#define PYBIND11_CORE_FRAGMENT_HPP +#ifndef PYHOLOSCAN_CORE_FRAGMENT_HPP +#define PYHOLOSCAN_CORE_FRAGMENT_HPP #include #include @@ -77,4 +77,4 @@ class PyFragment : public Fragment { } // namespace holoscan -#endif /* PYBIND11_CORE_FRAGMENT_HPP */ +#endif /* PYHOLOSCAN_CORE_FRAGMENT_HPP */ diff --git a/python/holoscan/core/gil_guarded_pyobject.hpp b/python/holoscan/core/gil_guarded_pyobject.hpp index e8883ce2..3dd75695 100644 --- a/python/holoscan/core/gil_guarded_pyobject.hpp +++ b/python/holoscan/core/gil_guarded_pyobject.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef PYBIND11_CORE_GIL_GUARDED_PYOBJECT_HPP -#define PYBIND11_CORE_GIL_GUARDED_PYOBJECT_HPP +#ifndef PYHOLOSCAN_CORE_GIL_GUARDED_PYOBJECT_HPP +#define PYHOLOSCAN_CORE_GIL_GUARDED_PYOBJECT_HPP #include @@ -56,4 +56,4 @@ class GILGuardedPyObject { } // namespace holoscan -#endif /* PYBIND11_CORE_GIL_GUARDED_PYOBJECT_HPP */ +#endif /* PYHOLOSCAN_CORE_GIL_GUARDED_PYOBJECT_HPP */ diff --git a/python/holoscan/core/io_context.cpp b/python/holoscan/core/io_context.cpp index 7edc51e1..9c6ff5d9 100644 --- a/python/holoscan/core/io_context.cpp +++ b/python/holoscan/core/io_context.cpp @@ -20,23 +20,29 @@ #include #include // needed for py::cast to work with std::vector types +#include +#include #include #include #include +#include #include +#include #include #include "../gxf/entity.hpp" #include "core.hpp" +#include "emitter_receiver_registry.hpp" +#include "emitter_receivers.hpp" #include "gxf/std/tensor.hpp" #include "holoscan/core/application.hpp" +#include "holoscan/core/domain/tensor.hpp" #include "holoscan/core/expected.hpp" #include "holoscan/core/io_context.hpp" -#include "holoscan/core/domain/tensor.hpp" #include "holoscan/operators/holoviz/holoviz.hpp" #include "io_context_pydoc.hpp" -#include "tensor.hpp" // for PyTensor #include "operator.hpp" // for PyOperator +#include "tensor.hpp" // for PyTensor using std::string_literals::operator""s; using pybind11::literals::operator""_a; @@ -45,6 +51,16 @@ namespace py = pybind11; namespace holoscan { +class PyRegistryContext { + public: + PyRegistryContext() = default; + + EmitterReceiverRegistry& registry() { return registry_; } + + private: + EmitterReceiverRegistry& registry_ = EmitterReceiverRegistry::get_instance(); +}; + template <> struct codec> { static expected serialize(std::shared_ptr value, @@ -66,15 +82,18 @@ struct codec> { py::bytes serialized = cloudpickle.attr("dumps")(value->obj()); serialized_string = serialized.cast(); } - return codec::serialize(serialized_string, endpoint); + // Move `serialized_string` because it will not be used again, and to prevent a copy. + CloudPickleSerializedObject serialized_object{std::move(serialized_string)}; + return codec::serialize(serialized_object, endpoint); } - static expected deserialize(Endpoint* endpoint) { + static expected deserialize(Endpoint* endpoint) { HOLOSCAN_LOG_TRACE( - "\tdeserialize std::string corresponding to std::shared_ptr"); - auto maybe_str = codec::deserialize(endpoint); - if (!maybe_str) { return forward_error(maybe_str); } - return maybe_str.value(); + "\tdeserialize CloudPickleSerializedObject corresponding to " + "std::shared_ptr"); + auto maybe_obj = codec::deserialize(endpoint); + if (!maybe_obj) { return forward_error(maybe_obj); } + return std::move(maybe_obj.value()); } }; @@ -84,126 +103,6 @@ static void register_py_object_codec() { "std::shared_ptr"s); } -py::tuple vector2pytuple(const std::vector>& vec) { - py::tuple result(vec.size()); - int counter = 0; - for (auto& arg_value : vec) { - // Increase the ref count of the Python object because we are going to store (copy) it in a - // tuple. - arg_value->obj().inc_ref(); - // We should not release arg_value->obj() because the Python object can be referenced by the - // input context of other operators - PyTuple_SET_ITEM(result.ptr(), counter++, arg_value->obj().ptr()); - } - return result; -} - -// TODO: can remove this unused version of vector2pytuple -py::tuple vector2pytuple(const std::vector& vec) { - py::tuple result(vec.size()); - int counter = 0; - for (auto& arg_value : vec) { - // Create a shared Entity - holoscan::PyEntity entity_wrapper(arg_value); - // Create a Python object from the shared Entity - py::object item = py::cast(entity_wrapper); - // Move the Python object into the tuple - PyTuple_SET_ITEM(result.ptr(), counter++, item.release().ptr()); - } - return result; -} - -py::object gxf_entity_to_py_object(holoscan::gxf::Entity in_entity) { - // Create a shared Entity (increase ref count) - holoscan::PyEntity entity_wrapper(in_entity); - - try { - auto components_expected = entity_wrapper.findAll(); - auto components = components_expected.value(); - auto n_components = components.size(); - - HOLOSCAN_LOG_DEBUG("py_receive: Entity Case"); - if ((n_components == 1) && (components[0]->name()[0] == '#')) { - HOLOSCAN_LOG_DEBUG("py_receive: SINGLE COMPONENT WITH # NAME"); - // special case for single non-TensorMap tensor - // (will have been serialized with a name starting with #numpy, #cupy or #holoscan) - auto component = components[0]; - std::string component_name = component->name(); - py::object holoscan_pytensor = entity_wrapper.py_get(component_name.c_str()); - - if (component_name.find("#numpy") != std::string::npos) { - HOLOSCAN_LOG_DEBUG("py_receive: name starting with #numpy"); - // cast the holoscan::Tensor to a NumPy array - py::module_ numpy; - try { - numpy = py::module_::import("numpy"); - } catch (const pybind11::error_already_set& e) { - if (e.matches(PyExc_ImportError)) { - throw pybind11::import_error( - fmt::format("Failed to import numpy to deserialize array with " - "__array_interface__ attribute: {}", - e.what())); - } else { - throw; - } - } - // py::object holoscan_pytensor_obj = py::cast(holoscan_pytensor); - py::object numpy_array = numpy.attr("asarray")(holoscan_pytensor); - return numpy_array; - } else if (component_name.find("#cupy") != std::string::npos) { - HOLOSCAN_LOG_DEBUG("py_receive: name starting with #cupy"); - // cast the holoscan::Tensor to a CuPy array - py::module_ cupy; - try { - cupy = py::module_::import("cupy"); - } catch (const pybind11::error_already_set& e) { - if (e.matches(PyExc_ImportError)) { - throw pybind11::import_error( - fmt::format("Failed to import cupy to deserialize array with " - "__cuda_array_interface__ attribute: {}", - e.what())); - } else { - throw; - } - } - py::object cupy_array = cupy.attr("asarray")(holoscan_pytensor); - return cupy_array; - } else if (component_name.find("#holoscan") != std::string::npos) { - HOLOSCAN_LOG_DEBUG("py_receive: name starting with #holoscan"); - return holoscan_pytensor; - } else { - throw std::runtime_error( - fmt::format("Invalid tensor name (if # is the first character in the name, the " - "name must start with #numpy, #cupy or #holoscan). Found: {}", - component_name)); - } - } else { - HOLOSCAN_LOG_DEBUG("py_receive: TensorMap case"); - py::dict dict_tensor; - for (size_t i = 0; i < n_components; i++) { - auto component = components[i]; - auto component_name = component->name(); - if (std::string(component_name).compare("message_label") == 0) { - // Skip checking for Tensor as it's the message label for flow tracking - continue; - } - if (std::string(component_name).compare("cuda_stream_id_") == 0) { - // Skip checking for Tensor as it's a stream ID from CudaStreamHandler - continue; - } - auto holoscan_pytensor = entity_wrapper.py_get(component_name); - if (holoscan_pytensor) { dict_tensor[component_name] = holoscan_pytensor; } - } - return dict_tensor; - } - } catch (const std::bad_any_cast& e) { - throw std::runtime_error( - fmt::format("Unable to cast the received data to the specified type (holoscan::gxf::" - "Entity): {}", - e.what())); - } -} - py::object PyInputContext::py_receive(const std::string& name) { auto py_op = py_op_.cast(); auto py_op_spec = py_op->py_shared_spec(); @@ -224,106 +123,32 @@ py::object PyInputContext::py_receive(const std::string& name) { // Check element type (querying the first element using the name '{name}:0') auto& element = any_result[0]; auto& element_type = element.type(); - - if (element_type == typeid(holoscan::gxf::Entity)) { - py::tuple result_tuple(any_result.size()); - int counter = 0; - try { - for (auto& any_item : any_result) { - if (any_item.type() == typeid(nullptr_t)) { - // add None to the tuple - PyTuple_SET_ITEM(result_tuple.ptr(), counter++, py::none().release().ptr()); - continue; - } - auto in_entity = std::any_cast(any_item); - - // Get the Python object from the entity - py::object in_obj = gxf_entity_to_py_object(in_entity); - - // Move the Python object into the tuple - PyTuple_SET_ITEM(result_tuple.ptr(), counter++, in_obj.release().ptr()); - } - } catch (const std::bad_any_cast& e) { - HOLOSCAN_LOG_ERROR( - "Unable to receive input (std::vector) with name " - "'{}' ({})", - name, - e.what()); - } - return result_tuple; - } else if (element_type == typeid(std::shared_ptr)) { - std::vector> result; - try { - for (auto& any_item : any_result) { - result.push_back(std::any_cast>(any_item)); - } - } catch (const std::bad_any_cast& e) { - HOLOSCAN_LOG_ERROR( - "Unable to receive input (std::vector>) with name " - "'{}' ({})", - name, - e.what()); - } - py::tuple result_tuple = vector2pytuple(result); - return result_tuple; - } else if (element_type == typeid(std::string)) { - std::vector> result; - try { - for (auto& any_item : any_result) { - std::string obj_str = std::any_cast(any_item); - - py::module_ cloudpickle; - try { - cloudpickle = py::module_::import("cloudpickle"); - } catch (const py::error_already_set& e) { - if (e.matches(PyExc_ImportError)) { - throw pybind11::import_error(fmt::format( - e.what() + - "\nThe cloudpickle module is required for Python distributed"s - " apps.\nPlease install it with `python -m pip install cloudpickle`"s)); - } else { - throw; - } - } - py::object deserialized = cloudpickle.attr("loads")(py::bytes(obj_str)); - - result.push_back(std::make_shared(deserialized)); + auto& registry = holoscan::EmitterReceiverRegistry::get_instance(); + const auto& receiver_func = registry.get_receiver(element_type); + + py::tuple result_tuple(any_result.size()); + int counter = 0; + try { + for (auto& any_item : any_result) { + if (any_item.type() == typeid(nullptr_t)) { + // add None to the tuple + PyTuple_SET_ITEM(result_tuple.ptr(), counter++, py::none().release().ptr()); + continue; } - } catch (const std::bad_any_cast& e) { - HOLOSCAN_LOG_ERROR( - "Unable to receive input (std::vector) with name " - "'{}' ({})", - name, - e.what()); - } - py::tuple result_tuple = vector2pytuple(result); - return result_tuple; - } else if (element_type == typeid(std::vector)) { - py::tuple result_tuple(any_result.size()); - int counter = 0; - try { - for (auto& any_item : any_result) { - HOLOSCAN_LOG_DEBUG("py_receive: HolovizOp::InputSpec case"); - - auto specs = std::any_cast>(any_item); + // Get the Python object from the entity + py::object in_obj = receiver_func(any_item, name, *this); - // cast vector to Python list of HolovizOp.InputSpec - py::object py_specs = py::cast(specs); - - // Move the Python object into the tuple - PyTuple_SET_ITEM(result_tuple.ptr(), counter++, py_specs.release().ptr()); - } - } catch (const std::bad_any_cast& e) { - HOLOSCAN_LOG_ERROR( - "Unable to receive input (std::vector) with name " - "'{}' ({})", - name, - e.what()); + // Move the Python object into the tuple + PyTuple_SET_ITEM(result_tuple.ptr(), counter++, in_obj.release().ptr()); } - return result_tuple; - } else { - return py::none(); + } catch (const std::bad_any_cast& e) { + HOLOSCAN_LOG_ERROR( + "Unable to receive input (std::vector) with name " + "'{}' ({})", + name, + e.what()); } + return result_tuple; } else { auto maybe_result = receive(name.c_str()); if (!maybe_result.has_value()) { @@ -332,59 +157,14 @@ py::object PyInputContext::py_receive(const std::string& name) { } auto result = maybe_result.value(); auto& result_type = result.type(); - if (result_type == typeid(holoscan::gxf::Entity)) { - auto in_entity = std::any_cast(result); - return gxf_entity_to_py_object(in_entity); - } else if (result_type == typeid(std::shared_ptr)) { - HOLOSCAN_LOG_DEBUG("py_receive: Python object case"); - auto in_message = std::any_cast>(result); - return in_message->obj(); - } else if (result_type == typeid(std::string)) { - HOLOSCAN_LOG_DEBUG("py_receive: cloudpickle string case"); - std::string obj_str = std::any_cast(result); - py::module_ cloudpickle; - try { - cloudpickle = py::module_::import("cloudpickle"); - } catch (const py::error_already_set& e) { - if (e.matches(PyExc_ImportError)) { - throw pybind11::import_error( - fmt::format(e.what() + - "\nThe cloudpickle module is required for Python distributed"s - " apps.\nPlease install it with `python -m pip install cloudpickle`"s)); - } else { - throw; - } - } - py::object deserialized = cloudpickle.attr("loads")(py::bytes(obj_str)); - - return deserialized; - } else if (result_type == typeid(std::vector)) { - HOLOSCAN_LOG_DEBUG("py_receive: HolovizOp::InputSpec case"); - // can directly return vector - auto specs = std::any_cast>(result); - py::object py_specs = py::cast(specs); - return py_specs; - } else if (result_type == typeid(std::shared_ptr>)) { - auto camera_pose = std::any_cast>>(result); - py::object py_camera_pose = py::cast(*camera_pose); - return py_camera_pose; - } else if (result_type == typeid(nullptr_t)) { - return py::none(); - } else { - HOLOSCAN_LOG_ERROR("Invalid message type: {}", result_type.name()); - return py::none(); - } + auto& registry = holoscan::EmitterReceiverRegistry::get_instance(); + const auto& receiver_func = registry.get_receiver(result_type); + return receiver_func(result, name, *this); } } -bool is_tensor_like(py::object value) { - return ((py::hasattr(value, "__dlpack__") && py::hasattr(value, "__dlpack_device__")) || - py::isinstance(value) || - py::hasattr(value, "__cuda_array_interface__") || - py::hasattr(value, "__array_interface__")); -} - -void PyOutputContext::py_emit(py::object& data, const std::string& name) { +void PyOutputContext::py_emit(py::object& data, const std::string& name, + const std::string& emitter_name) { // Note:: Issue 4206197 // In the UcxTransmitter::sync_io_abi(), while popping an entity from the queue, // Runtime::GxfEntityRefCountDec() on the entity can be called (which locks 'ref_count_mutex_'). @@ -404,12 +184,19 @@ void PyOutputContext::py_emit(py::object& data, const std::string& name) { HOLOSCAN_LOG_DEBUG("py_emit (operator name={}, port name={}):", op_name, name); #endif + auto& registry = holoscan::EmitterReceiverRegistry::get_instance(); + if (!emitter_name.empty()) { + HOLOSCAN_LOG_DEBUG("py_emit: emitting a {}", emitter_name); + const auto& emit_func = registry.get_emitter(emitter_name); + emit_func(data, name, *this); + return; + } + // If this is a PyEntity emit a gxf::Entity so that it can be consumed by non-Python operator. if (py::isinstance(data)) { - HOLOSCAN_LOG_DEBUG("py_emit: emitting a holoscan::gxf::Entity"); - py::gil_scoped_release release; - auto entity = gxf::Entity(static_cast(data.cast())); - emit(entity, name.c_str()); + HOLOSCAN_LOG_DEBUG("py_emit: emitting a holoscan::PyEntity"); + const auto& emit_func = registry.get_emitter(typeid(holoscan::PyEntity)); + emit_func(data, name, *this); return; } @@ -422,51 +209,19 @@ void PyOutputContext::py_emit(py::object& data, const std::string& name) { if (py::isinstance(seq[0])) { HOLOSCAN_LOG_DEBUG( "py_emit: emitting a std::vector object"); - auto input_spec = data.cast>(); - py::gil_scoped_release release; - emit>(input_spec, name.c_str()); + const auto& emit_func = + registry.get_emitter(typeid(std::vector)); + emit_func(data, name, *this); return; } } } - bool is_tensormap_like = false; - - // check if data is dict and all items in the dict are array-like/holoscan Tensor type + // handle pybind11::dict separately from other Python types for special TensorMap treatment if (py::isinstance(data)) { - is_tensormap_like = true; - auto dict_obj = data.cast(); - - // Check if all items in the dict are tensor-like - for (auto& item : dict_obj) { - auto& value = item.second; - // Check if item is tensor-like - if (!is_tensor_like(py::reinterpret_borrow(value))) { - is_tensormap_like = false; - break; - } - } - if (is_tensormap_like) { - HOLOSCAN_LOG_TRACE("py_emit: emitting dict of array-like objects as a tensormap"); - py::gil_scoped_release release; - auto entity = nvidia::gxf::Entity::New(gxf_context()); - if (!entity) { throw std::runtime_error("Failed to create entity"); } - auto py_entity = static_cast(entity.value()); - - py::gil_scoped_acquire acquire; - for (auto& item : dict_obj) { - std::string key = item.first.cast(); - auto& value = item.second; - auto value_obj = py::reinterpret_borrow(value); - auto py_tensor_obj = PyTensor::as_tensor(value_obj); - py_entity.py_add(py_tensor_obj, key.c_str()); - } - py::gil_scoped_release release2; - emit(py_entity, name.c_str()); - return; - } else { - HOLOSCAN_LOG_DEBUG("py_emit: dict, but not a tensormap"); - } + const auto& emit_func = registry.get_emitter(typeid(pybind11::dict)); + emit_func(data, name, *this); + return; } bool is_ucx_connector = false; @@ -501,39 +256,11 @@ void PyOutputContext::py_emit(py::object& data, const std::string& name) { // single fragment applications, where serialization of tensors is not necessary, so we guard // this loop in an `is_distributed_app` condition. This way single fragment applications will // still just directly pass the Python object. - if (is_distributed_app) { - // TensorMap case was already handled above - if (is_tensor_like(data)) { - HOLOSCAN_LOG_DEBUG("py_emit: tensor-like over UCX connector"); - // For tensor-like data, we should create an entity and transmit using the holoscan::Tensor - // serializer. cloudpickle fails to serialize PyTensor and we want to avoid using it anyways - // as it would be inefficient to serialize the tensor to a string. - py::gil_scoped_release release; - auto entity = nvidia::gxf::Entity::New(gxf_context()); - if (!entity) { throw std::runtime_error("Failed to create entity"); } - auto py_entity = static_cast(entity.value()); - - py::gil_scoped_acquire acquire; - auto py_tensor_obj = PyTensor::as_tensor(data); - if (py::hasattr(data, "__cuda_array_interface__")) { - // checking with __cuda_array_interface__ instead of - // if (py::isinstance(value, cupy.attr("ndarray"))) - - // This way we don't have to add try/except logic around importing the CuPy module. - // One consequence of this is that Non-CuPy arrays having __cuda_array_interface__ will be - // cast to CuPy arrays on deserialization. - py_entity.py_add(py_tensor_obj, "#cupy: tensor"); - } else if (py::hasattr(data, "__array_interface__")) { - // objects with __array_interface__ defined will be cast to NumPy array on - // deserialization. - py_entity.py_add(py_tensor_obj, "#numpy: tensor"); - } else { - py_entity.py_add(py_tensor_obj, "#holoscan: tensor"); - } - py::gil_scoped_release release2; - emit(py_entity, name.c_str()); - return; - } + if (is_distributed_app && is_tensor_like(data)) { + HOLOSCAN_LOG_DEBUG("py_emit: emitting a tensor-like object over a UCX connector"); + const auto& emit_func = registry.get_emitter(typeid(holoscan::Tensor)); + emit_func(data, name, *this); + return; } // Emit everything else as a Python object. @@ -543,9 +270,8 @@ void PyOutputContext::py_emit(py::object& data, const std::string& name) { // serialization will occur for distributed applications even in the case where an implicit // broadcast codelet was inserted. HOLOSCAN_LOG_DEBUG("py_emit: emitting a std::shared_ptr"); - auto data_ptr = std::make_shared(data); - py::gil_scoped_release release; - emit>(data_ptr, name.c_str()); + const auto& emit_func = registry.get_emitter(typeid(std::shared_ptr)); + emit_func(data, name, *this); return; } @@ -573,23 +299,63 @@ void init_io_context(py::module_& m) { py::class_>( m, "PyInputContext", R"doc(Input context class.)doc") - .def("receive", &PyInputContext::py_receive); + .def("receive", &PyInputContext::py_receive, "name"_a); py::class_>( m, "PyOutputContext", R"doc(Output context class.)doc") - .def("emit", &PyOutputContext::py_emit); + .def("emit", &PyOutputContext::py_emit, "data"_a, "name"_a, "emitter_name"_a = ""); // register a cloudpickle-based serializer for Python objects register_py_object_codec(); + + py::class_( + m, "EmitterReceiverRegistry", doc::EmitterReceiverRegistry::doc_EmitterReceiverRegistry) + .def("registered_types", + &EmitterReceiverRegistry::registered_types, + doc::EmitterReceiverRegistry::doc_registered_types); + + // See how this register_types method is called in __init__.py to register handling of these + // types. For user-defined operators that need to add additional types, the registry can be + // imported from holoscan.core. See the holoscan.operators.HolovizOp source for an example. + m.def("register_types", [](EmitterReceiverRegistry& registry) { + registry.add_emitter_receiver("nullptr_t"s, true); + registry.add_emitter_receiver("CloudPickleSerializedObject"s, + true); + registry.add_emitter_receiver("std::string"s, true); + registry.add_emitter_receiver>("PyObject"s, true); + + // receive-only case (emit occurs via holoscan::PyEntity instead) + registry.add_emitter_receiver("holoscan::gxf::Entity"s, true); + + // emitter-only cases (each of these is received as holoscan::gxf::Entity) + registry.add_emitter_receiver("holoscan::PyEntity"s, true); + registry.add_emitter_receiver("pybind11::dict"s, true); + registry.add_emitter_receiver("holoscan::Tensor"s, true); + }); + + m.def( + "registry", + []() { return EmitterReceiverRegistry::get_instance(); }, + py::return_value_policy::reference); + + auto registry = EmitterReceiverRegistry::get_instance(); + py::class_>( + m, "PyRegistryContext", "PyRegistryContext class") + .def(py::init<>()) + // return a reference to the C++ static registry object + .def("registry", + &PyRegistryContext::registry, + "Return a reference to the static EmitterReceiverRegistry", + py::return_value_policy::reference_internal); } PyInputContext::PyInputContext(ExecutionContext* execution_context, Operator* op, - std::unordered_map>& inputs, + std::unordered_map>& inputs, py::object py_op) : gxf::GXFInputContext::GXFInputContext(execution_context, op, inputs), py_op_(py_op) {} PyOutputContext::PyOutputContext(ExecutionContext* execution_context, Operator* op, - std::unordered_map>& outputs, + std::unordered_map>& outputs, py::object py_op) : gxf::GXFOutputContext::GXFOutputContext(execution_context, op, outputs), py_op_(py_op) {} diff --git a/python/holoscan/core/io_context.hpp b/python/holoscan/core/io_context.hpp index f018b898..f72d490b 100644 --- a/python/holoscan/core/io_context.hpp +++ b/python/holoscan/core/io_context.hpp @@ -15,14 +15,21 @@ * limitations under the License. */ -#ifndef PYBIND11_CORE_IO_CONTEXT_HPP -#define PYBIND11_CORE_IO_CONTEXT_HPP +#ifndef PYHOLOSCAN_CORE_IO_CONTEXT_HPP +#define PYHOLOSCAN_CORE_IO_CONTEXT_HPP #include +#include #include #include +#include +#include +#include +#include #include +#include +#include #include "gil_guarded_pyobject.hpp" #include "holoscan/core/execution_context.hpp" @@ -41,7 +48,7 @@ class PyInputContext : public gxf::GXFInputContext { /* Inherit the constructors */ using gxf::GXFInputContext::GXFInputContext; PyInputContext(ExecutionContext* execution_context, Operator* op, - std::unordered_map>& inputs, + std::unordered_map>& inputs, py::object py_op); py::object py_receive(const std::string& name); @@ -56,10 +63,10 @@ class PyOutputContext : public gxf::GXFOutputContext { using gxf::GXFOutputContext::GXFOutputContext; PyOutputContext(ExecutionContext* execution_context, Operator* op, - std::unordered_map>& outputs, + std::unordered_map>& outputs, py::object py_op); - void py_emit(py::object& data, const std::string& name); + void py_emit(py::object& data, const std::string& name, const std::string& emitter_name = ""); private: py::object py_op_ = py::none(); @@ -67,4 +74,4 @@ class PyOutputContext : public gxf::GXFOutputContext { } // namespace holoscan -#endif /* PYBIND11_CORE_IO_CONTEXT_HPP */ +#endif /* PYHOLOSCAN_CORE_IO_CONTEXT_HPP */ diff --git a/python/holoscan/core/io_context_pydoc.hpp b/python/holoscan/core/io_context_pydoc.hpp index 2337f43f..a397669e 100644 --- a/python/holoscan/core/io_context_pydoc.hpp +++ b/python/holoscan/core/io_context_pydoc.hpp @@ -55,6 +55,23 @@ Class representing an output context. } // namespace OutputContext +namespace EmitterReceiverRegistry { + +PYDOC(EmitterReceiverRegistry, R"doc( +Registry of methods to emit/receive different types. +)doc") + +PYDOC(registered_types, R"doc( +List of types with an emitter and/or receiver registered + +Returns +------- +names : list of str + The list of registered emitter/receiver names. +)doc") + +} // namespace EmitterReceiverRegistry + } // namespace holoscan::doc #endif // PYHOLOSCAN_CORE_IO_CONTEXT_PYDOC_HPP diff --git a/python/holoscan/core/io_spec.cpp b/python/holoscan/core/io_spec.cpp index f19f1aca..5d92068e 100644 --- a/python/holoscan/core/io_spec.cpp +++ b/python/holoscan/core/io_spec.cpp @@ -38,7 +38,8 @@ namespace py = pybind11; namespace holoscan { void init_io_spec(py::module_& m) { - py::class_ iospec(m, "IOSpec", R"doc(I/O specification class.)doc"); + py::class_> iospec( + m, "IOSpec", R"doc(I/O specification class.)doc"); py::enum_(iospec, "IOType", doc::IOType::doc_IOType) .value("INPUT", IOSpec::IOType::kInput) diff --git a/python/holoscan/core/io_spec.hpp b/python/holoscan/core/io_spec.hpp index 5b0c690d..5d4353f9 100644 --- a/python/holoscan/core/io_spec.hpp +++ b/python/holoscan/core/io_spec.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef PYBIND11_CORE_IO_SPEC_HPP -#define PYBIND11_CORE_IO_SPEC_HPP +#ifndef PYHOLOSCAN_CORE_IO_SPEC_HPP +#define PYHOLOSCAN_CORE_IO_SPEC_HPP #include @@ -43,4 +43,4 @@ static const std::unordered_map connector_ty } // namespace holoscan -#endif /* PYBIND11_CORE_IO_SPEC_HPP */ +#endif /* PYHOLOSCAN_CORE_IO_SPEC_HPP */ diff --git a/python/holoscan/core/kwarg_handling.cpp b/python/holoscan/core/kwarg_handling.cpp index e33a0b1e..e7271461 100644 --- a/python/holoscan/core/kwarg_handling.cpp +++ b/python/holoscan/core/kwarg_handling.cpp @@ -40,32 +40,56 @@ namespace py = pybind11; namespace holoscan { +/// Convert a py::object to a `YAML::Node` type. +template +inline static YAML::Node cast_to_yaml_node(const py::handle& obj) { + YAML::Node yaml_node; + yaml_node.push_back(obj.cast()); + return yaml_node[0]; +} + +// Specialization for uint8_t +template <> +inline YAML::Node cast_to_yaml_node(const py::handle& obj) { + YAML::Node yaml_node; + yaml_node.push_back(obj.cast()); + return yaml_node[0]; +} + +// Specialization for int8_t +template <> +inline YAML::Node cast_to_yaml_node(const py::handle& obj) { + YAML::Node yaml_node; + yaml_node.push_back(obj.cast()); + return yaml_node[0]; +} + void set_scalar_arg_via_dtype(const py::object& obj, const py::dtype& dt, Arg& out) { std::string dtype_name = dt.attr("name").cast(); if (dtype_name == "float16") { // currently promoting float16 scalars to float - out = obj.cast(); + out = cast_to_yaml_node(obj); } else if (dtype_name == "float32") { - out = obj.cast(); + out = cast_to_yaml_node(obj); } else if (dtype_name == "float64") { - out = obj.cast(); + out = cast_to_yaml_node(obj); } else if (dtype_name == "bool") { - out = obj.cast(); + out = cast_to_yaml_node(obj); } else if (dtype_name == "int8") { - out = obj.cast(); + out = cast_to_yaml_node(obj); } else if (dtype_name == "int16") { - out = obj.cast(); + out = cast_to_yaml_node(obj); } else if (dtype_name == "int32") { - out = obj.cast(); + out = cast_to_yaml_node(obj); } else if (dtype_name == "int64") { - out = obj.cast(); + out = cast_to_yaml_node(obj); } else if (dtype_name == "uint8") { - out = obj.cast(); + out = cast_to_yaml_node(obj); } else if (dtype_name == "uint16") { - out = obj.cast(); + out = cast_to_yaml_node(obj); } else if (dtype_name == "uint32") { - out = obj.cast(); + out = cast_to_yaml_node(obj); } else if (dtype_name == "uint64") { - out = obj.cast(); + out = cast_to_yaml_node(obj); } else { throw std::runtime_error("unsupported dtype: "s + dtype_name + ", leaving Arg uninitialized"s); } @@ -75,23 +99,21 @@ void set_scalar_arg_via_dtype(const py::object& obj, const py::dtype& dt, Arg& o template void set_vector_arg_via_numpy_array(const py::array& obj, Arg& out) { // not intended for images or other large tensors, just - // for short arrays containing parameter settings to operators + // for short arrays containing parameter settings to operators/resources if (obj.attr("ndim").cast() == 1) { - std::vector v; - v.reserve(obj.attr("size").cast()); - for (auto item : obj) v.push_back(item.cast()); - out = v; + YAML::Node yaml_node = YAML::Load("[]"); // Create an empty sequence + for (const auto& item : obj) yaml_node.push_back(cast_to_yaml_node(item)); + out = yaml_node; } else if (obj.attr("ndim").cast() == 2) { - std::vector> v; - std::vector shape = obj.attr("shape").cast>(); - v.reserve(static_cast(shape[0])); + YAML::Node yaml_node = YAML::Load("[]"); // Create an empty sequence for (auto item : obj) { - std::vector vv; - vv.reserve(static_cast(shape[1])); - for (auto inner_item : item) { vv.push_back(inner_item.cast()); } - v.push_back(vv); + YAML::Node inner_yaml_node = YAML::Load("[]"); // Create an empty sequence + for (const auto& inner_item : item) { + inner_yaml_node.push_back(cast_to_yaml_node(inner_item)); + } + if (inner_yaml_node.size() > 0) { yaml_node.push_back(inner_yaml_node); } } - out = v; + out = yaml_node; } else { throw std::runtime_error("Only 1d and 2d NumPy arrays are supported."); } @@ -100,27 +122,49 @@ void set_vector_arg_via_numpy_array(const py::array& obj, Arg& out) { template void set_vector_arg_via_py_sequence(const py::sequence& seq, Arg& out) { // not intended for images or other large tensors, just - // for short arrays containing parameter settings to operators + // for short arrays containing parameter settings to operators/resources - auto first_item = seq[0]; - if (py::isinstance(first_item) && !py::isinstance(first_item)) { - // Handle list of list and other sequence of sequence types. - std::vector> v; - v.reserve(static_cast(py::len(seq))); - for (auto item : seq) { - std::vector vv; - vv.reserve(static_cast(py::len(item))); - for (auto inner_item : item) { vv.push_back(inner_item.cast()); } - v.push_back(vv); + if constexpr (std::is_same_v> || + std::is_same_v>) { + auto first_item = seq[0]; + if (py::isinstance(first_item) && !py::isinstance(first_item)) { + // Handle list of list and other sequence of sequence types. + std::vector> v; + v.reserve(static_cast(py::len(seq))); + for (auto item : seq) { + std::vector vv; + vv.reserve(static_cast(py::len(item))); + for (auto inner_item : item) { vv.push_back(inner_item.cast()); } + v.push_back(vv); + } + out = v; + } else { + // 1d vector to handle a sequence of elements + std::vector v; + size_t length = py::len(seq); + v.reserve(length); + for (auto item : seq) v.push_back(item.cast()); + out = v; } - out = v; } else { - // 1d vector to handle a sequence of elements - std::vector v; - size_t length = py::len(seq); - v.reserve(length); - for (auto item : seq) v.push_back(item.cast()); - out = v; + auto first_item = seq[0]; + if (py::isinstance(first_item) && !py::isinstance(first_item)) { + // Handle list of list and other sequence of sequence types. + YAML::Node yaml_node = YAML::Load("[]"); // Create an empty sequence + for (auto item : seq) { + YAML::Node inner_yaml_node = YAML::Load("[]"); // Create an empty sequence + for (const auto& inner_item : item) { + inner_yaml_node.push_back(cast_to_yaml_node(inner_item)); + } + if (inner_yaml_node.size() > 0) { yaml_node.push_back(inner_yaml_node); } + } + out = yaml_node; + } else { + // 1d vector to handle a sequence of elements + YAML::Node yaml_node = YAML::Load("[]"); // Create an empty sequence + for (const auto& item : seq) yaml_node.push_back(cast_to_yaml_node(item)); + out = yaml_node; + } } } @@ -339,7 +383,7 @@ py::object arg_to_py_object(Arg& arg) { Arg py_object_to_arg(py::object obj, std::string name = "") { Arg out(name); if (py::isinstance(obj)) { - out = obj.cast(); + out = cast_to_yaml_node(obj); } else if (py::isinstance(obj)) { // handle numpy arrays py::dtype array_dtype = obj.cast().dtype(); @@ -350,11 +394,11 @@ Arg py_object_to_arg(py::object obj, std::string name = "") { // will work for any that can be cast to py::list set_vector_arg_via_iterable(obj, out); } else if (py::isinstance(obj)) { - out = obj.cast(); + out = cast_to_yaml_node(obj); } else if (py::isinstance(obj) || PyLong_Check(obj.ptr())) { - out = obj.cast(); + out = cast_to_yaml_node(obj); } else if (py::isinstance(obj)) { - out = obj.cast(); + out = cast_to_yaml_node(obj); } else if (PyComplex_Check(obj.ptr())) { throw std::runtime_error("complex value cannot be converted to Arg"); } else if (PyNumber_Check(obj.ptr())) { @@ -367,7 +411,7 @@ Arg py_object_to_arg(py::object obj, std::string name = "") { return out; } else { // cast any other unknown numeric type to double - out = obj.cast(); + out = cast_to_yaml_node(obj); } } else if (py::isinstance(obj)) { out = obj.cast>(); diff --git a/python/holoscan/core/kwarg_handling.hpp b/python/holoscan/core/kwarg_handling.hpp index c23dfc24..1e195c43 100644 --- a/python/holoscan/core/kwarg_handling.hpp +++ b/python/holoscan/core/kwarg_handling.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef HOLOSCAN_PYTHON_PYBIND11_CORE_KWARG_HANDLING_HPP -#define HOLOSCAN_PYTHON_PYBIND11_CORE_KWARG_HANDLING_HPP +#ifndef PYHOLOSCAN_CORE_KWARG_HANDLING_HPP +#define PYHOLOSCAN_CORE_KWARG_HANDLING_HPP #include // py::array, py::dtype #include @@ -50,4 +50,4 @@ py::dict arglist_to_kwargs(ArgList&); } // namespace holoscan -#endif /* HOLOSCAN_PYTHON_PYBIND11_CORE_KWARG_HANDLING_HPP */ +#endif /* PYHOLOSCAN_CORE_KWARG_HANDLING_HPP */ diff --git a/python/holoscan/core/operator.cpp b/python/holoscan/core/operator.cpp index e4b53cc5..efb34d3e 100644 --- a/python/holoscan/core/operator.cpp +++ b/python/holoscan/core/operator.cpp @@ -58,6 +58,10 @@ void init_operator(py::module_& m) { "name"_a, doc::OperatorSpec::doc_input_kwargs, py::return_value_policy::reference_internal) + .def_property_readonly("inputs", + &OperatorSpec::inputs, + doc::OperatorSpec::doc_inputs, + py::return_value_policy::reference_internal) .def("output", py::overload_cast<>(&OperatorSpec::output), doc::OperatorSpec::doc_output, @@ -67,6 +71,10 @@ void init_operator(py::module_& m) { "name"_a, doc::OperatorSpec::doc_output_kwargs, py::return_value_policy::reference_internal) + .def_property_readonly("outputs", + &OperatorSpec::outputs, + doc::OperatorSpec::doc_outputs, + py::return_value_policy::reference_internal) .def_property_readonly( "description", &OperatorSpec::description, doc::OperatorSpec::doc_description) .def( @@ -81,11 +89,6 @@ void init_operator(py::module_& m) { // defined from python via inheritance from the `Operator` class as defined in // core/__init__.py. - py::enum_(m, "ParameterFlag", doc::ParameterFlag::doc_ParameterFlag) - .value("NONE", ParameterFlag::kNone) - .value("OPTIONAL", ParameterFlag::kOptional) - .value("DYNAMIC", ParameterFlag::kDynamic); - py::class_>( m, "PyOperatorSpec", R"doc(Operator specification class.)doc") .def(py::init(), @@ -536,13 +539,16 @@ void PyOperator::set_py_tracing() { } void PyOperator::initialize() { - Operator::initialize(); // Get the initialize method of the Python Operator class and call it py::gil_scoped_acquire scope_guard; set_py_tracing(); py_initialize_.operator()(); + + // Call the parent class's initialize method after invoking the Python Operator's initialize + // method. + Operator::initialize(); } void PyOperator::start() { diff --git a/python/holoscan/core/operator.hpp b/python/holoscan/core/operator.hpp index 684f9204..ba6b7f53 100644 --- a/python/holoscan/core/operator.hpp +++ b/python/holoscan/core/operator.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef PYBIND11_CORE_OPERATOR_HPP -#define PYBIND11_CORE_OPERATOR_HPP +#ifndef PYHOLOSCAN_CORE_OPERATOR_HPP +#define PYHOLOSCAN_CORE_OPERATOR_HPP #include @@ -163,4 +163,4 @@ class PyOperator : public Operator { } // namespace holoscan -#endif /* PYBIND11_CORE_OPERATOR_HPP */ +#endif /* PYHOLOSCAN_CORE_OPERATOR_HPP */ diff --git a/python/holoscan/core/operator_pydoc.hpp b/python/holoscan/core/operator_pydoc.hpp index 1681f520..4706d8b3 100644 --- a/python/holoscan/core/operator_pydoc.hpp +++ b/python/holoscan/core/operator_pydoc.hpp @@ -63,6 +63,10 @@ name : str The name of the input port. )doc") +PYDOC(inputs, R"doc( +Return the reference of the input port map. +)doc") + PYDOC(output, R"doc( Add an outputput to the specification. )doc") @@ -76,6 +80,10 @@ name : str The name of the output port. )doc") +PYDOC(outputs, R"doc( +Return the reference of the output port map. +)doc") + PYDOC(param, R"doc( Add a parameter to the specification. @@ -134,7 +142,7 @@ Operator class. Can be initialized with any number of Python positional and keyword arguments. If a `name` keyword argument is provided, it must be a `str` and will be -used to set the name of the Operator. +used to set the name of the operator. `Condition` classes will be added to ``self.conditions``, `Resource` classes will be added to ``self.resources``, and any other arguments will be diff --git a/python/holoscan/core/resource_pydoc.hpp b/python/holoscan/core/resource_pydoc.hpp index d54186bd..c259d520 100644 --- a/python/holoscan/core/resource_pydoc.hpp +++ b/python/holoscan/core/resource_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -37,7 +37,7 @@ Class representing a resource. Can be initialized with any number of Python positional and keyword arguments. If a `name` keyword argument is provided, it must be a `str` and will be -used to set the name of the Operator. +used to set the name of the resource. If a `fragment` keyword argument is provided, it must be of type `holoscan.core.Fragment` (or diff --git a/python/holoscan/core/scheduler_pydoc.hpp b/python/holoscan/core/scheduler_pydoc.hpp index 2292d691..fe9811cc 100644 --- a/python/holoscan/core/scheduler_pydoc.hpp +++ b/python/holoscan/core/scheduler_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -37,7 +37,7 @@ Class representing a scheduler. Can be initialized with any number of Python positional and keyword arguments. If a `name` keyword argument is provided, it must be a `str` and will be -used to set the name of the Operator. +used to set the name of the scheduler. If a `fragment` keyword argument is provided, it must be of type `holoscan.core.Fragment` (or diff --git a/python/holoscan/core/tensor.cpp b/python/holoscan/core/tensor.cpp index 4825aa6e..9eb79fd1 100644 --- a/python/holoscan/core/tensor.cpp +++ b/python/holoscan/core/tensor.cpp @@ -99,8 +99,14 @@ void init_tensor(py::module_& m) { .def_property_readonly("dtype", &PyTensor::dtype, doc::Tensor::doc_dtype) .def_property_readonly("itemsize", &PyTensor::itemsize, doc::Tensor::doc_itemsize) .def_property_readonly("nbytes", &PyTensor::nbytes, doc::Tensor::doc_nbytes) - .def_property_readonly("data", &PyTensor::data, doc::Tensor::doc_data) + .def_property_readonly( + "data", + [](const Tensor& t) { + return static_cast(reinterpret_cast(t.data())); + }, + doc::Tensor::doc_data) .def_property_readonly("device", &PyTensor::device, doc::Tensor::doc_device) + .def("is_contiguous", &PyTensor::is_contiguous, doc::Tensor::doc_is_contiguous) // DLPack protocol .def("__dlpack__", &PyTensor::dlpack, "stream"_a = py::none(), doc::Tensor::doc_dlpack) .def("__dlpack_device__", &PyTensor::dlpack_device, doc::Tensor::doc_dlpack_device); @@ -599,4 +605,12 @@ py::tuple PyTensor::dlpack_device(const py::object& obj) { // raw pointer here instead. return py_dlpack_device(tensor.get()); } + +bool is_tensor_like(py::object value) { + return ((py::hasattr(value, "__dlpack__") && py::hasattr(value, "__dlpack_device__")) || + py::isinstance(value) || + py::hasattr(value, "__cuda_array_interface__") || + py::hasattr(value, "__array_interface__")); +} + } // namespace holoscan diff --git a/python/holoscan/core/tensor.hpp b/python/holoscan/core/tensor.hpp index 856e0e55..4f4c84d9 100644 --- a/python/holoscan/core/tensor.hpp +++ b/python/holoscan/core/tensor.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef PYBIND11_CORE_TENSOR_HPP -#define PYBIND11_CORE_TENSOR_HPP +#ifndef PYHOLOSCAN_CORE_TENSOR_HPP +#define PYHOLOSCAN_CORE_TENSOR_HPP #include @@ -182,6 +182,8 @@ class PyTensor : public Tensor { static py::tuple dlpack_device(const py::object& obj); }; +bool is_tensor_like(py::object value); + } // namespace holoscan -#endif /* PYBIND11_CORE_TENSOR_HPP */ +#endif /* PYHOLOSCAN_CORE_TENSOR_HPP */ diff --git a/python/holoscan/core/tensor_pydoc.hpp b/python/holoscan/core/tensor_pydoc.hpp index a139e065..e92cc531 100644 --- a/python/holoscan/core/tensor_pydoc.hpp +++ b/python/holoscan/core/tensor_pydoc.hpp @@ -59,17 +59,17 @@ PYDOC(as_tensor, R"doc( Convert a Python object to a Tensor. Parameters -========== +---------- object : array-like An object such as a NumPy array, CuPy array, PyTorch tensor, etc. supporting one of the supported protocols. Returns -======= +------- holocan.Tensor Notes -===== +----- For device arrays, this method first attempts to convert via ``__cuda_array_interface__`` [1]_, but falls back to the DLPack protocol [2]_, [3]_ if it is unavailable. @@ -77,7 +77,7 @@ For host arrays, this method first attempts to convert via the DLPack protocol, the ``__array_interface__`` [3]_ if it is unavailable. References -========== +---------- .. [1] https://numpy.org/doc/stable/reference/arrays.interface.html .. [2] https://dmlc.github.io/dlpack/latest/python_spec.html .. [3] https://data-apis.org/array-api/2022.12/API_specification/generated/array_api.array.__dlpack__.html @@ -88,17 +88,17 @@ PYDOC(from_dlpack, R"doc( Convert a Python object to a Tensor via the DLPack protocol [1]_, [2]_. Parameters -========== +---------- object : array-like An object such as a NumPy array, CuPy array, PyTorch tensor, etc. supporting one of the supported protocols. Returns -======= +------- holocan.Tensor References -========== +---------- .. [1] https://dmlc.github.io/dlpack/latest/python_spec.html .. [2] https://data-apis.org/array-api/2022.12/API_specification/generated/array_api.array.__dlpack__.html )doc") @@ -150,6 +150,15 @@ PYDOC(nbytes, R"doc( The size of the tensor's data in bytes. )doc") +PYDOC(is_contiguous, R"doc( +Determine whether the tensor has a contiguous, row-major memory layout. + +Returns +------- +bool + ``True`` if the tensor has a contiguous, row-major memory layout. ``False`` otherwise. +)doc") + PYDOC(dlpack, R"doc( Exports the array for consumption by ``from_dlpack()`` as a DLPack capsule. diff --git a/python/holoscan/executors/executors.cpp b/python/holoscan/executors/executors.cpp index fc8b969c..97db35dd 100644 --- a/python/holoscan/executors/executors.cpp +++ b/python/holoscan/executors/executors.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -37,21 +37,11 @@ namespace holoscan { PYBIND11_MODULE(_executors, m) { m.doc() = R"pbdoc( - Holoscan SDK Python Bindings - --------------------------------------- + Holoscan SDK Executor Python Bindings + ------------------------------------- .. currentmodule:: _executors - .. autosummary:: - :toctree: _generate - add - subtract )pbdoc"; -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif - py::class_>( m, "GXFExecutor", doc::GXFExecutor::doc_GXFExecutor) .def(py::init(), "app"_a, doc::GXFExecutor::doc_GXFExecutor_app); diff --git a/python/holoscan/graphs/graphs.cpp b/python/holoscan/graphs/graphs.cpp index 2538fbf8..5ed0a8ef 100644 --- a/python/holoscan/graphs/graphs.cpp +++ b/python/holoscan/graphs/graphs.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -159,21 +159,11 @@ using PyFragmentGraph = PYBIND11_MODULE(_graphs, m) { m.doc() = R"pbdoc( - Holoscan SDK Python Bindings - --------------------------------------- + Holoscan SDK Graph Python Bindings + ---------------------------------- .. currentmodule:: _graphs - .. autosummary:: - :toctree: _generate - add - subtract )pbdoc"; -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif - py::class_(m, "OperatorNodeType"); py::class_(m, "OperatorEdgeDataElementType"); py::class_(m, "OperatorEdgeDataType"); diff --git a/python/holoscan/gxf/CMakeLists.txt b/python/holoscan/gxf/CMakeLists.txt index 352f54f2..6249d6a9 100644 --- a/python/holoscan/gxf/CMakeLists.txt +++ b/python/holoscan/gxf/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,3 +23,15 @@ holoscan_pybind11_module(gxf ../core/execution_context.cpp ../core/tensor.cpp ) + +# Copy headers +install( + DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + DESTINATION include/holoscan/python + FILE_PERMISSIONS OWNER_READ OWNER_WRITE GROUP_READ WORLD_READ + DIRECTORY_PERMISSIONS OWNER_READ OWNER_EXECUTE OWNER_WRITE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE + COMPONENT "holoscan-python_libs" + FILES_MATCHING + PATTERN "*.hpp" + PATTERN "*pydoc.hpp" EXCLUDE +) diff --git a/python/holoscan/gxf/entity.hpp b/python/holoscan/gxf/entity.hpp index dbe15c46..fb808b70 100644 --- a/python/holoscan/gxf/entity.hpp +++ b/python/holoscan/gxf/entity.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef PYBIND11_GXF_ENTITY_HPP -#define PYBIND11_GXF_ENTITY_HPP +#ifndef PYHOLOSCAN_GXF_ENTITY_HPP +#define PYHOLOSCAN_GXF_ENTITY_HPP #include @@ -45,4 +45,4 @@ class PyEntity : public gxf::Entity { } // namespace holoscan -#endif /* PYBIND11_GXF_ENTITY_HPP */ +#endif /* PYHOLOSCAN_GXF_ENTITY_HPP */ diff --git a/python/holoscan/gxf/gxf.cpp b/python/holoscan/gxf/gxf.cpp index 28e72cb5..4cd06303 100644 --- a/python/holoscan/gxf/gxf.cpp +++ b/python/holoscan/gxf/gxf.cpp @@ -61,21 +61,11 @@ static const gxf_tid_t default_tid = {0, 0}; PYBIND11_MODULE(_gxf, m) { m.doc() = R"pbdoc( - Holoscan SDK Python Bindings - --------------------------------------- + Holoscan SDK GXF Python Bindings + -------------------------------- .. currentmodule:: _gxf - .. autosummary:: - :toctree: _generate - add - subtract )pbdoc"; -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif - init_entity(m); // TODO: This method can be removed once Executor::extension_manager(), diff --git a/python/holoscan/gxf/gxf_operator.cpp b/python/holoscan/gxf/gxf_operator.cpp index 7f4c3047..96e5d5f2 100644 --- a/python/holoscan/gxf/gxf_operator.cpp +++ b/python/holoscan/gxf/gxf_operator.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -64,6 +64,13 @@ void init_gxf_operator(py::module_& m) { py::overload_cast<>(&ops::GXFOperator::gxf_cid, py::const_), py::overload_cast(&ops::GXFOperator::gxf_cid), doc::GXFOperator::doc_gxf_cid) + .def("initialize", + &ops::GXFOperator::initialize, + doc::GXFOperator::doc_initialize) // note: virtual function + .def( + "setup", &ops::GXFOperator::setup, doc::GXFOperator::doc_setup) // note: virtual function + .def_property_readonly( + "description", &ops::GXFOperator::description, doc::GXFOperator::doc_description) .def( "__repr__", [](const py::object& obj) { diff --git a/python/holoscan/gxf/gxf_operator_pydoc.hpp b/python/holoscan/gxf/gxf_operator_pydoc.hpp index 6940db84..3eb3a0c3 100644 --- a/python/holoscan/gxf/gxf_operator_pydoc.hpp +++ b/python/holoscan/gxf/gxf_operator_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -61,10 +61,18 @@ PYDOC(gxf_cid, R"doc( The GXF component ID. )doc") +PYDOC(setup, R"doc( +Operator setup method. +)doc") + PYDOC(initialize, R"doc( Initialize the operator. )doc") +PYDOC(description, R"doc( +YAML formatted string describing the operator. +)doc") + } // namespace GXFOperator } // namespace holoscan::doc diff --git a/python/holoscan/logger/logger.cpp b/python/holoscan/logger/logger.cpp index 6547da31..705a0a4a 100644 --- a/python/holoscan/logger/logger.cpp +++ b/python/holoscan/logger/logger.cpp @@ -31,19 +31,11 @@ namespace holoscan { PYBIND11_MODULE(_logger, m) { m.doc() = R"pbdoc( - Holoscan SDK Python Bindings - --------------------------------------- + Holoscan SDK Logger Python Bindings + ----------------------------------- .. currentmodule:: _logger - .. autosummary:: - :toctree: _generate )pbdoc"; -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif - py::enum_(m, "LogLevel", doc::Logger::doc_LogLevel) .value("TRACE", LogLevel::TRACE) .value("DEBUG", LogLevel::DEBUG) diff --git a/python/holoscan/network_contexts/network_contexts.cpp b/python/holoscan/network_contexts/network_contexts.cpp index aa89189a..d5ee40c4 100644 --- a/python/holoscan/network_contexts/network_contexts.cpp +++ b/python/holoscan/network_contexts/network_contexts.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -70,21 +70,11 @@ class PyUcxContext : public UcxContext { PYBIND11_MODULE(_network_contexts, m) { m.doc() = R"pbdoc( - Holoscan SDK Python Bindings - --------------------------------------- + Holoscan SDK NetworkContext Python Bindings + ------------------------------------------- .. currentmodule:: _network_contexts - .. autosummary:: - :toctree: _generate - add - subtract )pbdoc"; -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif - py::class_ #include +#include "../operator_util.hpp" #include "./pydoc.hpp" #include "holoscan/core/fragment.hpp" @@ -54,7 +55,7 @@ class PyAJASourceOp : public AJASourceOp { using AJASourceOp::AJASourceOp; // Define a constructor that fully initializes the object. - PyAJASourceOp(Fragment* fragment, const std::string& device = "0"s, + PyAJASourceOp(Fragment* fragment, const py::args& args, const std::string& device = "0"s, NTV2Channel channel = NTV2Channel::NTV2_CHANNEL1, uint32_t width = 1920, uint32_t height = 1080, uint32_t framerate = 60, bool rdma = false, bool enable_overlay = false, @@ -69,6 +70,7 @@ class PyAJASourceOp : public AJASourceOp { Arg{"enable_overlay", enable_overlay}, Arg{"overlay_channel", overlay_channel}, Arg{"overlay_rdma", overlay_rdma}}) { + add_positional_condition_and_resource_args(this, args); name_ = name; fragment_ = fragment; spec_ = std::make_shared(fragment); @@ -80,21 +82,11 @@ class PyAJASourceOp : public AJASourceOp { PYBIND11_MODULE(_aja_source, m) { m.doc() = R"pbdoc( - Holoscan SDK Python Bindings + Holoscan SDK AJASourceOp Python Bindings --------------------------------------- .. currentmodule:: _aja_source - .. autosummary:: - :toctree: _generate - add - subtract )pbdoc"; -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif - py::enum_(m, "NTV2Channel") .value("NTV2_CHANNEL1", NTV2Channel::NTV2_CHANNEL1) .value("NTV2_CHANNEL2", NTV2Channel::NTV2_CHANNEL2) @@ -110,6 +102,7 @@ PYBIND11_MODULE(_aja_source, m) { py::class_>( m, "AJASourceOp", doc::AJASourceOp::doc_AJASourceOp) .def(py::init @@ -95,4 +95,4 @@ and uses a light-weight initialization. } // namespace holoscan::doc::AJASourceOp -#endif /* HOLOSCAN_OPERATORS_AJA_SOURCE_PYDOC_HPP */ +#endif /* PYHOLOSCAN_OPERATORS_AJA_SOURCE_PYDOC_HPP */ diff --git a/python/holoscan/operators/bayer_demosaic/bayer_demosaic.cpp b/python/holoscan/operators/bayer_demosaic/bayer_demosaic.cpp index e7854928..d36c8604 100644 --- a/python/holoscan/operators/bayer_demosaic/bayer_demosaic.cpp +++ b/python/holoscan/operators/bayer_demosaic/bayer_demosaic.cpp @@ -20,6 +20,7 @@ #include #include +#include "../operator_util.hpp" #include "./pydoc.hpp" #include "holoscan/core/fragment.hpp" @@ -55,7 +56,8 @@ class PyBayerDemosaicOp : public BayerDemosaicOp { using BayerDemosaicOp::BayerDemosaicOp; // Define a constructor that fully initializes the object. - PyBayerDemosaicOp(Fragment* fragment, std::shared_ptr pool, + PyBayerDemosaicOp(Fragment* fragment, const py::args& args, + std::shared_ptr pool, std::shared_ptr cuda_stream_pool = nullptr, const std::string& in_tensor_name = "", const std::string& out_tensor_name = "", int interpolation_mode = 0, int bayer_grid_pos = 2, bool generate_alpha = false, @@ -68,6 +70,7 @@ class PyBayerDemosaicOp : public BayerDemosaicOp { Arg{"generate_alpha", generate_alpha}, Arg{"alpha_value", alpha_value}}) { if (cuda_stream_pool) { this->add_arg(Arg{"cuda_stream_pool", cuda_stream_pool}); } + add_positional_condition_and_resource_args(this, args); name_ = name; fragment_ = fragment; spec_ = std::make_shared(fragment); @@ -79,25 +82,16 @@ class PyBayerDemosaicOp : public BayerDemosaicOp { PYBIND11_MODULE(_bayer_demosaic, m) { m.doc() = R"pbdoc( - Holoscan SDK Python Bindings - --------------------------------------- + Holoscan SDK BayerDemosaicOp Python Bindings + -------------------------------------------- .. currentmodule:: _bayer_demosaic - .. autosummary:: - :toctree: _generate - add - subtract )pbdoc"; -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif - py::class_>( m, "BayerDemosaicOp", doc::BayerDemosaicOp::doc_BayerDemosaicOp) .def(py::init<>(), doc::BayerDemosaicOp::doc_BayerDemosaicOp) .def(py::init, std::shared_ptr, const std::string&, @@ -116,7 +110,7 @@ PYBIND11_MODULE(_bayer_demosaic, m) { "bayer_grid_pos"_a = 2, "generate_alpha"_a = false, "alpha_value"_a = 255, - "name"_a = "format_converter"s, + "name"_a = "bayer_demosaic"s, doc::BayerDemosaicOp::doc_BayerDemosaicOp) .def("initialize", &BayerDemosaicOp::initialize, doc::BayerDemosaicOp::doc_initialize) .def("setup", &BayerDemosaicOp::setup, "spec"_a, doc::BayerDemosaicOp::doc_setup); diff --git a/python/holoscan/operators/bayer_demosaic/pydoc.hpp b/python/holoscan/operators/bayer_demosaic/pydoc.hpp index 389d21a7..bc60ae66 100644 --- a/python/holoscan/operators/bayer_demosaic/pydoc.hpp +++ b/python/holoscan/operators/bayer_demosaic/pydoc.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef HOLOSCAN_OPERATORS_BAYER_DEMOSAIC_PYDOC_HPP -#define HOLOSCAN_OPERATORS_BAYER_DEMOSAIC_PYDOC_HPP +#ifndef PYHOLOSCAN_OPERATORS_BAYER_DEMOSAIC_PYDOC_HPP +#define PYHOLOSCAN_OPERATORS_BAYER_DEMOSAIC_PYDOC_HPP #include @@ -32,11 +32,11 @@ Bayer Demosaic operator. receiver : nvidia::gxf::Tensor or nvidia::gxf::VideoBuffer The input video frame to process. If the input is a VideoBuffer it must be an 8-bit - unsigned grayscale video (nvidia::gxf::VideoFormat::GXF_VIDEO_FORMAT_GRAY). The video - buffer may be in either host or device memory (a host->device copy is performed if needed). - If a video buffer is not found, the input port message is searched for a tensor with the - name specified by ``in_tensor_name``. This must be a device tensor in either 8-bit or 16-bit - unsigned integer format. + unsigned grayscale video (`nvidia::gxf::VideoFormat::GXF_VIDEO_FORMAT_GRAY`). If a video + buffer is not found, the input port message is searched for a device + tensor with the name specified by `in_tensor_name`. The tensor must have either 8-bit or + 16-bit unsigned integer format. The tensor or video buffer may be in either host or device + memory (a host->device copy is performed if needed). **==Named Outputs==** @@ -46,6 +46,16 @@ Bayer Demosaic operator. be either 8-bit or 16-bit unsigned integer (matching the bit depth of the input). The name of the tensor that is output is controlled by ``out_tensor_name``. +**==Device Memory Requirements==** + + When using this operator with a ``holoscan.resources.BlockMemoryPool``, the minimum + ``block_size`` is ``(rows * columns * output_channels * element_size_bytes)`` where + ``output_channels`` is 4 when ``generate_alpha`` is ``True`` and 3 otherwise. If the input + tensor or video buffer is already on the device, only a single memory block is needed. However, + if the input is on the host, a second memory block will also be needed in order to make an + internal copy of the input to the device. The memory buffer must be on device + (``storage_type=1``). + Parameters ---------- fragment : holoscan.core.Fragment (constructor only) @@ -93,15 +103,6 @@ name : str, optional (constructor only) The name of the operator. Default value is ``"bayer_demosaic"``. )doc") -PYDOC(gxf_typename, R"doc( -The GXF type name of the resource. - -Returns -------- -str - The GXF type name of the resource -)doc") - PYDOC(initialize, R"doc( Initialize the operator. @@ -120,4 +121,4 @@ spec : holoscan.core.OperatorSpec } // namespace holoscan::doc::BayerDemosaicOp -#endif /* HOLOSCAN_OPERATORS_BAYER_DEMOSAIC_PYDOC_HPP */ +#endif /* PYHOLOSCAN_OPERATORS_BAYER_DEMOSAIC_PYDOC_HPP */ diff --git a/python/holoscan/operators/format_converter/format_converter.cpp b/python/holoscan/operators/format_converter/format_converter.cpp index e18f2644..43bdf703 100644 --- a/python/holoscan/operators/format_converter/format_converter.cpp +++ b/python/holoscan/operators/format_converter/format_converter.cpp @@ -23,6 +23,7 @@ #include #include +#include "../operator_util.hpp" #include "./pydoc.hpp" #include "holoscan/core/fragment.hpp" @@ -58,9 +59,9 @@ class PyFormatConverterOp : public FormatConverterOp { using FormatConverterOp::FormatConverterOp; // Define a constructor that fully initializes the object. - PyFormatConverterOp(Fragment* fragment, std::shared_ptr pool, - const std::string& out_dtype, const std::string& in_dtype = "", - const std::string& in_tensor_name = "", + PyFormatConverterOp(Fragment* fragment, const py::args& args, + std::shared_ptr pool, const std::string& out_dtype, + const std::string& in_dtype = "", const std::string& in_tensor_name = "", const std::string& out_tensor_name = "", float scale_min = 0.f, float scale_max = 1.f, uint8_t alpha_value = static_cast(255), int32_t resize_height = 0, int32_t resize_width = 0, int32_t resize_mode = 0, @@ -80,6 +81,7 @@ class PyFormatConverterOp : public FormatConverterOp { Arg{"out_channel_order", out_channel_order}, Arg{"pool", pool}}) { if (cuda_stream_pool) { this->add_arg(Arg{"cuda_stream_pool", cuda_stream_pool}); } + add_positional_condition_and_resource_args(this, args); name_ = name; fragment_ = fragment; spec_ = std::make_shared(fragment); @@ -91,24 +93,15 @@ class PyFormatConverterOp : public FormatConverterOp { PYBIND11_MODULE(_format_converter, m) { m.doc() = R"pbdoc( - Holoscan SDK Python Bindings - --------------------------------------- + Holoscan SDK FormatConverterOp Python Bindings + ---------------------------------------------- .. currentmodule:: _format_converter - .. autosummary:: - :toctree: _generate - add - subtract )pbdoc"; -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif - py::class_>( m, "FormatConverterOp", doc::FormatConverterOp::doc_FormatConverterOp) .def(py::init, const std::string&, const std::string&, diff --git a/python/holoscan/operators/format_converter/pydoc.hpp b/python/holoscan/operators/format_converter/pydoc.hpp index 90579dfc..101159c9 100644 --- a/python/holoscan/operators/format_converter/pydoc.hpp +++ b/python/holoscan/operators/format_converter/pydoc.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef HOLOSCAN_OPERATORS_FORMAT_CONVERTER_PYDOC_HPP -#define HOLOSCAN_OPERATORS_FORMAT_CONVERTER_PYDOC_HPP +#ifndef PYHOLOSCAN_OPERATORS_FORMAT_CONVERTER_PYDOC_HPP +#define PYHOLOSCAN_OPERATORS_FORMAT_CONVERTER_PYDOC_HPP #include @@ -32,12 +32,11 @@ Format conversion operator. source_video : nvidia::gxf::Tensor or nvidia::gxf::VideoBuffer The input video frame to process. If the input is a VideoBuffer it must be in format - GXF_VIDEO_FORMAT_RGBA, GXF_VIDEO_FORMAT_RGB or GXF_VIDEO_FORMAT_NV12. This video + GXF_VIDEO_FORMAT_RGBA, GXF_VIDEO_FORMAT_RGB or GXF_VIDEO_FORMAT_NV12. If a video buffer is + not found, the input port message is searched for a tensor with the name specified by + `in_tensor_name`. This must be a tensor in one of several supported formats (unsigned 8-bit + int or float32 graycale, unsigned 8-bit int RGB or RGBA YUV420 or NV12). The tensor or video buffer may be in either host or device memory (a host->device copy is performed if needed). - If a video buffer is not found, the input port message is searched for a tensor with the - name specified by ``in_tensor_name``. This must be a device tensor in one of several - supported formats (unsigned 8-bit int or float32 graycale, unsigned 8-bit int RGB or RGBA, - YUV420 or NV12). **==Named Outputs==** @@ -46,6 +45,25 @@ Format conversion operator. output tensor will depend on the specific parameters that were set for this operator. The name of the Tensor transmitted on this port is determined by ``out_tensor_name``. +**==Device Memory Requirements==** + + When using this operator with a ``holoscan.resources.BlockMemoryPool``, between 1 and 3 device + memory blocks (``storage_type=1``) will be required based on the input tensors and parameters: + + - 1.) In all cases there is a memory block needed for the output tensor. The size of this + block will be ``out_height * out_width * out_channels * out_element_size_bytes`` where + ``(out_height, out_width)`` will either be ``(in_height, in_width)`` (or + ``(resize_height, resize_width)`` a resize was specified). `out_element_size` is the + element size in bytes (e.g. 1 for RGB888 or 4 for Float32). + - 2.) If a resize is being done, another memory block is required for this. This block will + have size ``resize_height * resize_width * in_channels * in_element_size_bytes``. + - 3.) If the input tensor will be in host memory, a memory block is needed to copy the input + to the device. This block will have size + ``in_height * in_width * in_channels * in_element_size_bytes``. + + Thus when declaring the memory pool, `num_blocks` will be between 1 and 3 and `block_size` + must be greater or equal to the largest of the individual blocks sizes described above. + Parameters ---------- fragment : holoscan.core.Fragment (constructor only) @@ -134,4 +152,4 @@ spec : holoscan.core.OperatorSpec } // namespace holoscan::doc::FormatConverterOp -#endif /* HOLOSCAN_OPERATORS_FORMAT_CONVERTER_PYDOC_HPP */ +#endif /* PYHOLOSCAN_OPERATORS_FORMAT_CONVERTER_PYDOC_HPP */ diff --git a/python/holoscan/operators/gxf_codelet/CMakeLists.txt b/python/holoscan/operators/gxf_codelet/CMakeLists.txt new file mode 100644 index 00000000..f2008e64 --- /dev/null +++ b/python/holoscan/operators/gxf_codelet/CMakeLists.txt @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +holoscan_pybind11_module(gxf_codelet + gxf_codelet.cpp +) +target_link_libraries(gxf_codelet_python + PUBLIC holoscan::ops::gxf_codelet +) diff --git a/python/holoscan/operators/gxf_codelet/__init__.py b/python/holoscan/operators/gxf_codelet/__init__.py new file mode 100644 index 00000000..a5feac12 --- /dev/null +++ b/python/holoscan/operators/gxf_codelet/__init__.py @@ -0,0 +1,67 @@ +""" +SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" # noqa: E501 + +import holoscan.core # noqa: F401 +import holoscan.gxf # noqa: F401 +from holoscan.core import OperatorSpec, _Fragment + +from ._gxf_codelet import GXFCodeletOp as _GXFCodeletOp + +__all__ = ["GXFCodeletOp"] + + +class GXFCodeletOp(_GXFCodeletOp): + def __setattr__(self, name, value): + readonly_attributes = [ + "fragment", + "gxf_typename", + "conditions", + "resources", + "operator_type", + "description", + ] + if name in readonly_attributes: + raise AttributeError(f'cannot override read-only property "{name}"') + super().__setattr__(name, value) + + def __init__(self, fragment, *args, **kwargs): + if not isinstance(fragment, _Fragment): + raise ValueError( + "The first argument to an Operator's constructor must be the Fragment " + "(Application) to which it belongs." + ) + # It is recommended to not use super() + # (https://pybind11.readthedocs.io/en/stable/advanced/classes.html#overriding-virtual-functions-in-python) + _GXFCodeletOp.__init__(self, self, fragment, *args, **kwargs) + # Create a PyOperatorSpec object and pass it to the C++ API + spec = OperatorSpec(fragment=self.fragment, op=self) + self.spec = spec + # Call setup method in the derived class + self.setup(spec) + + def setup(self, spec: OperatorSpec): + # This method is invoked by the derived class to set up the operator. + super().setup(spec) + + def initialize(self): + # Place holder for initialize method + pass + + +# copy docstrings defined in pydoc.hpp +GXFCodeletOp.__doc__ = _GXFCodeletOp.__doc__ +GXFCodeletOp.__init__.__doc__ = _GXFCodeletOp.__init__.__doc__ diff --git a/python/holoscan/operators/gxf_codelet/gxf_codelet.cpp b/python/holoscan/operators/gxf_codelet/gxf_codelet.cpp new file mode 100644 index 00000000..728f57f3 --- /dev/null +++ b/python/holoscan/operators/gxf_codelet/gxf_codelet.cpp @@ -0,0 +1,120 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include + +#include "../operator_util.hpp" +#include "./pydoc.hpp" + +#include "holoscan/core/fragment.hpp" +#include "holoscan/core/operator.hpp" +#include "holoscan/core/operator_spec.hpp" +#include "holoscan/core/resources/gxf/allocator.hpp" +#include "holoscan/core/resources/gxf/cuda_stream_pool.hpp" +#include "holoscan/operators/gxf_codelet/gxf_codelet.hpp" + +using std::string_literals::operator""s; +using pybind11::literals::operator""_a; + +#define STRINGIFY(x) #x +#define MACRO_STRINGIFY(x) STRINGIFY(x) + +namespace py = pybind11; + +namespace holoscan::ops { + +/* Trampoline class for handling Python kwargs + * + * These add a constructor that takes a Fragment for which to initialize the operator. + * The explicit parameter list and default arguments take care of providing a Pythonic + * kwarg-based interface with appropriate default values matching the operator's + * default parameters in the C++ API `setup` method. + * + * The sequence of events in this constructor is based on Fragment::make_operator + */ + +class PyGXFCodeletOp : public GXFCodeletOp { + public: + /* Inherit the constructors */ + using GXFCodeletOp::GXFCodeletOp; + + // Define a constructor that fully initializes the object. + PyGXFCodeletOp(py::object op, Fragment* fragment, const std::string& gxf_typename, + const py::args& args, const std::string& name, const py::kwargs& kwargs) + : GXFCodeletOp(gxf_typename.c_str()) { + py_op_ = op; + py_initialize_ = py::getattr(op, "initialize"); // cache the initialize method + + add_positional_condition_and_resource_args(this, args); + add_kwargs(this, kwargs); + + name_ = name; + fragment_ = fragment; + } + + void initialize() override { + // Get the initialize method of the Python Operator class and call it + py::gil_scoped_acquire scope_guard; + + // TODO(gbae): setup tracing + // // set_py_tracing(); + + // Call the initialize method of the Python Operator class + py_initialize_.operator()(); + + // Call the parent class's initialize method after invoking the Python Operator's initialize + // method. + GXFCodeletOp::initialize(); + } + + protected: + py::object py_op_ = py::none(); + py::object py_initialize_ = py::none(); +}; + +/* The python module */ + +PYBIND11_MODULE(_gxf_codelet, m) { + m.doc() = R"pbdoc( + Holoscan SDK Python Bindings + --------------------------------------- + .. currentmodule:: _gxf_codelet + )pbdoc"; + + py::class_>( + m, "GXFCodeletOp", doc::GXFCodeletOp::doc_GXFCodeletOp) + .def(py::init<>(), doc::GXFCodeletOp::doc_GXFCodeletOp) + .def(py::init(), + "op"_a, + "fragment"_a, + "gxf_typename"_a, + "name"_a = "gxf_codelet"s, + doc::GXFCodeletOp::doc_GXFCodeletOp) + .def_property_readonly( + "gxf_typename", &GXFCodeletOp::gxf_typename, doc::GXFCodeletOp::doc_gxf_typename) + .def("initialize", &GXFCodeletOp::initialize, doc::GXFCodeletOp::doc_initialize) + .def("setup", &GXFCodeletOp::setup, doc::GXFCodeletOp::doc_setup); +} // PYBIND11_MODULE NOLINT +} // namespace holoscan::ops diff --git a/python/holoscan/operators/gxf_codelet/pydoc.hpp b/python/holoscan/operators/gxf_codelet/pydoc.hpp new file mode 100644 index 00000000..6171ec1a --- /dev/null +++ b/python/holoscan/operators/gxf_codelet/pydoc.hpp @@ -0,0 +1,89 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PYHOLOSCAN_OPERATORS_GXF_CODELET_PYDOC_HPP +#define PYHOLOSCAN_OPERATORS_GXF_CODELET_PYDOC_HPP + +#include + +#include "../../macros.hpp" + +namespace holoscan::doc::GXFCodeletOp { + +// PyGXFCodeletOp Constructor +PYDOC(GXFCodeletOp, R"doc( +GXF Codelet wrapper operator. + +**==Named Inputs==** + + Input ports are automatically defined based on the parameters of the underlying GXF Codelet + that include the ``nvidia::gxf::Receiver`` component handle. + + To view the information about the operator, refer to the `description` property of this object. + +**==Named Outputs==** + + Output ports are automatically defined based on the parameters of the underlying GXF Codelet + that include the ``nvidia::gxf::Transmitter`` component handle. + + To view the information about the operator, refer to the ``description`` property of this object. + +Parameters +---------- +fragment : holoscan.core.Fragment (constructor only) + The fragment that the operator belongs to. +gxf_typename : str + The GXF type name that identifies the specific GXF Codelet being wrapped. +*args : tuple + Additional positional arguments (``holoscan.core.Condition`` or ``holoscan.core.Resource``). +name : str, optional (constructor only) + The name of the operator. Default value is ``"gxf_codelet"``. +**kwargs : dict + The additional keyword arguments that can be passed depend on the underlying GXF Codelet. The + additional parameters are the parameters of the underlying GXF Codelet that are neither + specifically part of the ``nvidia::gxf::Receiver`` nor the ``nvidia::gxf::Transmitter`` + components. These parameters can provide further customization and functionality to the operator. +)doc") + +PYDOC(gxf_typename, R"doc( +The GXF type name of the resource. + +Returns +------- +str + The GXF type name of the resource +)doc") + +PYDOC(initialize, R"doc( +Initialize the operator. + +This method is called only once when the operator is created for the first time, +and uses a light-weight initialization. +)doc") + +PYDOC(setup, R"doc( +Define the operator specification. + +Parameters +---------- +spec : holoscan.core.OperatorSpec + The operator specification. +)doc") + +} // namespace holoscan::doc::GXFCodeletOp + +#endif /* PYHOLOSCAN_OPERATORS_GXF_CODELET_PYDOC_HPP */ diff --git a/python/holoscan/operators/holoviz/CMakeLists.txt b/python/holoscan/operators/holoviz/CMakeLists.txt index d1416806..17b09bfc 100644 --- a/python/holoscan/operators/holoviz/CMakeLists.txt +++ b/python/holoscan/operators/holoviz/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,6 +15,7 @@ holoscan_pybind11_module(holoviz holoviz.cpp + ../../core/emitter_receiver_registry.cpp ) target_link_libraries(holoviz_python PUBLIC holoscan::ops::holoviz diff --git a/python/holoscan/operators/holoviz/__init__.py b/python/holoscan/operators/holoviz/__init__.py index a9bcab1e..a6de09fc 100644 --- a/python/holoscan/operators/holoviz/__init__.py +++ b/python/holoscan/operators/holoviz/__init__.py @@ -17,10 +17,15 @@ from collections.abc import MutableMapping, Sequence -from holoscan.core import IOSpec +from holoscan.core import IOSpec, io_type_registry from holoscan.resources import Allocator, CudaStreamPool, UnboundedAllocator # noqa: F401 from ._holoviz import HolovizOp as _HolovizOp +from ._holoviz import Pose3D # noqa: F401 +from ._holoviz import register_types as _register_types + +# register methods for receiving or emitting list[HolovizOp.InputSpec] and camera pose types +_register_types(io_type_registry) _holoviz_str_to_input_type = { "unknown": _HolovizOp.InputType.UNKNOWN, @@ -53,6 +58,7 @@ class HolovizOp(_HolovizOp): def __init__( self, fragment, + *args, allocator=None, receivers=(), tensors=(), @@ -68,6 +74,10 @@ def __init__( enable_render_buffer_input=False, enable_render_buffer_output=False, enable_camera_pose_output=False, + camera_pose_output_type="projection_matrix", + camera_eye=(0.0, 0.0, 1.0), + camera_look_at=(0.0, 0.0, 0.0), + camera_up=(0.0, 1.0, 0.0), font_path="", cuda_stream_pool=None, name="holoviz_op", @@ -182,7 +192,8 @@ def __init__( tensor_input_specs.append(ispec) super().__init__( - fragment=fragment, + fragment, + *args, allocator=allocator, receivers=receiver_iospecs, tensors=tensor_input_specs, @@ -198,6 +209,10 @@ def __init__( enable_render_buffer_input=enable_render_buffer_input, enable_render_buffer_output=enable_render_buffer_output, enable_camera_pose_output=enable_camera_pose_output, + camera_pose_output_type=camera_pose_output_type, + camera_eye=camera_eye, + camera_look_at=camera_look_at, + camera_up=camera_up, font_path=font_path, cuda_stream_pool=cuda_stream_pool, name=name, diff --git a/python/holoscan/operators/holoviz/holoviz.cpp b/python/holoscan/operators/holoviz/holoviz.cpp index f6b91a54..f3e24444 100644 --- a/python/holoscan/operators/holoviz/holoviz.cpp +++ b/python/holoscan/operators/holoviz/holoviz.cpp @@ -18,18 +18,27 @@ #include #include // for vector +#include +#include + #include #include #include #include +#include "../operator_util.hpp" #include "./pydoc.hpp" +#include +#include "../../core/emitter_receiver_registry.hpp" // EmitterReceiverRegistry +#include "../../core/io_context.hpp" // PyOutputContext #include "holoscan/core/codec_registry.hpp" +#include "holoscan/core/condition.hpp" #include "holoscan/core/fragment.hpp" #include "holoscan/core/io_context.hpp" #include "holoscan/core/operator.hpp" #include "holoscan/core/operator_spec.hpp" +#include "holoscan/core/resource.hpp" #include "holoscan/core/resources/gxf/allocator.hpp" #include "holoscan/core/resources/gxf/cuda_stream_pool.hpp" #include "holoscan/operators/holoviz/codecs.hpp" @@ -43,6 +52,14 @@ using pybind11::literals::operator""_a; namespace py = pybind11; +namespace holoscan { + +// default emitter_receiver templates can handle std::vector +// default emitter_receiver templates can handle std::shared_ptr> +// default emitter_receiver templates can handle std::shared_ptr + +} // namespace holoscan + namespace holoscan::ops { /* Trampoline class for handling Python kwargs @@ -62,7 +79,7 @@ class PyHolovizOp : public HolovizOp { // Define a constructor that fully initializes the object. PyHolovizOp( - Fragment* fragment, std::shared_ptr<::holoscan::Allocator> allocator, + Fragment* fragment, const py::args& args, std::shared_ptr<::holoscan::Allocator> allocator, std::vector receivers = std::vector(), const std::vector& tensors = std::vector(), const std::vector>& color_lut = std::vector>(), @@ -70,7 +87,11 @@ class PyHolovizOp : public HolovizOp { uint32_t width = 1920, uint32_t height = 1080, float framerate = 60.f, bool use_exclusive_display = false, bool fullscreen = false, bool headless = false, bool enable_render_buffer_input = false, bool enable_render_buffer_output = false, - bool enable_camera_pose_output = false, const std::string& font_path = "", + bool enable_camera_pose_output = false, + const std::string& camera_pose_output_type = "projection_matrix", + const std::array& camera_eye = {0.f, 0.f, 1.f}, + const std::array& camera_look_at = {0.f, 0.f, 0.f}, + const std::array& camera_up = {0.f, 1.f, 1.f}, const std::string& font_path = "", std::shared_ptr cuda_stream_pool = nullptr, const std::string& name = "holoviz_op") : HolovizOp(ArgList{Arg{"allocator", allocator}, @@ -86,12 +107,17 @@ class PyHolovizOp : public HolovizOp { Arg{"enable_render_buffer_input", enable_render_buffer_input}, Arg{"enable_render_buffer_output", enable_render_buffer_output}, Arg{"enable_camera_pose_output", enable_camera_pose_output}, + Arg{"camera_pose_output_type", camera_pose_output_type}, + Arg{"camera_eye", camera_eye}, + Arg{"camera_look_at", camera_look_at}, + Arg{"camera_up", camera_up}, Arg{"font_path", font_path}}) { // only append tensors argument if it is not empty // avoids [holoscan] [error] [gxf_operator.hpp:126] Unable to handle parameter 'tensors' if (tensors.size() > 0) { this->add_arg(Arg{"tensors", tensors}); } if (receivers.size() > 0) { this->add_arg(Arg{"receivers", receivers}); } if (cuda_stream_pool) { this->add_arg(Arg{"cuda_stream_pool", cuda_stream_pool}); } + add_positional_condition_and_resource_args(this, args); name_ = name; fragment_ = fragment; spec_ = std::make_shared(fragment); @@ -103,25 +129,16 @@ class PyHolovizOp : public HolovizOp { PYBIND11_MODULE(_holoviz, m) { m.doc() = R"pbdoc( - Holoscan SDK Python Bindings - --------------------------------------- + Holoscan SDK HolovizOp Python Bindings + -------------------------------------- .. currentmodule:: _holoviz - .. autosummary:: - :toctree: _generate - add - subtract )pbdoc"; -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif - py::class_> holoviz_op( m, "HolovizOp", doc::HolovizOp::doc_HolovizOp); holoviz_op .def(py::init, std::vector, const std::vector&, @@ -138,6 +155,10 @@ PYBIND11_MODULE(_holoviz, m) { bool, bool, const std::string&, + const std::array&, + const std::array&, + const std::array&, + const std::string&, std::shared_ptr, const std::string&>(), "fragment"_a, @@ -156,6 +177,10 @@ PYBIND11_MODULE(_holoviz, m) { "enable_render_buffer_input"_a = false, "enable_render_buffer_output"_a = false, "enable_camera_pose_output"_a = false, + "camera_pose_output_type"_a = "projection_matrix", + "camera_eye"_a = std::array{0.f, 0.f, 1.f}, + "camera_look_at"_a = std::array{0.f, 0.f, 0.f}, + "camera_up"_a = std::array{0.f, 1.f, 1.f}, "font_path"_a = "", "cuda_stream_pool"_a = py::none(), "name"_a = "holoviz_op"s, @@ -215,10 +240,34 @@ PYBIND11_MODULE(_holoviz, m) { .def_readwrite("height", &HolovizOp::InputSpec::View::height_) .def_readwrite("matrix", &HolovizOp::InputSpec::View::matrix_); + // need to wrap nvidia::gxf::Pose3D for use of received Pose3D object from Python + py::class_ Pose3D(m, "Pose3D", doc::HolovizOp::Pose3D::doc_Pose3D); + Pose3D.def_readwrite("rotation", &nvidia::gxf::Pose3D::rotation) + .def_readwrite("translation", &nvidia::gxf::Pose3D::translation) + .def("__repr__", [](const nvidia::gxf::Pose3D& pose) { + return fmt::format( + "Pose3D(rotation: {}, translation: {})", pose.rotation, pose.translation); + }); + // Register the std::vector codec when the Python module is imported. // This is useful for, e.g. testing serialization with pytest without having to first create a // HolovizOp operator (which registers the type in its initialize method). CodecRegistry::get_instance().add_codec>( "std::vector>", true); + + // Import the emitter/receiver registry from holoscan.core and pass it to this function to + // register this new C++ type with the SDK. + m.def("register_types", [](EmitterReceiverRegistry& registry) { + registry.add_emitter_receiver>( + "std::vector"s); + // array camera pose object + registry.add_emitter_receiver>>( + "std::shared_ptr>"s); + // Pose3D camera pose object + registry.add_emitter_receiver>( + "std::shared_ptr"s); + // camera_eye_input, camera_look_at_input, camera_up_input + registry.add_emitter_receiver>("std::array"s); + }); } // PYBIND11_MODULE NOLINT } // namespace holoscan::ops diff --git a/python/holoscan/operators/holoviz/pydoc.hpp b/python/holoscan/operators/holoviz/pydoc.hpp index bd45e6d4..a26fd37b 100644 --- a/python/holoscan/operators/holoviz/pydoc.hpp +++ b/python/holoscan/operators/holoviz/pydoc.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef HOLOSCAN_OPERATORS_HOLOVIZ_PYDOC_HPP -#define HOLOSCAN_OPERATORS_HOLOVIZ_PYDOC_HPP +#ifndef PYHOLOSCAN_OPERATORS_HOLOVIZ_PYDOC_HPP +#define PYHOLOSCAN_OPERATORS_HOLOVIZ_PYDOC_HPP #include @@ -43,11 +43,11 @@ This is a Vulkan-based visualizer. must be found or an exception will be raised. Any extra, named tensors not present in the ``tensors`` parameter specification (or optional, dynamic ``input_specs`` input) will be ignored. - input_specs : list[holoscan.operators.HolovizOp.InputSpec] (optional) + input_specs : list[holoscan.operators.HolovizOp.InputSpec], optional A list of ``InputSpec`` objects. This port can be used to dynamically update the overlay specification at run time. No inputs are required on this port in order for the operator to ``compute``. - render_buffer_input : nvidia::gxf::VideoBuffer (optional) + render_buffer_input : nvidia::gxf::VideoBuffer, optional An empty render buffer can optionally be provided. The video buffer must have format GXF_VIDEO_FORMAT_RGBA and be in device memory. This input port only exists if ``enable_render_buffer_input`` was set to ``True``, in which case ``compute`` will only be @@ -55,16 +55,35 @@ This is a Vulkan-based visualizer. **==Named Outputs==** - render_buffer_output : nvidia::gxf::VideoBuffer (optional) + render_buffer_output : nvidia::gxf::VideoBuffer, optional Output for a filled render buffer. If an input render buffer is specified, it is using that one, else it allocates a new buffer. The video buffer will have format GXF_VIDEO_FORMAT_RGBA and will be in device memory. This output is useful for offline rendering or headless mode. This output port only exists if ``enable_render_buffer_output`` was set to ``True``. - camera_pose_output : std::array (optional) - The camera pose. The parameters returned represent the values of a 4x4 row major - projection matrix. This output port only exists if ``enable_camera_pose_output`` was set to - ``True``. + camera_pose_output : std::array or nvidia::gxf::Pose3D, optional + The camera pose. Depending on the value of ``camera_pose_output_type`` this outputs a 4x4 + row major projection matrix (type ``std::array``) or the camera extrinsics model + (type ``nvidia::gxf::Pose3D``). This output port only exists if + ``enable_camera_pose_output`` was set to ``True``. + +**==Device Memory Requirements==** + + If ``render_buffer_input`` is enabled, the provided buffer is used and no memory block will be + allocated. Otherwise, when using this operator with a ``holoscan.resources.BlockMemoryPool``, a + single device memory block is needed (``storage_type=1``). The size of this memory block can be + determined by rounding the width and height up to the nearest even size and then padding the + rows as needed so that the row stride is a multiple of 256 bytes. C++ code to calculate the + block size is as follows + +.. code-block:: python + + def get_block_size(height, width): + height_even = height + (height & 1) + width_even = width + (width & 1) + row_bytes = width_even * 4; # 4 bytes per pixel for 8-bit RGBA + row_stride = (row_bytes % 256 == 0) ? row_bytes : ((row_bytes // 256 + 1) * 256) + return height_even * row_stride Parameters ---------- @@ -84,7 +103,8 @@ color_lut : list of list of float, optional window_title : str, optional Title on window canvas. Default value is ``"Holoviz"``. display_name : str, optional - In exclusive mode, name of display to use as shown with xrandr. Default value is ``"DP-0"``. + In exclusive mode, name of display to use as shown with `xrandr` or `hwinfo --monitor`. Default + value is ``"DP-0"``. width : int, optional Window width or display resolution width if in exclusive or fullscreen mode. Default value is ``1920``. @@ -106,9 +126,18 @@ enable_render_buffer_input : bool, optional enable_render_buffer_output : bool, optional If ``True``, an additional output port, named ``"render_buffer_output"`` is added to the operator. Default value is ``False``. -enable_camera_pose_output : bool, optional. +enable_camera_pose_output : bool, optional If ``True``, an additional output port, named ``"camera_pose_output"`` is added to the operator. Default value is ``False``. +camera_pose_output_type : str, optional + Type of data output at ``"camera_pose_output"``. Supported values are ``projection_matrix`` and + ``extrinsics_model``. Default value is ``projection_matrix``. +camera_eye : sequence of three floats, optional + Initial camera eye position. Default value is ``(0.0, 0.0, 1.0)``. +camera_look_at : sequence of three floats, optional + Initial camera look at position. Default value is ``(0.0, 0.0, 0.0)``. +camera_up : sequence of three floats, optional + Initial camera up vector. Default value is ``(0.0, 1.0, 0.0)``. font_path : str, optional File path for the font used for rendering text. Default value is ``""``. cuda_stream_pool : holoscan.resources.CudaStreamPool, optional @@ -326,6 +355,21 @@ spec : holoscan.core.OperatorSpec } // namespace holoscan::doc::HolovizOp +namespace holoscan::doc::HolovizOp::Pose3D { + +PYDOC(Pose3D, R"doc( +nvidia::gxf::Pose3D object representing a camera pose + +Attributes +---------- +rotation : list of float + 9 floating point values representing a 3x3 rotation matrix. +translation : list of float + 3 floating point values representing a translation vector. +)doc") + +} // namespace holoscan::doc::HolovizOp::Pose3D + namespace holoscan::doc::HolovizOp::InputSpec { // HolovizOp.InputSpec Constructor @@ -405,4 +449,4 @@ viewport instead of the upper left corner. } // namespace holoscan::doc::HolovizOp::InputSpec -#endif /* HOLOSCAN_OPERATORS_HOLOVIZ_PYDOC_HPP */ +#endif /* PYHOLOSCAN_OPERATORS_HOLOVIZ_PYDOC_HPP */ diff --git a/python/holoscan/operators/inference/inference.cpp b/python/holoscan/operators/inference/inference.cpp index 32fb13eb..7c9b35cf 100644 --- a/python/holoscan/operators/inference/inference.cpp +++ b/python/holoscan/operators/inference/inference.cpp @@ -22,6 +22,7 @@ #include #include +#include "../operator_util.hpp" #include "./pydoc.hpp" #include "holoscan/core/fragment.hpp" @@ -73,12 +74,13 @@ class PyInferenceOp : public InferenceOp { using InferenceOp::InferenceOp; // Define a constructor that fully initializes the object. - PyInferenceOp(Fragment* fragment, const std::string& backend, + PyInferenceOp(Fragment* fragment, const py::args& args, const std::string& backend, std::shared_ptr<::holoscan::Allocator> allocator, py::dict inference_map, // InferenceOp::DataVecMap py::dict model_path_map, // InferenceOp::DataMap py::dict pre_processor_map, // InferenceOp::DataVecMap py::dict device_map, // InferenceOp::DataMap + py::dict temporal_map, // InferenceOp::DataMap py::dict backend_map, // InferenceOp::DataMap const std::vector& in_tensor_names, const std::vector& out_tensor_names, bool infer_on_cpu = false, @@ -101,6 +103,7 @@ class PyInferenceOp : public InferenceOp { Arg{"enable_fp16", enable_fp16}, Arg{"is_engine_path", is_engine_path}}) { if (cuda_stream_pool) { this->add_arg(Arg{"cuda_stream_pool", cuda_stream_pool}); } + add_positional_condition_and_resource_args(this, args); name_ = name; fragment_ = fragment; @@ -126,6 +129,12 @@ class PyInferenceOp : public InferenceOp { } } + py::dict temporal_map_infer = temporal_map.cast(); + for (auto& [key, value] : temporal_map_infer) { temporal_map_infer[key] = py::str(value); } + + py::dict device_map_infer = device_map.cast(); + for (auto& [key, value] : device_map_infer) { device_map_infer[key] = py::str(value); } + // convert from Python dict to InferenceOp::DataVecMap auto inference_map_datavecmap = _dict_to_inference_datavecmap(inference_map_dict); this->add_arg(Arg("inference_map", inference_map_datavecmap)); @@ -133,9 +142,12 @@ class PyInferenceOp : public InferenceOp { auto model_path_datamap = _dict_to_inference_datamap(model_path_map.cast()); this->add_arg(Arg("model_path_map", model_path_datamap)); - auto device_datamap = _dict_to_inference_datamap(device_map.cast()); + auto device_datamap = _dict_to_inference_datamap(device_map_infer); this->add_arg(Arg("device_map", device_datamap)); + auto temporal_datamap = _dict_to_inference_datamap(temporal_map_infer); + this->add_arg(Arg("temporal_map", temporal_datamap)); + auto backend_datamap = _dict_to_inference_datamap(backend_map.cast()); this->add_arg(Arg("backend_map", backend_datamap)); @@ -152,26 +164,17 @@ class PyInferenceOp : public InferenceOp { PYBIND11_MODULE(_inference, m) { m.doc() = R"pbdoc( - Holoscan SDK Python Bindings - --------------------------------------- + Holoscan SDK InferenceOp Python Bindings + ---------------------------------------- .. currentmodule:: _inference - .. autosummary:: - :toctree: _generate - add - subtract )pbdoc"; -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif - py::class_> inference_op( m, "InferenceOp", doc::InferenceOp::doc_InferenceOp); inference_op .def(py::init, py::dict, @@ -179,6 +182,7 @@ PYBIND11_MODULE(_inference, m) { py::dict, py::dict, py::dict, + py::dict, const std::vector&, const std::vector&, bool, @@ -197,6 +201,7 @@ PYBIND11_MODULE(_inference, m) { "model_path_map"_a, "pre_processor_map"_a, "device_map"_a = py::dict(), + "temporal_map"_a = py::dict(), "backend_map"_a = py::dict(), "in_tensor_names"_a = std::vector{}, "out_tensor_names"_a = std::vector{}, diff --git a/python/holoscan/operators/inference/pydoc.hpp b/python/holoscan/operators/inference/pydoc.hpp index e80ddb14..b26ab665 100644 --- a/python/holoscan/operators/inference/pydoc.hpp +++ b/python/holoscan/operators/inference/pydoc.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef HOLOSCAN_OPERATORS_INFERENCE_PYDOC_HPP -#define HOLOSCAN_OPERATORS_INFERENCE_PYDOC_HPP +#ifndef PYHOLOSCAN_OPERATORS_INFERENCE_PYDOC_HPP +#define PYHOLOSCAN_OPERATORS_INFERENCE_PYDOC_HPP #include @@ -42,6 +42,13 @@ Inference operator. be emitted. The names of the tensors transmitted correspond to those in ``out_tensor_names``. +**==Device Memory Requirements==** + + When using this operator with a ``holoscan.resources.BlockMemoryPool``, ``num_blocks`` must be + greater than or equal to the number of output tensors that will be produced. The ``block_size`` + in bytes must be greater than or equal to the largest output tensor (in bytes). If + ``output_on_cuda`` is ``True``, the blocks should be in device memory (``storage_type=1``), + otherwise they should be CUDA pinned host memory (``storage_type=0``). For more details on ``InferenceOp`` parameters, see [Customizing the Inference Operator](https://docs.nvidia.com/holoscan/sdk-user-guide/examples/byom.html#customizing-the-inference-operator) @@ -56,15 +63,17 @@ backend : {"trt", "onnxrt", "torch"} ``"onnxrt"`` for the ONNX runtime. allocator : holoscan.resources.Allocator Memory allocator to use for the output. -inference_map : holoscan.operators.InferenceOp.DataVecMap +inference_map : dict[str, List[str]] Tensor to model map. -model_path_map : holoscan.operators.InferenceOp.DataMap +model_path_map : dict[str, str] Path to the ONNX model to be loaded. -pre_processor_map : holoscan.operators.InferenceOp::DataVecMap +pre_processor_map : dict[str, List[str]] Pre processed data to model map. -device_map : holoscan.operators.InferenceOp.DataMap, optional +device_map : dict[str, int], optional Mapping of model to GPU ID for inference. -backend_map : holoscan.operators.InferenceOp.DataMap, optional +temporal_map : dict[str, int], optional + Mapping of model to frame delay for inference. +backend_map : dict[str, str], optional Mapping of model to backend type for inference. Backend options: ``"trt"`` or ``"torch"`` in_tensor_names : sequence of str, optional Input tensors. @@ -109,4 +118,4 @@ spec : holoscan.core.OperatorSpec } // namespace holoscan::doc::InferenceOp -#endif /* HOLOSCAN_OPERATORS_INFERENCE_PYDOC_HPP */ +#endif /* PYHOLOSCAN_OPERATORS_INFERENCE_PYDOC_HPP */ diff --git a/python/holoscan/operators/inference_processor/inference_processor.cpp b/python/holoscan/operators/inference_processor/inference_processor.cpp index d076f671..527eec19 100644 --- a/python/holoscan/operators/inference_processor/inference_processor.cpp +++ b/python/holoscan/operators/inference_processor/inference_processor.cpp @@ -22,6 +22,7 @@ #include #include +#include "../operator_util.hpp" #include "./pydoc.hpp" #include "holoscan/core/fragment.hpp" @@ -73,7 +74,8 @@ class PyInferenceProcessorOp : public InferenceProcessorOp { using InferenceProcessorOp::InferenceProcessorOp; // Define a constructor that fully initializes the object. - PyInferenceProcessorOp(Fragment* fragment, std::shared_ptr<::holoscan::Allocator> allocator, + PyInferenceProcessorOp(Fragment* fragment, const py::args& args, + std::shared_ptr<::holoscan::Allocator> allocator, py::dict process_operations, // InferenceProcessorOp::DataVecMap py::dict processed_map, // InferenceProcessorOp::DataVecMap const std::vector& in_tensor_names, @@ -92,6 +94,7 @@ class PyInferenceProcessorOp : public InferenceProcessorOp { Arg{"config_path", config_path}, Arg{"disable_transmitter", disable_transmitter}}) { if (cuda_stream_pool) { this->add_arg(Arg{"cuda_stream_pool", cuda_stream_pool}); } + add_positional_condition_and_resource_args(this, args); name_ = name; fragment_ = fragment; @@ -136,21 +139,11 @@ class PyInferenceProcessorOp : public InferenceProcessorOp { PYBIND11_MODULE(_inference_processor, m) { m.doc() = R"pbdoc( - Holoscan SDK Python Bindings - --------------------------------------- + Holoscan SDK InferenceProcessorOp Python Bindings + ------------------------------------------------- .. currentmodule:: _inference_processor - .. autosummary:: - :toctree: _generate - add - subtract )pbdoc"; -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif - py::class_, py::dict, py::dict, diff --git a/python/holoscan/operators/inference_processor/pydoc.hpp b/python/holoscan/operators/inference_processor/pydoc.hpp index 1cb0f482..171b39af 100644 --- a/python/holoscan/operators/inference_processor/pydoc.hpp +++ b/python/holoscan/operators/inference_processor/pydoc.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef HOLOSCAN_OPERATORS_INFERENCE_PROCESSOR_PYDOC_HPP -#define HOLOSCAN_OPERATORS_INFERENCE_PROCESSOR_PYDOC_HPP +#ifndef PYHOLOSCAN_OPERATORS_INFERENCE_PROCESSOR_PYDOC_HPP +#define PYHOLOSCAN_OPERATORS_INFERENCE_PROCESSOR_PYDOC_HPP #include @@ -43,6 +43,14 @@ Holoinfer Processing operator. be emitted. The names of the tensors transmitted correspond to those in ``out_tensor_names``. +**==Device Memory Requirements==** + + When using this operator with a ``holoscan.resources.BlockMemoryPool``, ``num_blocks`` must be + greater than or equal to the number of output tensors that will be produced. The ``block_size`` + in bytes must be greater than or equal to the largest output tensor (in bytes). If + ``output_on_cuda`` is ``True``, the blocks should be in device memory (``storage_type=1``), + otherwise they should be CUDA pinned host memory (``storage_type=0``). + Parameters ---------- fragment : holoscan.core.Fragment (constructor only) @@ -93,4 +101,4 @@ spec : holoscan.core.OperatorSpec } // namespace holoscan::doc::InferenceProcessorOp -#endif /* HOLOSCAN_OPERATORS_INFERENCE_PROCESSOR_PYDOC_HPP */ +#endif /* PYHOLOSCAN_OPERATORS_INFERENCE_PROCESSOR_PYDOC_HPP */ diff --git a/python/holoscan/operators/operator_util.hpp b/python/holoscan/operators/operator_util.hpp new file mode 100644 index 00000000..8033d33b --- /dev/null +++ b/python/holoscan/operators/operator_util.hpp @@ -0,0 +1,344 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef PYHOLOSCAN_OPERATORS_OPERATOR_UTIL_HPP +#define PYHOLOSCAN_OPERATORS_OPERATOR_UTIL_HPP + +#include // py::dtype, py::array +#include +#include // needed for py::cast to work with std::vector types + +#include +#include +#include + +#include "holoscan/core/condition.hpp" +#include "holoscan/core/gxf/gxf_resource.hpp" +#include "holoscan/core/operator.hpp" +#include "holoscan/core/resource.hpp" + +namespace py = pybind11; + +namespace holoscan { + +void add_positional_condition_and_resource_args(Operator* op, const py::args& args) { + for (auto it = args.begin(); it != args.end(); ++it) { + if (py::isinstance(*it)) { + op->add_arg(it->cast>()); + } else if (py::isinstance(*it)) { + op->add_arg(it->cast>()); + } else { + HOLOSCAN_LOG_WARN( + "Unhandled positional argument detected (only Condition and Resource objects can be " + "parsed positionally)"); + } + } +} + +//////////////////////////////////////////////////////////////////////////////// +// The following methods have been adapted from kwarg_handling.hpp to enhance +// the `kwargs_to_arglist()` method with these updates: +// - Remove dependency on "holoscan/operators/aja_source/ntv2channel.hpp" +// - Convert `py::object` to `Arg` using YAML::Node type, eliminating the need +// for a uniform type in the `Arg` class. +// - Except for `Resource` and `Condition` types, which are handled separately. +// +// `kwargs_to_arglist()` is used in `add_kwargs()` to convert Python kwargs to +// `ArgList` for use in the `ComponentBase::add_arg()` method. +//////////////////////////////////////////////////////////////////////////////// + +// TODO(gbae): Refactor these utility classes and implementations in kwarg_handling.hpp + +/// Convert a py::object to a `YAML::Node` type. +template +inline static YAML::Node cast_to_yaml_node(const py::handle& obj) { + YAML::Node yaml_node; + yaml_node.push_back(obj.cast()); + return yaml_node[0]; +} + +// Specialization for uint8_t +template <> +inline YAML::Node cast_to_yaml_node(const py::handle& obj) { + YAML::Node yaml_node; + yaml_node.push_back(obj.cast()); + return yaml_node[0]; +} + +// Specialization for int8_t +template <> +inline YAML::Node cast_to_yaml_node(const py::handle& obj) { + YAML::Node yaml_node; + yaml_node.push_back(obj.cast()); + return yaml_node[0]; +} + +template +void set_vector_arg_via_numpy_array(const py::array& obj, Arg& out) { + // not intended for images or other large tensors, just + // for short arrays containing parameter settings to operators/resources + if (obj.attr("ndim").cast() == 1) { + YAML::Node yaml_node = YAML::Load("[]"); // Create an empty sequence + for (const auto& item : obj) yaml_node.push_back(cast_to_yaml_node(item)); + out = yaml_node; + } else if (obj.attr("ndim").cast() == 2) { + YAML::Node yaml_node = YAML::Load("[]"); // Create an empty sequence + for (auto item : obj) { + YAML::Node inner_yaml_node = YAML::Load("[]"); // Create an empty sequence + for (const auto& inner_item : item) { + inner_yaml_node.push_back(cast_to_yaml_node(inner_item)); + } + if (inner_yaml_node.size() > 0) { yaml_node.push_back(inner_yaml_node); } + } + out = yaml_node; + } else { + throw std::runtime_error("Only 1d and 2d NumPy arrays are supported."); + } +} + +void set_vector_arg_via_dtype(const py::object& obj, const py::dtype& dt, Arg& out) { + std::string dtype_name = dt.attr("name").cast(); + if (dtype_name == "float16") { // currently promoting float16 scalars to float + set_vector_arg_via_numpy_array(obj, out); + } else if (dtype_name == "float32") { + set_vector_arg_via_numpy_array(obj, out); + } else if (dtype_name == "float64") { + set_vector_arg_via_numpy_array(obj, out); + } else if (dtype_name == "bool") { + set_vector_arg_via_numpy_array(obj, out); + } else if (dtype_name == "int8") { + set_vector_arg_via_numpy_array(obj, out); + } else if (dtype_name == "int16") { + set_vector_arg_via_numpy_array(obj, out); + } else if (dtype_name == "int32") { + set_vector_arg_via_numpy_array(obj, out); + } else if (dtype_name == "int64") { + set_vector_arg_via_numpy_array(obj, out); + } else if (dtype_name == "uint8") { + set_vector_arg_via_numpy_array(obj, out); + } else if (dtype_name == "uint16") { + set_vector_arg_via_numpy_array(obj, out); + } else if (dtype_name == "uint32") { + set_vector_arg_via_numpy_array(obj, out); + } else if (dtype_name == "uint64") { + set_vector_arg_via_numpy_array(obj, out); + } else if (dtype_name.find("str") == 0) { + py::list list_obj = obj.attr("tolist")().cast(); + // TODO(grelee): set_vector_arg_via_seqeuence(list_obj, out); + } else { + throw std::runtime_error("unsupported dtype: "s + dtype_name + ", leaving Arg uninitialized"s); + } + return; +} + +template +void set_vector_arg_via_py_sequence(const py::sequence& seq, Arg& out) { + // not intended for images or other large tensors, just + // for short arrays containing parameter settings to operators/resources + + if constexpr (std::is_same_v> || + std::is_same_v>) { + auto first_item = seq[0]; + if (py::isinstance(first_item) && !py::isinstance(first_item)) { + // Handle list of list and other sequence of sequence types. + std::vector> v; + v.reserve(static_cast(py::len(seq))); + for (auto item : seq) { + std::vector vv; + vv.reserve(static_cast(py::len(item))); + for (auto inner_item : item) { vv.push_back(inner_item.cast()); } + v.push_back(vv); + } + out = v; + } else { + // 1d vector to handle a sequence of elements + std::vector v; + size_t length = py::len(seq); + v.reserve(length); + for (auto item : seq) v.push_back(item.cast()); + out = v; + } + } else { + auto first_item = seq[0]; + if (py::isinstance(first_item) && !py::isinstance(first_item)) { + // Handle list of list and other sequence of sequence types. + YAML::Node yaml_node = YAML::Load("[]"); // Create an empty sequence + for (auto item : seq) { + YAML::Node inner_yaml_node = YAML::Load("[]"); // Create an empty sequence + for (const auto& inner_item : item) { + inner_yaml_node.push_back(cast_to_yaml_node(inner_item)); + } + if (inner_yaml_node.size() > 0) { yaml_node.push_back(inner_yaml_node); } + } + out = yaml_node; + } else { + // 1d vector to handle a sequence of elements + YAML::Node yaml_node = YAML::Load("[]"); // Create an empty sequence + for (const auto& item : seq) yaml_node.push_back(cast_to_yaml_node(item)); + out = yaml_node; + } + } +} + +void set_vector_arg_via_iterable(const py::object& obj, Arg& out) { + py::sequence seq; + if (py::isinstance(obj)) { + seq = obj; + } else { + // convert other iterables to a list first + seq = py::list(obj); + } + + if (py::len(seq) == 0) { throw std::runtime_error("sequences of length 0 are not supported."); } + + auto item0 = seq[0]; + if (py::isinstance(item0) && !py::isinstance(item0)) { + py::sequence inner_seq = item0; + if (py::len(inner_seq) == 0) { + throw std::runtime_error("sequences of length 0 are not supported."); + } + auto item = inner_seq[0]; + if (py::isinstance(item) && !py::isinstance(item)) { + throw std::runtime_error("Nested sequences of depth > 2 levels are not supported."); + } + if (py::isinstance(item)) { + set_vector_arg_via_py_sequence(seq, out); + } else if (py::isinstance(item)) { + set_vector_arg_via_py_sequence(seq, out); + } else if (py::isinstance(item)) { + set_vector_arg_via_py_sequence(seq, out); + } else if (py::isinstance(item)) { + set_vector_arg_via_py_sequence(seq, out); + } else { + throw std::runtime_error("Nested sequence of unsupported type."); + } + } else { + auto item = item0; + if (py::isinstance(item)) { + set_vector_arg_via_py_sequence(seq, out); + } else if (py::isinstance(item)) { + set_vector_arg_via_py_sequence(seq, out); + } else if (py::isinstance(item)) { + set_vector_arg_via_py_sequence(seq, out); + } else if (py::isinstance(item)) { + set_vector_arg_via_py_sequence(seq, out); + } else if (py::isinstance(item)) { + set_vector_arg_via_py_sequence>(seq, out); + } else if (py::isinstance(item)) { + set_vector_arg_via_py_sequence>(seq, out); + } + } + return; +} + +void set_scalar_arg_via_dtype(const py::object& obj, const py::dtype& dt, Arg& out) { + std::string dtype_name = dt.attr("name").cast(); + if (dtype_name == "float16") { // currently promoting float16 scalars to float + out = cast_to_yaml_node(obj); + } else if (dtype_name == "float32") { + out = cast_to_yaml_node(obj); + } else if (dtype_name == "float64") { + out = cast_to_yaml_node(obj); + } else if (dtype_name == "bool") { + out = cast_to_yaml_node(obj); + } else if (dtype_name == "int8") { + out = cast_to_yaml_node(obj); + } else if (dtype_name == "int16") { + out = cast_to_yaml_node(obj); + } else if (dtype_name == "int32") { + out = cast_to_yaml_node(obj); + } else if (dtype_name == "int64") { + out = cast_to_yaml_node(obj); + } else if (dtype_name == "uint8") { + out = cast_to_yaml_node(obj); + } else if (dtype_name == "uint16") { + out = cast_to_yaml_node(obj); + } else if (dtype_name == "uint32") { + out = cast_to_yaml_node(obj); + } else if (dtype_name == "uint64") { + out = cast_to_yaml_node(obj); + } else { + throw std::runtime_error("unsupported dtype: "s + dtype_name + ", leaving Arg uninitialized"s); + } + return; +} + +Arg py_object_to_arg(py::object obj, std::string name = "") { + Arg out(name); + if (py::isinstance(obj)) { + out = cast_to_yaml_node(obj); + } else if (py::isinstance(obj)) { + // handle numpy arrays + py::dtype array_dtype = obj.cast().dtype(); + set_vector_arg_via_dtype(obj, array_dtype, out); + return out; + } else if (py::isinstance(obj) && !py::isinstance(obj)) { + // does not handle every possible type of iterable (e.g. dict) + // will work for any that can be cast to py::list + set_vector_arg_via_iterable(obj, out); + } else if (py::isinstance(obj)) { + out = cast_to_yaml_node(obj); + } else if (py::isinstance(obj) || PyLong_Check(obj.ptr())) { + out = cast_to_yaml_node(obj); + } else if (py::isinstance(obj)) { + out = cast_to_yaml_node(obj); + } else if (PyComplex_Check(obj.ptr())) { + throw std::runtime_error("complex value cannot be converted to Arg"); + } else if (PyNumber_Check(obj.ptr())) { + py::module_ np = py::module_::import("numpy"); + auto numpy_generic = np.attr("generic"); + if (py::isinstance(obj, numpy_generic)) { + // cast numpy scalars to appropriate dtype + py::dtype dt = np.attr("dtype")(obj); + set_scalar_arg_via_dtype(obj, dt, out); + return out; + } else { + // cast any other unknown numeric type to double + out = cast_to_yaml_node(obj); + } + } else if (py::isinstance(obj)) { + out = obj.cast>(); + } else if (py::isinstance(obj)) { + out = obj.cast>(); + } else { + throw std::runtime_error("python object could not be converted to Arg"); + } + return out; +} + +ArgList kwargs_to_arglist(const py::kwargs& kwargs) { + // Note: scalars will be kNative while any iterables will have type kNative. + // There is currently no option to choose conversion to kArray instead of kNative. + ArgList arglist; + if (kwargs) { + for (auto& [name, handle] : kwargs) { + arglist.add(py_object_to_arg(handle.cast(), name.cast())); + } + /// .. do something with kwargs + } + return arglist; +} + +//////////////////////////////////////////////////////////////////////////////// + +void add_kwargs(ComponentBase* component, const py::kwargs& kwargs) { + ArgList arg_list = kwargs_to_arglist(kwargs); + component->add_arg(arg_list); +} + +} // namespace holoscan + +#endif /* PYHOLOSCAN_OPERATORS_OPERATOR_UTIL_HPP */ diff --git a/python/holoscan/operators/segmentation_postprocessor/pydoc.hpp b/python/holoscan/operators/segmentation_postprocessor/pydoc.hpp index 40faacd1..3d60ebad 100644 --- a/python/holoscan/operators/segmentation_postprocessor/pydoc.hpp +++ b/python/holoscan/operators/segmentation_postprocessor/pydoc.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef HOLOSCAN_OPERATORS_SEGMENTATION_POSTPROCESSOR_PYDOC_HPP -#define HOLOSCAN_OPERATORS_SEGMENTATION_POSTPROCESSOR_PYDOC_HPP +#ifndef PYHOLOSCAN_OPERATORS_SEGMENTATION_POSTPROCESSOR_PYDOC_HPP +#define PYHOLOSCAN_OPERATORS_SEGMENTATION_POSTPROCESSOR_PYDOC_HPP #include @@ -31,16 +31,21 @@ Operator carrying out post-processing operations on segmentation outputs. **==Named Inputs==** in_tensor : nvidia::gxf::Tensor - Expects a message containing a 32-bit floating point tensor with name ``in_tensor_name``. - The expected data layout of this tensor is HWC, NCHW or NHWC format as specified via - ``data_format``. + Expects a message containing a 32-bit floating point device tensor with name + ``in_tensor_name``. The expected data layout of this tensor is HWC, NCHW or NHWC format as + specified via ``data_format``. **==Named Outputs==** out_tensor : nvidia::gxf::Tensor - Emits a message containing a tensor named "out_tensor" that contains the segmentation + Emits a message containing a device tensor named "out_tensor" that contains the segmentation labels. This tensor will have unsigned 8-bit integer data type and shape (H, W, 1). +**==Device Memory Requirements==** + + When used with a ``holoscan.resources.BlockMemoryPool``, this operator requires only a single + device memory block (``storage_type=1``) of size ``height * width`` bytes. + Parameters ---------- fragment : holoscan.core.Fragment (constructor only) @@ -78,4 +83,4 @@ spec : holoscan.core.OperatorSpec } // namespace holoscan::doc::SegmentationPostprocessorOp -#endif /* HOLOSCAN_OPERATORS_SEGMENTATION_POSTPROCESSOR_PYDOC_HPP */ +#endif /* PYHOLOSCAN_OPERATORS_SEGMENTATION_POSTPROCESSOR_PYDOC_HPP */ diff --git a/python/holoscan/operators/segmentation_postprocessor/segmentation_postprocessor.cpp b/python/holoscan/operators/segmentation_postprocessor/segmentation_postprocessor.cpp index b936767b..ee5e8a37 100644 --- a/python/holoscan/operators/segmentation_postprocessor/segmentation_postprocessor.cpp +++ b/python/holoscan/operators/segmentation_postprocessor/segmentation_postprocessor.cpp @@ -20,6 +20,7 @@ #include #include +#include "../operator_util.hpp" #include "./pydoc.hpp" #include "holoscan/core/fragment.hpp" @@ -56,7 +57,7 @@ class PySegmentationPostprocessorOp : public SegmentationPostprocessorOp { // Define a constructor that fully initializes the object. PySegmentationPostprocessorOp( - Fragment* fragment, std::shared_ptr<::holoscan::Allocator> allocator, + Fragment* fragment, const py::args& args, std::shared_ptr<::holoscan::Allocator> allocator, const std::string& in_tensor_name = "", const std::string& network_output_type = "softmax"s, const std::string& data_format = "hwc"s, std::shared_ptr cuda_stream_pool = nullptr, @@ -66,6 +67,7 @@ class PySegmentationPostprocessorOp : public SegmentationPostprocessorOp { Arg{"data_format", data_format}, Arg{"allocator", allocator}}) { if (cuda_stream_pool) { this->add_arg(Arg{"cuda_stream_pool", cuda_stream_pool}); } + add_positional_condition_and_resource_args(this, args); name_ = name; fragment_ = fragment; spec_ = std::make_shared(fragment); @@ -77,21 +79,11 @@ class PySegmentationPostprocessorOp : public SegmentationPostprocessorOp { PYBIND11_MODULE(_segmentation_postprocessor, m) { m.doc() = R"pbdoc( - Holoscan SDK Python Bindings - --------------------------------------- + Holoscan SDK SegmentationPostprocessorOp Bindings + ------------------------------------------------- .. currentmodule:: _segmentation_postprocessor - .. autosummary:: - :toctree: _generate - add - subtract )pbdoc"; -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif - py::class_, const std::string&, const std::string&, diff --git a/python/holoscan/operators/v4l2_video_capture/pydoc.hpp b/python/holoscan/operators/v4l2_video_capture/pydoc.hpp index d062dbe1..1cfb346f 100644 --- a/python/holoscan/operators/v4l2_video_capture/pydoc.hpp +++ b/python/holoscan/operators/v4l2_video_capture/pydoc.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef HOLOSCAN_OPERATORS_V4L2_VIDEO_CAPTURE_PYDOC_HPP -#define HOLOSCAN_OPERATORS_V4L2_VIDEO_CAPTURE_PYDOC_HPP +#ifndef PYHOLOSCAN_OPERATORS_V4L2_VIDEO_CAPTURE_PYDOC_HPP +#define PYHOLOSCAN_OPERATORS_V4L2_VIDEO_CAPTURE_PYDOC_HPP #include @@ -33,7 +33,7 @@ Operator to get a video stream from a V4L2 source. Inputs a video stream from a V4L2 node, including USB cameras and HDMI IN. - Input stream is on host. If no pixel format is specified in the yaml configuration file, the - pixel format will be automatically selected. However, only ``AB24`` and ``YUYV`` are then + pixel format will be automatically selected. However, only ``AB24``, ``YUYV``, and ``MJPG`` are then supported. If a pixel format is specified in the yaml file, then this format will be used. However, note that the operator then expects that this format can be encoded as RGBA32. If not, the behavior @@ -47,6 +47,23 @@ Use ``holoscan.operators.FormatConverterOp`` to move data from the host to a GPU signal : nvidia::gxf::VideoBuffer A message containing a video buffer on the host with format GXF_VIDEO_FORMAT_RGBA. +**==Device Memory Requirements==** + + When using this operator with a ``holoscan.resources.BlockMemoryPool``, a single device memory + block is needed (``storage_type=1``). The size of this memory block can be determined by + rounding the width and height up to the nearest even size and then padding the rows as needed + so that the row stride is a multiple of 256 bytes. C++ code to calculate the block size is as + follows: + +.. code-block:: python + + def get_block_size(height, width): + height_even = height + (height & 1) + width_even = width + (width & 1) + row_bytes = width_even * 4; # 4 bytes per pixel for 8-bit RGBA + row_stride = (row_bytes % 256 == 0) ? row_bytes : ((row_bytes // 256 + 1) * 256) + return height_even * row_stride + Parameters ---------- fragment : Fragment (constructor only) @@ -103,4 +120,4 @@ and uses a light-weight initialization. } // namespace holoscan::doc::V4L2VideoCaptureOp -#endif /* HOLOSCAN_OPERATORS_V4L2_VIDEO_CAPTURE_PYDOC_HPP */ +#endif /* PYHOLOSCAN_OPERATORS_V4L2_VIDEO_CAPTURE_PYDOC_HPP */ diff --git a/python/holoscan/operators/v4l2_video_capture/v4l2_video_capture.cpp b/python/holoscan/operators/v4l2_video_capture/v4l2_video_capture.cpp index f21f89f9..de14fff9 100644 --- a/python/holoscan/operators/v4l2_video_capture/v4l2_video_capture.cpp +++ b/python/holoscan/operators/v4l2_video_capture/v4l2_video_capture.cpp @@ -22,6 +22,7 @@ #include #include +#include "../operator_util.hpp" #include "./pydoc.hpp" #include "holoscan/core/fragment.hpp" @@ -57,7 +58,8 @@ class PyV4L2VideoCaptureOp : public V4L2VideoCaptureOp { using V4L2VideoCaptureOp::V4L2VideoCaptureOp; // Define a constructor that fully initializes the object. - PyV4L2VideoCaptureOp(Fragment* fragment, std::shared_ptr<::holoscan::Allocator> allocator, + PyV4L2VideoCaptureOp(Fragment* fragment, const py::args& args, + std::shared_ptr<::holoscan::Allocator> allocator, const std::string& device = "/dev/video0"s, uint32_t width = 0, uint32_t height = 0, uint32_t num_buffers = 4, const std::string& pixel_format = "auto", @@ -76,7 +78,7 @@ class PyV4L2VideoCaptureOp : public V4L2VideoCaptureOp { if (gain.has_value()) { this->add_arg(Arg{"gain", gain.value() }); } - + add_positional_condition_and_resource_args(this, args); name_ = name; fragment_ = fragment; spec_ = std::make_shared(fragment); @@ -86,27 +88,18 @@ class PyV4L2VideoCaptureOp : public V4L2VideoCaptureOp { PYBIND11_MODULE(_v4l2_video_capture, m) { m.doc() = R"pbdoc( - Holoscan SDK Python Bindings - --------------------------------------- + Holoscan SDK V4L2VideoCaptureOp Python Bindings + ----------------------------------------------- .. currentmodule:: _v4l2_video_capture - .. autosummary:: - :toctree: _generate - add - subtract )pbdoc"; -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif - py::class_>( m, "V4L2VideoCaptureOp", doc::V4L2VideoCaptureOp::doc_V4L2VideoCaptureOp) .def(py::init, const std::string&, uint32_t, diff --git a/python/holoscan/operators/video_stream_recorder/pydoc.hpp b/python/holoscan/operators/video_stream_recorder/pydoc.hpp index 70ae4fd9..919158c1 100644 --- a/python/holoscan/operators/video_stream_recorder/pydoc.hpp +++ b/python/holoscan/operators/video_stream_recorder/pydoc.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef HOLOSCAN_OPERATORS_VIDEO_STREAM_RECORDER_PYDOC_HPP -#define HOLOSCAN_OPERATORS_VIDEO_STREAM_RECORDER_PYDOC_HPP +#ifndef PYHOLOSCAN_OPERATORS_VIDEO_STREAM_RECORDER_PYDOC_HPP +#define PYHOLOSCAN_OPERATORS_VIDEO_STREAM_RECORDER_PYDOC_HPP #include @@ -68,4 +68,4 @@ spec : holoscan.core.OperatorSpec } // namespace holoscan::doc::VideoStreamRecorderOp -#endif /* HOLOSCAN_OPERATORS_VIDEO_STREAM_RECORDER_PYDOC_HPP */ +#endif /* PYHOLOSCAN_OPERATORS_VIDEO_STREAM_RECORDER_PYDOC_HPP */ diff --git a/python/holoscan/operators/video_stream_recorder/video_stream_recorder.cpp b/python/holoscan/operators/video_stream_recorder/video_stream_recorder.cpp index df0ebee9..55b5a126 100644 --- a/python/holoscan/operators/video_stream_recorder/video_stream_recorder.cpp +++ b/python/holoscan/operators/video_stream_recorder/video_stream_recorder.cpp @@ -20,6 +20,7 @@ #include #include +#include "../operator_util.hpp" #include "./pydoc.hpp" #include "holoscan/core/fragment.hpp" @@ -53,12 +54,13 @@ class PyVideoStreamRecorderOp : public VideoStreamRecorderOp { using VideoStreamRecorderOp::VideoStreamRecorderOp; // Define a constructor that fully initializes the object. - PyVideoStreamRecorderOp(Fragment* fragment, const std::string& directory, + PyVideoStreamRecorderOp(Fragment* fragment, const py::args& args, const std::string& directory, const std::string& basename, bool flush_on_tick_ = false, const std::string& name = "video_stream_recorder") : VideoStreamRecorderOp(ArgList{Arg{"directory", directory}, Arg{"basename", basename}, Arg{"flush_on_tick", flush_on_tick_}}) { + add_positional_condition_and_resource_args(this, args); name_ = name; fragment_ = fragment; spec_ = std::make_shared(fragment); @@ -70,27 +72,22 @@ class PyVideoStreamRecorderOp : public VideoStreamRecorderOp { PYBIND11_MODULE(_video_stream_recorder, m) { m.doc() = R"pbdoc( - Holoscan SDK Python Bindings - --------------------------------------- + Holoscan SDK VideoStreamRecorderOp Python Bindings + -------------------------------------------------- .. currentmodule:: _video_stream_recorder - .. autosummary:: - :toctree: _generate - add - subtract )pbdoc"; -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif - py::class_>( m, "VideoStreamRecorderOp", doc::VideoStreamRecorderOp::doc_VideoStreamRecorderOp) - .def(py::init(), + .def(py::init(), "fragment"_a, "directory"_a, "basename"_a, diff --git a/python/holoscan/operators/video_stream_replayer/pydoc.hpp b/python/holoscan/operators/video_stream_replayer/pydoc.hpp index cd7dee32..118daefc 100644 --- a/python/holoscan/operators/video_stream_replayer/pydoc.hpp +++ b/python/holoscan/operators/video_stream_replayer/pydoc.hpp @@ -15,8 +15,8 @@ * limitations under the License. */ -#ifndef HOLOSCAN_OPERATORS_VIDEO_STREAM_REPLAYER_PYDOC_HPP -#define HOLOSCAN_OPERATORS_VIDEO_STREAM_REPLAYER_PYDOC_HPP +#ifndef PYHOLOSCAN_OPERATORS_VIDEO_STREAM_REPLAYER_PYDOC_HPP +#define PYHOLOSCAN_OPERATORS_VIDEO_STREAM_REPLAYER_PYDOC_HPP #include @@ -81,4 +81,4 @@ spec : holoscan.core.OperatorSpec } // namespace holoscan::doc::VideoStreamReplayerOp -#endif /* HOLOSCAN_OPERATORS_VIDEO_STREAM_REPLAYER_PYDOC_HPP */ +#endif /* PYHOLOSCAN_OPERATORS_VIDEO_STREAM_REPLAYER_PYDOC_HPP */ diff --git a/python/holoscan/operators/video_stream_replayer/video_stream_replayer.cpp b/python/holoscan/operators/video_stream_replayer/video_stream_replayer.cpp index 01ee5217..73712be9 100644 --- a/python/holoscan/operators/video_stream_replayer/video_stream_replayer.cpp +++ b/python/holoscan/operators/video_stream_replayer/video_stream_replayer.cpp @@ -20,6 +20,7 @@ #include #include +#include "../operator_util.hpp" #include "./pydoc.hpp" #include "holoscan/core/fragment.hpp" @@ -53,7 +54,7 @@ class PyVideoStreamReplayerOp : public VideoStreamReplayerOp { using VideoStreamReplayerOp::VideoStreamReplayerOp; // Define a constructor that fully initializes the object. - PyVideoStreamReplayerOp(Fragment* fragment, const std::string& directory, + PyVideoStreamReplayerOp(Fragment* fragment, const py::args& args, const std::string& directory, const std::string& basename, size_t batch_size = 1UL, bool ignore_corrupted_entities = true, float frame_rate = 0.f, bool realtime = true, bool repeat = false, uint64_t count = 0UL, @@ -66,6 +67,7 @@ class PyVideoStreamReplayerOp : public VideoStreamReplayerOp { Arg{"realtime", realtime}, Arg{"repeat", repeat}, Arg{"count", count}}) { + add_positional_condition_and_resource_args(this, args); name_ = name; fragment_ = fragment; spec_ = std::make_shared(fragment); @@ -77,27 +79,18 @@ class PyVideoStreamReplayerOp : public VideoStreamReplayerOp { PYBIND11_MODULE(_video_stream_replayer, m) { m.doc() = R"pbdoc( - Holoscan SDK Python Bindings - --------------------------------------- + Holoscan SDK VideoStreamReplayerOp Python Bindings + -------------------------------------------------- .. currentmodule:: _video_stream_replayer - .. autosummary:: - :toctree: _generate - add - subtract )pbdoc"; -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif - py::class_>( m, "VideoStreamReplayerOp", doc::VideoStreamReplayerOp::doc_VideoStreamReplayerOp) .def(py::init(), "fragment"_a, "name"_a = "unbounded_allocator"s, - doc::UnboundedAllocator::doc_UnboundedAllocator_python) + doc::UnboundedAllocator::doc_UnboundedAllocator) .def_property_readonly("gxf_typename", &UnboundedAllocator::gxf_typename, doc::UnboundedAllocator::doc_gxf_typename) diff --git a/python/holoscan/resources/allocators_pydoc.hpp b/python/holoscan/resources/allocators_pydoc.hpp index 30d203cc..bcd485e9 100644 --- a/python/holoscan/resources/allocators_pydoc.hpp +++ b/python/holoscan/resources/allocators_pydoc.hpp @@ -89,13 +89,6 @@ namespace BlockMemoryPool { PYDOC(BlockMemoryPool, R"doc( Block memory pool resource. -Provides a maximum number of equally sized blocks of memory. -)doc") - -// Constructor -PYDOC(BlockMemoryPool_python, R"doc( -Block memory pool resource. - Provides a maximum number of equally sized blocks of memory. Parameters @@ -138,11 +131,6 @@ namespace CudaStreamPool { PYDOC(CudaStreamPool, R"doc( CUDA stream pool. -)doc") - -// Constructor -PYDOC(CudaStreamPool_python, R"doc( -CUDA stream pool. Parameters ---------- @@ -197,13 +185,6 @@ namespace UnboundedAllocator { PYDOC(UnboundedAllocator, R"doc( Unbounded allocator. -This allocator uses dynamic memory allocation without an upper bound. -)doc") - -// Constructor -PYDOC(UnboundedAllocator_python, R"doc( -Unbounded allocator. - This allocator uses dynamic memory allocation without an upper bound. Parameters diff --git a/python/holoscan/resources/clocks.cpp b/python/holoscan/resources/clocks.cpp index c21d043d..dbf96d22 100644 --- a/python/holoscan/resources/clocks.cpp +++ b/python/holoscan/resources/clocks.cpp @@ -149,7 +149,7 @@ void init_clocks(py::module_& m) { "initial_time_scale"_a = 1.0, "use_time_since_epoch"_a = false, "name"_a = "realtime_clock"s, - doc::RealtimeClock::doc_RealtimeClock_python) + doc::RealtimeClock::doc_RealtimeClock) .def_property_readonly( "gxf_typename", &RealtimeClock::gxf_typename, doc::RealtimeClock::doc_gxf_typename) .def("setup", &RealtimeClock::setup, "spec"_a, doc::RealtimeClock::doc_setup) @@ -178,7 +178,7 @@ void init_clocks(py::module_& m) { "fragment"_a, "initial_timestamp"_a = 0LL, "name"_a = "realtime_clock"s, - doc::ManualClock::doc_ManualClock_python) + doc::ManualClock::doc_ManualClock) .def_property_readonly( "gxf_typename", &ManualClock::gxf_typename, doc::ManualClock::doc_gxf_typename) .def("setup", &ManualClock::setup, "spec"_a, doc::ManualClock::doc_setup) diff --git a/python/holoscan/resources/clocks_pydoc.hpp b/python/holoscan/resources/clocks_pydoc.hpp index 6e9716f6..60e3f983 100644 --- a/python/holoscan/resources/clocks_pydoc.hpp +++ b/python/holoscan/resources/clocks_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -71,11 +71,6 @@ target_time_ns : int namespace RealtimeClock { PYDOC(RealtimeClock, R"doc( -Real-time clock class. -)doc") - -// Constructor -PYDOC(RealtimeClock_python, R"doc( Realtime clock. Parameters @@ -127,11 +122,6 @@ time_scale : float, optional namespace ManualClock { PYDOC(ManualClock, R"doc( -Manual clock class. -)doc") - -// Constructor -PYDOC(ManualClock_python, R"doc( Manual clock. Parameters @@ -143,6 +133,7 @@ initial_timestamp : int, optional name : str, optional The name of the clock. )doc") + PYDOC(gxf_typename, R"doc( The GXF type name of the resource. diff --git a/python/holoscan/resources/component_serializers.cpp b/python/holoscan/resources/component_serializers.cpp index 8246c3bd..299ab7b5 100644 --- a/python/holoscan/resources/component_serializers.cpp +++ b/python/holoscan/resources/component_serializers.cpp @@ -97,7 +97,7 @@ void init_component_serializers(py::module_& m) { .def(py::init(), "fragment"_a, "name"_a = "standard_component_serializer"s, - doc::StdComponentSerializer::doc_StdComponentSerializer_python) + doc::StdComponentSerializer::doc_StdComponentSerializer) .def_property_readonly("gxf_typename", &StdComponentSerializer::gxf_typename, doc::StdComponentSerializer::doc_gxf_typename) @@ -116,7 +116,7 @@ void init_component_serializers(py::module_& m) { "fragment"_a, "allocator"_a = py::none(), "name"_a = "ucx_component_serializer"s, - doc::UcxComponentSerializer::doc_UcxComponentSerializer_python) + doc::UcxComponentSerializer::doc_UcxComponentSerializer) .def_property_readonly("gxf_typename", &UcxComponentSerializer::gxf_typename, doc::UcxComponentSerializer::doc_gxf_typename) diff --git a/python/holoscan/resources/component_serializers_pydoc.hpp b/python/holoscan/resources/component_serializers_pydoc.hpp index f28d6762..4b2fac54 100644 --- a/python/holoscan/resources/component_serializers_pydoc.hpp +++ b/python/holoscan/resources/component_serializers_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,11 +28,6 @@ namespace StdComponentSerializer { PYDOC(StdComponentSerializer, R"doc( Serializer for GXF Timestamp and Tensor components. -)doc") - -// Constructor -PYDOC(StdComponentSerializer_python, R"doc( -Serializer for GXF Timestamp and Tensor components. Parameters ---------- @@ -73,11 +68,6 @@ namespace UcxComponentSerializer { PYDOC(UcxComponentSerializer, R"doc( UCX component serializer. -)doc") - -// Constructor -PYDOC(UcxComponentSerializer_python, R"doc( -UCX component serializer. Parameters ---------- diff --git a/python/holoscan/resources/entity_serializers.cpp b/python/holoscan/resources/entity_serializers.cpp index 19179062..79f55611 100644 --- a/python/holoscan/resources/entity_serializers.cpp +++ b/python/holoscan/resources/entity_serializers.cpp @@ -70,7 +70,7 @@ void init_entity_serializers(py::module_& m) { // "component_serializers"_a = std::vector>{}, "verbose_warning"_a = false, "name"_a = "ucx_entity_serializer"s, - doc::UcxEntitySerializer::doc_UcxEntitySerializer_python) + doc::UcxEntitySerializer::doc_UcxEntitySerializer) .def_property_readonly("gxf_typename", &UcxEntitySerializer::gxf_typename, doc::UcxEntitySerializer::doc_gxf_typename) diff --git a/python/holoscan/resources/entity_serializers_pydoc.hpp b/python/holoscan/resources/entity_serializers_pydoc.hpp index 4185661a..ee6179a3 100644 --- a/python/holoscan/resources/entity_serializers_pydoc.hpp +++ b/python/holoscan/resources/entity_serializers_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,11 +28,6 @@ namespace UcxEntitySerializer { PYDOC(UcxEntitySerializer, R"doc( UCX entity serializer. -)doc") - -// Constructor -PYDOC(UcxEntitySerializer_python, R"doc( -UCX entity serializer. Parameters ---------- diff --git a/python/holoscan/resources/gxf_component_resource.cpp b/python/holoscan/resources/gxf_component_resource.cpp new file mode 100644 index 00000000..d084ce70 --- /dev/null +++ b/python/holoscan/resources/gxf_component_resource.cpp @@ -0,0 +1,116 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include + +#include "./gxf_component_resource_pydoc.hpp" +#include "holoscan/core/component_spec.hpp" +#include "holoscan/core/fragment.hpp" +#include "holoscan/core/gxf/gxf_resource.hpp" +#include "holoscan/core/resources/gxf/gxf_component_resource.hpp" + +#include "../operators/operator_util.hpp" + +using std::string_literals::operator""s; +using pybind11::literals::operator""_a; + +namespace py = pybind11; + +namespace holoscan { + +// PyGXFComponentResource trampoline class: provides override for virtual function is_available + +class PyGXFComponentResource : public GXFComponentResource { + public: + /* Inherit the constructors */ + using GXFComponentResource::GXFComponentResource; + + // Define a constructor that fully initializes the object. + PyGXFComponentResource(py::object component, Fragment* fragment, const std::string& gxf_typename, + const std::string& name, const py::kwargs& kwargs) + : GXFComponentResource(gxf_typename.c_str()) { + py_component_ = component; + py_initialize_ = py::getattr(component, "initialize"); // cache the initialize method + + // We don't need to call `add_positional_condition_and_resource_args(this, args);` because + // Holoscan resources don't accept the positional arguments for Condition and Resource. + add_kwargs(this, kwargs); + + name_ = name; + fragment_ = fragment; + } + + void initialize() override { + // Get the initialize method of the Python Resource class and call it + py::gil_scoped_acquire scope_guard; + + // TODO(gbae): setup tracing + // // set_py_tracing(); + + // Call the initialize method of the Python Resource class + py_initialize_.operator()(); + + // Call the parent class's initialize method after invoking the Python Resource's initialize + // method. + GXFComponentResource::initialize(); + } + + protected: + py::object py_component_ = py::none(); + py::object py_initialize_ = py::none(); +}; + +/* Trampoline classes for handling Python kwargs + * + * These add a constructor that takes a Fragment for which to initialize the resource. + * The explicit parameter list and default arguments take care of providing a Pythonic + * kwarg-based interface with appropriate default values matching the resource's + * default parameters in the C++ API `setup` method. + * + * The sequence of events in this constructor is based on Fragment::make_resource + */ +void init_gxf_component_resource(py::module_& m) { + py::class_>( + m, "GXFComponentResource", doc::GXFComponentResource::doc_GXFComponentResource) + .def(py::init<>(), doc::GXFComponentResource::doc_GXFComponentResource) + .def(py::init(), + "component"_a, + "fragment"_a, + "gxf_typename"_a, + py::kw_only(), + "name"_a = "gxf_component"s, + doc::GXFComponentResource::doc_GXFComponentResource) + .def_property_readonly("gxf_typename", + &GXFComponentResource::gxf_typename, + doc::GXFComponentResource::doc_gxf_typename) + .def("initialize", + &GXFComponentResource::initialize, + doc::GXFComponentResource::doc_initialize) + .def("setup", &GXFComponentResource::setup, "spec"_a, doc::GXFComponentResource::doc_setup); +} +} // namespace holoscan diff --git a/python/holoscan/resources/gxf_component_resource_pydoc.hpp b/python/holoscan/resources/gxf_component_resource_pydoc.hpp new file mode 100644 index 00000000..d69610fa --- /dev/null +++ b/python/holoscan/resources/gxf_component_resource_pydoc.hpp @@ -0,0 +1,74 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PYHOLOSCAN_RESOURCES_GXF_COMPONENT_RESOURCE_PYDOC_HPP +#define PYHOLOSCAN_RESOURCES_GXF_COMPONENT_RESOURCE_PYDOC_HPP + +#include + +#include "../macros.hpp" + +namespace holoscan::doc { + +namespace GXFComponentResource { + +PYDOC(GXFComponentResource, R"doc( +Class that wraps a GXF Component as a Holoscan Resource. + +Parameters +---------- +fragment : holoscan.core.Fragment (constructor only) + The fragment that the resource belongs to. +gxf_typename : str + The GXF type name that identifies the specific GXF Component being wrapped. +name : str, optional (constructor only) + The name of the resource. Default value is ``"gxf_component"``. +**kwargs : dict + The additional keyword arguments that can be passed depend on the underlying GXF Component. + These parameters can provide further customization and functionality to the resource. +)doc") + +PYDOC(gxf_typename, R"doc( +The GXF type name of the resource. + +Returns +------- +str + The GXF type name of the resource +)doc") + +PYDOC(initialize, R"doc( +Initialize the resource. + +This method is called only once when the resource is created for the first time, +and uses a light-weight initialization. +)doc") + +PYDOC(setup, R"doc( +Define the resource specification. + +Parameters +---------- +spec : holoscan.core.ComponentSpec + The resource specification. +)doc") + +} // namespace GXFComponentResource + +} // namespace holoscan::doc + +#endif /* PYHOLOSCAN_RESOURCES_GXF_COMPONENT_RESOURCE_PYDOC_HPP */ diff --git a/python/holoscan/resources/receivers.cpp b/python/holoscan/resources/receivers.cpp index acab5bdf..d45035bd 100644 --- a/python/holoscan/resources/receivers.cpp +++ b/python/holoscan/resources/receivers.cpp @@ -91,7 +91,7 @@ void init_receivers(py::module_& m) { "capacity"_a = 1UL, "policy"_a = 2UL, "name"_a = "double_buffer_receiver"s, - doc::DoubleBufferReceiver::doc_DoubleBufferReceiver_python) + doc::DoubleBufferReceiver::doc_DoubleBufferReceiver) .def_property_readonly("gxf_typename", &DoubleBufferReceiver::gxf_typename, doc::DoubleBufferReceiver::doc_gxf_typename) @@ -113,7 +113,7 @@ void init_receivers(py::module_& m) { "address"_a = std::string("0.0.0.0"), "port"_a = kDefaultUcxPort, "name"_a = "ucx_receiver"s, - doc::UcxReceiver::doc_UcxReceiver_python) + doc::UcxReceiver::doc_UcxReceiver) .def_property_readonly( "gxf_typename", &UcxReceiver::gxf_typename, doc::UcxReceiver::doc_gxf_typename) .def("setup", &UcxReceiver::setup, "spec"_a, doc::UcxReceiver::doc_setup); diff --git a/python/holoscan/resources/receivers_pydoc.hpp b/python/holoscan/resources/receivers_pydoc.hpp index 08e31dfd..668cfb70 100644 --- a/python/holoscan/resources/receivers_pydoc.hpp +++ b/python/holoscan/resources/receivers_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -46,13 +46,6 @@ namespace DoubleBufferReceiver { PYDOC(DoubleBufferReceiver, R"doc( Receiver using a double-buffered queue. -New messages are first pushed to a back stage. -)doc") - -// Constructor -PYDOC(DoubleBufferReceiver_python, R"doc( -Receiver using a double-buffered queue. - New messages are first pushed to a back stage. Parameters @@ -92,13 +85,6 @@ namespace UcxReceiver { PYDOC(UcxReceiver, R"doc( UCX network receiver using a double-buffered queue. -New messages are first pushed to a back stage. -)doc") - -// Constructor -PYDOC(UcxReceiver_python, R"doc( -UCX network receiver using a double-buffered queue. - New messages are first pushed to a back stage. Parameters diff --git a/python/holoscan/resources/resources.cpp b/python/holoscan/resources/resources.cpp index a524fed5..ddf8f792 100644 --- a/python/holoscan/resources/resources.cpp +++ b/python/holoscan/resources/resources.cpp @@ -29,6 +29,7 @@ void init_allocators(py::module_&); void init_receivers(py::module_&); void init_transmitters(py::module_&); void init_clocks(py::module_&); +void init_gxf_component_resource(py::module_&); void init_serialization_buffers(py::module_&); void init_component_serializers(py::module_&); void init_entity_serializers(py::module_&); @@ -36,25 +37,16 @@ void init_std_entity_serializer(py::module_&); PYBIND11_MODULE(_resources, m) { m.doc() = R"pbdoc( - Holoscan SDK Python Bindings - --------------------------------------- + Holoscan SDK Resources Python Bindings + -------------------------------------- .. currentmodule:: _resources - .. autosummary:: - :toctree: _generate - add - subtract )pbdoc"; -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif - init_allocators(m); init_receivers(m); init_transmitters(m); init_clocks(m); + init_gxf_component_resource(m); init_serialization_buffers(m); init_component_serializers(m); init_entity_serializers(m); diff --git a/python/holoscan/resources/serialization_buffers.cpp b/python/holoscan/resources/serialization_buffers.cpp index 1cb0b8bd..28e38b05 100644 --- a/python/holoscan/resources/serialization_buffers.cpp +++ b/python/holoscan/resources/serialization_buffers.cpp @@ -88,7 +88,7 @@ void init_serialization_buffers(py::module_& m) { "allocator"_a = py::none(), "buffer_size"_a = kDefaultSerializationBufferSize, "name"_a = "serialization_buffer"s, - doc::SerializationBuffer::doc_SerializationBuffer_python) + doc::SerializationBuffer::doc_SerializationBuffer) .def_property_readonly("gxf_typename", &SerializationBuffer::gxf_typename, doc::SerializationBuffer::doc_gxf_typename) @@ -104,7 +104,7 @@ void init_serialization_buffers(py::module_& m) { "allocator"_a = py::none(), "buffer_size"_a = kDefaultSerializationBufferSize, "name"_a = "serialization_buffer"s, - doc::UcxSerializationBuffer::doc_UcxSerializationBuffer_python) + doc::UcxSerializationBuffer::doc_UcxSerializationBuffer) .def_property_readonly("gxf_typename", &UcxSerializationBuffer::gxf_typename, doc::UcxSerializationBuffer::doc_gxf_typename) diff --git a/python/holoscan/resources/serialization_buffers_pydoc.hpp b/python/holoscan/resources/serialization_buffers_pydoc.hpp index e12405df..1e01d52b 100644 --- a/python/holoscan/resources/serialization_buffers_pydoc.hpp +++ b/python/holoscan/resources/serialization_buffers_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -28,11 +28,6 @@ namespace SerializationBuffer { PYDOC(SerializationBuffer, R"doc( Serialization Buffer. -)doc") - -// Constructor -PYDOC(SerializationBuffer_python, R"doc( -Serialization Buffer. Parameters ---------- @@ -70,11 +65,6 @@ namespace UcxSerializationBuffer { PYDOC(UcxSerializationBuffer, R"doc( UCX serialization buffer. -)doc") - -// Constructor -PYDOC(UcxSerializationBuffer_python, R"doc( -UCX serialization buffer. Parameters ---------- diff --git a/python/holoscan/resources/std_entity_serializer.cpp b/python/holoscan/resources/std_entity_serializer.cpp index 8f1f5a45..dd7ef6cb 100644 --- a/python/holoscan/resources/std_entity_serializer.cpp +++ b/python/holoscan/resources/std_entity_serializer.cpp @@ -59,7 +59,7 @@ void init_std_entity_serializer(py::module_& m) { .def(py::init(), "fragment"_a, "name"_a = "std_entity_serializer"s, - doc::StdEntitySerializer::doc_StdEntitySerializer_python) + doc::StdEntitySerializer::doc_StdEntitySerializer) .def_property_readonly("gxf_typename", &StdEntitySerializer::gxf_typename, doc::StdEntitySerializer::doc_gxf_typename) diff --git a/python/holoscan/resources/std_entity_serializer_pydoc.hpp b/python/holoscan/resources/std_entity_serializer_pydoc.hpp index 6905c317..0e440499 100644 --- a/python/holoscan/resources/std_entity_serializer_pydoc.hpp +++ b/python/holoscan/resources/std_entity_serializer_pydoc.hpp @@ -28,11 +28,6 @@ namespace StdEntitySerializer { PYDOC(StdEntitySerializer, R"doc( Default serializer for GXF entities. -)doc") - -// Constructor -PYDOC(StdEntitySerializer_python, R"doc( -Default serializer for GXF entities. Parameters ---------- diff --git a/python/holoscan/resources/transmitters.cpp b/python/holoscan/resources/transmitters.cpp index 0a9629c8..81575de5 100644 --- a/python/holoscan/resources/transmitters.cpp +++ b/python/holoscan/resources/transmitters.cpp @@ -97,7 +97,7 @@ void init_transmitters(py::module_& m) { "capacity"_a = 1UL, "policy"_a = 2UL, "name"_a = "double_buffer_transmitter"s, - doc::DoubleBufferTransmitter::doc_DoubleBufferTransmitter_python) + doc::DoubleBufferTransmitter::doc_DoubleBufferTransmitter) .def_property_readonly("gxf_typename", &DoubleBufferTransmitter::gxf_typename, doc::DoubleBufferTransmitter::doc_gxf_typename) @@ -128,7 +128,7 @@ void init_transmitters(py::module_& m) { "local_port"_a = static_cast(0), "maximum_connection_retries"_a = 10, "name"_a = "ucx_transmitter"s, - doc::UcxTransmitter::doc_UcxTransmitter_python) + doc::UcxTransmitter::doc_UcxTransmitter) .def_property_readonly( "gxf_typename", &UcxTransmitter::gxf_typename, doc::UcxTransmitter::doc_gxf_typename) .def("setup", &UcxTransmitter::setup, "spec"_a, doc::UcxTransmitter::doc_setup); diff --git a/python/holoscan/resources/transmitters_pydoc.hpp b/python/holoscan/resources/transmitters_pydoc.hpp index fa7e97c0..f4bd5ebf 100644 --- a/python/holoscan/resources/transmitters_pydoc.hpp +++ b/python/holoscan/resources/transmitters_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -43,14 +43,8 @@ str namespace DoubleBufferTransmitter { -PYDOC(DoubleBufferTransmitter, R"doc( -Transmitter using a double-buffered queue. - -Messages are pushed to a back stage after they are published. -)doc") - // Constructor -PYDOC(DoubleBufferTransmitter_python, R"doc( +PYDOC(DoubleBufferTransmitter, R"doc( Transmitter using a double-buffered queue. Messages are pushed to a back stage after they are published. @@ -92,13 +86,6 @@ namespace UcxTransmitter { PYDOC(UcxTransmitter, R"doc( UCX network transmitter using a double-buffered queue. -Messages are pushed to a back stage after they are published. -)doc") - -// Constructor -PYDOC(UcxTransmitter_python, R"doc( -UCX network transmitter using a double-buffered queue. - Messages are pushed to a back stage after they are published. Parameters diff --git a/python/holoscan/schedulers/event_based_scheduler.cpp b/python/holoscan/schedulers/event_based_scheduler.cpp index d0b09497..a55c9a75 100644 --- a/python/holoscan/schedulers/event_based_scheduler.cpp +++ b/python/holoscan/schedulers/event_based_scheduler.cpp @@ -99,7 +99,7 @@ void init_event_based_scheduler(py::module_& m) { "max_duration_ms"_a = -1LL, "stop_on_deadlock_timeout"_a = 0LL, "name"_a = "multithread_scheduler"s, - doc::EventBasedScheduler::doc_EventBasedScheduler_python) + doc::EventBasedScheduler::doc_EventBasedScheduler) .def_property_readonly("clock", &EventBasedScheduler::clock) .def_property_readonly("worker_thread_number", &EventBasedScheduler::worker_thread_number) .def_property_readonly("max_duration_ms", &EventBasedScheduler::max_duration_ms) diff --git a/python/holoscan/schedulers/event_based_scheduler_pydoc.hpp b/python/holoscan/schedulers/event_based_scheduler_pydoc.hpp index 18a9cceb..e489048c 100644 --- a/python/holoscan/schedulers/event_based_scheduler_pydoc.hpp +++ b/python/holoscan/schedulers/event_based_scheduler_pydoc.hpp @@ -26,13 +26,7 @@ namespace holoscan::doc { namespace EventBasedScheduler { -// Constructor PYDOC(EventBasedScheduler, R"doc( -Event-based multi-thread scheduler class. -)doc") - -// PyEventBasedScheduler Constructor -PYDOC(EventBasedScheduler_python, R"doc( Event-based multi-thread scheduler Parameters diff --git a/python/holoscan/schedulers/greedy_scheduler.cpp b/python/holoscan/schedulers/greedy_scheduler.cpp index 91632717..371ecb8a 100644 --- a/python/holoscan/schedulers/greedy_scheduler.cpp +++ b/python/holoscan/schedulers/greedy_scheduler.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -98,7 +98,7 @@ void init_greedy_scheduler(py::module_& m) { "check_recession_period_ms"_a = 0.0, "stop_on_deadlock_timeout"_a = 0LL, "name"_a = "greedy_scheduler"s, - doc::GreedyScheduler::doc_GreedyScheduler_python) + doc::GreedyScheduler::doc_GreedyScheduler) .def_property_readonly("clock", &GreedyScheduler::clock) .def_property_readonly("max_duration_ms", &GreedyScheduler::max_duration_ms) .def_property_readonly("stop_on_deadlock", &GreedyScheduler::stop_on_deadlock) diff --git a/python/holoscan/schedulers/greedy_scheduler_pydoc.hpp b/python/holoscan/schedulers/greedy_scheduler_pydoc.hpp index 37ea8cec..626655f2 100644 --- a/python/holoscan/schedulers/greedy_scheduler_pydoc.hpp +++ b/python/holoscan/schedulers/greedy_scheduler_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,13 +26,7 @@ namespace holoscan::doc { namespace GreedyScheduler { -// Constructor PYDOC(GreedyScheduler, R"doc( -GreedyScheduler scheduler class. -)doc") - -// PyGreedyScheduler Constructor -PYDOC(GreedyScheduler_python, R"doc( Greedy scheduler Parameters diff --git a/python/holoscan/schedulers/multithread_scheduler.cpp b/python/holoscan/schedulers/multithread_scheduler.cpp index dc5a84aa..59348327 100644 --- a/python/holoscan/schedulers/multithread_scheduler.cpp +++ b/python/holoscan/schedulers/multithread_scheduler.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -103,7 +103,7 @@ void init_multithread_scheduler(py::module_& m) { "max_duration_ms"_a = -1LL, "stop_on_deadlock_timeout"_a = 0LL, "name"_a = "multithread_scheduler"s, - doc::MultiThreadScheduler::doc_MultiThreadScheduler_python) + doc::MultiThreadScheduler::doc_MultiThreadScheduler) .def_property_readonly("clock", &MultiThreadScheduler::clock) .def_property_readonly("worker_thread_number", &MultiThreadScheduler::worker_thread_number) .def_property_readonly("max_duration_ms", &MultiThreadScheduler::max_duration_ms) diff --git a/python/holoscan/schedulers/multithread_scheduler_pydoc.hpp b/python/holoscan/schedulers/multithread_scheduler_pydoc.hpp index 67548261..fb55ddad 100644 --- a/python/holoscan/schedulers/multithread_scheduler_pydoc.hpp +++ b/python/holoscan/schedulers/multithread_scheduler_pydoc.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,13 +26,7 @@ namespace holoscan::doc { namespace MultiThreadScheduler { -// Constructor PYDOC(MultiThreadScheduler, R"doc( -Multi-thread scheduler class. -)doc") - -// PyMultiThreadScheduler Constructor -PYDOC(MultiThreadScheduler_python, R"doc( Multi-thread scheduler Parameters diff --git a/python/holoscan/schedulers/schedulers.cpp b/python/holoscan/schedulers/schedulers.cpp index 46053127..65a3499e 100644 --- a/python/holoscan/schedulers/schedulers.cpp +++ b/python/holoscan/schedulers/schedulers.cpp @@ -43,21 +43,11 @@ void init_multithread_scheduler(py::module_&); PYBIND11_MODULE(_schedulers, m) { m.doc() = R"pbdoc( - Holoscan SDK Python Bindings + Holoscan SDK Schedulers Python Bindings --------------------------------------- .. currentmodule:: _schedulers - .. autosummary:: - :toctree: _generate - add - subtract )pbdoc"; -#ifdef VERSION_INFO - m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); -#else - m.attr("__version__") = "dev"; -#endif - init_event_based_scheduler(m); init_greedy_scheduler(m); init_multithread_scheduler(m); diff --git a/python/tests/system/test_application_async_ping.py b/python/tests/system/test_application_async_ping.py new file mode 100644 index 00000000..b7986dac --- /dev/null +++ b/python/tests/system/test_application_async_ping.py @@ -0,0 +1,175 @@ +""" + SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" # noqa: E501 + +import datetime +import time +from concurrent.futures import Future, ThreadPoolExecutor + +import pytest + +from holoscan.conditions import ( + AsynchronousCondition, + AsynchronousEventState, + CountCondition, + PeriodicCondition, +) +from holoscan.core import Application, Operator, OperatorSpec +from holoscan.operators import PingRxOp +from holoscan.schedulers import EventBasedScheduler, GreedyScheduler, MultiThreadScheduler + + +class AsyncPingTxOp(Operator): + """Asynchronous transmit operator. + + This operator sends a message asynchronously, where the delay between when the async event will + be set to EVENT_DONE is specified by `delay`. After a total count of 10, the statue will be set + to EVENT_NEVER, and the operator will stop sending messages. + """ + + def __init__(self, fragment, *args, delay=0.2, count=10, **kwargs): + self.index = 0 + + # counter to keep track of number of times compute was called + self.iter = 0 + self.count = count + self.delay = delay + + # add an asynchronous condition + self.async_cond_ = AsynchronousCondition(fragment, name="async_cond") + + # thread pool with 1 worker to run async_send + self.executor_ = ThreadPoolExecutor(max_workers=1) + self.future_ = None # will be set during start() + + # Need to call the base class constructor last + # Note: It is essential that we pass self.async_cond_ to the parent + # class constructor here. + super().__init__(fragment, self.async_cond_, *args, **kwargs) + + def aysnc_send(self): + """Function to be submitted to self.executor by start() + + When the condition's even_state is EVENT_WAITING, set to EVENT_DONE. This function will + only exit once the condition is set to EVENT_NEVER. + """ + while True: + time.sleep(self.delay) + print("in async_send") + if self.async_cond_.event_state == AsynchronousEventState.EVENT_WAITING: + self.async_cond_.event_state = AsynchronousEventState.EVENT_DONE + elif self.async_cond_.event_state == AsynchronousEventState.EVENT_NEVER: + break + return + + def setup(self, spec: OperatorSpec): + spec.output("out") + + def start(self): + self.future_ = self.executor_.submit(self.aysnc_send) + assert isinstance(self.future_, Future) + + def compute(self, op_input, op_output, context): + print("in compute") + self.iter += 1 + if self.iter < self.count: + self.async_cond_.event_state = AsynchronousEventState.EVENT_WAITING + else: + self.async_cond_.event_state = AsynchronousEventState.EVENT_NEVER + + op_output.emit(self.iter, "out") + + def stop(self): + self.async_cond_.event_state = AsynchronousEventState.EVENT_NEVER + self.future_.result() + + +class MyAsyncPingApp(Application): + def __init__(self, *args, count=10, delay=0.1, ext_count=None, ext_delay=None, **kwargs): + self.count = count + self.delay = delay + self.ext_count = ext_count + self.ext_delay = ext_delay + super().__init__(*args, **kwargs) + + def compose(self): + tx_args = [] + if self.ext_count: + # add a count condition external to the count on the operator itself + tx_args.append(CountCondition(self, count=self.ext_count)) + if self.ext_delay: + # add a count condition external to the count on the operator itself + tx_args.append(PeriodicCondition(self, recess_period=self.ext_delay)) + + tx = AsyncPingTxOp( + self, + *tx_args, + count=self.count, + delay=self.delay, + name="tx", + ) + rx = PingRxOp(self, name="rx") + self.add_flow(tx, rx) + + +@pytest.mark.parametrize( + "scheduler_class", [GreedyScheduler, MultiThreadScheduler, EventBasedScheduler] +) +@pytest.mark.parametrize("extra_count_condition", [False, True]) +@pytest.mark.parametrize("extra_periodic_condition", [False, True]) +def test_my_ping_async_multicondition_event_wait_app( + scheduler_class, extra_count_condition, extra_periodic_condition, capfd +): + count = 8 + delay = 0.025 + + if extra_count_condition: + # add another CountCondition that would terminate the app earlier + ext_count = max(count // 2, 1) + expected_count = ext_count + else: + ext_count = None + expected_count = count + + if extra_periodic_condition: + # add another PeriodicCondition that would cause a longer period than delay + expected_delay = 2 * delay + ext_delay = datetime.timedelta(seconds=expected_delay) + else: + expected_delay = delay + ext_delay = None + + app = MyAsyncPingApp( + count=count, + delay=delay, + ext_count=ext_count, + ext_delay=ext_delay, + ) + + # set scheduler (using its default options) + app.scheduler(scheduler_class(app)) + + t_start = time.time() + app.run() + duration = time.time() - t_start + + # overall duration must be longer than delays caused by async_send + assert duration > expected_delay * (expected_count - 1) + + # assert that the expected number of messages were received + captured = capfd.readouterr() + assert f"Rx message value: {expected_count}" in captured.out + assert f"Rx message value: {expected_count + 1}" not in captured.out diff --git a/python/tests/system/test_application_video_stream.py b/python/tests/system/test_application_video_stream.py new file mode 100644 index 00000000..1b0b5dc0 --- /dev/null +++ b/python/tests/system/test_application_video_stream.py @@ -0,0 +1,217 @@ +""" + SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" # noqa: E501 +import math +import os +import shutil +import tempfile + +import cupy as cp +import numpy as np +import pytest + +from holoscan.conditions import CountCondition +from holoscan.core import Application, Operator, OperatorSpec +from holoscan.operators import VideoStreamRecorderOp, VideoStreamReplayerOp + + +def get_frame(xp, height, width, channels, fortran_ordered): + shape = (height, width, channels) + size = math.prod(shape) + frame = xp.arange(size, dtype=np.uint8).reshape(shape) + if fortran_ordered: + frame = xp.asfortranarray(frame) + return frame + + +class FrameGeneratorOp(Operator): + def __init__( + self, fragment, *args, width=800, height=640, on_host=True, fortran_ordered=False, **kwargs + ): + self.height = height + self.width = width + self.on_host = on_host + self.fortran_ordered = fortran_ordered + # Need to call the base class constructor last + super().__init__(fragment, *args, **kwargs) + + def setup(self, spec: OperatorSpec): + spec.output("frame") + + def compute(self, op_input, op_output, context): + xp = np if self.on_host else cp + frame = get_frame(xp, self.height, self.width, 3, self.fortran_ordered) + op_output.emit(dict(frame=frame), "frame") + + +class FrameValidationRxOp(Operator): + def __init__( + self, fragment, *args, expected_width=800, expected_height=640, on_host=True, **kwargs + ): + self.expected_height = expected_height + self.expected_width = expected_width + self.on_host = on_host + self.count = 0 + # Need to call the base class constructor last + super().__init__(fragment, *args, **kwargs) + + def setup(self, spec: OperatorSpec): + spec.input("in") + + def compute(self, op_input, op_output, context): + tensormap = op_input.receive("in") + assert "frame" in tensormap + tensor = tensormap["frame"] + self.count += 1 + print(f"received frame {self.count}") + assert tensor.shape == (self.expected_height, self.expected_width, 3) + if self.on_host: + xp = np + assert hasattr(tensor, "__array_interface__") + + else: + xp = cp + assert hasattr(tensor, "__cuda_array_interface__") + assert xp.asarray(tensor).dtype == xp.uint8 + expected_frame = get_frame(xp, self.expected_height, self.expected_width, 3, False) + xp.testing.assert_array_equal(expected_frame, xp.ascontiguousarray(xp.asarray(tensor))) + + +class VideoRecorderApp(Application): + def __init__( + self, + *args, + count=10, + width=800, + height=640, + on_host=True, + fortran_ordered=False, + directory="/tmp", + basename="test_video", + **kwargs, + ): + self.count = count + self.directory = directory + self.basename = basename + self.width = width + self.height = height + self.on_host = on_host + self.fortran_ordered = fortran_ordered + super().__init__(*args, **kwargs) + + def compose(self): + source = FrameGeneratorOp( + self, + CountCondition(self, count=self.count), + width=self.width, + height=self.height, + on_host=self.on_host, + fortran_ordered=self.fortran_ordered, + name="video_source", + ) + recorder = VideoStreamRecorderOp( + self, directory=self.directory, basename=self.basename, name="recorder" + ) + self.add_flow(source, recorder) + + +class VideoReplayerApp(Application): + def __init__( + self, + *args, + count=10, + expected_width=800, + expected_height=640, + on_host=True, + directory="/tmp", + basename="test_video", + **kwargs, + ): + self.count = count + self.directory = directory + self.basename = basename + self.expected_width = expected_width + self.expected_height = expected_height + self.on_host = on_host + super().__init__(*args, **kwargs) + + def compose(self): + replayer = VideoStreamReplayerOp( + self, + directory=self.directory, + basename=self.basename, + repeat=False, + realtime=False, + name="replayer", + ) + frame_validator = FrameValidationRxOp( + self, + expected_width=self.expected_width, + expected_height=self.expected_height, + on_host=self.on_host, + name="frame_validator", + ) + self.add_flow(replayer, frame_validator) + + +@pytest.mark.parametrize("on_host", [True, False]) +@pytest.mark.parametrize("fortran_ordered", [True, False]) +def test_recorder_and_replayer_roundtrip(fortran_ordered, on_host, capfd): + """Test the functionality of VideoStreamRecorderOp and VideoStreamReplayerOp. + + This test case runs two applications back to back + + 1.) recorder_app : serializes synthetic data frames to disk + 2.) replayer_app : deserializes frames from disk and validates them + """ + directory = tempfile.mkdtemp() + try: + count = 10 + width = 800 + height = 640 + basename = "test_video_cpu" if on_host else "test_video_gpu" + recorder_app = VideoRecorderApp( + count=count, + directory=directory, + basename=basename, + width=width, + height=height, + on_host=on_host, + fortran_ordered=fortran_ordered, + ) + recorder_app.run() + + # verify that expected files were generated + out_files = os.listdir(directory) + assert len(out_files) == 2 + assert basename + ".gxf_entities" in out_files + assert basename + ".gxf_index" in out_files + + # verify that deserialized frames match the expected shape + replayer_app = VideoReplayerApp( + directory=directory, + basename=basename, + expected_width=width, + expected_height=height, + on_host=on_host, + ) + replayer_app.run() + + # assert that replayer_app received all frames + captured = capfd.readouterr() + assert f"received frame {count}" in captured.out + finally: + shutil.rmtree(directory) diff --git a/python/tests/system/test_flow_tracking_with_tensormap.py b/python/tests/system/test_flow_tracking_with_tensormap.py index d54e4083..6bef6edc 100644 --- a/python/tests/system/test_flow_tracking_with_tensormap.py +++ b/python/tests/system/test_flow_tracking_with_tensormap.py @@ -78,13 +78,13 @@ def setup(self, spec: OperatorSpec): spec.param("sigma") def compute(self, op_input, op_output, context): - # in_message is of dict + # in_message is a dict of tensors in_message = op_input.receive("input_tensor") # smooth along first two axes, but not the color channels sigma = (self.sigma, self.sigma, 0) - # out_message is of dict + # out_message will be a dict of tensors out_message = dict() for key, value in in_message.items(): diff --git a/python/tests/system/test_format_converter_invalid_input.py b/python/tests/system/test_format_converter_invalid_input.py new file mode 100644 index 00000000..8a00a9c0 --- /dev/null +++ b/python/tests/system/test_format_converter_invalid_input.py @@ -0,0 +1,178 @@ +""" + SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" # noqa: E501 +import cupy as cp +import numpy as np +import pytest + +from holoscan.conditions import CountCondition +from holoscan.core import Application, Operator, OperatorSpec +from holoscan.operators import FormatConverterOp, PingRxOp +from holoscan.resources import UnboundedAllocator + + +class TxRGBA(Operator): + """Transmit an RGBA device tensor of user-specified shape. + + The tensor must be on device for use with FormatConverterOp. + """ + + def __init__( + self, + fragment, + *args, + shape=(32, 64), + channels=4, + fortran_ordered=False, + padded=False, + memory_location=False, + **kwargs, + ): + self.shape = shape + self.channels = channels + self.fortran_ordered = fortran_ordered + self.memory_location = memory_location + self.padded = padded + super().__init__(fragment, *args, **kwargs) + + def setup(self, spec: OperatorSpec): + spec.output("out") + + def compute(self, op_input, op_output, context): + shape = self.shape if self.channels is None else self.shape + (self.channels,) + order = "F" if self.fortran_ordered else "C" + if self.memory_location == "device": + img = cp.zeros(shape, dtype=cp.uint8, order=order) + elif self.memory_location == "cpu": + img = np.zeros(shape, dtype=cp.uint8, order=order) + if self.padded: + # slice so reduce shape[1] without changing stride[0] + # This matches a common case where padding is added to make CUDA kernels more efficient + img = img[:, :-2, ...] + op_output.emit(dict(frame=img), "out") + + +class MyFormatConverterApp(Application): + """Test passing conditions positionally to a wrapped C++ operator (FormatConverterOp).""" + + def __init__( + self, + *args, + count=10, + shape=(32, 16), + channels=4, + padded=False, + input_memory_location="device", + in_dtype="rgba8888", + out_dtype="rgb888", + fortran_ordered=False, + **kwargs, + ): + self.count = count + self.shape = shape + self.channels = channels + self.in_dtype = in_dtype + self.out_dtype = out_dtype + self.input_memory_location = input_memory_location + self.padded = padded + self.fortran_ordered = fortran_ordered + super().__init__(*args, **kwargs) + + def compose(self): + tx = TxRGBA( + self, + shape=self.shape, + channels=self.channels, + fortran_ordered=self.fortran_ordered, + padded=self.padded, + memory_location=self.input_memory_location, + name="tx", + ) + converter = FormatConverterOp( + self, + CountCondition(self, self.count), + pool=UnboundedAllocator(self), + in_tensor_name="frame", + in_dtype=self.in_dtype, + out_dtype=self.out_dtype, + ) + rx = PingRxOp(self, name="rx") + self.add_flow(tx, converter) + self.add_flow(converter, rx) + + +@pytest.mark.parametrize("padded", [False, True]) +def test_format_converter_invalid_memory_layout(padded, capfd): + count = 10 + app = MyFormatConverterApp( + count=count, channels=4, padded=padded, fortran_ordered=True, input_memory_location="device" + ) + + with pytest.raises(RuntimeError): + app.run() + captured = capfd.readouterr() + assert "error" in captured.err + assert "Tensor is expected to be in a C-contiguous memory layout" in captured.err + + +@pytest.mark.parametrize("shape", [(4,), (4, 4, 4, 4)]) +def test_format_converter_invalid_rank(shape, capfd): + count = 10 + app = MyFormatConverterApp( + count=count, + shape=shape, + channels=None, + fortran_ordered=False, + input_memory_location="device", + ) + + with pytest.raises(RuntimeError): + app.run() + captured = capfd.readouterr() + assert "error" in captured.err + assert "Expected a tensor with 2 or 3 dimensions" in captured.err + + +@pytest.mark.parametrize("channels", [1, 2, 3, 4, 5]) +def test_format_converter_invalid_num_channels(channels, capfd): + count = 10 + # For in_dtype rgba8888, must have 4 channels + app = MyFormatConverterApp( + count=count, channels=channels, fortran_ordered=False, input_memory_location="device" + ) + + if channels == 4: + app.run() + captured = capfd.readouterr() + assert "error" not in captured.err + else: + with pytest.raises(RuntimeError): + app.run() + captured = capfd.readouterr() + assert "error" in captured.err + assert "Failed to verify the channels for the expected input dtype" in captured.err + + +def test_format_converter_host_tensor(capfd): + count = 10 + # For in_dtype rgba8888, must have 4 channels + app = MyFormatConverterApp( + count=count, channels=4, fortran_ordered=False, input_memory_location="cpu" + ) + + app.run() + captured = capfd.readouterr() + assert "error" not in captured.err diff --git a/python/tests/system/test_holoviz_camera_emit_receive.py b/python/tests/system/test_holoviz_camera_emit_receive.py new file mode 100644 index 00000000..d7f29ce9 --- /dev/null +++ b/python/tests/system/test_holoviz_camera_emit_receive.py @@ -0,0 +1,174 @@ +""" + SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" # noqa: E501 +import math + +import cupy as cp +import numpy as np +import pytest + +from holoscan.conditions import CountCondition +from holoscan.core import Application, Operator, OperatorSpec +from holoscan.operators import HolovizOp, PingRxOp +from holoscan.operators.holoviz import Pose3D + + +def get_frame(xp, height, width, channels, dtype=np.uint8): + shape = (height, width, channels) if channels else (height, width) + size = math.prod(shape) + frame = xp.arange(size, dtype=dtype).reshape(shape) + return frame + + +class FrameGeneratorOp(Operator): + def __init__( + self, + fragment, + *args, + width=800, + height=640, + channels=3, + on_host=False, + dtype=np.uint8, + **kwargs, + ): + self.height = height + self.width = width + self.channels = channels + self.on_host = on_host + self.dtype = dtype + # Need to call the base class constructor last + super().__init__(fragment, *args, **kwargs) + + def setup(self, spec: OperatorSpec): + spec.output("frame") + spec.output("camera_eye") + spec.output("camera_look_at") + spec.output("camera_up") + + def compute(self, op_input, op_output, context): + xp = np if self.on_host else cp + frame = get_frame(xp, self.height, self.width, self.channels, self.dtype) + print(f"Emitting frame with shape: {frame.shape}") + op_output.emit(dict(frame=frame), "frame") + # emit camera pose vectors + op_output.emit([0.0, 0.0, 1.0], "camera_eye", "std::array") + op_output.emit([0.0, 0.0, 0.0], "camera_look_at", "std::array") + op_output.emit([0.0, 1.0, 0.0], "camera_up", "std::array") + + +class CameraPoseForwardingOp(Operator): + def setup(self, spec: OperatorSpec): + spec.input("camera_pose_input") + spec.input("camera_up_input") + spec.output("camera_pose_output") + + def compute(self, op_input, op_output, context): + # verify that values match output of FrameGeneratorOp + camera_up = op_input.receive("camera_up_input") + assert isinstance(camera_up, list) + assert camera_up == [0.0, 1.0, 0.0] + + camera_pose = op_input.receive("camera_pose_input") + # verify that received type matches expected types (depends on camera_pose_output_type) + assert isinstance(camera_pose, (list, Pose3D)) + op_output.emit(camera_pose, "camera_pose_output") + + +class HolovizHeadlessApp(Application): + def __init__( + self, + *args, + count=10, + width=800, + height=640, + on_host=False, + enable_camera_pose_output=False, + camera_pose_output_type="projection_matrix", + **kwargs, + ): + self.count = count + self.width = width + self.height = height + self.on_host = on_host + self.enable_camera_pose_output = enable_camera_pose_output + self.camera_pose_output_type = camera_pose_output_type + super().__init__(*args, **kwargs) + + def compose(self): + source = FrameGeneratorOp( + self, + CountCondition(self, count=self.count), + width=self.width, + height=self.height, + on_host=self.on_host, + dtype=np.uint8, + name="video_source", + ) + vizualizer = HolovizOp( + self, + headless=True, + name="visualizer", + width=self.width, + height=self.height, + enable_camera_pose_output=self.enable_camera_pose_output, + camera_pose_output_type=self.camera_pose_output_type, + tensors=[ + # name="" here to match the output of FrameGenerationOp + dict(name="frame", type="color", opacity=0.5, priority=0), + ], + ) + self.add_flow(source, vizualizer, {("frame", "receivers")}) + self.add_flow(source, vizualizer, {("camera_eye", "camera_eye_input")}) + self.add_flow(source, vizualizer, {("camera_look_at", "camera_look_at_input")}) + self.add_flow(source, vizualizer, {("camera_up", "camera_up_input")}) + if self.enable_camera_pose_output: + pose_forwarder = CameraPoseForwardingOp(self, name="pose_forwarder") + camera_pose_rx = PingRxOp(self, name="camera_pose_rx") + self.add_flow(vizualizer, pose_forwarder, {("camera_pose_output", "camera_pose_input")}) + self.add_flow(source, pose_forwarder, {("camera_up", "camera_up_input")}) + self.add_flow(pose_forwarder, camera_pose_rx, {("camera_pose_output", "in")}) + + +@pytest.mark.parametrize("camera_pose_output_type", ["projection_matrix", "extrinsics_model"]) +def test_holovizop_camera_inputs(camera_pose_output_type, capfd): + """Test HolovizOp with valid (row-major) and invalid (column-major) memory layouts.""" + count = 3 + width = 800 + height = 640 + holoviz_app = HolovizHeadlessApp( + count=count, + width=width, + height=height, + on_host=False, + enable_camera_pose_output=True, + camera_pose_output_type=camera_pose_output_type, + ) + + holoviz_app.run() + + captured = capfd.readouterr() + + # assert that replayer_app emitted all frames + assert captured.out.count("Emitting frame") == count + + # check that receive printed the expected type of object + if camera_pose_output_type == "extrinsics_model": + pose_repr = "Pose3D(rotation: [1, 0, 0, 0, 1, 0, 0, 0, 1], translation: [-0, -0, -1])" + assert captured.out.count(f"Rx message value: {pose_repr}") == count + elif camera_pose_output_type == "projection_matrix": + # just check start of output value to avoid issues with floating point precision + assert captured.out.count("Rx message value: [1.73205") == count diff --git a/python/tests/system/test_operator_tensor_validation.py b/python/tests/system/test_operator_tensor_validation.py new file mode 100644 index 00000000..32cd2113 --- /dev/null +++ b/python/tests/system/test_operator_tensor_validation.py @@ -0,0 +1,360 @@ +""" + SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" # noqa: E501 +import math + +import cupy as cp +import numpy as np +import pytest + +from holoscan.conditions import CountCondition +from holoscan.core import Application, Operator, OperatorSpec +from holoscan.operators import BayerDemosaicOp, HolovizOp, SegmentationPostprocessorOp +from holoscan.resources import UnboundedAllocator + + +def get_frame(xp, height, width, channels, fortran_ordered, dtype=np.uint8): + shape = (height, width, channels) if channels else (height, width) + size = math.prod(shape) + frame = xp.arange(size, dtype=dtype).reshape(shape) + if fortran_ordered: + frame = xp.asfortranarray(frame) + return frame + + +class FrameGeneratorOp(Operator): + def __init__( + self, + fragment, + *args, + width=800, + height=640, + channels=3, + on_host=True, + fortran_ordered=False, + dtype=np.uint8, + **kwargs, + ): + self.height = height + self.width = width + self.channels = channels + self.on_host = on_host + self.fortran_ordered = fortran_ordered + self.dtype = dtype + # Need to call the base class constructor last + super().__init__(fragment, *args, **kwargs) + + def setup(self, spec: OperatorSpec): + spec.output("frame") + + def compute(self, op_input, op_output, context): + xp = np if self.on_host else cp + frame = get_frame( + xp, self.height, self.width, self.channels, self.fortran_ordered, self.dtype + ) + print(f"Emitting frame with shape: {frame.shape}") + op_output.emit(dict(frame=frame), "frame") + + +class HolovizHeadlessApp(Application): + def __init__( + self, + *args, + count=10, + width=800, + height=640, + on_host=True, + fortran_ordered=False, + **kwargs, + ): + self.count = count + self.width = width + self.height = height + self.on_host = on_host + self.fortran_ordered = fortran_ordered + super().__init__(*args, **kwargs) + + def compose(self): + source = FrameGeneratorOp( + self, + CountCondition(self, count=self.count), + width=self.width, + height=self.height, + on_host=self.on_host, + fortran_ordered=self.fortran_ordered, + dtype=np.uint8, + name="video_source", + ) + vizualizer = HolovizOp( + self, + headless=True, + name="visualizer", + width=self.width, + height=self.height, + enable_camera_pose_output=False, + tensors=[ + # name="" here to match the output of FrameGenerationOp + dict(name="frame", type="color", opacity=0.5, priority=0), + ], + ) + self.add_flow(source, vizualizer, {("frame", "receivers")}) + + +@pytest.mark.parametrize("fortran_ordered", [True, False]) +def test_holovizop_memory_layout(fortran_ordered, capfd): + """Test HolovizOp with valid (row-major) and invalid (column-major) memory layouts.""" + count = 3 + width = 800 + height = 640 + holoviz_app = HolovizHeadlessApp( + count=count, + width=width, + height=height, + on_host=True, + fortran_ordered=fortran_ordered, + ) + + if fortran_ordered: + with pytest.raises(RuntimeError): + holoviz_app.run() + captured = capfd.readouterr() + + # assert that app raised exception on the first frame + assert captured.out.count("Emitting frame") == 1 + assert "Tensor must have a row-major memory layout" in captured.err + + else: + holoviz_app.run() + + captured = capfd.readouterr() + + # assert that replayer_app received all frames + assert captured.out.count("Emitting frame") == count + + +class BayerDemosaicApp(Application): + def __init__( + self, + *args, + count=10, + width=800, + height=640, + channels=3, + on_host=False, + fortran_ordered=False, + **kwargs, + ): + self.count = count + self.width = width + self.height = height + self.channels = channels + self.on_host = on_host + self.fortran_ordered = fortran_ordered + super().__init__(*args, **kwargs) + + def compose(self): + source = FrameGeneratorOp( + self, + CountCondition(self, count=self.count), + width=self.width, + height=self.height, + channels=self.channels, + on_host=self.on_host, # BayerDemosaicOp expects device tensor + fortran_ordered=self.fortran_ordered, + dtype=np.uint8, + name="video_source", + ) + demosaic = BayerDemosaicOp( + self, + in_tensor_name="frame", + out_tensor_name="frame_rgb", + generate_alpha=False, + bayer_grid_pos=2, + interpolation_mode=0, + pool=UnboundedAllocator(self, "device_pool"), + name="demosaic", + ) + vizualizer = HolovizOp( + self, + headless=True, + name="visualizer", + width=self.width, + height=self.height, + enable_camera_pose_output=False, + tensors=[ + # name="" here to match the output of FrameGenerationOp + dict(name="frame_rgb", type="color", opacity=0.5, priority=0), + ], + ) + self.add_flow(source, demosaic, {("frame", "receiver")}) + self.add_flow(demosaic, vizualizer, {("transmitter", "receivers")}) + + +@pytest.mark.parametrize("fortran_ordered, on_host", [(False, False), (True, False), (False, True)]) +@pytest.mark.parametrize("channels", [3, None]) +def test_bayer_demosaic_memory_layout(fortran_ordered, on_host, channels, capfd): + """Test HolovizOp with valid (row-major) and invalid (column-major) memory layouts.""" + count = 3 + width = 800 + height = 640 + demosaic_app = BayerDemosaicApp( + count=count, + width=width, + height=height, + channels=channels, + on_host=on_host, + fortran_ordered=fortran_ordered, + ) + if channels is None: + with pytest.raises(RuntimeError): + demosaic_app.run() + captured = capfd.readouterr() + + # assert that app raised exception on the first frame + assert captured.out.count("Emitting frame") == 1 + assert "Input tensor has 2 dimensions. Expected a tensor with 3 dimensions" in captured.err + + else: + if fortran_ordered: + with pytest.raises(RuntimeError): + demosaic_app.run() + captured = capfd.readouterr() + + # assert that app raised exception on the first frame + assert captured.out.count("Emitting frame") == 1 + assert "Tensor must have a row-major memory layout" in captured.err + + else: + demosaic_app.run() + + captured = capfd.readouterr() + + # assert that replayer_app received all frames + assert captured.out.count("Emitting frame") == count + + +class SegmentationPostprocessorApp(Application): + def __init__( + self, + *args, + count=10, + width=800, + height=640, + channels=1, + on_host=False, + fortran_ordered=False, + network_output_type="softmax", + dtype=np.float32, + **kwargs, + ): + self.count = count + self.width = width + self.height = height + self.channels = channels + self.on_host = on_host + self.fortran_ordered = fortran_ordered + self.network_output_type = network_output_type + self.dtype = dtype + super().__init__(*args, **kwargs) + + def compose(self): + source = FrameGeneratorOp( + self, + CountCondition(self, count=self.count), + width=self.width, + height=self.height, + channels=self.channels, + on_host=self.on_host, # SegmentationPostprocessorOp expects device tensor + fortran_ordered=self.fortran_ordered, + dtype=self.dtype, + name="video_source", + ) + postprocessor = SegmentationPostprocessorOp( + self, + allocator=UnboundedAllocator(self, "device_allocator"), + in_tensor_name="frame", + data_format="hwc", + network_output_type=self.network_output_type, + name="postprocessor", + ) + vizualizer = HolovizOp( + self, + headless=True, + name="visualizer", + width=self.width, + height=self.height, + enable_camera_pose_output=False, + tensors=[ + # name="" here to match the output of FrameGenerationOp + dict(name="out_tensor", type="color", opacity=0.5, priority=0), + ], + ) + self.add_flow(source, postprocessor, {("frame", "in_tensor")}) + self.add_flow(postprocessor, vizualizer, {("out_tensor", "receivers")}) + + +@pytest.mark.parametrize( + "fortran_ordered, on_host, dtype", + [ + (False, False, np.float32), # valid values + (True, False, np.float32), # invalid memory order + (False, True, np.float32), # tensor not on device + (False, False, np.uint8), # input is not 32-bit floating point + ], +) +@pytest.mark.parametrize("channels", [1, 5]) +@pytest.mark.parametrize("network_output_type", ["softmax", "sigmoid"]) +def test_segmentation_postproceesor_memory_layout( + fortran_ordered, on_host, dtype, channels, network_output_type, capfd +): + """Test HolovizOp with valid (row-major) and invalid (column-major) memory layouts.""" + count = 3 + width = 800 + height = 640 + postprocessor_app = SegmentationPostprocessorApp( + count=count, + width=width, + height=height, + channels=channels, + on_host=on_host, + fortran_ordered=fortran_ordered, + dtype=dtype, + network_output_type=network_output_type, + ) + if on_host or fortran_ordered or dtype != np.float32: + with pytest.raises(RuntimeError): + postprocessor_app.run() + captured = capfd.readouterr() + + # assert that app raised exception on the first frame + assert captured.out.count("Emitting frame") == 1 + if on_host: + assert "Input tensor must be in CUDA device or pinned host memory" in captured.err + elif fortran_ordered: + assert "Input tensor must have row-major memory layout" in captured.err + elif dtype != np.float32: + assert "Input tensor must be of type float32" in captured.err + else: + postprocessor_app.run() + + captured = capfd.readouterr() + + # assert that replayer_app received all frames + assert captured.out.count("Emitting frame") == count + + # multi-channel input to sigmoid warns + warning_count = int(network_output_type == "sigmoid" and channels > 1) + assert captured.err.count("Only the first channel will be used") == warning_count diff --git a/python/tests/system/test_positional_condition_args.py b/python/tests/system/test_positional_condition_args.py new file mode 100644 index 00000000..7c9fea8d --- /dev/null +++ b/python/tests/system/test_positional_condition_args.py @@ -0,0 +1,87 @@ +""" + SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + SPDX-License-Identifier: Apache-2.0 + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +""" # noqa: E501 +import time + +import cupy as cp + +from holoscan.conditions import CountCondition, PeriodicCondition +from holoscan.core import Application, Operator, OperatorSpec +from holoscan.operators import FormatConverterOp, PingRxOp +from holoscan.resources import UnboundedAllocator + + +class TxRGBA(Operator): + """Transmit an RGBA device tensor of user-specified shape. + + The tensor must be on device for use with FormatConverterOp. + """ + + def __init__(self, fragment, *args, shape=(32, 64), **kwargs): + self.shape = shape + super().__init__(fragment, *args, **kwargs) + + def setup(self, spec: OperatorSpec): + spec.output("rgba_out") + + def compute(self, op_input, op_output, context): + img = cp.zeros(self.shape + (4,), dtype=cp.uint8) + op_output.emit(dict(rgba=img), "rgba_out") + + +class MyFormatConverterApp(Application): + """Test passing conditions positionally to a wrapped C++ operator (FormatConverterOp).""" + + def __init__(self, *args, count=10, period=None, explicitly_set_connectors=False, **kwargs): + self.count = count + self.period = period + self.explicitly_set_connectors = explicitly_set_connectors + super().__init__(*args, **kwargs) + + def compose(self): + tx = TxRGBA(self, shape=(32, 16), name="tx") + converter = FormatConverterOp( + self, + CountCondition(self, self.count), + PeriodicCondition(self, recess_period=self.period), + pool=UnboundedAllocator(self), + in_tensor_name="rgba", + in_dtype="rgba8888", + out_dtype="rgb888", + ) + rx = PingRxOp(self, name="rx") + self.add_flow(tx, converter) + self.add_flow(converter, rx) + + +def test_format_converter_app(capfd): + count = 10 + period = 50_000_000 # 50 ms per frame + app = MyFormatConverterApp(count=count, period=period) + + tstart = time.time() + app.run() + duration = time.time() - tstart + + # assert that the expected number of messages were received + captured = capfd.readouterr() + + # verify that only the expected number of messages were received + assert captured.out.count("Rx message value") == count + + # verify that run time was not faster than the specified period + min_duration_seconds = (count - 1) * (period / 1.0e9) + assert duration > min_duration_seconds diff --git a/python/tests/unit/test_conditions.py b/python/tests/unit/test_conditions.py index ef389108..b3c65500 100644 --- a/python/tests/unit/test_conditions.py +++ b/python/tests/unit/test_conditions.py @@ -20,6 +20,8 @@ import pytest from holoscan.conditions import ( + AsynchronousCondition, + AsynchronousEventState, BooleanCondition, CountCondition, DownstreamMessageAffordableCondition, @@ -70,6 +72,43 @@ def test_positional_initialization(self, app): BooleanCondition(app, False, "bool") +@pytest.mark.parametrize("name", ["READY", "WAIT", "EVENT_WAITING", "EVENT_DONE", "EVENT_NEVER"]) +def test_aynchronous_event_state_enum(name): + assert hasattr(AsynchronousEventState, name) + + +class TestAsynchronousCondition: + def test_kwarg_based_initialization(self, app, capfd): + name = "async" + cond = AsynchronousCondition(fragment=app, name=name) + assert isinstance(cond, GXFCondition) + assert isinstance(cond, Condition) + assert cond.gxf_typename == "nvidia::gxf::AsynchronousSchedulingTerm" + + assert ( + f""" +name: {name} +fragment: "" +args: + [] +""" + in repr(cond) + ) + + # assert no warnings or errors logged + captured = capfd.readouterr() + assert "error" not in captured.err + assert "warning" not in captured.err + + def test_event_state(self, app, capfd): + cond = AsynchronousCondition(fragment=app, name="async") + assert cond.event_state == AsynchronousEventState.READY + cond.event_state = AsynchronousEventState.EVENT_NEVER + + def test_default_initialization(self, app): + AsynchronousCondition(app) + + class TestCountCondition: def test_kwarg_based_initialization(self, app, capfd): name = "count" diff --git a/python/tests/unit/test_core.py b/python/tests/unit/test_core.py index aa63aa9f..fd50e930 100644 --- a/python/tests/unit/test_core.py +++ b/python/tests/unit/test_core.py @@ -45,6 +45,7 @@ Resource, Scheduler, _Fragment, + io_type_registry, py_object_to_arg, ) from holoscan.core._core import OperatorSpec as OperatorSpecBase @@ -991,3 +992,23 @@ def test_as_tensor_stride_workaround(self, use_cupy): tensor_f = as_tensor(coords_f) assert tensor_f.strides == coords.strides + + +class TestIOTypeRegistry: + def test_registery_entries(self): + registered_types = io_type_registry.registered_types() + + # not an exhaustive list, just a few examples + assert "holoscan::Tensor" in registered_types + assert "CloudPickleSerializedObject" in registered_types + assert "std::string" in registered_types + assert "PyObject" in registered_types + + def test_holoviz_registered_types(self): + # import HolovizOp to ensure its associated types will have been registered + from holoscan.operators import HolovizOp # noqa: F401 + + registered_types = io_type_registry.registered_types() + assert "std::shared_ptr" in registered_types + assert "std::shared_ptr>" in registered_types + assert "std::vector" in registered_types diff --git a/python/tests/unit/test_kwargs.py b/python/tests/unit/test_kwargs.py index fa016d6b..cd45cb86 100644 --- a/python/tests/unit/test_kwargs.py +++ b/python/tests/unit/test_kwargs.py @@ -39,50 +39,50 @@ "value, container_type, element_type", [ # Python float - (5.0, ArgContainerType.NATIVE, ArgElementType.FLOAT64), + (5.0, ArgContainerType.NATIVE, ArgElementType.YAML_NODE), # strings and sequences of strings - ("abcd", ArgContainerType.NATIVE, ArgElementType.STRING), + ("abcd", ArgContainerType.NATIVE, ArgElementType.YAML_NODE), # tuple, list set also work - (("ab", "cd"), ArgContainerType.VECTOR, ArgElementType.STRING), - (["ab", "cd"], ArgContainerType.VECTOR, ArgElementType.STRING), - ({"ab", "cd"}, ArgContainerType.VECTOR, ArgElementType.STRING), + (("ab", "cd"), ArgContainerType.NATIVE, ArgElementType.YAML_NODE), + (["ab", "cd"], ArgContainerType.NATIVE, ArgElementType.YAML_NODE), + ({"ab", "cd"}, ArgContainerType.NATIVE, ArgElementType.YAML_NODE), # Python int gets cast to signed 64-bit int - (5, ArgContainerType.NATIVE, ArgElementType.INT64), + (5, ArgContainerType.NATIVE, ArgElementType.YAML_NODE), # range works - (range(8), ArgContainerType.VECTOR, ArgElementType.INT64), + (range(8), ArgContainerType.NATIVE, ArgElementType.YAML_NODE), # generator works - ((a + 1 for a in range(3)), ArgContainerType.VECTOR, ArgElementType.INT64), - ((3, 5, -1), ArgContainerType.VECTOR, ArgElementType.INT64), - (False, ArgContainerType.NATIVE, ArgElementType.BOOLEAN), - (True, ArgContainerType.NATIVE, ArgElementType.BOOLEAN), - ((False, True), ArgContainerType.VECTOR, ArgElementType.BOOLEAN), + ((a + 1 for a in range(3)), ArgContainerType.NATIVE, ArgElementType.YAML_NODE), + ((3, 5, -1), ArgContainerType.NATIVE, ArgElementType.YAML_NODE), + (False, ArgContainerType.NATIVE, ArgElementType.YAML_NODE), + (True, ArgContainerType.NATIVE, ArgElementType.YAML_NODE), + ((False, True), ArgContainerType.NATIVE, ArgElementType.YAML_NODE), # numpy dtypes get cast to the respective C++ types - (np.uint8(3), ArgContainerType.NATIVE, ArgElementType.UNSIGNED8), - (np.uint16(3), ArgContainerType.NATIVE, ArgElementType.UNSIGNED16), - (np.uint32(3), ArgContainerType.NATIVE, ArgElementType.UNSIGNED32), - (np.uint64(3), ArgContainerType.NATIVE, ArgElementType.UNSIGNED64), - (np.int8(3), ArgContainerType.NATIVE, ArgElementType.INT8), - (np.int16(3), ArgContainerType.NATIVE, ArgElementType.INT16), - (np.int32(3), ArgContainerType.NATIVE, ArgElementType.INT32), - (np.int64(3), ArgContainerType.NATIVE, ArgElementType.INT64), - (np.float16(3), ArgContainerType.NATIVE, ArgElementType.FLOAT32), - (np.float32(3), ArgContainerType.NATIVE, ArgElementType.FLOAT32), - (np.float64(3), ArgContainerType.NATIVE, ArgElementType.FLOAT64), - (np.bool_(3), ArgContainerType.NATIVE, ArgElementType.BOOLEAN), + (np.uint8(3), ArgContainerType.NATIVE, ArgElementType.YAML_NODE), + (np.uint16(3), ArgContainerType.NATIVE, ArgElementType.YAML_NODE), + (np.uint32(3), ArgContainerType.NATIVE, ArgElementType.YAML_NODE), + (np.uint64(3), ArgContainerType.NATIVE, ArgElementType.YAML_NODE), + (np.int8(3), ArgContainerType.NATIVE, ArgElementType.YAML_NODE), + (np.int16(3), ArgContainerType.NATIVE, ArgElementType.YAML_NODE), + (np.int32(3), ArgContainerType.NATIVE, ArgElementType.YAML_NODE), + (np.int64(3), ArgContainerType.NATIVE, ArgElementType.YAML_NODE), + (np.float16(2.5), ArgContainerType.NATIVE, ArgElementType.YAML_NODE), + (np.float32(2.5), ArgContainerType.NATIVE, ArgElementType.YAML_NODE), + (np.float64(2.5), ArgContainerType.NATIVE, ArgElementType.YAML_NODE), + (np.bool_(3), ArgContainerType.NATIVE, ArgElementType.YAML_NODE), ( - np.arange(5, dtype=np.float16), - ArgContainerType.VECTOR, - ArgElementType.FLOAT32, # float16 will be promoted to float32 + np.full(5, 2.5, dtype=np.float16), + ArgContainerType.NATIVE, + ArgElementType.YAML_NODE, # float16 will be promoted to float32 ), ( - np.arange(5, dtype=np.float32), - ArgContainerType.VECTOR, - ArgElementType.FLOAT32, + np.full(5, 2.5, dtype=np.float32), + ArgContainerType.NATIVE, + ArgElementType.YAML_NODE, ), ( - np.arange(5, dtype=np.uint8), - ArgContainerType.VECTOR, - ArgElementType.UNSIGNED8, + np.full(5, 3, dtype=np.uint8), + ArgContainerType.NATIVE, + ArgElementType.YAML_NODE, ), ], ) @@ -156,80 +156,80 @@ def test_py_object_to_arg(value, container_type, element_type): [ # list of lists ( - [[3.0, 3.0, 3.0], [4.0, 4.0, 5.0]], - ArgContainerType.VECTOR, - ArgElementType.FLOAT64, + [[2.5, 2.5, 2.5], [2.5, 2.5, 2.5]], + ArgContainerType.NATIVE, + ArgElementType.YAML_NODE, ), - ([[3, 3, 3], [4, 4, 5]], ArgContainerType.VECTOR, ArgElementType.INT64), + ([[3, 3, 3], [4, 4, 5]], ArgContainerType.NATIVE, ArgElementType.YAML_NODE), ( [[False, True], [True, True]], - ArgContainerType.VECTOR, - ArgElementType.BOOLEAN, + ArgContainerType.NATIVE, + ArgElementType.YAML_NODE, ), # list of tuple - ([(3, 3, 3), (4, 4, 5)], ArgContainerType.VECTOR, ArgElementType.INT64), + ([(3, 3, 3), (4, 4, 5)], ArgContainerType.NATIVE, ArgElementType.YAML_NODE), # sequence of sequence of mixed type - (([3, 3, 3], (4, 4, 5)), ArgContainerType.VECTOR, ArgElementType.INT64), + (([3, 3, 3], (4, 4, 5)), ArgContainerType.NATIVE, ArgElementType.YAML_NODE), # 2d array cases ( np.ones((5, 8), dtype=bool), - ArgContainerType.VECTOR, - ArgElementType.BOOLEAN, + ArgContainerType.NATIVE, + ArgElementType.YAML_NODE, ), ( - np.arange(5 * 8, dtype=np.float32).reshape(5, 8), - ArgContainerType.VECTOR, - ArgElementType.FLOAT32, + np.full((5, 8), 2.5, dtype=np.float32), + ArgContainerType.NATIVE, + ArgElementType.YAML_NODE, ), # noqa ( - np.arange(5 * 8, dtype=np.float64).reshape(5, 8), - ArgContainerType.VECTOR, - ArgElementType.FLOAT64, + np.full((5, 8), 2.5, dtype=np.float64), + ArgContainerType.NATIVE, + ArgElementType.YAML_NODE, ), # noqa ( np.arange(5 * 8, dtype=np.int8).reshape(5, 8), - ArgContainerType.VECTOR, - ArgElementType.INT8, + ArgContainerType.NATIVE, + ArgElementType.YAML_NODE, ), # noqa ( np.arange(5 * 8, dtype=np.int16).reshape(5, 8), - ArgContainerType.VECTOR, - ArgElementType.INT16, + ArgContainerType.NATIVE, + ArgElementType.YAML_NODE, ), # noqa ( np.arange(5 * 8, dtype=np.int32).reshape(5, 8), - ArgContainerType.VECTOR, - ArgElementType.INT32, + ArgContainerType.NATIVE, + ArgElementType.YAML_NODE, ), # noqa ( np.arange(5 * 8, dtype=np.int64).reshape(5, 8), - ArgContainerType.VECTOR, - ArgElementType.INT64, + ArgContainerType.NATIVE, + ArgElementType.YAML_NODE, ), # noqa ( np.arange(5 * 8, dtype=np.uint8).reshape(5, 8), - ArgContainerType.VECTOR, - ArgElementType.UNSIGNED8, + ArgContainerType.NATIVE, + ArgElementType.YAML_NODE, ), # noqa ( np.arange(5 * 8, dtype=np.uint16).reshape(5, 8), - ArgContainerType.VECTOR, - ArgElementType.UNSIGNED16, + ArgContainerType.NATIVE, + ArgElementType.YAML_NODE, ), # noqa ( np.arange(5 * 8, dtype=np.uint32).reshape(5, 8), - ArgContainerType.VECTOR, - ArgElementType.UNSIGNED32, + ArgContainerType.NATIVE, + ArgElementType.YAML_NODE, ), # noqa ( np.arange(5 * 8, dtype=np.uint64).reshape(5, 8), - ArgContainerType.VECTOR, - ArgElementType.UNSIGNED64, + ArgContainerType.NATIVE, + ArgElementType.YAML_NODE, ), # noqa ( [["str1", "string2"], ["s3", "st4"]], - ArgContainerType.VECTOR, - ArgElementType.STRING, + ArgContainerType.NATIVE, + ArgElementType.YAML_NODE, ), ], ) @@ -302,10 +302,10 @@ def test_unsupported_numpy_dtype_raises(): def test_unknown_scalar_numeric_type(): - """Test that unknown scalar types get cast to double.""" + """Test that unknown scalar types get cast to YAML Node.""" arg = py_object_to_arg(decimal.Decimal(3)) assert arg.arg_type.container_type == ArgContainerType.NATIVE - assert arg.arg_type.element_type == ArgElementType.FLOAT64 + assert arg.arg_type.element_type == ArgElementType.YAML_NODE @pytest.mark.parametrize( @@ -413,33 +413,23 @@ def test_arglist_from_kwargs(): == """name: arglist args: - name: alpha - type: double + type: YAML::Node value: 5 - name: beta - type: float + type: YAML::Node value: 3 - name: offsets - type: std::vector - value: - - 1 - - 0 - - 3 + type: YAML::Node + value: [1, 0, 3] - name: names - type: std::vector - value: - - abc - - def + type: YAML::Node + value: [abc, def] - name: verbose - type: bool + type: YAML::Node value: false - name: flags - type: std::vector - value: - - false - - true - - true - - false - - true""" + type: YAML::Node + value: [false, true, true, false, true]""" ) diff --git a/python/tests/unit/test_network_context.py b/python/tests/unit/test_network_context.py index 58536bb0..24d74c8a 100644 --- a/python/tests/unit/test_network_context.py +++ b/python/tests/unit/test_network_context.py @@ -26,7 +26,9 @@ def test_default_init(self, app): e = UcxContext(app) assert isinstance(e, GXFNetworkContext) assert isinstance(e, NetworkContext) - assert isinstance(e.spec, ComponentSpec) + # The 'e.spec' is from the binder (ComponentSpec), and 'ComponentSpec' actually + # derives from PyComponentSpec, which inherits from ComponentSpec. + assert issubclass(ComponentSpec, type(e.spec)) def test_init_kwargs(self, app): entity_serializer = UcxEntitySerializer( diff --git a/python/tests/unit/test_operators_native.py b/python/tests/unit/test_operators_native.py index 47056ea1..7edaf00e 100644 --- a/python/tests/unit/test_operators_native.py +++ b/python/tests/unit/test_operators_native.py @@ -254,7 +254,7 @@ def _check_tensor_property_values(self, t, arr, cuda=False): assert t.shape == arr.shape assert t.strides == arr.strides - assert type(t.data).__name__ == "PyCapsule" + assert isinstance(t.data, int) @pytest.mark.parametrize( "dtype", unsigned_dtypes + signed_dtypes + float_dtypes + complex_dtypes diff --git a/python/tests/unit/test_schedulers.py b/python/tests/unit/test_schedulers.py index 71124b3b..0981b33b 100644 --- a/python/tests/unit/test_schedulers.py +++ b/python/tests/unit/test_schedulers.py @@ -28,7 +28,9 @@ def test_default_init(self, app): scheduler = GreedyScheduler(app) assert isinstance(scheduler, GXFScheduler) assert isinstance(scheduler, Scheduler) - assert isinstance(scheduler.spec, ComponentSpec) + # The 'scheduler.spec' is from the binder (ComponentSpec), and 'ComponentSpec' actually + # derives from PyComponentSpec, which inherits from ComponentSpec. + assert issubclass(ComponentSpec, type(scheduler.spec)) @pytest.mark.parametrize("ClockClass", [ManualClock, RealtimeClock]) def test_init_kwargs(self, app, ClockClass): # noqa: N803 @@ -81,7 +83,9 @@ def test_default_init(self, app): scheduler = MultiThreadScheduler(app) assert isinstance(scheduler, GXFScheduler) assert isinstance(scheduler, Scheduler) - assert isinstance(scheduler.spec, ComponentSpec) + # The 'scheduler.spec' is from the binder (ComponentSpec), and 'ComponentSpec' actually + # derives from PyComponentSpec, which inherits from ComponentSpec. + assert issubclass(ComponentSpec, type(scheduler.spec)) @pytest.mark.parametrize("ClockClass", [ManualClock, RealtimeClock]) def test_init_kwargs(self, app, ClockClass): # noqa: N803 @@ -142,7 +146,9 @@ def test_default_init(self, app): scheduler = EventBasedScheduler(app) assert isinstance(scheduler, GXFScheduler) assert isinstance(scheduler, Scheduler) - assert isinstance(scheduler.spec, ComponentSpec) + # The 'scheduler.spec' is from the binder (ComponentSpec), and 'ComponentSpec' actually + # derives from PyComponentSpec, which inherits from ComponentSpec. + assert issubclass(ComponentSpec, type(scheduler.spec)) @pytest.mark.parametrize("ClockClass", [ManualClock, RealtimeClock]) def test_init_kwargs(self, app, ClockClass): # noqa: N803 diff --git a/run b/run index c53130e5..7921895d 100755 --- a/run +++ b/run @@ -865,9 +865,9 @@ test() { # local run_headless=$([ -z ${DISPLAY-} ] && echo "xvfb-run -a") launch $(get_build_dir) \ - ${extra_args[@]} \ --init \ --cap-drop=NET_BIND_SERVICE \ + ${extra_args[@]} \ --run-cmd "$run_headless ctest . $test_regex_flag --timeout $timeout --output-on-failure $verbose $options" } @@ -885,6 +885,7 @@ Arguments: --mount-point - Specifies the mount point (default is the directory of this script) --run-cmd - Specifies a command to run in the container instead of running interactively. This is the equivalent of what you would put after `bash -c` with `docker run`. + --ssh-x11 : Enable X11 forwarding of graphical HoloHub applications over SSH ' } launch() { @@ -898,6 +899,7 @@ launch() { local mount_device_opt="" local extra_args="" local run_cmd="bash" + local ssh_x11=0 # Skip the first argument to pass the remaining arguments to the docker command. local working_dir=${1:-$(get_build_dir)} @@ -918,6 +920,10 @@ launch() { shift shift ;; + --ssh-x11) + ssh_x11=1 + shift + ;; *) extra_args+=("$1") shift @@ -928,12 +934,6 @@ launch() { local host_mount_to_top=$(realpath --relative-to="$mount_point" "$TOP") local container_top="${container_mount}/${host_mount_to_top}" - # Allow the docker group to access X11. - # Note: not necessary for WSL2 (`SI:localuser:wslg` is added by default) - if [ -v DISPLAY ] && command -v xhost >/dev/null; then - run_command xhost +local:docker - fi - # Mount V4L2 device nodes for video_dev in $(find /dev -regex '/dev/video[0-9]+'); do mount_device_opt+=" --device $video_dev" @@ -976,6 +976,40 @@ launch() { # Add docker group to enable DooD groups+=" --group-add $(get_group_id docker)" + # display - use XDG_SESSION_TYPE to detect X11 or Wayland + # see https://www.freedesktop.org/software/systemd/man/latest/pam_systemd.html#%24XDG_SESSION_TYPE + if [ -n "${XDG_SESSION_TYPE-}" ]; then + display_server_opt="-e XDG_SESSION_TYPE" + if [ "${XDG_SESSION_TYPE}" == "wayland" ]; then + display_server_opt+=" -e WAYLAND_DISPLAY" + fi + fi + if [ -n "${XDG_RUNTIME_DIR-}" ]; then + display_server_opt+=" -e XDG_RUNTIME_DIR" + if [ -d ${XDG_RUNTIME_DIR} ]; then + display_server_opt+=" -v ${XDG_RUNTIME_DIR}:${XDG_RUNTIME_DIR}" + fi + fi + # if XDG_SESSION_TYPE is not set or set to tty or x11, use X11 + if [ -z "${XDG_SESSION_TYPE-}" ] || [ "${XDG_SESSION_TYPE}" == "x11" ] || [ "${XDG_SESSION_TYPE}" == "tty" ]; then + # Allow the docker group to access X11. + # Note: not necessary for WSL2 (`SI:localuser:wslg` is added by default) + if [ -v DISPLAY ] && command -v xhost >/dev/null; then + run_command xhost +local:docker + fi + display_server_opt+=" -v /tmp/.X11-unix:/tmp/.X11-unix" + display_server_opt+=" -e DISPLAY" + fi + # Allow X11 forwarding over SSH + if [[ $ssh_x11 -gt 0 ]]; then + XAUTH=/tmp/.docker.xauth + xauth nlist $DISPLAY | sed -e 's/^..../ffff/' | xauth -f $XAUTH nmerge - + chmod 777 $XAUTH + + display_server_opt+=" -v $XAUTH:$XAUTH" + display_server_opt+=" -e XAUTHORITY=$XAUTH" + fi + # DOCKER PARAMETERS # # --rm @@ -1000,7 +1034,7 @@ launch() { # Run the container as a non-root user. See details above for `groups` variable. # # -v /var/run/docker.sock:/var/run/docker.sock - # Enable the use of Holoscan CLI with Docker outside of Docker (DooD) for packaging and + # Enable the use of Holoscan CLI with Docker outside of Docker (DooD) for packaging and # running applications inside the container. # # -v ${TOP}:/workspace/holoscan-sdk @@ -1010,12 +1044,10 @@ launch() { # Start in the build or install directory # # --runtime=nvidia \ - # -e NVIDIA_DRIVER_CAPABILITIES=all # Enable GPU acceleration # - # -v /tmp/.X11-unix:/tmp/.X11-unix - # -e DISPLAY - # Enable graphical applications + # ${display_server_opt} + # Enable graphical applications (X11 or Wayland) # # ${mount_device_opt} # --device /dev/video${i} @@ -1052,12 +1084,9 @@ launch() { ${groups} \ -v /var/run/docker.sock:/var/run/docker.sock \ -v ${mount_point}:${container_mount} \ - ${extra_args[@]} \ -w ${container_top}/${working_dir} \ --runtime=nvidia \ - -e NVIDIA_DRIVER_CAPABILITIES=all \ - -v /tmp/.X11-unix:/tmp/.X11-unix \ - -e DISPLAY \ + ${display_server_opt} \ ${mount_device_opt} \ -e PYTHONPATH=${container_top}/${working_dir}/python/lib \ -e HOLOSCAN_LIB_PATH=${container_top}/${working_dir}/lib \ @@ -1068,6 +1097,7 @@ launch() { --cap-add=CAP_SYS_PTRACE \ --ulimit memlock=-1 \ --ulimit stack=67108864 \ + ${extra_args[@]} \ $img -c "export PATH=\$PATH:${container_top}/${working_dir}/bin; $run_cmd" # Append the Holoscan bin folder to the existing `PATH` before running } @@ -1248,7 +1278,6 @@ run_html_builder() { --entrypoint=bash \ -u $(id -u):$(id -g) \ --runtime=nvidia \ - -e NVIDIA_DRIVER_CAPABILITIES=all \ -v ${TOP}:/workspace/holoscan-sdk \ -w /workspace/holoscan-sdk/${DOCS_SRC_DIR} \ -e PYTHONPATH=/workspace/holoscan-sdk/$(get_install_dir)/python/lib \ diff --git a/runtime_docker/Dockerfile b/runtime_docker/Dockerfile index 2bd0124c..eef651a6 100644 --- a/runtime_docker/Dockerfile +++ b/runtime_docker/Dockerfile @@ -33,6 +33,7 @@ ARG MAX_PROC=32 ARG INSTALL_PATH=/opt/nvidia/holoscan ARG GPU_TYPE=dgpu ARG DEBIAN_FRONTEND=noninteractive +ENV NVIDIA_DRIVER_CAPABILITIES=all ############################################################ # Runtime C++ (no MKL) @@ -47,6 +48,8 @@ FROM base as runtime_cpp_no_mkl # libx* - X packages # libvulkan1 - for Vulkan apps (Holoviz) # libegl1 - to run headless Vulkan apps +# libwayland-client0, libwayland-egl1, libxkbcommon0 - GLFW runtime dependency for Wayland +# libdecor-0-plugin-1-cairo - GLFW runtime dependency for Wayland window decorations # libopenblas0 - libtorch dependency # libnuma1 - libtorch dependency # libgomp1 - libtorch & CuPy dependency @@ -76,6 +79,10 @@ RUN apt-get update \ libxrandr2="2:1.5.2-*" \ libvulkan1="1.3.204.1-*" \ libegl1="1.4.0-*" \ + libwayland-client0="1.20.0-*" \ + libwayland-egl1="1.20.0-*" \ + libxkbcommon0="1.4.0-*" \ + libdecor-0-plugin-1-cairo="0.1.0-*" \ libopenblas0="0.3.20+ds-*" \ libnuma1="2.0.14-*" \ libgomp1="12.3.0-*" \ @@ -95,8 +102,8 @@ RUN apt-get update \ libcusolver-12-2 \ cuda-cupti-12-2 \ cuda-nvtx-12-2 \ - && rm -rf /var/lib/apt/lists/* \ - && rm -f /usr/lib/*/libcudnn*train.so* + && rm -rf /var/lib/apt/lists/* \ + && rm -f /usr/lib/*/libcudnn*train.so* # Copy ONNX Runtime ARG ONNX_RUNTIME_VERSION=1.15.1_23.08 diff --git a/scripts/download_example_data b/scripts/download_example_data index ae64c1f3..55e2d954 100644 --- a/scripts/download_example_data +++ b/scripts/download_example_data @@ -14,12 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This script downloads sample datasets for the examples. -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +set -e + +# This script downloads a sample dataset for some of the examples. # Creates the data directory +# Note: we assume this script is installed in the `examples` folder +SCRIPT_DIR="$(dirname $(realpath $0))" DATA_DIR=${SCRIPT_DIR}/../data -mkdir ${DATA_DIR} +mkdir -p ${DATA_DIR} # Download the racerx sample data racerx_version="20231009" diff --git a/scripts/download_ngc_data b/scripts/download_ngc_data index c9148cb7..b8163fa9 100755 --- a/scripts/download_ngc_data +++ b/scripts/download_ngc_data @@ -13,6 +13,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +set -e + POSITIONAL_ARGS=() # Default values diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1ba73e5b..030191e4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -110,6 +110,8 @@ set(CORE_PROTO_FILES grpc_generate_cpp(CORE_GRPC_SRCS CORE_GRPC_HDRS ${CORE_PROTO_FILES}) add_holoscan_library(core + core/analytics/csv_data_exporter.cpp + core/analytics/data_exporter.cpp core/app_driver.cpp core/app_worker.cpp core/application.cpp @@ -139,6 +141,7 @@ add_holoscan_library(core core/graphs/flow_graph.cpp core/gxf/entity.cpp core/gxf/gxf_component.cpp + core/gxf/gxf_component_info.cpp core/gxf/gxf_condition.cpp core/gxf/gxf_execution_context.cpp core/gxf/gxf_extension_manager.cpp @@ -165,6 +168,7 @@ add_holoscan_library(core core/resources/gxf/double_buffer_receiver.cpp core/resources/gxf/double_buffer_transmitter.cpp core/resources/gxf/dfft_collector.cpp + core/resources/gxf/gxf_component_resource.cpp core/resources/gxf/manual_clock.cpp core/resources/gxf/realtime_clock.cpp core/resources/gxf/receiver.cpp diff --git a/src/core/analytics/csv_data_exporter.cpp b/src/core/analytics/csv_data_exporter.cpp new file mode 100644 index 00000000..d9aa4a33 --- /dev/null +++ b/src/core/analytics/csv_data_exporter.cpp @@ -0,0 +1,84 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "holoscan/core/analytics/csv_data_exporter.hpp" + +#include +#include +#include +#include + +#include + +namespace holoscan { +namespace { +constexpr const char* kAnalyticsDataFileNameEnvVarName = "HOLOSCAN_ANALYTICS_DATA_FILE_NAME"; +} // namespace + +CsvDataExporter::CsvDataExporter(const std::string& app_name, + const std::vector& columns) + : DataExporter(app_name), columns_(columns) { + auto data_file_env = CsvDataExporter::get_analytics_data_file_name_env(); + file_name_ = data_file_env ? data_file_env.value() : kAnalyticsOutputFileName; + std::filesystem::path file_path = std::filesystem::path(directory_name_) / file_name_; + auto absolute_path = std::filesystem::absolute(file_path); + file_ = std::ofstream(file_path, std::ios::app); + + if (file_.is_open()) { + write_row(columns_); + } else { + HOLOSCAN_LOG_ERROR("Error: unable to open file '{}'", absolute_path.string()); + } +} + +CsvDataExporter::~CsvDataExporter() { + file_.close(); +} + +void CsvDataExporter::export_data(const std::vector& data) { + if (data.size() != columns_.size()) { + HOLOSCAN_LOG_ERROR("Error: the number of values ({}) does not match the number of columns ({})", + data.size(), + columns_.size()); + } + write_row(data); +} + +expected CsvDataExporter::get_analytics_data_file_name_env() { + const char* value = std::getenv(kAnalyticsDataFileNameEnvVarName); + if (value && value[0]) { + return value; + } else { + return make_unexpected(ErrorCode::kNotFound); + } +} + +void CsvDataExporter::write_row(const std::vector& data) { + if (file_.is_open()) { + if (!data.empty()) { + auto it = begin(data); + file_ << *it; + ++it; + for (; it != end(data); ++it) { file_ << "," << *it; } + } + file_ << "\n"; + } else { + HOLOSCAN_LOG_ERROR("Error: unable to open file"); + } +} + +} // namespace holoscan diff --git a/src/core/analytics/data_exporter.cpp b/src/core/analytics/data_exporter.cpp new file mode 100644 index 00000000..3d7e2113 --- /dev/null +++ b/src/core/analytics/data_exporter.cpp @@ -0,0 +1,80 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "holoscan/core/analytics/data_exporter.hpp" + +#include +#include +#include +#include +#include + +#include + +namespace holoscan { +namespace { +constexpr const char* kAnalyticsDataDirectoryEnvVarName = "HOLOSCAN_ANALYTICS_DATA_DIRECTORY"; +} // namespace + +DataExporter::DataExporter(const std::string& app_name) : app_name_(app_name) { + create_data_directory_with_timestamp(); +} + +void DataExporter::create_data_directory_with_timestamp() { + auto data_directory_env = DataExporter::get_analytics_data_directory_env(); + const std::string current_dir = std::filesystem::current_path().string(); + std::string data_directory = data_directory_env ? data_directory_env.value() : current_dir; + + auto now = std::chrono::system_clock::now(); + auto local_time = std::chrono::system_clock::to_time_t(now); + auto local_time_str = std::put_time(std::localtime(&local_time), "%Y%m%d%H%M%S"); + + std::ostringstream dir_name; + dir_name << data_directory << "/" << app_name_ << "/" << local_time_str; + directory_name_ = dir_name.str(); + + if (!std::filesystem::exists(directory_name_)) { + std::filesystem::create_directories(directory_name_); + } +} + +expected DataExporter::get_analytics_data_directory_env() { + const char* value = std::getenv(kAnalyticsDataDirectoryEnvVarName); + if (value && value[0]) { + return value; + } else { + return make_unexpected(ErrorCode::kNotFound); + } +} + +void DataExporter::cleanup_data_directory() { + try { + std::filesystem::path dir_path(directory_name_); + dir_path = std::filesystem::absolute(dir_path).parent_path(); + std::uintmax_t removed = std::filesystem::remove_all(dir_path); + if (removed > 0) { + HOLOSCAN_LOG_INFO("Cleaned up {} files", removed); + } else { + HOLOSCAN_LOG_ERROR("Error: the directory ({}) does not exist or remove operation failed", + directory_name_); + } + } catch (const std::filesystem::filesystem_error& e) { + HOLOSCAN_LOG_ERROR("Error: {}", e.what()); + } +} + +} // namespace holoscan diff --git a/src/core/app_driver.cpp b/src/core/app_driver.cpp index c2f8495d..b8353228 100644 --- a/src/core/app_driver.cpp +++ b/src/core/app_driver.cpp @@ -65,6 +65,7 @@ bool AppDriver::get_bool_env_var(const char* name, bool default_value) { value.begin(), value.end(), value.begin(), [](unsigned char c) { return std::tolower(c); }); if (value == "true" || value == "1" || value == "on") { return true; } + if (value == "false" || value == "0" || value == "off") { return false; } } return default_value; @@ -638,6 +639,11 @@ bool AppDriver::collect_connections(holoscan::FragmentGraph& fragment_graph) { std::unordered_set visited_nodes; visited_nodes.reserve(fragments.size()); + // Initialize connection_map_ for each fragment, regardless of whether it has connections + for (auto& node : fragments) { + connection_map_[node] = std::vector>(); + } + // Initialize the indegrees of all nodes in the graph and add root fragments to the worklist. for (auto& node : fragments) { indegrees[node] = fragment_graph.get_previous_nodes(node).size(); diff --git a/src/core/arg.cpp b/src/core/arg.cpp index c85ea11f..8f3223c4 100644 --- a/src/core/arg.cpp +++ b/src/core/arg.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -121,6 +121,10 @@ YAML::Node Arg::to_yaml_node() const { return node; } +YAML::Node Arg::value_to_yaml_node() const { + return any_as_node(value_, arg_type_); +} + std::string Arg::description() const { YAML::Emitter emitter; emitter << to_yaml_node(); diff --git a/src/core/component_spec.cpp b/src/core/component_spec.cpp index e5228e4d..41c6ba1d 100644 --- a/src/core/component_spec.cpp +++ b/src/core/component_spec.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -46,8 +46,8 @@ YAML::Node ComponentSpec::to_yaml_node() const { YAML::Node param_node; param_node["name"] = name; param_node["type"] = type; - param_node["description"] = param->description(); - param_node["flag"] = parameterflag_namemap[param->flag()]; + param_node["description"] = param ? param->description() : ""; + param_node["flag"] = parameterflag_namemap[param ? param->flag() : ParameterFlag::kNone]; node["params"].push_back(param_node); } return node; diff --git a/src/core/domain/tensor.cpp b/src/core/domain/tensor.cpp index b12bef18..a3531a42 100644 --- a/src/core/domain/tensor.cpp +++ b/src/core/domain/tensor.cpp @@ -34,6 +34,20 @@ Tensor::Tensor(DLManagedTensor* dl_managed_tensor_ptr) { dl_managed_tensor = *dl_managed_tensor_ptr; } +bool Tensor::is_contiguous() const { + int32_t r = static_cast(ndim()); // rank + int64_t expected_stride = itemsize(); // size of a single element + auto tensor_strides = strides(); + auto tensor_shape = shape(); + for (int32_t i = r - 1; i >= 0; --i) { + int64_t s = tensor_strides[i]; // stride + int64_t d = static_cast(tensor_shape[i]); // dimension + if (s != expected_stride) { return false; } + expected_stride *= d; + } + return true; +} + DLManagedTensor* Tensor::to_dlpack() { auto dl_managed_tensor_ctx = new DLManagedTensorContext; auto& dl_managed_tensor = dl_managed_tensor_ctx->tensor; diff --git a/src/core/executors/gxf/gxf_executor.cpp b/src/core/executors/gxf/gxf_executor.cpp index 0604501c..a7f6fffb 100644 --- a/src/core/executors/gxf/gxf_executor.cpp +++ b/src/core/executors/gxf/gxf_executor.cpp @@ -68,6 +68,7 @@ #include "holoscan/core/signal_handler.hpp" #include "gxf/app/arg.hpp" +#include "gxf/std/clock.hpp" #include "gxf/std/default_extension.hpp" #include "gxf/std/extension_factory_helper.hpp" #include "gxf/std/monitor.hpp" @@ -1017,6 +1018,9 @@ void GXFExecutor::connect_broadcast_to_previous_op( // counter to ensure unique broadcast component names as required by nvidia::gxf::GraphEntity static uint32_t btx_count = 0; + HOLOSCAN_LOG_DEBUG( + "connecting broadcast codelet from previous_op {} to op {}", prev_op->name(), op->name()); + // A Broadcast component was added for prev_op for (const auto& [port_name, broadcast_entity] : broadcast_entities.at(prev_op)) { // Find the Broadcast component's source port name in the port-map. @@ -1188,97 +1192,96 @@ void GXFExecutor::create_broadcast_components(holoscan::OperatorGraph::NodeType if (target_ports.empty()) { HOLOSCAN_LOG_ERROR("No target component found for source_id: {}", source_cid); continue; + } else if (target_ports.size() == 1) { + continue; } - // Insert GXF's Broadcast component if source port is connected to multiple targets - if (target_ports.size() > 1) { - std::string rx_type_name; - - uint64_t curr_min_size = 1; - uint64_t curr_connector_capacity = 1; - uint64_t curr_connector_policy = 2; // fault - - // Create a corresponding condition of the op's output port and set it as the - // receiver's condition for the broadcast entity. - auto& op_io_spec = op->spec()->outputs()[source_cname]; - - // 1. Find the output port's condition. - // (ConditionType::kDownstreamMessageAffordable) - std::shared_ptr curr_condition; - for (auto& [condition_type, condition] : op_io_spec->conditions()) { - if (condition_type == ConditionType::kDownstreamMessageAffordable) { - curr_condition = condition; - break; - } - } - // 2. If it exists, store min_size parameter of the condition. - if (curr_condition) { - auto curr_downstream_condition = - std::dynamic_pointer_cast(curr_condition); - curr_min_size = curr_downstream_condition->min_size(); - } - - auto broadcast_entity = std::make_shared(); - auto broadcast_entity_name = - fmt::format("{}_broadcast_{}_{}", entity_prefix, op_name, source_cname); - auto maybe = broadcast_entity->setup(context, broadcast_entity_name.c_str()); - if (!maybe) { - throw std::runtime_error( - fmt::format("Failed to create broadcast entity: '{}'", broadcast_entity_name)); + std::string rx_type_name; + + uint64_t curr_min_size = 1; + uint64_t curr_connector_capacity = 1; + uint64_t curr_connector_policy = 2; // fault + + // Create a corresponding condition of the op's output port and set it as the + // receiver's condition for the broadcast entity. + auto& op_io_spec = op->spec()->outputs()[source_cname]; + + // 1. Find the output port's condition. + // (ConditionType::kDownstreamMessageAffordable) + std::shared_ptr curr_condition; + for (auto& [condition_type, condition] : op_io_spec->conditions()) { + if (condition_type == ConditionType::kDownstreamMessageAffordable) { + curr_condition = condition; + break; } - // Add the broadcast_entity to the list of implicit broadcast entities - implicit_broadcast_entities_.push_back(broadcast_entity); + } + // 2. If it exists, store min_size parameter of the condition. + if (curr_condition) { + auto curr_downstream_condition = + std::dynamic_pointer_cast(curr_condition); + curr_min_size = curr_downstream_condition->min_size(); + } - // Add the broadcast_entity for the current operator and the source port name - broadcast_entities[op][source_cname] = broadcast_entity; + auto broadcast_entity = std::make_shared(); + auto broadcast_entity_name = + fmt::format("{}_broadcast_{}_{}", entity_prefix, op_name, source_cname); + auto maybe = broadcast_entity->setup(context, broadcast_entity_name.c_str()); + if (!maybe) { + throw std::runtime_error( + fmt::format("Failed to create broadcast entity: '{}'", broadcast_entity_name)); + } + // Add the broadcast_entity to the list of implicit broadcast entities + implicit_broadcast_entities_.push_back(broadcast_entity); - switch (connector_type) { - case IOSpec::ConnectorType::kDefault: - case IOSpec::ConnectorType::kDoubleBuffer: - case IOSpec::ConnectorType::kUCX: // In any case, need to add doubleBufferReceiver. - { - // We don't create a holoscan::AnnotatedDoubleBufferReceiver even if data flow - // tracking is on because we don't want to mark annotations for the Broadcast - // component. - rx_type_name = "nvidia::gxf::DoubleBufferReceiver"; - auto curr_tx_handle = - op->graph_entity()->get(source_cname.c_str()); - if (curr_tx_handle.is_null()) { - HOLOSCAN_LOG_ERROR( - "Failed to get nvidia::gxf::DoubleBufferTransmitter, a default receive capacity " - "and policy will be used for the inserted broadcast component."); - } else { - HOLOSCAN_LOG_TRACE("getting capacity and policy from curr_tx_handle"); - auto p = get_capacity_and_policy(curr_tx_handle); - curr_connector_capacity = p.first; - curr_connector_policy = p.second; - } - } break; - default: - HOLOSCAN_LOG_ERROR("Unrecognized connector_type '{}' for source name '{}'", - static_cast(connector_type), - source_cname); - } - auto broadcast_component_name = - fmt::format("{}_broadcast_component_{}_{}", entity_prefix, op_name, source_cname); - auto broadcast_codelet = - broadcast_entity->addCodelet("nvidia::gxf::Broadcast", broadcast_component_name.c_str()); - if (broadcast_codelet.is_null()) { - HOLOSCAN_LOG_ERROR("Failed to create broadcast codelet for entity: {}", - broadcast_entity->name()); - } - // Broadcast component's receiver Parameter is named "source" so have to use that here - auto broadcast_rx = broadcast_entity->addReceiver(rx_type_name.c_str(), "source"); - if (broadcast_rx.is_null()) { - HOLOSCAN_LOG_ERROR("Failed to create receiver for broadcast component: {}", - broadcast_entity->name()); - } - broadcast_entity->configReceiver( - "source", curr_connector_capacity, curr_connector_policy, curr_min_size); + // Add the broadcast_entity for the current operator and the source port name + broadcast_entities[op][source_cname] = broadcast_entity; - // Connect Broadcast entity's receiver with the transmitter of the current operator - add_connection(source_cid, broadcast_rx->cid()); + switch (connector_type) { + case IOSpec::ConnectorType::kDefault: + case IOSpec::ConnectorType::kDoubleBuffer: + case IOSpec::ConnectorType::kUCX: // In any case, need to add doubleBufferReceiver. + { + // We don't create a holoscan::AnnotatedDoubleBufferReceiver even if data flow + // tracking is on because we don't want to mark annotations for the Broadcast + // component. + rx_type_name = "nvidia::gxf::DoubleBufferReceiver"; + auto curr_tx_handle = + op->graph_entity()->get(source_cname.c_str()); + if (curr_tx_handle.is_null()) { + HOLOSCAN_LOG_ERROR( + "Failed to get nvidia::gxf::DoubleBufferTransmitter, a default receive capacity " + "and policy will be used for the inserted broadcast component."); + } else { + HOLOSCAN_LOG_TRACE("getting capacity and policy from curr_tx_handle"); + auto p = get_capacity_and_policy(curr_tx_handle); + curr_connector_capacity = p.first; + curr_connector_policy = p.second; + } + } break; + default: + HOLOSCAN_LOG_ERROR("Unrecognized connector_type '{}' for source name '{}'", + static_cast(connector_type), + source_cname); + } + auto broadcast_component_name = + fmt::format("{}_broadcast_component_{}_{}", entity_prefix, op_name, source_cname); + auto broadcast_codelet = + broadcast_entity->addCodelet("nvidia::gxf::Broadcast", broadcast_component_name.c_str()); + if (broadcast_codelet.is_null()) { + HOLOSCAN_LOG_ERROR("Failed to create broadcast codelet for entity: {}", + broadcast_entity->name()); + } + // Broadcast component's receiver Parameter is named "source" so have to use that here + auto broadcast_rx = broadcast_entity->addReceiver(rx_type_name.c_str(), "source"); + if (broadcast_rx.is_null()) { + HOLOSCAN_LOG_ERROR("Failed to create receiver for broadcast component: {}", + broadcast_entity->name()); } + broadcast_entity->configReceiver( + "source", curr_connector_capacity, curr_connector_policy, curr_min_size); + + // Connect Broadcast entity's receiver with the transmitter of the current operator + add_connection(source_cid, broadcast_rx->cid()); } } @@ -1388,8 +1391,8 @@ bool GXFExecutor::initialize_fragment() { const auto& port_map_val = port_map.value(); // If the previous operator is found to be one that is connected to the current operator via - // the Broadcast component, then add the connection between the Broadcast component and the - // current operator's input port. + // a Broadcast component, then add the connection between the Broadcast component and the + // current operator's input port. Only ports with inter-fragment connections use broadcasting. if (broadcast_entities.find(prev_op) != broadcast_entities.end()) { // Add transmitter to the prev_op's broadcast component and connect it to op's input port. // Any connected ports are removed from port_map_val. @@ -1418,33 +1421,34 @@ bool GXFExecutor::initialize_fragment() { source_cid = source_gxf_resource->gxf_cid(); } - if (target_ports.size() > 1) { - HOLOSCAN_LOG_ERROR( - "Source port is connected to multiple target ports without Broadcast component. " - "Op: {} source name: {}", - op->name(), - source_port); - return false; - } // GXF Connection component should not be added for types using a NetworkContext auto connector_type = prev_op->spec()->outputs()[source_port]->connector_type(); if (connector_type != IOSpec::ConnectorType::kUCX) { - const auto& target_port = target_ports.begin(); - auto target_gxf_resource = std::dynamic_pointer_cast( - op->spec()->inputs()[*target_port]->connector()); - // For cycles, a previous operator may not have been initialized yet, so we don't - // connect them here. We connect them as a forward/downstream connection when we - // visit the operator which is connected to the current operator. - if (prev_op->id() != -1) { - gxf_uid_t target_cid = target_gxf_resource->gxf_cid(); - add_connection(source_cid, target_cid); - HOLOSCAN_LOG_DEBUG( - "Connected directly source : {} -> target : {}", source_port, *target_port); - } else { - HOLOSCAN_LOG_DEBUG("Connection source: {} -> target: {} will be added later", - source_port, - *target_port); + // const auto& target_port = target_ports.begin(); + for (const auto& target_port : target_ports) { + auto target_gxf_resource = std::dynamic_pointer_cast( + op->spec()->inputs()[target_port]->connector()); + // For cycles, a previous operator may not have been initialized yet, so we don't + // connect them here. We connect them as a forward/downstream connection when we + // visit the operator which is connected to the current operator. + if (prev_op->id() != -1) { + gxf_uid_t target_cid = target_gxf_resource->gxf_cid(); + add_connection(source_cid, target_cid); + HOLOSCAN_LOG_DEBUG( + "Connected directly source : {} -> target : {}", source_port, target_port); + } else { + HOLOSCAN_LOG_DEBUG("Connection source: {} -> target: {} will be added later", + source_port, + target_port); + } } + } else if (target_ports.size() > 1) { + HOLOSCAN_LOG_ERROR( + "Source port with UCX connector is connected to multiple target ports without a " + "Broadcast component. Op: {} source name: {}", + op->name(), + source_port); + return false; } } } @@ -1503,23 +1507,24 @@ bool GXFExecutor::initialize_fragment() { } // Iterate through downstream connections and find the direct ones to connect, only if // downstream operator is already initialized. This is to handle cycles in the graph. + bool target_op_has_ucx_connector = false; for (auto [source_cid, target_info] : connections) { auto& [source_cname, connector_type, target_ports] = target_info; - if (target_ports.size() == 1) { - // There is a direct connection without Broadcast - // Check if next op is already initialized, that means, it's a cycle and we can add the - // connection now - auto tmp_next_op = target_ports.begin()->first; - if (tmp_next_op->id() != -1 && connector_type != IOSpec::ConnectorType::kUCX) { + if (connector_type == IOSpec::ConnectorType::kUCX) { + target_op_has_ucx_connector = true; + continue; // Connection components are only for non-UCX connections + } + for (auto& [tmp_next_op, target_port_name] : target_ports) { + if (tmp_next_op->id() != -1) { // Operator is already initialized HOLOSCAN_LOG_DEBUG("next op {} is already initialized, due to a cycle.", tmp_next_op->name()); auto target_gxf_resource = std::dynamic_pointer_cast( - tmp_next_op->spec()->inputs()[target_ports.begin()->second]->connector()); + tmp_next_op->spec()->inputs()[target_port_name]->connector()); gxf_uid_t target_cid = target_gxf_resource->gxf_cid(); add_connection(source_cid, target_cid); HOLOSCAN_LOG_TRACE( - "Next Op {} is connected to the current Op {} as a downstream connection due to a " + "Next Op {} is connected to the current Op {} as a downstream connection due to a " "cycle.", tmp_next_op->name(), op_name); @@ -1527,27 +1532,41 @@ bool GXFExecutor::initialize_fragment() { } } - // Create the Broadcast components and add their IDs to broadcast_entities, but do not add any - // transmitter to the Broadcast entity. The transmitters will be added later when the incoming - // edges to the respective operators are processed. - create_broadcast_components(op, broadcast_entities, connections); - if (op_type != Operator::OperatorType::kVirtual) { + if (!target_op_has_ucx_connector) { for (auto& next_op : graph.get_next_nodes(op)) { - if (next_op->id() != -1 && next_op->operator_type() != Operator::OperatorType::kVirtual) { - HOLOSCAN_LOG_DEBUG( - "next_op of {} is {}. It is already initialized.", op_name, next_op->name()); - // next operator is already initialized, so connect the broadcast component to the next - // operator's input port, if there is any - auto port_map = graph.get_port_map(op, next_op); - if (!port_map.has_value()) { - HOLOSCAN_LOG_ERROR("Could not find port map for {} -> {}", op_name, next_op->name()); - return false; - } - if (broadcast_entities.find(op) != broadcast_entities.end()) { - connect_broadcast_to_previous_op(broadcast_entities, next_op, op, port_map.value()); + if (next_op->operator_type() == Operator::OperatorType::kVirtual) { + target_op_has_ucx_connector = true; + break; + } + } + } + + if (target_op_has_ucx_connector) { + HOLOSCAN_LOG_DEBUG("At least one target of op {} has a UCX connector.", op_name); + // Create the Broadcast components and add their IDs to broadcast_entities, but do not add + // any transmitter to the Broadcast entity. The transmitters will be added later when the + // incoming edges to the respective operators are processed. + create_broadcast_components(op, broadcast_entities, connections); + if (op_type != Operator::OperatorType::kVirtual) { + for (auto& next_op : graph.get_next_nodes(op)) { + if (next_op->id() != -1 && next_op->operator_type() != Operator::OperatorType::kVirtual) { + HOLOSCAN_LOG_DEBUG( + "next_op of {} is {}. It is already initialized.", op_name, next_op->name()); + // next operator is already initialized, so connect the broadcast component to the + // next operator's input port, if there is any + auto port_map = graph.get_port_map(op, next_op); + if (!port_map.has_value()) { + HOLOSCAN_LOG_ERROR("Could not find port map for {} -> {}", op_name, next_op->name()); + return false; + } + if (broadcast_entities.find(op) != broadcast_entities.end()) { + connect_broadcast_to_previous_op(broadcast_entities, next_op, op, port_map.value()); + } } } } + } else { + HOLOSCAN_LOG_DEBUG("No target of op {} has a UCX connector.", op_name); } } return true; @@ -1701,6 +1720,42 @@ bool GXFExecutor::initialize_gxf_graph(OperatorGraph& graph) { "Failed to create entity to hold nvidia::gxf::Connection components."); } + bool job_stats_enabled = + AppDriver::get_bool_env_var("HOLOSCAN_ENABLE_GXF_JOB_STATISTICS", false); + if (job_stats_enabled) { + auto clock = util_entity_->add("jobstats_clock", {}); + if (clock) { + bool codelet_statistics = + AppDriver::get_bool_env_var("HOLOSCAN_GXF_JOB_STATISTICS_CODELET", false); + uint32_t event_history_count = + AppDriver::get_int_env_var("HOLOSCAN_GXF_JOB_STATISTICS_COUNT", 100u); + + // GXF issue 4552622: can't create FilePath Arg, so we call setParameter below instead + std::vector jobstats_args{ + nvidia::gxf::Arg("clock", clock), + nvidia::gxf::Arg("codelet_statistics", codelet_statistics), + nvidia::gxf::Arg("event_history_count", event_history_count)}; + + std::string json_file_path = ""; // default value is no JSON output + const char* path_env_value = std::getenv("HOLOSCAN_GXF_JOB_STATISTICS_PATH"); + if (path_env_value && path_env_value[0] != '\0') { + jobstats_args.push_back( + nvidia::gxf::Arg("json_file_path", nvidia::gxf::FilePath(path_env_value))); + } + + HOLOSCAN_LOG_DEBUG("GXF JobStatistics enabled with:"); + HOLOSCAN_LOG_DEBUG(" codelet report: {}", codelet_statistics); + HOLOSCAN_LOG_DEBUG(" event_history_count: {}", event_history_count); + HOLOSCAN_LOG_DEBUG(" json_file_path: {}", json_file_path); + auto stats = + util_entity_->addComponent("nvidia::gxf::JobStatistics", "jobstats", jobstats_args); + if (!stats) { HOLOSCAN_LOG_ERROR("Failed to create JobStatistics component."); } + } else { + HOLOSCAN_LOG_ERROR( + "Failed to create clock for job statistics (statistics will not be collected)."); + } + } + auto scheduler = std::dynamic_pointer_cast(fragment_->scheduler()); scheduler->initialize(); // will call GXFExecutor::initialize_scheduler @@ -2066,7 +2121,12 @@ bool GXFExecutor::add_condition_to_graph_entity( if (condition && graph_entity) { add_component_args_to_graph_entity(condition->args(), graph_entity); auto gxf_condition = std::dynamic_pointer_cast(condition); - + if (!gxf_condition) { + // Non-GXF condition isn't supported, so log an error if this unexpected path is reached. + HOLOSCAN_LOG_ERROR("Failed to cast condition '{}' to holoscan::gxf::GXFCondition", + condition->name()); + return false; + } // do not overwrite previous graph entity if this condition is already associated with one if (gxf_condition && !gxf_condition->gxf_graph_entity()) { HOLOSCAN_LOG_TRACE( @@ -2075,10 +2135,6 @@ bool GXFExecutor::add_condition_to_graph_entity( gxf_condition->gxf_graph_entity(graph_entity); // Don't have to call initialize() here, ArgumentSetter already calls it later. return true; - } else { - // Non-GXF condition isn't supported, so log an error if this unexpected path is reached. - HOLOSCAN_LOG_ERROR("Failed to cast condition '{}' to holoscan::gxf::GXFCondition", - condition->name()); } } return false; @@ -2090,6 +2146,7 @@ bool GXFExecutor::add_resource_to_graph_entity( add_component_args_to_graph_entity(resource->args(), graph_entity); // Native Resources will not be added to the GraphEntity auto gxf_resource = std::dynamic_pointer_cast(resource); + // don't raise error if the pointer cast failed as that is expected for native Resource types // do not overwrite previous graph entity if this resource is already associated with one // (e.g. sometimes the same allocator may be used across multiple operators) diff --git a/src/core/gxf/gxf_component_info.cpp b/src/core/gxf/gxf_component_info.cpp new file mode 100644 index 00000000..48e4a448 --- /dev/null +++ b/src/core/gxf/gxf_component_info.cpp @@ -0,0 +1,212 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "holoscan/core/gxf/gxf_component_info.hpp" + +#include +#include +#include + +#include "holoscan/core/gxf/gxf_utils.hpp" + +namespace holoscan::gxf { + +ComponentInfo::ComponentInfo(gxf_context_t context, gxf_tid_t tid) + : gxf_context_(context), component_tid_(tid) { + // Initialize component info + component_info_.parameters = new const char*[MAX_PARAM_COUNT]; + component_info_.num_parameters = MAX_PARAM_COUNT; + // This call will set the true number of parameters (will error if it exceeds MAX_PARAM_COUNT) + HOLOSCAN_GXF_CALL_FATAL(GxfComponentInfo(gxf_context_, component_tid_, &component_info_)); + + // Get the number of parameters + const auto num_parameters = component_info_.num_parameters; + + // Allocate space for parameter info + parameter_infos_.resize(num_parameters); + parameter_info_map_.reserve(num_parameters); + transmitter_parameters_.reserve(num_parameters); + receiver_parameters_.reserve(num_parameters); + normal_parameters_.reserve(num_parameters); + + // Process each parameter + for (size_t i = 0; i < num_parameters; ++i) { + // Add the parameter key to the list + parameter_keys_.push_back(component_info_.parameters[i]); + + // Get the parameter info and add it to the map + HOLOSCAN_GXF_CALL_FATAL(GxfGetParameterInfo( + gxf_context_, component_tid_, component_info_.parameters[i], ¶meter_infos_[i])); + parameter_info_map_[std::string(component_info_.parameters[i])] = parameter_infos_[i]; + + // Determine if the parameter is a transmitter or receiver + bool is_transmitter = false; + bool is_receiver = false; + if (parameter_infos_[i].type == GXF_PARAMETER_TYPE_HANDLE) { + // Check if the parameter is a transmitter + if (parameter_infos_[i].handle_tid == transmitter_tid()) { + is_transmitter = true; + } else { + HOLOSCAN_GXF_CALL_FATAL(GxfComponentIsBase( + gxf_context_, parameter_infos_[i].handle_tid, transmitter_tid(), &is_transmitter)); + } + + // If not a transmitter, check if the parameter is a receiver + if (!is_transmitter && parameter_infos_[i].handle_tid == receiver_tid()) { + is_receiver = true; + } else if (!is_transmitter) { + HOLOSCAN_GXF_CALL_FATAL(GxfComponentIsBase( + gxf_context_, parameter_infos_[i].handle_tid, receiver_tid(), &is_receiver)); + } + } + + // Add the parameter to the appropriate list + if (is_transmitter) { + transmitter_parameters_.push_back(component_info_.parameters[i]); + } else if (is_receiver) { + receiver_parameters_.push_back(component_info_.parameters[i]); + } else { + normal_parameters_.push_back(component_info_.parameters[i]); + } + } +} + +ComponentInfo::~ComponentInfo() { + if (component_info_.parameters) { delete[] component_info_.parameters; } +} + +ArgType ComponentInfo::get_arg_type(const gxf_parameter_info_t& param) { + ArgElementType element_type = ArgElementType::kCustom; + + switch (param.type) { + case GXF_PARAMETER_TYPE_CUSTOM: + element_type = ArgElementType::kCustom; + break; + case GXF_PARAMETER_TYPE_HANDLE: + element_type = ArgElementType::kResource; + break; + case GXF_PARAMETER_TYPE_STRING: + element_type = ArgElementType::kString; + break; + case GXF_PARAMETER_TYPE_INT64: + element_type = ArgElementType::kInt64; + break; + case GXF_PARAMETER_TYPE_UINT64: + element_type = ArgElementType::kUnsigned64; + break; + case GXF_PARAMETER_TYPE_FLOAT64: + element_type = ArgElementType::kFloat64; + break; + case GXF_PARAMETER_TYPE_BOOL: + element_type = ArgElementType::kBoolean; + break; + case GXF_PARAMETER_TYPE_INT32: + element_type = ArgElementType::kInt32; + break; + case GXF_PARAMETER_TYPE_FILE: + element_type = ArgElementType::kString; + break; + case GXF_PARAMETER_TYPE_INT8: + element_type = ArgElementType::kInt8; + break; + case GXF_PARAMETER_TYPE_INT16: + element_type = ArgElementType::kInt16; + break; + case GXF_PARAMETER_TYPE_UINT8: + element_type = ArgElementType::kUnsigned8; + break; + case GXF_PARAMETER_TYPE_UINT16: + element_type = ArgElementType::kUnsigned16; + break; + case GXF_PARAMETER_TYPE_UINT32: + element_type = ArgElementType::kUnsigned32; + break; + case GXF_PARAMETER_TYPE_FLOAT32: + element_type = ArgElementType::kFloat32; + break; + case GXF_PARAMETER_TYPE_COMPLEX64: + element_type = ArgElementType::kComplex64; + break; + case GXF_PARAMETER_TYPE_COMPLEX128: + element_type = ArgElementType::kComplex128; + break; + default: + element_type = ArgElementType::kCustom; + break; + } + ArgContainerType container_type = ArgContainerType::kNative; + int32_t dimension = param.rank; + if (dimension > 0) { + if (param.shape[0] != -1) { + container_type = ArgContainerType::kArray; + } else { + container_type = ArgContainerType::kVector; + } + } + return ArgType{element_type, container_type, dimension}; +} + +gxf_tid_t ComponentInfo::receiver_tid() const { + static gxf_tid_t receiver_tid = GxfTidNull(); + if (receiver_tid == GxfTidNull()) { + HOLOSCAN_GXF_CALL_FATAL( + GxfComponentTypeId(gxf_context_, "nvidia::gxf::Receiver", &receiver_tid)); + } + + return receiver_tid; +} + +gxf_tid_t ComponentInfo::transmitter_tid() const { + static gxf_tid_t transmitter_tid = GxfTidNull(); + if (transmitter_tid == GxfTidNull()) { + HOLOSCAN_GXF_CALL_FATAL( + GxfComponentTypeId(gxf_context_, "nvidia::gxf::Transmitter", &transmitter_tid)); + } + + return transmitter_tid; +} + +const gxf_component_info_t& ComponentInfo::component_info() const { + return component_info_; +} + +const std::vector& ComponentInfo::parameter_keys() const { + return parameter_keys_; +} + +const std::vector& ComponentInfo::parameter_infos() const { + return parameter_infos_; +} + +const std::unordered_map& ComponentInfo::parameter_info_map() + const { + return parameter_info_map_; +} + +const std::vector& ComponentInfo::receiver_parameters() const { + return receiver_parameters_; +} + +const std::vector& ComponentInfo::transmitter_parameters() const { + return transmitter_parameters_; +} + +const std::vector& ComponentInfo::normal_parameters() const { + return normal_parameters_; +} + +} // namespace holoscan::gxf diff --git a/src/core/gxf/gxf_io_context.cpp b/src/core/gxf/gxf_io_context.cpp index f6dbccb9..f998b022 100644 --- a/src/core/gxf/gxf_io_context.cpp +++ b/src/core/gxf/gxf_io_context.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -31,7 +31,7 @@ namespace holoscan::gxf { -nvidia::gxf::Receiver* get_gxf_receiver(const std::unique_ptr& input_spec) { +nvidia::gxf::Receiver* get_gxf_receiver(const std::shared_ptr& input_spec) { auto connector = input_spec->connector(); auto gxf_resource = std::dynamic_pointer_cast(connector); if (gxf_resource == nullptr) { @@ -51,7 +51,7 @@ GXFInputContext::GXFInputContext(ExecutionContext* execution_context, Operator* : InputContext(execution_context, op) {} GXFInputContext::GXFInputContext(ExecutionContext* execution_context, Operator* op, - std::unordered_map>& inputs) + std::unordered_map>& inputs) : InputContext(execution_context, op, inputs) {} gxf_context_t GXFInputContext::gxf_context() const { @@ -139,7 +139,7 @@ GXFOutputContext::GXFOutputContext(ExecutionContext* execution_context, Operator GXFOutputContext::GXFOutputContext( ExecutionContext* execution_context, Operator* op, - std::unordered_map>& outputs) + std::unordered_map>& outputs) : OutputContext(execution_context, op, outputs) {} gxf_context_t GXFOutputContext::gxf_context() const { @@ -191,7 +191,7 @@ void GXFOutputContext::emit_impl(std::any data, const char* name, OutputType out } } - const std::unique_ptr& output_spec = it->second; + const std::shared_ptr& output_spec = it->second; auto connector = output_spec->connector(); auto gxf_resource = std::dynamic_pointer_cast(connector); diff --git a/src/core/gxf/gxf_operator.cpp b/src/core/gxf/gxf_operator.cpp index ec3fc3d4..e11f027d 100644 --- a/src/core/gxf/gxf_operator.cpp +++ b/src/core/gxf/gxf_operator.cpp @@ -29,15 +29,15 @@ void GXFOperator::initialize() { gxf_uid_t GXFOperator::add_codelet_to_graph_entity() { HOLOSCAN_LOG_TRACE("calling graph_entity()->addCodelet for {}", name_); if (!graph_entity_) { throw std::runtime_error("graph entity is not initialized"); } - auto codelet_handle = graph_entity_->addCodelet(gxf_typename(), name_.c_str()); - if (!codelet_handle) { + codelet_handle_ = graph_entity_->addCodelet(gxf_typename(), name_.c_str()); + if (!codelet_handle_) { throw std::runtime_error("Failed to add codelet of type " + std::string(gxf_typename())); } - gxf_uid_t codelet_cid = codelet_handle->cid(); + gxf_uid_t codelet_cid = codelet_handle_->cid(); gxf_eid_ = graph_entity_->eid(); gxf_cid_ = codelet_cid; gxf_context_ = graph_entity_->context(); - HOLOSCAN_LOG_TRACE("\tadded codelet with cid = {}", codelet_handle->cid()); + HOLOSCAN_LOG_TRACE("\tadded codelet with cid = {}", codelet_handle_->cid()); return codelet_cid; } diff --git a/src/core/gxf/gxf_resource.cpp b/src/core/gxf/gxf_resource.cpp index 546ca0c1..4db484b6 100644 --- a/src/core/gxf/gxf_resource.cpp +++ b/src/core/gxf/gxf_resource.cpp @@ -49,10 +49,8 @@ void GXFResource::initialize() { return; } - // Set resource type before calling Resource::initialize() + // Set resource type and gxf context before calling Resource::initialize() resource_type_ = holoscan::Resource::ResourceType::kGXF; - - Resource::initialize(); auto& executor = fragment()->executor(); auto gxf_executor = dynamic_cast(&executor); if (gxf_executor == nullptr) { @@ -91,14 +89,32 @@ void GXFResource::initialize() { return; } - update_params_from_args(); + // Resource::initialize() is called after GXFComponent::gxf_initialize() to ensure that the + // component is initialized before setting parameters. + Resource::initialize(); +} - static gxf_tid_t allocator_tid = GxfTidNull(); // issue 4336947 +void GXFResource::add_to_graph_entity(Operator* op) { + if (gxf_context_ == nullptr) { + // cannot reassign to a different graph entity if the resource was already initialized with GXF + if (gxf_graph_entity_ && is_initialized_) { return; } + + gxf_graph_entity_ = op->graph_entity(); + fragment_ = op->fragment(); + if (gxf_graph_entity_) { + gxf_context_ = gxf_graph_entity_->context(); + gxf_eid_ = gxf_graph_entity_->eid(); + } + } + this->initialize(); +} + +void GXFResource::set_parameters() { + update_params_from_args(); // Set Handler parameters for (auto& [key, param_wrap] : spec_->params()) { // Issue 4336947: dev_id parameter for allocator needs to be handled manually - bool dev_id_handled = false; if (key.compare(std::string("dev_id")) == 0) { if (!gxf_graph_entity_) { HOLOSCAN_LOG_ERROR( @@ -107,92 +123,83 @@ void GXFResource::initialize() { "will be used"); continue; } - gxf_tid_t derived_tid = GxfTidNull(); - bool is_derived = false; - gxf_result_t tid_result; - tid_result = GxfComponentTypeId(gxf_context_, gxf_typename(), &derived_tid); - if (tid_result != GXF_SUCCESS) { - HOLOSCAN_LOG_ERROR( - "Unable to get component type id of '{}': {}", gxf_typename(), tid_result); - } - if (GxfTidIsNull(allocator_tid)) { - tid_result = GxfComponentTypeId(gxf_context_, "nvidia::gxf::Allocator", &allocator_tid); - if (tid_result != GXF_SUCCESS) { - HOLOSCAN_LOG_ERROR("Unable to get component type id of 'nvidia::gxf::Allocator': {}", - tid_result); - } - } - tid_result = GxfComponentIsBase(gxf_context_, derived_tid, allocator_tid, &is_derived); - if (tid_result != GXF_SUCCESS) { - HOLOSCAN_LOG_ERROR( - "Unable to get determine if '{}' is derived from 'nvidia::gxf::Allocator': {}", - gxf_typename(), - tid_result); - } - if (is_derived) { - HOLOSCAN_LOG_DEBUG( - "The dev_id parameter is deprecated by GXF and will be removed from " - "Holoscan SDK in the future."); - + std::optional dev_id_value; + try { auto dev_id_param = *std::any_cast*>(param_wrap.value()); - if (dev_id_param.has_value()) { - int32_t device_id = dev_id_param.get(); - - auto devices = gxf_graph_entity_->findAll(); - if (devices.size() > 0) { - HOLOSCAN_LOG_WARN("Existing entity already has a GPUDevice resource"); - } - - // Create an EntityGroup to associate the GPUDevice with this resource - std::string entity_group_name = - fmt::format("{}_eid{}_dev_id{}_group", name(), gxf_eid_, device_id); - auto entity_group_gid = - ::holoscan::gxf::add_entity_group(gxf_context_, entity_group_name); - - // Add GPUDevice component to the same entity as this resource - // TODO (GXF4): requested an addResource method to handle nvidia::gxf::ResourceBase types - std::string device_component_name = - fmt::format("{}_eid{}_gpu_device_id{}_component", name(), gxf_eid_, device_id); - auto dev_handle = - gxf_graph_entity_->addComponent("nvidia::gxf::GPUDevice", - device_component_name.c_str(), - {nvidia::gxf::Arg("dev_id", device_id)}); - if (dev_handle.is_null()) { - HOLOSCAN_LOG_ERROR("Failed to create GPUDevice for resource '{}'", name_); - } else { - // TODO: warn and handle case if the resource was already in a different entity group - - // The GPUDevice and this resource have the same eid. - // Make their eid is added to the newly created entity group. - GXF_ASSERT_SUCCESS(GxfUpdateEntityGroup(gxf_context_, entity_group_gid, gxf_eid_)); - } - dev_id_handled = true; - } + if (dev_id_param.has_value()) { dev_id_value = dev_id_param.get(); } + } catch (const std::bad_any_cast& e) { + HOLOSCAN_LOG_ERROR("Cannot cast dev_id argument to int32_t: {}}", e.what()); } + bool dev_id_handled = handle_dev_id(dev_id_value); + if (dev_id_handled) { continue; } } - HOLOSCAN_LOG_TRACE( - "GXF component '{}' of type '{}': setting GXF parameter '{}'", name_, gxf_typename(), key); - if (!dev_id_handled) { set_gxf_parameter(name_, key, param_wrap); } - // TODO: handle error - HOLOSCAN_LOG_TRACE("GXFResource '{}':: setting GXF parameter '{}'", name(), key); - } - is_initialized_ = true; + set_gxf_parameter(name_, key, param_wrap); + } } -void GXFResource::add_to_graph_entity(Operator* op) { - if (gxf_context_ == nullptr) { - // cannot reassign to a different graph entity if the resource was already initialized with GXF - if (gxf_graph_entity_ && is_initialized_) { return; } +bool GXFResource::handle_dev_id(std::optional& dev_id_value) { + static gxf_tid_t allocator_tid = GxfTidNull(); // issue 4336947 - gxf_graph_entity_ = op->graph_entity(); - fragment_ = op->fragment(); - if (gxf_graph_entity_) { - gxf_context_ = gxf_graph_entity_->context(); - gxf_eid_ = gxf_graph_entity_->eid(); + gxf_tid_t derived_tid = GxfTidNull(); + bool is_derived = false; + gxf_result_t tid_result; + tid_result = GxfComponentTypeId(gxf_context_, gxf_typename(), &derived_tid); + if (tid_result != GXF_SUCCESS) { + HOLOSCAN_LOG_ERROR("Unable to get component type id of '{}': {}", gxf_typename(), tid_result); + } + if (GxfTidIsNull(allocator_tid)) { + tid_result = GxfComponentTypeId(gxf_context_, "nvidia::gxf::Allocator", &allocator_tid); + if (tid_result != GXF_SUCCESS) { + HOLOSCAN_LOG_ERROR("Unable to get component type id of 'nvidia::gxf::Allocator': {}", + tid_result); } } - this->initialize(); + tid_result = GxfComponentIsBase(gxf_context_, derived_tid, allocator_tid, &is_derived); + if (tid_result != GXF_SUCCESS) { + HOLOSCAN_LOG_ERROR( + "Unable to get determine if '{}' is derived from 'nvidia::gxf::Allocator': {}", + gxf_typename(), + tid_result); + } + if (is_derived) { + HOLOSCAN_LOG_DEBUG( + "The dev_id parameter is deprecated by GXF and will be removed from " + "Holoscan SDK in the future."); + + if (dev_id_value.has_value()) { + int32_t device_id = dev_id_value.value(); + + auto devices = gxf_graph_entity_->findAll(); + if (devices.size() > 0) { + HOLOSCAN_LOG_WARN("Existing entity already has a GPUDevice resource"); + } + + // Create an EntityGroup to associate the GPUDevice with this resource + std::string entity_group_name = + fmt::format("{}_eid{}_dev_id{}_group", name(), gxf_eid_, device_id); + auto entity_group_gid = ::holoscan::gxf::add_entity_group(gxf_context_, entity_group_name); + + // Add GPUDevice component to the same entity as this resource + // TODO (GXF4): requested an addResource method to handle nvidia::gxf::ResourceBase types + std::string device_component_name = + fmt::format("{}_eid{}_gpu_device_id{}_component", name(), gxf_eid_, device_id); + auto dev_handle = gxf_graph_entity_->addComponent("nvidia::gxf::GPUDevice", + device_component_name.c_str(), + {nvidia::gxf::Arg("dev_id", device_id)}); + if (dev_handle.is_null()) { + HOLOSCAN_LOG_ERROR("Failed to create GPUDevice for resource '{}'", name_); + } else { + // TODO: warn and handle case if the resource was already in a different entity group + + // The GPUDevice and this resource have the same eid. + // Make their eid is added to the newly created entity group. + GXF_ASSERT_SUCCESS(GxfUpdateEntityGroup(gxf_context_, entity_group_gid, gxf_eid_)); + } + return true; + } + } + return false; } } // namespace holoscan::gxf diff --git a/src/core/io_spec.cpp b/src/core/io_spec.cpp index 0698d4f5..71f61c23 100644 --- a/src/core/io_spec.cpp +++ b/src/core/io_spec.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -38,6 +38,16 @@ YAML::Node IOSpec::to_yaml_node() const { {ConnectorType::kUCX, "kUCX"s}, }; + std::unordered_map conditiontype_namemap{ + {ConditionType::kNone, "kNone"s}, + {ConditionType::kMessageAvailable, "kMessageAvailable"s}, + {ConditionType::kDownstreamMessageAffordable, "kDownstreamMessageAffordable"s}, + {ConditionType::kCount, "kCount"s}, + {ConditionType::kBoolean, "kBoolean"s}, + {ConditionType::kPeriodic, "kPeriodic"s}, + {ConditionType::kAsynchronous, "kAsynchronous"s}, + }; + node["name"] = name(); node["io_type"] = iotype_namemap[io_type()]; node["typeinfo_name"] = std::string{typeinfo()->name()}; @@ -46,7 +56,17 @@ YAML::Node IOSpec::to_yaml_node() const { if (conn) { node["connector"] = conn->to_yaml_node(); } node["conditions"] = YAML::Node(YAML::NodeType::Sequence); for (const auto& c : conditions_) { - if (c.second) { node["conditions"].push_back(c.second->to_yaml_node()); } + if (c.first == ConditionType::kNone) { + YAML::Node none_condition = YAML::Node(YAML::NodeType::Map); + none_condition["type"] = conditiontype_namemap[c.first]; + node["conditions"].push_back(none_condition); + } else { + if (c.second) { + YAML::Node condition = c.second->to_yaml_node(); + condition["type"] = conditiontype_namemap[c.first]; + node["conditions"].push_back(condition); + } + } } return node; } diff --git a/src/core/operator.cpp b/src/core/operator.cpp index 084d2090..3595e7e8 100644 --- a/src/core/operator.cpp +++ b/src/core/operator.cpp @@ -276,7 +276,7 @@ void Operator::reset_graph_entities() { }; auto reset_iospec = [reset_resource, - reset_condition](const std::unordered_map>& io_specs) { + reset_condition](const std::unordered_map>& io_specs) { for (auto& [_, io_spec] : io_specs) { reset_resource(io_spec->connector()); for (auto& [_, condition] : io_spec->conditions()) { reset_condition(condition); } diff --git a/src/core/resource.cpp b/src/core/resource.cpp index 265b4306..73f33814 100644 --- a/src/core/resource.cpp +++ b/src/core/resource.cpp @@ -34,21 +34,7 @@ void Resource::initialize() { return; } - auto& spec = *spec_; - - // Set arguments - auto& params = spec.params(); - update_params_from_args(params); - - // Set default values for unspecified arguments if the resource is native - if (resource_type_ == ResourceType::kNative) { - // Set only default parameter values - for (auto& [key, param_wrap] : params) { - // If no value is specified, the default value will be used by setting an empty argument. - Arg empty_arg(""); - ArgumentSetter::set_param(param_wrap, empty_arg); - } - } + set_parameters(); is_initialized_ = true; } @@ -64,4 +50,22 @@ YAML::Node Resource::to_yaml_node() const { return node; } +void Resource::update_params_from_args() { + update_params_from_args(spec_->params()); +} + +void Resource::set_parameters() { + update_params_from_args(); + + // Set default values for unspecified arguments if the resource is native + if (resource_type_ == ResourceType::kNative) { + // Set only default parameter values + for (auto& [key, param_wrap] : spec_->params()) { + // If no value is specified, the default value will be used by setting an empty argument. + Arg empty_arg(""); + ArgumentSetter::set_param(param_wrap, empty_arg); + } + } +} + } // namespace holoscan diff --git a/src/core/resources/gxf/annotated_double_buffer_transmitter.cpp b/src/core/resources/gxf/annotated_double_buffer_transmitter.cpp index b208e8c3..d93bbcdd 100644 --- a/src/core/resources/gxf/annotated_double_buffer_transmitter.cpp +++ b/src/core/resources/gxf/annotated_double_buffer_transmitter.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -32,21 +32,40 @@ gxf_result_t AnnotatedDoubleBufferTransmitter::publish_abi(gxf_uid_t uid) { auto gxf_entity = nvidia::gxf::Entity::Shared(context(), uid); gxf_entity->deactivate(); // GXF Entity might be activated by the caller; so deactivate it to // add MessageLabel - auto buffer = gxf_entity.value().add("message_label"); - - // We do not activate the GXF Entity because these message entities are not supposed to be - // activated by default. - - if (!buffer) { - // Fail early if we cannot add the MessageLabel - HOLOSCAN_LOG_ERROR(GxfResultStr(buffer.error())); - return buffer.error(); - } - MessageLabel m; m = op()->get_consolidated_input_label(); m.update_last_op_publish(); - *buffer.value() = m; + + // Check if a message_label component already exists in the entity + static gxf_tid_t message_label_tid = GxfTidNull(); + if (message_label_tid == GxfTidNull()) { + GxfComponentTypeId(context(), "holoscan::MessageLabel", &message_label_tid); + } + // If a message_label component already exists in the entity, just update the value of the + // MessageLabel + if (gxf::has_component(context(), uid, message_label_tid, "message_label")) { + HOLOSCAN_LOG_DEBUG( + "Found a message label already inside the entity. Replacing the original with a new " + "one with timestamp."); + auto maybe_buffer = gxf_entity.value().get("message_label"); + if (!maybe_buffer) { + // Fail early if we cannot add the MessageLabel + HOLOSCAN_LOG_ERROR(GxfResultStr(maybe_buffer.error())); + return maybe_buffer.error(); + } + *maybe_buffer.value() = m; + } else { // if no message_label component exists in the entity, add a new one + auto maybe_buffer = gxf_entity.value().add("message_label"); + if (!maybe_buffer) { + // Fail early if we cannot add the MessageLabel + HOLOSCAN_LOG_ERROR(GxfResultStr(maybe_buffer.error())); + return maybe_buffer.error(); + } + *maybe_buffer.value() = m; + } + + // We do not activate the GXF Entity because these message entities are not supposed to be + // activated by default. } // Call the Base class' publish_abi now diff --git a/src/core/resources/gxf/gxf_component_resource.cpp b/src/core/resources/gxf/gxf_component_resource.cpp new file mode 100644 index 00000000..06c7562d --- /dev/null +++ b/src/core/resources/gxf/gxf_component_resource.cpp @@ -0,0 +1,112 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "holoscan/core/resources/gxf/gxf_component_resource.hpp" + +#include +#include + +#include "holoscan/core/fragment.hpp" + +namespace holoscan { + +const char* GXFComponentResource::gxf_typename() const { + return gxf_typename_.c_str(); +} + +void GXFComponentResource::setup([[maybe_unused]] ComponentSpec& spec) { + // Get the GXF context from the executor + gxf_context_t gxf_context = fragment()->executor().context(); + + // Get the type ID of the component + gxf_tid_t component_tid; + gxf_result_t result = GxfComponentTypeId(gxf_context, gxf_typename(), &component_tid); + if (result != GXF_SUCCESS) { + HOLOSCAN_LOG_ERROR("Unable to find the GXF type name ('{}') for the component", gxf_typename()); + throw std::runtime_error( + fmt::format("Unable to find the GXF type name ('{}') for the component", gxf_typename())); + } + + // Create a new ComponentInfo object for the component + gxf_component_info_ = std::make_shared(gxf_context, component_tid); + + // Set fake parameters for the component to be able to show the resource description + // through the `ComponentSpec::description()` method + auto& params = spec.params(); + auto& gxf_parameter_map = gxf_component_info_->parameter_info_map(); + + for (const auto& key : gxf_component_info_->normal_parameters()) { + const auto& gxf_param = gxf_parameter_map.at(key); + ParameterFlag flag = static_cast(gxf_param.flags); + + parameters_.emplace_back(nullptr, key, gxf_param.headline, gxf_param.description, flag); + auto& parameter = parameters_.back(); + + ParameterWrapper&& parameter_wrapper{ + ¶meter, &typeid(void), gxf::ComponentInfo::get_arg_type(gxf_param), ¶meter}; + params.try_emplace(key, parameter_wrapper); + } +} + +void GXFComponentResource::set_parameters() { + // Here, we don't call update_params_from_args(). + // Instead, we set the parameters manually using the GXF API. + + // Set parameter values if they are specified in the arguments + auto& parameter_map = gxf_component_info_->parameter_info_map(); + bool has_dev_id_param = parameter_map.find("dev_id") != parameter_map.end(); + std::optional dev_id_value; + for (auto& arg : args_) { + // Issue 4336947: dev_id parameter for allocator needs to be handled manually + if (arg.name().compare(std::string("dev_id")) == 0 && has_dev_id_param) { + const auto& arg_type = arg.arg_type(); + if (arg_type.element_type() == holoscan::ArgElementType::kInt32 && + arg_type.container_type() == holoscan::ArgContainerType::kNative) { + try { + dev_id_value = std::any_cast(arg.value()); + continue; + } catch (const std::bad_any_cast& e) { + HOLOSCAN_LOG_ERROR("Cannot cast dev_id argument to int32_t: {}}", e.what()); + } + } + } + // Set the parameter if it is found in the parameter map + if (parameter_map.find(arg.name()) != parameter_map.end()) { + holoscan::gxf::GXFParameterAdaptor::set_param( + gxf_context(), gxf_cid(), arg.name().c_str(), arg.arg_type(), arg.value()); + } + } + if (has_dev_id_param) { + if (!gxf_graph_entity_) { + HOLOSCAN_LOG_ERROR( + "`dev_id` parameter found, but gxf_graph_entity_ was not initialized so it could not " + "be added to the entity group. This parameter will be ignored and default GPU device " + "0 " + "will be used"); + } else { + // Get default value if not set by arguments + if (!dev_id_value.has_value()) { + // Get parameter value for dev_id + auto& parameter_info = parameter_map.at("dev_id"); + dev_id_value = *static_cast(parameter_info.default_value); + } + handle_dev_id(dev_id_value); + } + } +} + +} // namespace holoscan diff --git a/src/operators/CMakeLists.txt b/src/operators/CMakeLists.txt index 18494c67..7c6c00ee 100644 --- a/src/operators/CMakeLists.txt +++ b/src/operators/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,6 +18,7 @@ add_subdirectory(async_ping_rx) add_subdirectory(async_ping_tx) add_subdirectory(bayer_demosaic) add_subdirectory(format_converter) +add_subdirectory(gxf_codelet) add_subdirectory(holoviz) add_subdirectory(inference) add_subdirectory(inference_processor) diff --git a/src/operators/aja_source/aja_source.cpp b/src/operators/aja_source/aja_source.cpp index 5e21eb4f..b6f74d6b 100644 --- a/src/operators/aja_source/aja_source.cpp +++ b/src/operators/aja_source/aja_source.cpp @@ -217,9 +217,9 @@ AJAStatus AJASourceOp::SetupVideo() { return AJA_STATUS_UNSUPPORTED; } device_.Connect(NTV2_XptFrameBuffer1Input, NTV2_Xpt425Mux1ARGB); - device_.Connect(NTV2_XptFrameBuffer1BInput, NTV2_Xpt425Mux1BRGB); + device_.Connect(NTV2_XptFrameBuffer1DS2Input, NTV2_Xpt425Mux1BRGB); device_.Connect(NTV2_XptFrameBuffer2Input, NTV2_Xpt425Mux2ARGB); - device_.Connect(NTV2_XptFrameBuffer2BInput, NTV2_Xpt425Mux2BRGB); + device_.Connect(NTV2_XptFrameBuffer2DS2Input, NTV2_Xpt425Mux2BRGB); device_.Connect(NTV2_Xpt425Mux1AInput, NTV2_XptCSC1VidRGB); device_.Connect(NTV2_Xpt425Mux1BInput, NTV2_XptCSC2VidRGB); device_.Connect(NTV2_Xpt425Mux2AInput, NTV2_XptCSC3VidRGB); @@ -230,9 +230,9 @@ AJAStatus AJASourceOp::SetupVideo() { device_.Connect(NTV2_XptCSC4VidInput, NTV2_XptHDMIIn1Q4); } else { device_.Connect(NTV2_XptFrameBuffer1Input, NTV2_Xpt425Mux1ARGB); - device_.Connect(NTV2_XptFrameBuffer1BInput, NTV2_Xpt425Mux1BRGB); + device_.Connect(NTV2_XptFrameBuffer1DS2Input, NTV2_Xpt425Mux1BRGB); device_.Connect(NTV2_XptFrameBuffer2Input, NTV2_Xpt425Mux2ARGB); - device_.Connect(NTV2_XptFrameBuffer2BInput, NTV2_Xpt425Mux2BRGB); + device_.Connect(NTV2_XptFrameBuffer2DS2Input, NTV2_Xpt425Mux2BRGB); device_.Connect(NTV2_Xpt425Mux1AInput, NTV2_XptHDMIIn1RGB); device_.Connect(NTV2_Xpt425Mux1BInput, NTV2_XptHDMIIn1Q2RGB); device_.Connect(NTV2_Xpt425Mux2AInput, NTV2_XptHDMIIn1Q3RGB); diff --git a/src/operators/bayer_demosaic/bayer_demosaic.cpp b/src/operators/bayer_demosaic/bayer_demosaic.cpp index e3816cf5..00b03810 100644 --- a/src/operators/bayer_demosaic/bayer_demosaic.cpp +++ b/src/operators/bayer_demosaic/bayer_demosaic.cpp @@ -182,7 +182,7 @@ void BayerDemosaicOp::compute(InputContext& op_input, OutputContext& op_output, input_data_ptr = frame->pointer(); // if the input tensor is not coming from device then move it to device - if (input_memory_type != nvidia::gxf::MemoryStorageType::kDevice) { + if (input_memory_type == nvidia::gxf::MemoryStorageType::kSystem) { size_t buffer_size = rows * columns * in_channels * element_size; if (buffer_size > device_scratch_buffer_.size()) { @@ -213,8 +213,59 @@ void BayerDemosaicOp::compute(InputContext& op_input, OutputContext& op_output, // Get needed information from the tensor // cast Holoscan::Tensor to nvidia::gxf::Tensor so attribute access code can remain as-is nvidia::gxf::Tensor in_tensor_gxf{in_tensor->dl_ctx()}; + auto in_rank = in_tensor_gxf.rank(); + if (in_rank != 3) { + throw std::runtime_error( + fmt::format("Input tensor has {} dimensions. Expected a tensor with 3 dimensions " + "(corresponding to an RGB or RGBA image).", + in_rank)); + } + + DLDevice dev = in_tensor->device(); + if ((dev.device_type != kDLCUDA) && (dev.device_type != kDLCPU) && + (dev.device_type != kDLCUDAHost)) { + throw std::runtime_error( + "Input tensor must be in CUDA device memory, CUDA pinned memory or on the CPU."); + } + + // Originally had: + // auto is_contiguous = in_tensor_gxf.isContiguous().value(); + // but there was a bug in GXF 4.0's isContiguous(), so added is_contiguous to holoscan::Tensor + // instead. + if (!in_tensor->is_contiguous()) { + throw std::runtime_error( + fmt::format("Tensor must have a row-major memory layout (values along the last axis " + " are adjacent in memory). Detected shape:({}, {}, {}), " + "strides: ({}, {}, {})", + in_tensor_gxf.shape().dimension(0), + in_tensor_gxf.shape().dimension(1), + in_tensor_gxf.shape().dimension(2), + in_tensor_gxf.stride(0), + in_tensor_gxf.stride(1), + in_tensor_gxf.stride(2))); + } + + if (dev.device_type == kDLCPU) { + size_t buffer_size = rows * columns * in_channels * element_size; + + if (buffer_size > device_scratch_buffer_.size()) { + device_scratch_buffer_.resize( + pool.value(), buffer_size, nvidia::gxf::MemoryStorageType::kDevice); + if (!device_scratch_buffer_.pointer()) { + throw std::runtime_error( + fmt::format("Failed to allocate device scratch buffer ({} bytes)", buffer_size)); + } + } + + CUDA_TRY(cudaMemcpy(static_cast(device_scratch_buffer_.pointer()), + static_cast(in_tensor_gxf.pointer()), + buffer_size, + cudaMemcpyHostToDevice)); + input_data_ptr = device_scratch_buffer_.pointer(); + } else { + input_data_ptr = in_tensor_gxf.pointer(); + } - input_data_ptr = in_tensor_gxf.pointer(); if (input_data_ptr == nullptr) { // This should never happen, but just in case... HOLOSCAN_LOG_ERROR("Unable to get tensor data pointer. nullptr returned."); diff --git a/src/operators/format_converter/format_converter.cpp b/src/operators/format_converter/format_converter.cpp index d3933589..b6ebf8a5 100644 --- a/src/operators/format_converter/format_converter.cpp +++ b/src/operators/format_converter/format_converter.cpp @@ -47,6 +47,24 @@ _holoscan_cuda_err; \ }) +namespace { + +bool has_c_ordered_memory_layout(const nvidia::gxf::Tensor& in_tensor_gxf, int rank, + int n_channels) { + // Could use in_tensor_gxf.isContiguous() to check contiguity, but we want to support cases + // with some padding between rows, so do this custom check to support that case. + // All but first axis must be contiguous in memory. + if (rank == 3) { + return (in_tensor_gxf.stride(1) == n_channels * in_tensor_gxf.stride(2)) || + (in_tensor_gxf.stride(2) == in_tensor_gxf.bytes_per_element()); + } else if (rank == 2) { + return in_tensor_gxf.stride(1) == in_tensor_gxf.bytes_per_element(); + } else { + throw std::runtime_error("rank must be 2 or 3"); + } +} +} // namespace + namespace holoscan::ops { // Note that currently "uint8" for the `input_dtype` or `output_dtype` arguments to the operator @@ -318,8 +336,9 @@ void FormatConverterOp::compute(InputContext& op_input, OutputContext& op_output // If the buffer is in host memory, copy it to a device (GPU) buffer // as needed for the NPP resize/convert operations. - if (in_memory_storage_type == nvidia::gxf::MemoryStorageType::kHost) { - size_t buffer_size = rows * columns * in_channels; + if (in_memory_storage_type == nvidia::gxf::MemoryStorageType::kSystem) { + uint32_t element_size = nvidia::gxf::PrimitiveTypeSize(in_primitive_type); + size_t buffer_size = rows * columns * in_channels * element_size; if (buffer_size > device_scratch_buffer_->size()) { device_scratch_buffer_->resize( pool.value(), buffer_size, nvidia::gxf::MemoryStorageType::kDevice); @@ -351,14 +370,60 @@ void FormatConverterOp::compute(InputContext& op_input, OutputContext& op_output in_tensor_data = in_tensor_gxf.pointer(); if (in_tensor_data == nullptr) { // This should never happen, but just in case... - HOLOSCAN_LOG_ERROR("Unable to get tensor data pointer. nullptr returned."); + throw std::runtime_error(fmt::format("Unable to get tensor data pointer. nullptr returned.")); } in_primitive_type = in_tensor_gxf.element_type(); in_memory_storage_type = in_tensor_gxf.storage_type(); + auto in_rank = in_tensor_gxf.rank(); + if (in_rank < 2 || in_rank > 3) { + throw std::runtime_error( + fmt::format("Input tensor has {} dimensions. Expected a tensor with 2 or 3 dimensions " + "(rows, columns[, channels]).", + in_rank)); + } rows = in_tensor_gxf.shape().dimension(0); columns = in_tensor_gxf.shape().dimension(1); in_channels = in_tensor_gxf.shape().dimension(2); out_channels = in_channels; + + if (!has_c_ordered_memory_layout(in_tensor_gxf, in_rank, in_channels)) { + std::string stride_string; + if (in_rank == 2) { + stride_string = fmt::format("({}, {})", in_tensor_gxf.stride(0), in_tensor_gxf.stride(1)); + } else { + stride_string = fmt::format("({}, {}, {})", + in_tensor_gxf.stride(0), + in_tensor_gxf.stride(1), + in_tensor_gxf.stride(2)); + } + throw std::runtime_error( + fmt::format("Tensor is expected to be in a C-contiguous memory layout " + "(values along the last axis are adjacent in memory). " + "Detected strides: {}", + stride_string)); + } + + if (in_memory_storage_type == nvidia::gxf::MemoryStorageType::kSystem) { + uint32_t element_size = nvidia::gxf::PrimitiveTypeSize(in_primitive_type); + size_t buffer_size = rows * columns * in_channels * element_size; + + if (buffer_size > device_scratch_buffer_->size()) { + device_scratch_buffer_->resize( + pool.value(), buffer_size, nvidia::gxf::MemoryStorageType::kDevice); + if (!device_scratch_buffer_->pointer()) { + throw std::runtime_error( + fmt::format("Failed to allocate device scratch buffer ({} bytes)", buffer_size)); + } + } + + CUDA_TRY(cudaMemcpy(static_cast(device_scratch_buffer_->pointer()), + static_cast(in_tensor_gxf.pointer()), + buffer_size, + cudaMemcpyHostToDevice)); + in_tensor_data = device_scratch_buffer_->pointer(); + in_memory_storage_type = nvidia::gxf::MemoryStorageType::kDevice; + } + in_color_planes.resize(1); in_color_planes[0].width = columns; in_color_planes[0].height = rows; @@ -366,9 +431,10 @@ void FormatConverterOp::compute(InputContext& op_input, OutputContext& op_output in_color_planes[0].size = in_color_planes[0].height * in_color_planes[0].stride; } - if (in_memory_storage_type != nvidia::gxf::MemoryStorageType::kDevice) { + if (in_memory_storage_type != nvidia::gxf::MemoryStorageType::kDevice && + in_memory_storage_type != nvidia::gxf::MemoryStorageType::kHost) { throw std::runtime_error(fmt::format( - "Tensor('{}') or VideoBuffer is not allocated on device.\n", in_tensor_name_.get())); + "Tensor('{}') or VideoBuffer was not allocated via CUDA APIs.\n", in_tensor_name_.get())); } if (in_dtype_ == FormatDType::kUnknown) { diff --git a/src/operators/gxf_codelet/CMakeLists.txt b/src/operators/gxf_codelet/CMakeLists.txt new file mode 100644 index 00000000..94579b07 --- /dev/null +++ b/src/operators/gxf_codelet/CMakeLists.txt @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +add_holoscan_operator(gxf_codelet gxf_codelet.cpp) + +target_link_libraries(op_gxf_codelet + PUBLIC + holoscan::core +) diff --git a/src/operators/gxf_codelet/gxf_codelet.cpp b/src/operators/gxf_codelet/gxf_codelet.cpp new file mode 100644 index 00000000..0ecce80f --- /dev/null +++ b/src/operators/gxf_codelet/gxf_codelet.cpp @@ -0,0 +1,119 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "holoscan/operators/gxf_codelet/gxf_codelet.hpp" + +#include +#include + +#include "gxf/core/common_expected_macro.hpp" +#include "gxf/core/expected_macro.hpp" +#include "gxf/core/parameter_registrar.hpp" +#include "holoscan/core/fragment.hpp" +#include "holoscan/core/gxf/gxf_utils.hpp" + +namespace holoscan::ops { + +const char* GXFCodeletOp::gxf_typename() const { + return gxf_typename_.c_str(); +} + +void GXFCodeletOp::setup(OperatorSpec& spec) { + // Get the GXF context from the executor + gxf_context_t gxf_context = fragment()->executor().context(); + + // Get the type ID of the codelet + gxf_tid_t codelet_tid; + gxf_result_t result = GxfComponentTypeId(gxf_context, gxf_typename(), &codelet_tid); + if (result != GXF_SUCCESS) { + HOLOSCAN_LOG_ERROR("Unable to find the GXF type name ('{}') for the codelet", gxf_typename()); + throw std::runtime_error( + fmt::format("Unable to find the GXF type name ('{}') for the codelet", gxf_typename())); + } + + // Create a new ComponentInfo object for the codelet + gxf_component_info_ = std::make_shared(gxf_context, codelet_tid); + + // Specify the input and output ports of the operator + for (const auto& param : gxf_component_info_->receiver_parameters()) { + spec.input(param); + } + for (const auto& param : gxf_component_info_->transmitter_parameters()) { + spec.output(param); + } + + // Set fake parameters for the codelet to be able to show the operator description + // through the `OperatorSpec::description()` method + auto& params = spec.params(); + auto& gxf_parameter_map = gxf_component_info_->parameter_info_map(); + + for (const auto& key : gxf_component_info_->normal_parameters()) { + const auto& gxf_param = gxf_parameter_map.at(key); + ParameterFlag flag = static_cast(gxf_param.flags); + + parameters_.emplace_back(nullptr, key, gxf_param.headline, gxf_param.description, flag); + auto& parameter = parameters_.back(); + + ParameterWrapper&& parameter_wrapper{ + ¶meter, &typeid(void), gxf::ComponentInfo::get_arg_type(gxf_param), ¶meter}; + params.try_emplace(key, parameter_wrapper); + } +} + +void GXFCodeletOp::set_parameters() { + // Here, we don't call update_params_from_args(). + // Instead, we set the parameters manually using the GXF API. + + // Set the handle values for the receiver parameters + auto& inputs = spec_->inputs(); + for (const auto& param_key : gxf_component_info_->receiver_parameters()) { + // Get the parameter info and the receiver handle + auto param_info = codelet_handle_->getParameterInfo(param_key).value(); + auto connector = inputs[param_key]->connector(); + auto gxf_connector = std::dynamic_pointer_cast(connector); + auto receiver_handle = nvidia::gxf::Handle::Create( + gxf_connector->gxf_context(), gxf_connector->gxf_cid()); + + // Set the parameter with the handle value + codelet_handle_->setParameter(param_key, receiver_handle.value()); + } + + // Set the handle values for the transmitter parameters + auto& outputs = spec_->outputs(); + for (const auto& param_key : gxf_component_info_->transmitter_parameters()) { + // Get the parameter info and the transmitter handle + auto param_info = codelet_handle_->getParameterInfo(param_key).value(); + auto connector = outputs[param_key]->connector(); + auto gxf_connector = std::dynamic_pointer_cast(connector); + auto transmitter_handle = nvidia::gxf::Handle::Create( + gxf_connector->gxf_context(), gxf_connector->gxf_cid()); + + // Set the parameter with the handle value + codelet_handle_->setParameter(param_key, transmitter_handle.value()); + } + + // Set parameter values if they are specified in the arguments + auto& parameter_map = gxf_component_info_->parameter_info_map(); + for (auto& arg : args_) { + if (parameter_map.find(arg.name()) != parameter_map.end()) { + holoscan::gxf::GXFParameterAdaptor::set_param( + gxf_context(), gxf_cid(), arg.name().c_str(), arg.arg_type(), arg.value()); + } + } +} + +} // namespace holoscan::ops diff --git a/src/operators/holoviz/buffer_info.cpp b/src/operators/holoviz/buffer_info.cpp index 0524926d..a59853c9 100644 --- a/src/operators/holoviz/buffer_info.cpp +++ b/src/operators/holoviz/buffer_info.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -27,6 +27,15 @@ namespace holoscan::ops { gxf_result_t BufferInfo::init(const nvidia::gxf::Handle& tensor) { rank = tensor->rank(); + // validate row-major memory layout + int32_t last_axis = rank - 1; + for (auto axis = last_axis; axis > 0; --axis) { + if (tensor->stride(axis) > tensor->stride(axis - 1)) { + HOLOSCAN_LOG_ERROR("Tensor must have a row-major memory layout (C-contiguous memory order)."); + return GXF_INVALID_DATA_FORMAT; + } + } + /// @todo this is assuming HWC, should either auto-detect (if possible) or make user /// configurable components = tensor->shape().dimension(tensor->rank() - 1); diff --git a/src/operators/holoviz/holoviz.cpp b/src/operators/holoviz/holoviz.cpp index 591b4a5e..fe56542e 100644 --- a/src/operators/holoviz/holoviz.cpp +++ b/src/operators/holoviz/holoviz.cpp @@ -40,6 +40,7 @@ #include "holoscan/operators/holoviz/buffer_info.hpp" #include "holoscan/operators/holoviz/codecs.hpp" +#include "gxf/multimedia/camera.hpp" #include "gxf/multimedia/video.hpp" #include "gxf/std/scheduling_terms.hpp" #include "gxf/std/tensor.hpp" @@ -379,6 +380,9 @@ void HolovizOp::setup(OperatorSpec& spec) { spec.param(receivers_, "receivers", "Input Receivers", "List of input receivers.", {}); spec.input("input_specs").condition(ConditionType::kNone); + spec.input>("camera_eye_input").condition(ConditionType::kNone); + spec.input>("camera_look_at_input").condition(ConditionType::kNone); + spec.input>("camera_up_input").condition(ConditionType::kNone); auto& render_buffer_input = spec.input("render_buffer_input").condition(ConditionType::kNone); @@ -395,12 +399,11 @@ void HolovizOp::setup(OperatorSpec& spec) { "using that one, else it allocates a new buffer.", &render_buffer_output); - auto& camera_pose_output = spec.output>("camera_pose_output"); + auto& camera_pose_output = spec.output("camera_pose_output"); spec.param(camera_pose_output_, "camera_pose_output", "CameraPoseOutput", - "Output the camera pose. The camera parameters are returned in a 4x4 row major " - "projection matrix.", + "Output the camera extrinsics model.", &camera_pose_output); spec.param( @@ -423,11 +426,12 @@ void HolovizOp::setup(OperatorSpec& spec) { "Window title", "Title on window canvas", DEFAULT_WINDOW_TITLE); - spec.param(display_name_, - "display_name", - "Display name", - "In exclusive mode, name of display to use as shown with xrandr.", - DEFAULT_DISPLAY_NAME); + spec.param( + display_name_, + "display_name", + "Display name", + "In exclusive mode, name of display to use as shown with `xrandr` or `hwinfo --monitor`.", + DEFAULT_DISPLAY_NAME); spec.param(width_, "width", "Width", @@ -457,7 +461,7 @@ void HolovizOp::setup(OperatorSpec& spec) { "headless", "Headless", "Enable headless mode. No window is opened, the render buffer is output to " - "‘render_buffer_output’.", + "`render_buffer_output`.", DEFAULT_HEADLESS); spec.param(window_close_scheduling_term_, "window_close_scheduling_term", @@ -472,6 +476,19 @@ void HolovizOp::setup(OperatorSpec& spec) { "FontPath", "File path for the font used for rendering text", std::string()); + spec.param(camera_pose_output_type_, + "camera_pose_output_type", + "Camera Pose Output Type", + "Type of data output at `camera_pose_output`. Supported values are " + "`projection_matrix` and `extrinsics_model`.", + std::string("projection_matrix")); + spec.param(camera_eye_, "camera_eye", "Camera Eye", "Camera eye position", {{0.f, 0.f, 1.f}}); + spec.param(camera_look_at_, + "camera_look_at", + "Camera Look At", + "Camera look at position", + {{0.f, 0.f, 0.f}}); + spec.param(camera_up_, "camera_up", "Camera Up", "Camera up vector", {{0.f, 1.f, 0.f}}); cuda_stream_handler_.define_params(spec); } @@ -626,6 +643,350 @@ void HolovizOp::set_input_spec_geometry(const InputSpec& input_spec) { viz::LineWidth(input_spec.line_width_); } +void HolovizOp::render_color_image(const InputSpec& input_spec, BufferInfo& buffer_info) { + // sanity checks + if (buffer_info.rank != 3) { + throw std::runtime_error(fmt::format("Expected rank 3 for tensor '{}', type '{}', but got {}", + buffer_info.name, + inputTypeToString(input_spec.type_), + buffer_info.rank)); + } + if (buffer_info.image_format == static_cast(-1)) { + std::runtime_error( + fmt::format("Color image: element type {} and channel count {} not supported", + static_cast(buffer_info.element_type), + buffer_info.components)); + } + + viz::ImageFormat image_format; + if (input_spec.type_ == InputType::COLOR_LUT) { + if (buffer_info.components != 1) { + throw std::runtime_error( + fmt::format("Expected one channel for tensor '{}' when using lookup table, but got {}", + buffer_info.name, + buffer_info.components)); + } + if (lut_.empty()) { + throw std::runtime_error(fmt::format( + "Type of tensor '{}' is '{}', but a color lookup table has not been specified", + buffer_info.name, + inputTypeToString(input_spec.type_))); + } + + // when using a LUT, the unorm formats are handled as single channel int formats + switch (buffer_info.image_format) { + case viz::ImageFormat::R8_UNORM: + image_format = viz::ImageFormat::R8_UINT; + break; + case viz::ImageFormat::R8_SNORM: + image_format = viz::ImageFormat::R8_SINT; + break; + case viz::ImageFormat::R16_UNORM: + image_format = viz::ImageFormat::R16_UINT; + break; + case viz::ImageFormat::R16_SNORM: + image_format = viz::ImageFormat::R16_SINT; + break; + default: + image_format = buffer_info.image_format; + break; + } + } else { + image_format = buffer_info.image_format; + } + + // start an image layer + viz::BeginImageLayer(); + set_input_spec(input_spec); + + if (input_spec.type_ == InputType::COLOR_LUT) { + viz::LUT(lut_.size() / 4, + viz::ImageFormat::R32G32B32A32_SFLOAT, + lut_.size() * sizeof(float), + lut_.data()); + } + + viz::ImageComponentMapping(buffer_info.component_swizzle[0], + buffer_info.component_swizzle[1], + buffer_info.component_swizzle[2], + buffer_info.component_swizzle[3]); + if (buffer_info.storage_type == nvidia::gxf::MemoryStorageType::kDevice) { + // if it's the device convert to `CUDeviceptr` + const auto cu_buffer_ptr = reinterpret_cast(buffer_info.buffer_ptr); + viz::ImageCudaDevice( + buffer_info.width, buffer_info.height, image_format, cu_buffer_ptr, buffer_info.stride[0]); + } else { + // convert to void * if using the system/host + const auto host_buffer_ptr = reinterpret_cast(buffer_info.buffer_ptr); + viz::ImageHost(buffer_info.width, + buffer_info.height, + image_format, + host_buffer_ptr, + buffer_info.stride[0]); + } + viz::EndLayer(); +} + +void HolovizOp::render_geometry(const ExecutionContext& context, const InputSpec& input_spec, + BufferInfo& buffer_info) { + if ((buffer_info.element_type != nvidia::gxf::PrimitiveType::kFloat32) && + (buffer_info.element_type != nvidia::gxf::PrimitiveType::kFloat64)) { + throw std::runtime_error( + fmt::format("Expected gxf::PrimitiveType::kFloat32 or gxf::PrimitiveType::kFloat64 " + "element type for coordinates, but got element type {}", + static_cast(buffer_info.element_type))); + } + + // get pointer to tensor buffer + std::vector host_buffer; + if (buffer_info.storage_type == nvidia::gxf::MemoryStorageType::kDevice) { + host_buffer.resize(buffer_info.bytes_size); + + // copy from device to host + CUDA_TRY(cudaMemcpyAsync(static_cast(host_buffer.data()), + static_cast(buffer_info.buffer_ptr), + buffer_info.bytes_size, + cudaMemcpyDeviceToHost, + cuda_stream_handler_.get_cuda_stream(context.context()))); + // wait for the CUDA memory copy to finish + CUDA_TRY(cudaStreamSynchronize(cuda_stream_handler_.get_cuda_stream(context.context()))); + + buffer_info.buffer_ptr = host_buffer.data(); + } + + // start a geometry layer + viz::BeginGeometryLayer(); + set_input_spec_geometry(input_spec); + + const auto coordinates = buffer_info.width; + + if (input_spec.type_ == InputType::TEXT) { + // text is defined by the top left coordinate and the size (x, y, s) per string, text + // strings are define by InputSpec::text_ + if ((buffer_info.components < 2) || (buffer_info.components > 3)) { + throw std::runtime_error(fmt::format("Expected two or three values per text, but got '{}'", + buffer_info.components)); + } + if (input_spec.text_.empty()) { + throw std::runtime_error( + fmt::format("No text has been specified by input spec '{}'.", input_spec.tensor_name_)); + } + uintptr_t src_coord = reinterpret_cast(buffer_info.buffer_ptr); + constexpr uint32_t values_per_coordinate = 3; + float coords[values_per_coordinate]{0.f, 0.f, 0.05f}; + for (uint32_t index = 0; index < coordinates; ++index) { + uint32_t component_index = 0; + // copy from source array + while (component_index < buffer_info.components) { + switch (buffer_info.element_type) { + case nvidia::gxf::PrimitiveType::kFloat32: + coords[component_index] = reinterpret_cast(src_coord)[component_index]; + break; + case nvidia::gxf::PrimitiveType::kFloat64: + coords[component_index] = reinterpret_cast(src_coord)[component_index]; + break; + default: + throw std::runtime_error("Unhandled element type"); + } + ++component_index; + } + src_coord += buffer_info.stride[1]; + viz::Text( + coords[0], + coords[1], + coords[2], + input_spec.text_[std::min(index, static_cast(input_spec.text_.size()) - 1)] + .c_str()); + } + } else { + std::vector coords; + viz::PrimitiveTopology topology; + uint32_t primitive_count; + uint32_t coordinate_count; + uint32_t values_per_coordinate; + std::vector default_coord; + switch (input_spec.type_) { + case InputType::POINTS: + // point primitives, one coordinate (x, y) per primitive + if (buffer_info.components != 2) { + throw std::runtime_error( + fmt::format("Expected two values per point, but got '{}'", buffer_info.components)); + } + topology = viz::PrimitiveTopology::POINT_LIST; + primitive_count = coordinates; + coordinate_count = primitive_count; + values_per_coordinate = 2; + default_coord = {0.f, 0.f}; + break; + case InputType::LINES: + // line primitives, two coordinates (x0, y0) and (x1, y1) per primitive + if (buffer_info.components != 2) { + throw std::runtime_error(fmt::format("Expected two values per line vertex, but got '{}'", + buffer_info.components)); + } + topology = viz::PrimitiveTopology::LINE_LIST; + primitive_count = coordinates / 2; + coordinate_count = primitive_count * 2; + values_per_coordinate = 2; + default_coord = {0.f, 0.f}; + break; + case InputType::LINE_STRIP: + // line strip primitive, a line primitive i is defined by each coordinate (xi, yi) and + // the following (xi+1, yi+1) + if (buffer_info.components != 2) { + throw std::runtime_error(fmt::format( + "Expected two values per line strip vertex, but got '{}'", buffer_info.components)); + } + topology = viz::PrimitiveTopology::LINE_STRIP; + primitive_count = coordinates - 1; + coordinate_count = coordinates; + values_per_coordinate = 2; + default_coord = {0.f, 0.f}; + break; + case InputType::TRIANGLES: + // triangle primitive, three coordinates (x0, y0), (x1, y1) and (x2, y2) per primitive + if (buffer_info.components != 2) { + throw std::runtime_error(fmt::format( + "Expected two values per triangle vertex, but got '{}'", buffer_info.components)); + } + topology = viz::PrimitiveTopology::TRIANGLE_LIST; + primitive_count = coordinates / 3; + coordinate_count = primitive_count * 3; + values_per_coordinate = 2; + default_coord = {0.f, 0.f}; + break; + case InputType::CROSSES: + // cross primitive, a cross is defined by the center coordinate and the size (xi, yi, + // si) + if ((buffer_info.components < 2) || (buffer_info.components > 3)) { + throw std::runtime_error(fmt::format( + "Expected two or three values per cross, but got '{}'", buffer_info.components)); + } + + topology = viz::PrimitiveTopology::CROSS_LIST; + primitive_count = coordinates; + coordinate_count = primitive_count; + values_per_coordinate = 3; + default_coord = {0.f, 0.f, 0.05f}; + break; + case InputType::RECTANGLES: + // axis aligned rectangle primitive, each rectangle is defined by two coordinates (xi, + // yi) and (xi+1, yi+1) + if (buffer_info.components != 2) { + throw std::runtime_error(fmt::format( + "Expected two values per rectangle vertex, but got '{}'", buffer_info.components)); + } + topology = viz::PrimitiveTopology::RECTANGLE_LIST; + primitive_count = coordinates / 2; + coordinate_count = primitive_count * 2; + values_per_coordinate = 2; + default_coord = {0.f, 0.f}; + break; + case InputType::OVALS: + // oval primitive, an oval primitive is defined by the center coordinate and the axis + // sizes (xi, yi, sxi, syi) + if ((buffer_info.components < 2) || (buffer_info.components > 4)) { + throw std::runtime_error(fmt::format( + "Expected two, three or four values per oval, but got '{}'", buffer_info.components)); + } + topology = viz::PrimitiveTopology::OVAL_LIST; + primitive_count = coordinates; + coordinate_count = primitive_count; + values_per_coordinate = 4; + default_coord = {0.f, 0.f, 0.05f, 0.05f}; + break; + case InputType::POINTS_3D: + // point primitives, one coordinate (x, y, z) per primitive + if (buffer_info.components != 3) { + throw std::runtime_error(fmt::format("Expected three values per 3D point, but got '{}'", + buffer_info.components)); + } + topology = viz::PrimitiveTopology::POINT_LIST_3D; + primitive_count = coordinates; + coordinate_count = primitive_count; + values_per_coordinate = 3; + default_coord = {0.f, 0.f, 0.f}; + break; + case InputType::LINES_3D: + // line primitives, two coordinates (x0, y0, z0) and (x1, y1, z1) per primitive + if (buffer_info.components != 3) { + throw std::runtime_error(fmt::format( + "Expected three values per 3D line vertex, but got '{}'", buffer_info.components)); + } + topology = viz::PrimitiveTopology::LINE_LIST_3D; + primitive_count = coordinates / 2; + coordinate_count = primitive_count * 2; + values_per_coordinate = 3; + default_coord = {0.f, 0.f, 0.f}; + break; + case InputType::LINE_STRIP_3D: + // line primitives, two coordinates (x0, y0, z0) and (x1, y1, z1) per primitive + if (buffer_info.components != 3) { + throw std::runtime_error( + fmt::format("Expected three values per 3D line strip vertex, but got '{}'", + buffer_info.components)); + } + topology = viz::PrimitiveTopology::LINE_STRIP_3D; + primitive_count = coordinates - 1; + coordinate_count = coordinates; + values_per_coordinate = 3; + default_coord = {0.f, 0.f}; + break; + case InputType::TRIANGLES_3D: + // triangle primitive, three coordinates (x0, y0, z0), (x1, y1, z1) and (x2, y2, z2) + // per primitive + if (buffer_info.components != 3) { + throw std::runtime_error( + fmt::format("Expected three values per 3D triangle vertex, but got '{}'", + buffer_info.components)); + } + topology = viz::PrimitiveTopology::TRIANGLE_LIST_3D; + primitive_count = coordinates / 3; + coordinate_count = primitive_count * 3; + values_per_coordinate = 3; + default_coord = {0.f, 0.f}; + break; + default: + throw std::runtime_error( + fmt::format("Unhandled tensor type '{}'", inputTypeToString(input_spec.type_))); + } + + // copy coordinates + uintptr_t src_coord = reinterpret_cast(buffer_info.buffer_ptr); + coords.reserve(coordinate_count * values_per_coordinate); + + for (uint32_t index = 0; index < coordinate_count; ++index) { + uint32_t component_index = 0; + // copy from source array + while (component_index < std::min(buffer_info.components, values_per_coordinate)) { + switch (buffer_info.element_type) { + case nvidia::gxf::PrimitiveType::kFloat32: + coords.push_back(reinterpret_cast(src_coord)[component_index]); + break; + case nvidia::gxf::PrimitiveType::kFloat64: + coords.push_back(reinterpret_cast(src_coord)[component_index]); + break; + default: + throw std::runtime_error("Unhandled element type"); + } + ++component_index; + } + // fill from default array + while (component_index < values_per_coordinate) { + coords.push_back(default_coord[component_index]); + ++component_index; + } + src_coord += buffer_info.stride[1]; + } + + if (primitive_count) { + viz::Primitive(topology, primitive_count, coords.size(), coords.data()); + } + } + + viz::EndLayer(); +} + void HolovizOp::render_depth_map(InputSpec* const input_spec_depth_map, const BufferInfo& buffer_info_depth_map, InputSpec* const input_spec_depth_map_color, @@ -692,6 +1053,7 @@ void HolovizOp::render_depth_map(InputSpec* const input_spec_depth_map, void HolovizOp::initialize() { register_converter>(); + register_converter>(); register_codec>("std::vector", true); // Set up prerequisite parameters before calling Operator::initialize() @@ -742,6 +1104,21 @@ void HolovizOp::start() { viz::Init(width_, height_, window_title_.get().c_str(), init_flags); } + // initialize the camera with the provided parameters + camera_eye_cur_ = camera_eye_.get(); + camera_look_at_cur_ = camera_look_at_.get(); + camera_up_cur_ = camera_up_.get(); + viz::SetCamera(camera_eye_cur_[0], + camera_eye_cur_[1], + camera_eye_cur_[2], + camera_look_at_cur_[0], + camera_look_at_cur_[1], + camera_look_at_cur_[2], + camera_up_cur_[0], + camera_up_cur_[1], + camera_up_cur_[2], + false); + // get the color lookup table const auto& color_lut = color_lut_.get(); lut_.reserve(color_lut.size() * 4); @@ -772,17 +1149,15 @@ void HolovizOp::stop() { void HolovizOp::compute(InputContext& op_input, OutputContext& op_output, ExecutionContext& context) { - std::vector messages_h = - op_input.receive>("receivers").value(); + // receive input messages + auto receivers_messages = op_input.receive>("receivers").value(); - // create vector of nvidia::gxf::Entity as expected by the code below - std::vector messages; - messages.reserve(messages_h.size()); - for (auto& message_h : messages_h) { - // cast each holoscan::gxf:Entity to its base class - nvidia::gxf::Entity message = static_cast(message_h); - messages.push_back(message); - } + auto input_specs_messages = + op_input.receive>("input_specs"); + + auto camera_eye_message = op_input.receive>("camera_eye_input"); + auto camera_look_at_message = op_input.receive>("camera_look_at_input"); + auto camera_up_message = op_input.receive>("camera_up_input"); // make instance current ScopedPushInstance scoped_instance(instance_); @@ -796,14 +1171,40 @@ void HolovizOp::compute(InputContext& op_input, OutputContext& op_output, // nothing to do if minimized if (viz::WindowIsMinimized()) { return; } + // create vector of nvidia::gxf::Entity as expected by the code below + std::vector messages; + messages.reserve(receivers_messages.size()); + for (auto& receivers_message : receivers_messages) { + // cast each holoscan::gxf:Entity to its base class + nvidia::gxf::Entity message = static_cast(receivers_message); + messages.push_back(message); + } + + // handle camera messages + if (camera_eye_message || camera_eye_message || camera_up_message) { + if (camera_eye_message) { camera_eye_cur_ = camera_eye_message.value(); } + if (camera_look_at_message) { camera_look_at_cur_ = camera_look_at_message.value(); } + if (camera_up_message) { camera_up_cur_ = camera_up_message.value(); } + // set the camera + viz::SetCamera(camera_eye_cur_[0], + camera_eye_cur_[1], + camera_eye_cur_[2], + camera_look_at_cur_[0], + camera_look_at_cur_[1], + camera_look_at_cur_[2], + camera_up_cur_[0], + camera_up_cur_[1], + camera_up_cur_[2], + true); + } + // build the input spec list std::vector input_spec_list(initial_input_spec_); // check the messages for input specs, they are added to the list - if (!op_input.empty("input_specs")) { - auto msg_input_specs = - op_input.receive>("input_specs").value(); - input_spec_list.insert(input_spec_list.end(), msg_input_specs.begin(), msg_input_specs.end()); + if (input_specs_messages) { + input_spec_list.insert( + input_spec_list.end(), input_specs_messages->begin(), input_specs_messages->end()); } // then get all tensors and video buffers of all messages, check if an input spec for the tensor @@ -929,95 +1330,10 @@ void HolovizOp::compute(InputContext& op_input, OutputContext& op_output, switch (input_spec.type_) { case InputType::COLOR: - case InputType::COLOR_LUT: { + case InputType::COLOR_LUT: // 2D color image - - // sanity checks - if (buffer_info.rank != 3) { - throw std::runtime_error( - fmt::format("Expected rank 3 for tensor '{}', type '{}', but got {}", - buffer_info.name, - inputTypeToString(input_spec.type_), - buffer_info.rank)); - } - if (buffer_info.image_format == static_cast(-1)) { - std::runtime_error( - fmt::format("Color image: element type {} and channel count {} not supported", - static_cast(buffer_info.element_type), - buffer_info.components)); - } - - viz::ImageFormat image_format; - if (input_spec.type_ == InputType::COLOR_LUT) { - if (buffer_info.components != 1) { - throw std::runtime_error(fmt::format( - "Expected one channel for tensor '{}' when using lookup table, but got {}", - buffer_info.name, - buffer_info.components)); - } - if (lut_.empty()) { - throw std::runtime_error(fmt::format( - "Type of tensor '{}' is '{}', but a color lookup table has not been specified", - buffer_info.name, - inputTypeToString(input_spec.type_))); - } - - // when using a LUT, the unorm formats are handled as single channel int formats - switch (buffer_info.image_format) { - case viz::ImageFormat::R8_UNORM: - image_format = viz::ImageFormat::R8_UINT; - break; - case viz::ImageFormat::R8_SNORM: - image_format = viz::ImageFormat::R8_SINT; - break; - case viz::ImageFormat::R16_UNORM: - image_format = viz::ImageFormat::R16_UINT; - break; - case viz::ImageFormat::R16_SNORM: - image_format = viz::ImageFormat::R16_SINT; - break; - default: - image_format = buffer_info.image_format; - break; - } - } else { - image_format = buffer_info.image_format; - } - - // start an image layer - viz::BeginImageLayer(); - set_input_spec(input_spec); - - if (input_spec.type_ == InputType::COLOR_LUT) { - viz::LUT(lut_.size() / 4, - viz::ImageFormat::R32G32B32A32_SFLOAT, - lut_.size() * sizeof(float), - lut_.data()); - } - - viz::ImageComponentMapping(buffer_info.component_swizzle[0], - buffer_info.component_swizzle[1], - buffer_info.component_swizzle[2], - buffer_info.component_swizzle[3]); - if (buffer_info.storage_type == nvidia::gxf::MemoryStorageType::kDevice) { - // if it's the device convert to `CUDeviceptr` - const auto cu_buffer_ptr = reinterpret_cast(buffer_info.buffer_ptr); - viz::ImageCudaDevice(buffer_info.width, - buffer_info.height, - image_format, - cu_buffer_ptr, - buffer_info.stride[0]); - } else { - // convert to void * if using the system/host - const auto host_buffer_ptr = reinterpret_cast(buffer_info.buffer_ptr); - viz::ImageHost(buffer_info.width, - buffer_info.height, - image_format, - host_buffer_ptr, - buffer_info.stride[0]); - } - viz::EndLayer(); - } break; + render_color_image(input_spec, buffer_info); + break; case InputType::POINTS: case InputType::LINES: @@ -1030,274 +1346,11 @@ void HolovizOp::compute(InputContext& op_input, OutputContext& op_output, case InputType::POINTS_3D: case InputType::LINES_3D: case InputType::LINE_STRIP_3D: - case InputType::TRIANGLES_3D: { + case InputType::TRIANGLES_3D: // geometry layer - if ((buffer_info.element_type != nvidia::gxf::PrimitiveType::kFloat32) && - (buffer_info.element_type != nvidia::gxf::PrimitiveType::kFloat64)) { - throw std::runtime_error( - fmt::format("Expected gxf::PrimitiveType::kFloat32 or gxf::PrimitiveType::kFloat64 " - "element type for coordinates, but got element type {}", - static_cast(buffer_info.element_type))); - } - - // get pointer to tensor buffer - std::vector host_buffer; - if (buffer_info.storage_type == nvidia::gxf::MemoryStorageType::kDevice) { - host_buffer.resize(buffer_info.bytes_size); - - // copy from device to host - CUDA_TRY(cudaMemcpyAsync(static_cast(host_buffer.data()), - static_cast(buffer_info.buffer_ptr), - buffer_info.bytes_size, - cudaMemcpyDeviceToHost, - cuda_stream_handler_.get_cuda_stream(context.context()))); - // wait for the CUDA memory copy to finish - CUDA_TRY(cudaStreamSynchronize(cuda_stream_handler_.get_cuda_stream(context.context()))); - - buffer_info.buffer_ptr = host_buffer.data(); - } - - // start a geometry layer - viz::BeginGeometryLayer(); - set_input_spec_geometry(input_spec); - - const auto coordinates = buffer_info.width; - - if (input_spec.type_ == InputType::TEXT) { - // text is defined by the top left coordinate and the size (x, y, s) per string, text - // strings are define by InputSpec::text_ - if ((buffer_info.components < 2) || (buffer_info.components > 3)) { - throw std::runtime_error(fmt::format( - "Expected two or three values per text, but got '{}'", buffer_info.components)); - } - if (input_spec.text_.empty()) { - throw std::runtime_error(fmt::format("No text has been specified by input spec '{}'.", - input_spec.tensor_name_)); - } - uintptr_t src_coord = reinterpret_cast(buffer_info.buffer_ptr); - constexpr uint32_t values_per_coordinate = 3; - float coords[values_per_coordinate]{0.f, 0.f, 0.05f}; - for (uint32_t index = 0; index < coordinates; ++index) { - uint32_t component_index = 0; - // copy from source array - while (component_index < buffer_info.components) { - switch (buffer_info.element_type) { - case nvidia::gxf::PrimitiveType::kFloat32: - coords[component_index] = - reinterpret_cast(src_coord)[component_index]; - break; - case nvidia::gxf::PrimitiveType::kFloat64: - coords[component_index] = - reinterpret_cast(src_coord)[component_index]; - break; - default: - throw std::runtime_error("Unhandled element type"); - } - ++component_index; - } - src_coord += buffer_info.stride[1]; - viz::Text( - coords[0], - coords[1], - coords[2], - input_spec - .text_[std::min(index, static_cast(input_spec.text_.size()) - 1)] - .c_str()); - } - } else { - std::vector coords; - viz::PrimitiveTopology topology; - uint32_t primitive_count; - uint32_t coordinate_count; - uint32_t values_per_coordinate; - std::vector default_coord; - switch (input_spec.type_) { - case InputType::POINTS: - // point primitives, one coordinate (x, y) per primitive - if (buffer_info.components != 2) { - throw std::runtime_error(fmt::format("Expected two values per point, but got '{}'", - buffer_info.components)); - } - topology = viz::PrimitiveTopology::POINT_LIST; - primitive_count = coordinates; - coordinate_count = primitive_count; - values_per_coordinate = 2; - default_coord = {0.f, 0.f}; - break; - case InputType::LINES: - // line primitives, two coordinates (x0, y0) and (x1, y1) per primitive - if (buffer_info.components != 2) { - throw std::runtime_error(fmt::format( - "Expected two values per line vertex, but got '{}'", buffer_info.components)); - } - topology = viz::PrimitiveTopology::LINE_LIST; - primitive_count = coordinates / 2; - coordinate_count = primitive_count * 2; - values_per_coordinate = 2; - default_coord = {0.f, 0.f}; - break; - case InputType::LINE_STRIP: - // line strip primitive, a line primitive i is defined by each coordinate (xi, yi) and - // the following (xi+1, yi+1) - if (buffer_info.components != 2) { - throw std::runtime_error( - fmt::format("Expected two values per line strip vertex, but got '{}'", - buffer_info.components)); - } - topology = viz::PrimitiveTopology::LINE_STRIP; - primitive_count = coordinates - 1; - coordinate_count = coordinates; - values_per_coordinate = 2; - default_coord = {0.f, 0.f}; - break; - case InputType::TRIANGLES: - // triangle primitive, three coordinates (x0, y0), (x1, y1) and (x2, y2) per primitive - if (buffer_info.components != 2) { - throw std::runtime_error( - fmt::format("Expected two values per triangle vertex, but got '{}'", - buffer_info.components)); - } - topology = viz::PrimitiveTopology::TRIANGLE_LIST; - primitive_count = coordinates / 3; - coordinate_count = primitive_count * 3; - values_per_coordinate = 2; - default_coord = {0.f, 0.f}; - break; - case InputType::CROSSES: - // cross primitive, a cross is defined by the center coordinate and the size (xi, yi, - // si) - if ((buffer_info.components < 2) || (buffer_info.components > 3)) { - throw std::runtime_error( - fmt::format("Expected two or three values per cross, but got '{}'", - buffer_info.components)); - } - - topology = viz::PrimitiveTopology::CROSS_LIST; - primitive_count = coordinates; - coordinate_count = primitive_count; - values_per_coordinate = 3; - default_coord = {0.f, 0.f, 0.05f}; - break; - case InputType::RECTANGLES: - // axis aligned rectangle primitive, each rectangle is defined by two coordinates (xi, - // yi) and (xi+1, yi+1) - if (buffer_info.components != 2) { - throw std::runtime_error( - fmt::format("Expected two values per rectangle vertex, but got '{}'", - buffer_info.components)); - } - topology = viz::PrimitiveTopology::RECTANGLE_LIST; - primitive_count = coordinates / 2; - coordinate_count = primitive_count * 2; - values_per_coordinate = 2; - default_coord = {0.f, 0.f}; - break; - case InputType::OVALS: - // oval primitive, an oval primitive is defined by the center coordinate and the axis - // sizes (xi, yi, sxi, syi) - if ((buffer_info.components < 2) || (buffer_info.components > 4)) { - throw std::runtime_error( - fmt::format("Expected two, three or four values per oval, but got '{}'", - buffer_info.components)); - } - topology = viz::PrimitiveTopology::OVAL_LIST; - primitive_count = coordinates; - coordinate_count = primitive_count; - values_per_coordinate = 4; - default_coord = {0.f, 0.f, 0.05f, 0.05f}; - break; - case InputType::POINTS_3D: - // point primitives, one coordinate (x, y, z) per primitive - if (buffer_info.components != 3) { - throw std::runtime_error(fmt::format( - "Expected three values per 3D point, but got '{}'", buffer_info.components)); - } - topology = viz::PrimitiveTopology::POINT_LIST_3D; - primitive_count = coordinates; - coordinate_count = primitive_count; - values_per_coordinate = 3; - default_coord = {0.f, 0.f, 0.f}; - break; - case InputType::LINES_3D: - // line primitives, two coordinates (x0, y0, z0) and (x1, y1, z1) per primitive - if (buffer_info.components != 3) { - throw std::runtime_error( - fmt::format("Expected three values per 3D line vertex, but got '{}'", - buffer_info.components)); - } - topology = viz::PrimitiveTopology::LINE_LIST_3D; - primitive_count = coordinates / 2; - coordinate_count = primitive_count * 2; - values_per_coordinate = 3; - default_coord = {0.f, 0.f, 0.f}; - break; - case InputType::LINE_STRIP_3D: - // line primitives, two coordinates (x0, y0, z0) and (x1, y1, z1) per primitive - if (buffer_info.components != 3) { - throw std::runtime_error( - fmt::format("Expected three values per 3D line strip vertex, but got '{}'", - buffer_info.components)); - } - topology = viz::PrimitiveTopology::LINE_STRIP_3D; - primitive_count = coordinates - 1; - coordinate_count = coordinates; - values_per_coordinate = 3; - default_coord = {0.f, 0.f}; - break; - case InputType::TRIANGLES_3D: - // triangle primitive, three coordinates (x0, y0, z0), (x1, y1, z1) and (x2, y2, z2) - // per primitive - if (buffer_info.components != 3) { - throw std::runtime_error( - fmt::format("Expected three values per 3D triangle vertex, but got '{}'", - buffer_info.components)); - } - topology = viz::PrimitiveTopology::TRIANGLE_LIST_3D; - primitive_count = coordinates / 3; - coordinate_count = primitive_count * 3; - values_per_coordinate = 3; - default_coord = {0.f, 0.f}; - break; - default: - throw std::runtime_error( - fmt::format("Unhandled tensor type '{}'", inputTypeToString(input_spec.type_))); - } - - // copy coordinates - uintptr_t src_coord = reinterpret_cast(buffer_info.buffer_ptr); - coords.reserve(coordinate_count * values_per_coordinate); - - for (uint32_t index = 0; index < coordinate_count; ++index) { - uint32_t component_index = 0; - // copy from source array - while (component_index < std::min(buffer_info.components, values_per_coordinate)) { - switch (buffer_info.element_type) { - case nvidia::gxf::PrimitiveType::kFloat32: - coords.push_back(reinterpret_cast(src_coord)[component_index]); - break; - case nvidia::gxf::PrimitiveType::kFloat64: - coords.push_back(reinterpret_cast(src_coord)[component_index]); - break; - default: - throw std::runtime_error("Unhandled element type"); - } - ++component_index; - } - // fill from default array - while (component_index < values_per_coordinate) { - coords.push_back(default_coord[component_index]); - ++component_index; - } - src_coord += buffer_info.stride[1]; - } - - if (primitive_count) { - viz::Primitive(topology, primitive_count, coords.size(), coords.data()); - } - } + render_geometry(context, input_spec, buffer_info); + break; - viz::EndLayer(); - } break; case InputType::DEPTH_MAP: { // 2D depth map if (buffer_info.element_type != nvidia::gxf::PrimitiveType::kUnsigned8) { @@ -1367,11 +1420,26 @@ void HolovizOp::compute(InputContext& op_input, OutputContext& op_output, // check if the the camera pose should be output if (camera_pose_output_enabled_) { - auto camera_pose_output = std::make_shared>(); + if (camera_pose_output_type_.get() == "projection_matrix") { + auto camera_pose_output = std::make_shared>(); + + viz::GetCameraPose(camera_pose_output->size(), camera_pose_output->data()); + + op_output.emit(camera_pose_output, "camera_pose_output"); + } else if (camera_pose_output_type_.get() == "extrinsics_model") { + float rotation[9]; + float translation[3]; - viz::GetCameraPose(camera_pose_output->size(), camera_pose_output->data()); + viz::GetCameraPose(rotation, translation); - op_output.emit(camera_pose_output, "camera_pose_output"); + auto pose = std::make_shared(); + std::copy(std::begin(rotation), std::end(rotation), std::begin(pose->rotation)); + std::copy(std::begin(translation), std::end(translation), std::begin(pose->translation)); + op_output.emit(pose, "camera_pose_output"); + } else { + throw std::runtime_error(fmt::format("Unhandled camera pose output type type '{}'", + camera_pose_output_type_.get())); + } } if (is_first_tick_) { diff --git a/src/operators/inference/inference.cpp b/src/operators/inference/inference.cpp index 728206af..6b6e7486 100644 --- a/src/operators/inference/inference.cpp +++ b/src/operators/inference/inference.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -134,6 +134,11 @@ void InferenceOp::setup(OperatorSpec& spec) { "Model Keyword with associated device", "Device ID on which model will do inference.", DataMap()); + spec.param(temporal_map_, + "temporal_map", + "Model Keyword with associated frame execution delay", + "Frame delay for model inference.", + DataMap()); spec.param(pre_processor_map_, "pre_processor_map", "Pre processor setting per model", @@ -193,6 +198,7 @@ void InferenceOp::start() { pre_processor_map_.get().get_map(), inference_map_.get().get_map(), device_map_.get().get_map(), + temporal_map_.get().get_map(), is_engine_path_.get(), infer_on_cpu_.get(), parallel_inference_.get(), diff --git a/src/operators/segmentation_postprocessor/segmentation_postprocessor.cpp b/src/operators/segmentation_postprocessor/segmentation_postprocessor.cpp index ca074f92..fa345c65 100644 --- a/src/operators/segmentation_postprocessor/segmentation_postprocessor.cpp +++ b/src/operators/segmentation_postprocessor/segmentation_postprocessor.cpp @@ -90,7 +90,20 @@ void SegmentationPostprocessorOp::compute(InputContext& op_input, OutputContext& throw std::runtime_error(fmt::format("Tensor '{}' not found in message", in_tensor_name)); } } + auto in_tensor = maybe_tensor; + // validate tensor format + DLDevice dev = in_tensor->device(); + if (dev.device_type != kDLCUDA && dev.device_type != kDLCUDAHost) { + throw std::runtime_error("Input tensor must be in CUDA device or pinned host memory."); + } + DLDataType dtype = in_tensor->dtype(); + if (dtype.code != kDLFloat || dtype.bits != 32) { + throw std::runtime_error("Input tensor must be of type float32."); + } + if (!in_tensor->is_contiguous()) { + throw std::runtime_error("Input tensor must have row-major memory layout."); + } // get the CUDA stream from the input message gxf_result_t stream_handler_result = @@ -107,11 +120,13 @@ void SegmentationPostprocessorOp::compute(InputContext& op_input, OutputContext& shape.channels = in_tensor->shape()[2]; } break; case DataFormat::kNCHW: { + if (in_tensor->shape()[0] != 1) { throw std::runtime_error("Batch size must be 1"); } shape.channels = in_tensor->shape()[1]; shape.height = in_tensor->shape()[2]; shape.width = in_tensor->shape()[3]; } break; case DataFormat::kNHWC: { + if (in_tensor->shape()[0] != 1) { throw std::runtime_error("Batch size must be 1"); } shape.height = in_tensor->shape()[1]; shape.width = in_tensor->shape()[2]; shape.channels = in_tensor->shape()[3]; @@ -123,6 +138,16 @@ void SegmentationPostprocessorOp::compute(InputContext& op_input, OutputContext& "Input channel count larger than allowed: {} > {}", shape.channels, kMaxChannelCount)); } + if ((network_output_type_value_ == NetworkOutputType::kSigmoid) && (shape.channels > 1)) { + static bool warned = false; + if (!warned) { + warned = true; + HOLOSCAN_LOG_WARN( + "Multi-channel input provided, but network_output_type is 'sigmoid'. Only the first " + "channel will be used."); + } + } + // Create a new message (nvidia::gxf::Entity) auto out_message = nvidia::gxf::Entity::New(context.context()); diff --git a/src/operators/v4l2_video_capture/CMakeLists.txt b/src/operators/v4l2_video_capture/CMakeLists.txt index 1b0a8c88..204b4f1c 100644 --- a/src/operators/v4l2_video_capture/CMakeLists.txt +++ b/src/operators/v4l2_video_capture/CMakeLists.txt @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,4 +21,5 @@ target_link_libraries(op_v4l2 PRIVATE GXF::multimedia V4L2 -) \ No newline at end of file + jpeg +) diff --git a/src/operators/v4l2_video_capture/v4l2_video_capture.cpp b/src/operators/v4l2_video_capture/v4l2_video_capture.cpp index e25e09ee..2d1bfe3c 100644 --- a/src/operators/v4l2_video_capture/v4l2_video_capture.cpp +++ b/src/operators/v4l2_video_capture/v4l2_video_capture.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -143,6 +144,15 @@ void V4L2VideoCaptureOp::compute(InputContext& op_input, OutputContext& op_outpu // Convert YUYV to RGBA output buffer YUYVToRGBA(read_buf.ptr, video_buffer.value()->pointer(), width_use_, height_use_); + // Return (queue) the buffer. + if (ioctl(fd_, VIDIOC_QBUF, &buf) < 0) { + throw std::runtime_error( + fmt::format("Failed to queue buffer {} on {}", buf.index, device_.get().c_str())); + } + } else if (pixel_format_use_ == V4L2_PIX_FMT_MJPEG) { + // Convert MJPG to RGBA output buffer + MJPEGToRGBA(read_buf.ptr, video_buffer.value()->pointer(), width_use_, height_use_); + // Return (queue) the buffer. if (ioctl(fd_, VIDIOC_QBUF, &buf) < 0) { throw std::runtime_error( @@ -270,17 +280,20 @@ void V4L2VideoCaptureOp::v4l2_check_formats() { // Update format with valid user-given format pixel_format_use_ = pixel_format; } else if (pixel_format_.get() == "auto") { - // Currently, AB24 and YUYV are supported in auto mode + // Currently, AB24, YUYV, and MJPG are supported in auto mode uint32_t ab24 = v4l2_fourcc('A', 'B', '2', '4'); uint32_t yuyv = v4l2_fourcc('Y', 'U', 'Y', 'V'); + uint32_t mjpg = v4l2_fourcc('M', 'J', 'P', 'G'); if (pixel_format_supported(fd_, ab24)) { pixel_format_use_ = ab24; } else if (pixel_format_supported(fd_, yuyv)) { pixel_format_use_ = yuyv; + } else if (pixel_format_supported(fd_, mjpg)) { + pixel_format_use_ = mjpg; } else { throw std::runtime_error( - "Automatic setting of pixel format failed: device does not support AB24 or YUYV. " + "Automatic setting of pixel format failed: device does not support AB24, YUYV, or MJPG. " " If you are sure that the device pixel format is RGBA, please specify the pixel format " "in the yaml configuration file."); } @@ -539,4 +552,49 @@ void V4L2VideoCaptureOp::YUYVToRGBA(const void* yuyv, void* rgba, size_t width, } } +// Support for MJPEG format +// Each frame is a JPEG image so use libjpeg to decompress the image and modify it to +// add alpha channel +void V4L2VideoCaptureOp::MJPEGToRGBA(const void* mjpg, void* rgba, size_t width, size_t height) { + struct jpeg_decompress_struct cinfo; + struct jpeg_error_mgr jerr; + // Size of image is width * height * 3 (RGB) + unsigned long jpg_size = width * height * 3; + int row_stride; + + cinfo.err = jpeg_std_error(&jerr); + jpeg_create_decompress(&cinfo); + + const unsigned char* src_buf = + const_cast(static_cast(mjpg)); + unsigned char* dest_buf = static_cast(rgba); + jpeg_mem_src(&cinfo, src_buf, jpg_size); + int rc = jpeg_read_header(&cinfo, TRUE); + + if (rc != 1) { throw std::runtime_error("Failed to read jpeg header"); } + + jpeg_start_decompress(&cinfo); + + // Each row has width * 4 pixels (RGBA) + row_stride = width * 4; + + while (cinfo.output_scanline < cinfo.output_height) { + unsigned char* buffer_array[1]; + buffer_array[0] = dest_buf + (cinfo.output_scanline) * row_stride; + // Decompress jpeg image and write it to buffer_arary + jpeg_read_scanlines(&cinfo, buffer_array, 1); + unsigned char* buf = buffer_array[0]; + // Modify image to add alpha channel with values set to 255 + // start from the end so we don't overwrite existing values + for (int i = (int)width * 3 - 1, j = row_stride - 1; i > 0; i -= 3, j -= 4) { + buf[j] = 255; + buf[j - 1] = buf[i]; + buf[j - 2] = buf[i - 1]; + buf[j - 3] = buf[i - 2]; + } + } + jpeg_finish_decompress(&cinfo); + jpeg_destroy_decompress(&cinfo); +} + } // namespace holoscan::ops diff --git a/src/operators/video_stream_recorder/video_stream_recorder.cpp b/src/operators/video_stream_recorder/video_stream_recorder.cpp index 3808473f..53836f1e 100644 --- a/src/operators/video_stream_recorder/video_stream_recorder.cpp +++ b/src/operators/video_stream_recorder/video_stream_recorder.cpp @@ -53,7 +53,7 @@ void VideoStreamRecorderOp::setup(OperatorSpec& spec) { } void VideoStreamRecorderOp::initialize() { - // Set up prerequisite parameters before calling GXFOperator::initialize() + // Set up prerequisite parameters before calling Operator::initialize() auto frag = fragment(); auto entity_serializer = frag->make_resource("recorder__std_entity_serializer"); diff --git a/src/operators/video_stream_replayer/video_stream_replayer.cpp b/src/operators/video_stream_replayer/video_stream_replayer.cpp index 3cead809..48639d68 100644 --- a/src/operators/video_stream_replayer/video_stream_replayer.cpp +++ b/src/operators/video_stream_replayer/video_stream_replayer.cpp @@ -89,7 +89,7 @@ void VideoStreamReplayerOp::setup(OperatorSpec& spec) { } void VideoStreamReplayerOp::initialize() { - // Set up prerequisite parameters before calling GXFOperator::initialize() + // Set up prerequisite parameters before calling Operator::initialize() auto frag = fragment(); auto entity_serializer = frag->make_resource("replayer__std_entity_serializer"); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c771c71d..82e3b621 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -70,6 +70,7 @@ ConfigureTest( core/condition.cpp core/condition_classes.cpp core/config.cpp + core/data_exporter.cpp core/dataflow_tracker.cpp core/fragment.cpp core/fragment_allocation.cpp @@ -126,6 +127,8 @@ ConfigureTest( system/env_wrapper.cpp system/exception_handling.cpp system/demosaic_op_app.cpp + system/format_converter_op_apps.cpp + system/jobstatistics_app.cpp system/holoviz_op_apps.cpp system/multithreaded_app.cpp system/native_async_operator_ping_app.cpp @@ -157,6 +160,7 @@ ConfigureTest( system/distributed/holoscan_ucx_ports_env.cpp system/distributed/ping_message_rx_op.cpp system/distributed/ping_message_tx_op.cpp + system/distributed/standalone_fragments.cpp system/distributed/ucx_message_serialization_ping_app.cpp system/env_wrapper.cpp system/ping_tensor_rx_op.cpp @@ -169,15 +173,26 @@ target_link_libraries(SYSTEM_DISTRIBUTED_TEST ) # set environment variables used by distributed applications in the tests -# - HOLOSCAN_STOP_ON_DEADLOCK_TIMEOUT=2500 : Set deadlock timeout for distributed app -# - HOLOSCAN_MAX_DURATION_MS=2500 : Set max duration for distributed app +# - HOLOSCAN_STOP_ON_DEADLOCK_TIMEOUT=3000 : Set deadlock timeout for distributed app +# - HOLOSCAN_MAX_DURATION_MS=3000 : Set max duration for distributed app +if(${CMAKE_SYSTEM_PROCESSOR} STREQUAL "aarch64") set(CMAKE_SYSTEM_DISTRIBUTED_TEST_FLAGS "\ -HOLOSCAN_STOP_ON_DEADLOCK_TIMEOUT=2500;\ -HOLOSCAN_MAX_DURATION_MS=2500;\ +HOLOSCAN_STOP_ON_DEADLOCK_TIMEOUT=6000;\ +HOLOSCAN_MAX_DURATION_MS=6000;\ HOLOSCAN_DISTRIBUTED_APP_SCHEDULER=multi_thread\ " ) +else() + set(CMAKE_SYSTEM_DISTRIBUTED_TEST_FLAGS +"\ +HOLOSCAN_STOP_ON_DEADLOCK_TIMEOUT=3000;\ +HOLOSCAN_MAX_DURATION_MS=3000;\ +HOLOSCAN_DISTRIBUTED_APP_SCHEDULER=multi_thread\ +" + ) +endif() + set_tests_properties( SYSTEM_DISTRIBUTED_TEST PROPERTIES ENVIRONMENT "${CMAKE_SYSTEM_DISTRIBUTED_TEST_FLAGS}" ) @@ -189,6 +204,7 @@ ConfigureTest( system/distributed/distributed_app.cpp system/distributed/distributed_demosaic_op_app.cpp system/distributed/holoscan_ucx_ports_env.cpp + system/distributed/standalone_fragments.cpp system/env_wrapper.cpp system/ping_tensor_rx_op.cpp system/ping_tensor_tx_op.cpp @@ -201,8 +217,8 @@ target_link_libraries(SYSTEM_DISTRIBUTED_EBS_TEST set(CMAKE_SYSTEM_DISTRIBUTED_EBS_TEST_FLAGS "\ -HOLOSCAN_STOP_ON_DEADLOCK_TIMEOUT=2500;\ -HOLOSCAN_MAX_DURATION_MS=2500;\ +HOLOSCAN_STOP_ON_DEADLOCK_TIMEOUT=5000;\ +HOLOSCAN_MAX_DURATION_MS=5000;\ HOLOSCAN_DISTRIBUTED_APP_SCHEDULER=event_based\ " ) @@ -248,13 +264,12 @@ target_link_libraries(HOLOINFER_TEST holoinfer ) -add_dependencies(HOLOINFER_TEST multiai_ultrasound_data) - # ################################################################################################## # * Flow Tracking tests ---------------------------------------------------------------------------------- ConfigureTest( FLOW_TRACKING_TEST flow_tracking/flow_tracking_cycle.cpp + flow_tracking/entity_passthrough.cpp ) target_link_libraries(FLOW_TRACKING_TEST PRIVATE diff --git a/tests/core/app_driver.cpp b/tests/core/app_driver.cpp index af817eb2..2992f597 100644 --- a/tests/core/app_driver.cpp +++ b/tests/core/app_driver.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,6 +20,7 @@ #include #include +#include #include @@ -112,4 +113,26 @@ TEST(AppDriver, TestExcludeCudaIpcTransportOnIgpu) { } } +TEST(AppDriver, TestGetBoolEnvVar) { + // Test cases for get_bool_env_var (issue 4616525) + std::vector> test_cases{ + {"tRue", true}, + {"False", false}, + {"1", true}, + {"0", false}, + {"On", true}, + {"Off", false}, + {"", true}, + {"invalid", true}, + }; + + const char* env_name = "HOLOSCAN_TEST_BOOL_ENV_VAR"; + for (const auto& [env_value, expected_result] : test_cases) { + setenv(env_name, env_value.c_str(), 1); + bool result = AppDriver::get_bool_env_var(env_name, true); + EXPECT_EQ(result, expected_result); + unsetenv(env_name); + } +} + } // namespace holoscan diff --git a/tests/core/data_exporter.cpp b/tests/core/data_exporter.cpp new file mode 100644 index 00000000..e0d5b06a --- /dev/null +++ b/tests/core/data_exporter.cpp @@ -0,0 +1,200 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include + +#include "common/assert.hpp" +#include "holoscan/core/analytics/csv_data_exporter.hpp" +#include "holoscan/core/analytics/data_exporter.hpp" + +namespace holoscan { + +TEST(DataExporterAPI, TestDataExporterConstructor) { + const std::string app_name = "test_app1"; + const std::vector columns = {"column1", "column2", "column3"}; + CsvDataExporter csv_data_exporter(app_name, columns); + + ASSERT_EQ(csv_data_exporter.app_name(), app_name); + ASSERT_EQ(csv_data_exporter.columns().size(), columns.size()); + ASSERT_EQ(csv_data_exporter.output_file_name(), kAnalyticsOutputFileName); + + csv_data_exporter.cleanup_data_directory(); +} + +TEST(DataExporterAPI, TestDataExporterDefaultDirectory) { + const std::string app_name = "test_app2"; + const std::vector columns = {"column1", "column2", "column3"}; + CsvDataExporter csv_data_exporter(app_name, columns); + + auto data_dir = csv_data_exporter.data_directory(); + ASSERT_TRUE(std::filesystem::exists(data_dir)); + + std::filesystem::path path1(data_dir); + std::filesystem::path path2(std::filesystem::current_path()); + path2 = path2 / app_name; + std::filesystem::path abs_path1 = std::filesystem::absolute(path1); + std::filesystem::path abs_path2 = std::filesystem::absolute(path2); + ASSERT_EQ(abs_path2, abs_path1.parent_path()); + + csv_data_exporter.cleanup_data_directory(); +} + +TEST(DataExporterAPI, TestDataExporterDirectory) { + const std::string app_name = "test_app3"; + const std::vector columns = {"column1", "column2", "column3"}; + auto path = std::filesystem::current_path(); + path = path / "test_dir"; + const std::string root_data_dir = path.string(); + setenv("HOLOSCAN_ANALYTICS_DATA_DIRECTORY", root_data_dir.c_str(), 1); + CsvDataExporter csv_data_exporter(app_name, columns); + + auto data_dir = csv_data_exporter.data_directory(); + ASSERT_TRUE(std::filesystem::exists(data_dir)); + + std::filesystem::path path1(data_dir); + std::filesystem::path path2(root_data_dir); + path2 = path2 / app_name; + std::filesystem::path abs_path1 = std::filesystem::absolute(path1); + std::filesystem::path abs_path2 = std::filesystem::absolute(path2); + ASSERT_EQ(abs_path2, abs_path1.parent_path()); + + unsetenv("HOLOSCAN_ANALYTICS_DATA_DIRECTORY"); + csv_data_exporter.cleanup_data_directory(); + // Explicitly remove root directory as we have created the new one. + std::filesystem::remove_all(path); +} + +TEST(DataExporterAPI, TestDataExporterDirectoryWithTimestamp) { + const std::string app_name = "test_app4"; + const std::vector columns = {"column1", "column2", "column3"}; + CsvDataExporter csv_data_exporter(app_name, columns); + + auto now = std::chrono::system_clock::now(); + auto local_time = std::chrono::system_clock::to_time_t(now); + std::ostringstream local_time_ss; + local_time_ss << std::put_time(std::localtime(&local_time), "%Y%m%d"); + auto data_dir = csv_data_exporter.data_directory(); + std::filesystem::path data_dir_path(data_dir); + auto file_name_str = data_dir_path.filename().string(); + ASSERT_TRUE(file_name_str.find(local_time_ss.str()) != std::string::npos); + + csv_data_exporter.cleanup_data_directory(); +} + +TEST(DataExporterAPI, TestDataExporterDirectoryWithTimestampOnEachRun) { + const std::string app_name = "test_app5"; + const std::vector columns = {"column1", "column2", "column3"}; + CsvDataExporter csv_data_exporter1(app_name, columns); + + const std::string app_name2 = "test_app_2"; + CsvDataExporter csv_data_exporter2(app_name2, columns); + + auto data_dir_1 = csv_data_exporter1.data_directory(); + auto data_dir_2 = csv_data_exporter2.data_directory(); + + ASSERT_TRUE(data_dir_1 != data_dir_2); + + csv_data_exporter1.cleanup_data_directory(); + csv_data_exporter2.cleanup_data_directory(); +} + +TEST(DataExporterAPI, TestDataExporterDataFile) { + const std::string app_name = "test_app6"; + const std::vector columns = {"column1", "column2", "column3"}; + CsvDataExporter csv_data_exporter(app_name, columns); + + std::string file_name = csv_data_exporter.data_directory() + "/" + kAnalyticsOutputFileName; + ASSERT_TRUE(std::filesystem::exists(file_name)); + ASSERT_EQ(csv_data_exporter.output_file_name(), kAnalyticsOutputFileName); + + csv_data_exporter.cleanup_data_directory(); +} + +TEST(DataExporterAPI, TestDataExporterCustomDataFile) { + const std::string app_name = "test_app7"; + const std::vector columns = {"column1", "column2", "column3"}; + const std::string custom_data_file = "output.csv"; + setenv("HOLOSCAN_ANALYTICS_DATA_FILE_NAME", custom_data_file.c_str(), 1); + CsvDataExporter csv_data_exporter(app_name, columns); + + std::string file_name = csv_data_exporter.data_directory() + "/" + custom_data_file; + ASSERT_TRUE(std::filesystem::exists(file_name)); + ASSERT_EQ(csv_data_exporter.output_file_name(), custom_data_file); + + unsetenv("HOLOSCAN_ANALYTICS_DATA_FILE_NAME"); + csv_data_exporter.cleanup_data_directory(); +} + +TEST(DataExporterAPI, TestDataExporterCsvFileColumns) { + const std::string app_name = "test_app8"; + const std::vector columns = {"column1", "column2", "column3"}; + std::string file_name; + std::string directory; + { + CsvDataExporter csv_data_exporter(app_name, columns); + directory = csv_data_exporter.data_directory(); + file_name = directory + "/" + csv_data_exporter.output_file_name(); + ASSERT_TRUE(std::filesystem::exists(file_name)); + } + + std::ifstream file(file_name); + ASSERT_TRUE(file.is_open()); + std::string line; + std::getline(file, line); + file.close(); + ASSERT_EQ(line, "column1,column2,column3"); + + // Explicitly remove directory as DatExporter's cleanup function cannot be called here. + std::filesystem::path abs_path = std::filesystem::absolute(directory); + std::filesystem::remove_all(abs_path.parent_path()); +} + +TEST(DataExporterAPI, TestDataExporterCsvData) { + const std::string app_name = "test_app9"; + const std::vector columns = {"column1", "column2", "column3"}; + std::string file_name; + std::string directory; + { + CsvDataExporter csv_data_exporter(app_name, columns); + directory = csv_data_exporter.data_directory(); + file_name = directory + "/" + csv_data_exporter.output_file_name(); + ASSERT_TRUE(std::filesystem::exists(file_name)); + const std::vector csv_data = {"1", "2", "3"}; + csv_data_exporter.export_data(csv_data); + } + + std::ifstream file(file_name); + ASSERT_TRUE(file.is_open()); + std::string line; + std::getline(file, line); + ASSERT_EQ(line, "column1,column2,column3"); + std::getline(file, line); + ASSERT_EQ(line, "1,2,3"); + file.close(); + + // Explicitly remove directory as DatExporter's cleanup function cannot be called here. + std::filesystem::path abs_path = std::filesystem::absolute(directory); + std::filesystem::remove_all(abs_path.parent_path()); +} + +} // namespace holoscan diff --git a/tests/core/io_spec.cpp b/tests/core/io_spec.cpp index c1f33ca2..07c2e78d 100644 --- a/tests/core/io_spec.cpp +++ b/tests/core/io_spec.cpp @@ -354,7 +354,8 @@ connector_type: kDoubleBuffer - name: min_size type: uint64_t value: 5 - spec: ~)", + spec: ~ + type: kMessageAvailable)", entity_typename); EXPECT_EQ(spec.description(), description); } diff --git a/tests/core/parameter.cpp b/tests/core/parameter.cpp index 9c7267b2..917c9403 100644 --- a/tests/core/parameter.cpp +++ b/tests/core/parameter.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2022-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2022-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,8 +15,8 @@ * limitations under the License. */ -#include "holoscan/core/parameter.hpp" +#include "holoscan/core/parameter.hpp" #include #include @@ -163,4 +163,28 @@ TEST(Parameter, TestAsteriskOperator) { ASSERT_EQ(*param_int_shared_ptr, 10); } +// simple test for formatter feature +TEST(Parameter, TestMetaParameterFormatter) { + MetaParameter p = MetaParameter(5); + std::string format_message = fmt::format("{}", p); + EXPECT_EQ(format_message, std::string{"5"}); + // capture output so that we can check that the expected value was logged + testing::internal::CaptureStderr(); + HOLOSCAN_LOG_INFO("Formatted parameter value: {}", p); + std::string log_output = testing::internal::GetCapturedStderr(); + EXPECT_TRUE(log_output.find("5") != std::string::npos); +} + +// simple test with format option for formatter feature +TEST(Parameter, TestMetaParameterFormatterSyntax) { + MetaParameter seconds = MetaParameter(1.32); + std::string format_message = fmt::format("{:0.3f}", seconds); + EXPECT_EQ(format_message, std::string{"1.320"}); + // capture output so that we can check that the expected value was logged + testing::internal::CaptureStderr(); + HOLOSCAN_LOG_INFO("Formatted parameter value: {:0.3f}", seconds); + std::string log_output = testing::internal::GetCapturedStderr(); + EXPECT_TRUE(log_output.find("1.320") != std::string::npos); +} + } // namespace holoscan diff --git a/tests/data/validation_frames/holoviz_geometry/cpp_holoviz_geometry.patch b/tests/data/validation_frames/holoviz_geometry/cpp_holoviz_geometry.patch index 0f0ee81a..a4f5ccdf 100644 --- a/tests/data/validation_frames/holoviz_geometry/cpp_holoviz_geometry.patch +++ b/tests/data/validation_frames/holoviz_geometry/cpp_holoviz_geometry.patch @@ -1,11 +1,11 @@ diff --git a/public/examples/holoviz/cpp/holoviz_geometry.cpp b/public/examples/holoviz/cpp/holoviz_geometry.cpp -index 2ac25f3b8..449ae0b68 100644 +index 1eb86c2ed..9be565f8a 100644 --- a/public/examples/holoviz/cpp/holoviz_geometry.cpp +++ b/public/examples/holoviz/cpp/holoviz_geometry.cpp @@ -26,6 +26,12 @@ - + #include - + +#ifdef RECORD_OUTPUT + #include + #include @@ -13,15 +13,15 @@ index 2ac25f3b8..449ae0b68 100644 +#endif + namespace holoscan::ops { - - class CameraPoseRxOp : public Operator { -@@ -336,6 +342,9 @@ class HolovizGeometryApp : public holoscan::Application { + + /** +@@ -316,6 +322,9 @@ class HolovizGeometryApp : public holoscan::Application { + add_flow(source, visualizer, {{"outputs", "receivers"}}); add_flow(source, visualizer, {{"output_specs", "input_specs"}}); add_flow(replayer, visualizer, {{"output", "receivers"}}); - add_flow(visualizer, camera_pose_rx, {{"camera_pose_output", "in"}}); + + // Recorder to validate the video output + RECORDER(visualizer); } - + private: diff --git a/tests/data/validation_frames/holoviz_geometry/python_holoviz_geometry.patch b/tests/data/validation_frames/holoviz_geometry/python_holoviz_geometry.patch index 2c4ee07e..b545a038 100644 --- a/tests/data/validation_frames/holoviz_geometry/python_holoviz_geometry.patch +++ b/tests/data/validation_frames/holoviz_geometry/python_holoviz_geometry.patch @@ -1,50 +1,50 @@ diff --git a/public/examples/holoviz/python/holoviz_geometry.py b/public/examples/holoviz/python/holoviz_geometry.py -index 04726bc1f..2ba16ed0d 100644 +index 2b00e3dcb..821a74664 100644 --- a/public/examples/holoviz/python/holoviz_geometry.py +++ b/public/examples/holoviz/python/holoviz_geometry.py -@@ -22,7 +22,10 @@ from argparse import ArgumentParser +@@ -22,7 +22,13 @@ from argparse import ArgumentParser import numpy as np - + from holoscan.core import Application, Operator, OperatorSpec -from holoscan.operators import HolovizOp, VideoStreamReplayerOp +from holoscan.operators import ( -+ FormatConverterOp, HolovizOp, VideoStreamRecorderOp, VideoStreamReplayerOp ++ FormatConverterOp, ++ HolovizOp, ++ VideoStreamRecorderOp, ++ VideoStreamReplayerOp, +) +from holoscan.resources import UnboundedAllocator - + sample_data_path = os.environ.get("HOLOSCAN_INPUT_PATH", "../data") - -@@ -331,6 +333,7 @@ class MyVideoProcessingApp(Application): + +@@ -302,11 +308,30 @@ class MyVideoProcessingApp(Application): text=["label_1", "label_2"], ), ], + enable_render_buffer_output=True, ) - # Since we specified `enable_camera_pose_output=True` for the visualizer, we can connect - # this output port to a receiver to print the camera pose. This receiver will just print -@@ -342,6 +345,25 @@ class MyVideoProcessingApp(Application): + self.add_flow(source, visualizer, {("output", "receivers")}) + self.add_flow(image_processing, visualizer, {("outputs", "receivers")}) self.add_flow(image_processing, visualizer, {("output_specs", "input_specs")}) - self.add_flow(visualizer, rx, {("camera_pose_output", "in")}) - + + recorder_format_converter = FormatConverterOp( + self, + name="recorder_format_converter", + in_dtype="rgba8888", + out_dtype="rgb888", -+ pool=UnboundedAllocator(self, name="pool") ++ pool=UnboundedAllocator(self, name="pool"), + ) + recorder = VideoStreamRecorderOp( -+ self, -+ name="recorder", -+ directory="RECORDING_DIR", -+ basename="SOURCE_VIDEO_BASENAME" ++ self, name="recorder", directory="RECORDING_DIR", basename="SOURCE_VIDEO_BASENAME" + ) + + visualizer.add_arg(allocator=UnboundedAllocator(self, name="allocator")) + -+ self.add_flow(visualizer, recorder_format_converter, {("render_buffer_output", "source_video")}) ++ self.add_flow( ++ visualizer, recorder_format_converter, {("render_buffer_output", "source_video")} ++ ) + self.add_flow(recorder_format_converter, recorder) + - + def main(config_count): app = MyVideoProcessingApp(config_count=config_count) diff --git a/tests/flow_tracking/entity_passthrough.cpp b/tests/flow_tracking/entity_passthrough.cpp new file mode 100644 index 00000000..f2762dc0 --- /dev/null +++ b/tests/flow_tracking/entity_passthrough.cpp @@ -0,0 +1,136 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include + +#include +#include + +namespace holoscan { + +// Do not pollute holoscan namespace with utility classes +namespace { + +/////////////////////////////////////////////////////////////////////////////// +// Flow Tracking Passthrough Operators +/////////////////////////////////////////////////////////////////////////////// + +class OneInOp : public Operator { + public: + HOLOSCAN_OPERATOR_FORWARD_ARGS(OneInOp) + + OneInOp() = default; + + void setup(OperatorSpec& spec) override { spec.input("in"); } + + void compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) override { + auto in_message = op_input.receive("in"); + + HOLOSCAN_LOG_INFO("OneInOp count {}", count_++); + } + + private: + int count_ = 1; +}; + +class OneOutOp : public Operator { + public: + HOLOSCAN_OPERATOR_FORWARD_ARGS(OneOutOp) + + OneOutOp() = default; + + void setup(OperatorSpec& spec) override { spec.output("out"); } + + void compute(InputContext&, OutputContext& op_output, ExecutionContext& context) override { + auto out_message = gxf::Entity::New(&context); + op_output.emit(out_message); + + HOLOSCAN_LOG_INFO("{} count {}", name(), count_++); + } + + private: + int count_ = 1; +}; + +class OneInOneOutOp : public Operator { + public: + HOLOSCAN_OPERATOR_FORWARD_ARGS(OneInOneOutOp) + + OneInOneOutOp() = default; + + void setup(OperatorSpec& spec) override { + spec.input("in"); + spec.output("out"); + } + + void compute(InputContext& op_input, OutputContext& op_output, + ExecutionContext& context) override { + auto in_message = op_input.receive("in"); + op_output.emit(in_message.value()); + HOLOSCAN_LOG_INFO("{} count {}", name(), count_++); + } + + private: + int count_ = 1; +}; + +/////////////////////////////////////////////////////////////////////////////// +// Flow Tracking Passthrough Application +/////////////////////////////////////////////////////////////////////////////// + +/* PassthroughApp + * + * OneOut--->OneInOneOut--->OneIn + */ +class PassthroughApp : public holoscan::Application { + public: + using Application::Application; + + void compose() override { + using namespace holoscan; + auto one_out = + make_operator("OneOut", make_condition("count-condition", 3)); + auto one_in = make_operator("OneIn"); + auto one_in_one_out = make_operator("OneInOneOut"); + + add_flow(one_out, one_in_one_out, {{"out", "in"}}); + add_flow(one_in_one_out, one_in, {{"out", "in"}}); + } +}; + +} // namespace + +TEST(Graphs, TestFlowTrackingWithEntityPassthrough) { + auto app = make_application(); + auto& tracker = app->track(0, 0, 0); + + // capture output so that we can check that the expected value is present + testing::internal::CaptureStdout(); + + app->run(); + + tracker.print(); + + std::string log_output = testing::internal::GetCapturedStdout(); + EXPECT_TRUE(log_output.find("OneOut,OneInOneOut,OneIn") != std::string::npos); +} + +} // namespace holoscan diff --git a/tests/holoinfer/inference/test_core.cpp b/tests/holoinfer/inference/test_core.cpp index 93a0899e..3df1307b 100644 --- a/tests/holoinfer/inference/test_core.cpp +++ b/tests/holoinfer/inference/test_core.cpp @@ -116,6 +116,7 @@ void HoloInferTests::setup_specifications() { pre_processor_map, inference_map, device_map, + temporal_map, is_engine_path, infer_on_cpu, parallel_inference, diff --git a/tests/holoinfer/inference/test_core.hpp b/tests/holoinfer/inference/test_core.hpp index 9ad79a76..48ef8e3b 100644 --- a/tests/holoinfer/inference/test_core.hpp +++ b/tests/holoinfer/inference/test_core.hpp @@ -57,27 +57,36 @@ class HoloInferTests { unsigned int pass_test_count = 0, fail_test_count = 0, total_test_count = 0; std::string backend = "trt"; - std::vector in_tensor_names = {"bmode_pre_proc", "aortic_pre_proc"}; - std::vector out_tensor_names = {"bmode_infer", "aortic_infer"}; + + std::vector in_tensor_names = {"m1_pre_proc", "m2_pre_proc"}; + std::vector out_tensor_names = {"m1_infer", "m2_infer"}; + + std::string model_folder = "../examples/bring_your_own_model/model/"; std::map model_path_map = { - {"bmode_perspective", "../data/multiai_ultrasound/models/bmode_perspective.onnx"}, - {"aortic_stenosis", "../data/multiai_ultrasound/models/aortic_stenosis.onnx"}, + {"model_1", model_folder + "identity_model.onnx"}, + {"model_2", model_folder + "identity_model.onnx"}, }; - std::map device_map = {{"bmode_perspective", "0"}, - {"aortic_stenosis", "0"}}; + std::map device_map = {{"model_1", "0"}, {"model_2", "0"}}; + + std::map temporal_map = {{"model_1", "1"}, {"model_2", "1"}}; std::map backend_map; std::map> pre_processor_map = { - {"bmode_perspective", {"bmode_pre_proc"}}, - {"aortic_stenosis", {"aortic_pre_proc"}}, + {"model_1", {"m1_pre_proc"}}, + {"model_2", {"m2_pre_proc"}}, }; std::map> inference_map = { - {"bmode_perspective", {"bmode_infer"}}, - {"aortic_stenosis", {"aortic_infer"}}, + {"model_1", {"m1_infer"}}, + {"model_2", {"m2_infer"}}, + }; + + const std::map> in_tensor_dimensions = { + {"m1_pre_proc", {3, 256, 256}}, + {"m2_pre_proc", {3, 256, 256}}, }; bool parallel_inference = true; @@ -87,11 +96,6 @@ class HoloInferTests { bool output_on_cuda = true; bool is_engine_path = false; - const std::map> in_tensor_dimensions = { - {"bmode_pre_proc", {320, 240, 3}}, - {"aortic_pre_proc", {300, 300, 3}}, - }; - /// Pointer to inference context. std::unique_ptr holoscan_infer_context_; diff --git a/tests/holoinfer/inference/test_inference.cpp b/tests/holoinfer/inference/test_inference.cpp index 49a1ecc7..49ac2f93 100644 --- a/tests/holoinfer/inference/test_inference.cpp +++ b/tests/holoinfer/inference/test_inference.cpp @@ -42,58 +42,58 @@ void HoloInferTests::inference_tests() { // Test: TRT backend, Missing input tensor status = prepare_for_inference(); - auto dm = std::move(inference_specs_->data_per_tensor_.at("bmode_pre_proc")); - inference_specs_->data_per_tensor_.erase("bmode_pre_proc"); + auto dm = std::move(inference_specs_->data_per_tensor_.at("m1_pre_proc")); + inference_specs_->data_per_tensor_.erase("m1_pre_proc"); status = do_inference(); holoinfer_assert( status, test_module, 3, test_identifier_infer.at(3), HoloInfer::holoinfer_code::H_ERROR); - inference_specs_->data_per_tensor_.insert({"bmode_pre_proc", dm}); + inference_specs_->data_per_tensor_.insert({"m1_pre_proc", dm}); // Test: TRT backend, Missing output tensor status = prepare_for_inference(); - dm = std::move(inference_specs_->output_per_model_.at("aortic_infer")); - inference_specs_->output_per_model_.erase("aortic_infer"); + dm = std::move(inference_specs_->output_per_model_.at("m2_infer")); + inference_specs_->output_per_model_.erase("m2_infer"); status = do_inference(); holoinfer_assert( status, test_module, 4, test_identifier_infer.at(4), HoloInfer::holoinfer_code::H_ERROR); - inference_specs_->output_per_model_.insert({"aortic_infer", dm}); + inference_specs_->output_per_model_.insert({"m2_infer", dm}); // Test: TRT backend, Empty input cuda buffer 1 - auto dbs = inference_specs_->data_per_tensor_.at("bmode_pre_proc")->device_buffer->size(); - inference_specs_->data_per_tensor_.at("bmode_pre_proc")->device_buffer->resize(0); - inference_specs_->data_per_tensor_.at("bmode_pre_proc")->device_buffer = nullptr; + auto dbs = inference_specs_->data_per_tensor_.at("m1_pre_proc")->device_buffer->size(); + inference_specs_->data_per_tensor_.at("m1_pre_proc")->device_buffer->resize(0); + inference_specs_->data_per_tensor_.at("m1_pre_proc")->device_buffer = nullptr; status = do_inference(); holoinfer_assert( status, test_module, 5, test_identifier_infer.at(5), HoloInfer::holoinfer_code::H_ERROR); - inference_specs_->data_per_tensor_.at("bmode_pre_proc")->device_buffer = + inference_specs_->data_per_tensor_.at("m1_pre_proc")->device_buffer = std::make_shared(); // Test: TRT backend, Empty input cuda buffer 2 status = do_inference(); holoinfer_assert( status, test_module, 6, test_identifier_infer.at(6), HoloInfer::holoinfer_code::H_ERROR); - inference_specs_->data_per_tensor_.at("bmode_pre_proc")->device_buffer->resize(dbs); + inference_specs_->data_per_tensor_.at("m1_pre_proc")->device_buffer->resize(dbs); // Test: TRT backend, Empty output cuda buffer 1 - dbs = inference_specs_->output_per_model_.at("aortic_infer")->device_buffer->size(); - inference_specs_->output_per_model_.at("aortic_infer")->device_buffer->resize(0); + dbs = inference_specs_->output_per_model_.at("m2_infer")->device_buffer->size(); + inference_specs_->output_per_model_.at("m2_infer")->device_buffer->resize(0); status = do_inference(); holoinfer_assert( status, test_module, 7, test_identifier_infer.at(7), HoloInfer::holoinfer_code::H_ERROR); // Test: TRT backend, Empty output cuda buffer 2 - inference_specs_->output_per_model_.at("aortic_infer")->device_buffer = nullptr; + inference_specs_->output_per_model_.at("m2_infer")->device_buffer = nullptr; status = do_inference(); holoinfer_assert( status, test_module, 8, test_identifier_infer.at(8), HoloInfer::holoinfer_code::H_ERROR); - inference_specs_->output_per_model_.at("aortic_infer")->device_buffer = + inference_specs_->output_per_model_.at("m2_infer")->device_buffer = std::make_shared(); // Test: TRT backend, Empty output cuda buffer 3 status = do_inference(); holoinfer_assert( status, test_module, 9, test_identifier_infer.at(9), HoloInfer::holoinfer_code::H_ERROR); - inference_specs_->output_per_model_.at("aortic_infer")->device_buffer->resize(dbs); + inference_specs_->output_per_model_.at("m2_infer")->device_buffer->resize(dbs); // Test: TRT backend, Basic end-to-end cuda inference status = do_inference(); @@ -133,20 +133,20 @@ void HoloInferTests::inference_tests() { // Test: TRT backend, Empty host input size_t re_dbs = 0; - dbs = inference_specs_->data_per_tensor_.at("bmode_pre_proc")->host_buffer.size(); - inference_specs_->data_per_tensor_.at("bmode_pre_proc")->host_buffer.resize(re_dbs); + dbs = inference_specs_->data_per_tensor_.at("m1_pre_proc")->host_buffer.size(); + inference_specs_->data_per_tensor_.at("m1_pre_proc")->host_buffer.resize(re_dbs); status = do_inference(); holoinfer_assert( status, test_module, 15, test_identifier_infer.at(15), HoloInfer::holoinfer_code::H_ERROR); - inference_specs_->data_per_tensor_.at("bmode_pre_proc")->host_buffer.resize(dbs); + inference_specs_->data_per_tensor_.at("m1_pre_proc")->host_buffer.resize(dbs); // Test: TRT backend, Empty host output - dbs = inference_specs_->output_per_model_.at("aortic_infer")->host_buffer.size(); - inference_specs_->output_per_model_.at("aortic_infer")->host_buffer.resize(re_dbs); + dbs = inference_specs_->output_per_model_.at("m2_infer")->host_buffer.size(); + inference_specs_->output_per_model_.at("m2_infer")->host_buffer.resize(re_dbs); status = do_inference(); holoinfer_assert( status, test_module, 16, test_identifier_infer.at(16), HoloInfer::holoinfer_code::H_ERROR); - inference_specs_->output_per_model_.at("aortic_infer")->host_buffer.resize(dbs); + inference_specs_->output_per_model_.at("m2_infer")->host_buffer.resize(dbs); if (use_onnxruntime) { // Test: ONNX backend, Basic parallel inference on CPU @@ -194,26 +194,26 @@ void HoloInferTests::inference_tests() { HoloInfer::holoinfer_code::H_SUCCESS); // Test: ONNX backend, Empty host input - dbs = inference_specs_->data_per_tensor_.at("bmode_pre_proc")->host_buffer.size(); - inference_specs_->data_per_tensor_.at("bmode_pre_proc")->host_buffer.resize(0); + dbs = inference_specs_->data_per_tensor_.at("m1_pre_proc")->host_buffer.size(); + inference_specs_->data_per_tensor_.at("m1_pre_proc")->host_buffer.resize(0); status = do_inference(); holoinfer_assert(status, test_module, 21, test_identifier_infer.at(21), HoloInfer::holoinfer_code::H_ERROR); - inference_specs_->data_per_tensor_.at("bmode_pre_proc")->host_buffer.resize(dbs); + inference_specs_->data_per_tensor_.at("m1_pre_proc")->host_buffer.resize(dbs); // Test: ONNX backend, Empty host output - dbs = inference_specs_->output_per_model_.at("aortic_infer")->host_buffer.size(); - inference_specs_->output_per_model_.at("aortic_infer")->host_buffer.resize(0); + dbs = inference_specs_->output_per_model_.at("m2_infer")->host_buffer.size(); + inference_specs_->output_per_model_.at("m2_infer")->host_buffer.resize(0); status = do_inference(); holoinfer_assert(status, test_module, 22, test_identifier_infer.at(22), HoloInfer::holoinfer_code::H_ERROR); - inference_specs_->output_per_model_.at("aortic_infer")->host_buffer.resize(dbs); + inference_specs_->output_per_model_.at("m2_infer")->host_buffer.resize(dbs); } else { // Test: ONNX backend on ARM, Basic sequential inference on GPU infer_on_cpu = false; @@ -230,7 +230,7 @@ void HoloInferTests::inference_tests() { auto dev_id = 1; backend = "trt"; auto cstatus = cudaGetDeviceProperties(&device_prop, dev_id); - device_map.at("bmode_perspective") = "1"; + device_map.at("model_1") = "1"; if (cstatus == cudaSuccess) { // Test: TRT backend, Basic sequential inference on multi-GPU @@ -288,10 +288,10 @@ void HoloInferTests::inference_tests() { test_identifier_infer.at(31), HoloInfer::holoinfer_code::H_SUCCESS); } - device_map.at("bmode_perspective") = "0"; + device_map.at("model_1") = "0"; if (is_x86_64) { - device_map.at("aortic_stenosis") = "1"; + device_map.at("model_2") = "1"; if (cstatus == cudaSuccess) { // Test: ONNX backend, Basic sequential inference on multi-GPU status = prepare_for_inference(); @@ -320,7 +320,7 @@ void HoloInferTests::inference_tests() { test_identifier_infer.at(25), HoloInfer::holoinfer_code::H_ERROR); } - device_map.at("aortic_stenosis") = "0"; + device_map.at("model_2") = "0"; } } } diff --git a/tests/holoinfer/inference/test_parameters.cpp b/tests/holoinfer/inference/test_parameters.cpp index 7eac0a1d..eed82b03 100644 --- a/tests/holoinfer/inference/test_parameters.cpp +++ b/tests/holoinfer/inference/test_parameters.cpp @@ -34,7 +34,7 @@ void HoloInferTests::parameter_test_inference() { status, test_module, 1, test_identifier_params.at(1), HoloInfer::holoinfer_code::H_ERROR); // Test: Parameters, model_path_map: key mismatch with pre_processor_map - model_path_map.at("test-dummy") = model_path_map.at("bmode_perspective"); + model_path_map.at("test-dummy") = model_path_map.at("model_1"); status = call_parameter_check_inference(); holoinfer_assert( status, test_module, 2, test_identifier_params.at(2), HoloInfer::holoinfer_code::H_ERROR); @@ -49,26 +49,26 @@ void HoloInferTests::parameter_test_inference() { // Test: Parameters, pre_processor_map empty value vector check pre_processor_map.erase("test-dummy"); - auto str_value = pre_processor_map.at("bmode_perspective")[0]; - pre_processor_map.at("bmode_perspective").pop_back(); + auto str_value = pre_processor_map.at("model_1")[0]; + pre_processor_map.at("model_1").pop_back(); status = call_parameter_check_inference(); holoinfer_assert( status, test_module, 4, test_identifier_params.at(4), HoloInfer::holoinfer_code::H_ERROR); - pre_processor_map.at("bmode_perspective").push_back(str_value); + pre_processor_map.at("model_1").push_back(str_value); // Test: Parameters, pre_processor_map empty tensor name check - pre_processor_map.at("bmode_perspective").push_back(""); + pre_processor_map.at("model_1").push_back(""); status = call_parameter_check_inference(); holoinfer_assert( status, test_module, 5, test_identifier_params.at(5), HoloInfer::holoinfer_code::H_ERROR); - pre_processor_map.at("bmode_perspective").pop_back(); + pre_processor_map.at("model_1").pop_back(); // Test: Parameters, pre_processor_map duplicate tensor name check - pre_processor_map.at("bmode_perspective").push_back(str_value); + pre_processor_map.at("model_1").push_back(str_value); status = call_parameter_check_inference(); holoinfer_assert( status, test_module, 6, test_identifier_params.at(6), HoloInfer::holoinfer_code::H_ERROR); - pre_processor_map.at("bmode_perspective").pop_back(); + pre_processor_map.at("model_1").pop_back(); // input tensor names test // Test: Parameters, input_tensor exist in pre_processor_map @@ -94,12 +94,12 @@ void HoloInferTests::parameter_test_inference() { inference_map.erase("test-dummy"); // Test: Parameters, inference_map duplicate entry check - str_value = inference_map.at("bmode_perspective")[0]; - inference_map.at("bmode_perspective").push_back(str_value); + str_value = inference_map.at("model_1")[0]; + inference_map.at("model_1").push_back(str_value); status = call_parameter_check_inference(); holoinfer_assert( status, test_module, 10, test_identifier_params.at(10), HoloInfer::holoinfer_code::H_ERROR); - inference_map.at("bmode_perspective").pop_back(); + inference_map.at("model_1").pop_back(); // output tensor names test // Test: Parameters, output_tensor exist in inference_map @@ -136,7 +136,7 @@ void HoloInferTests::parameter_setup_test() { status, test_module, 14, test_identifier_params.at(14), HoloInfer::holoinfer_code::H_ERROR); // Test: TRT backend, Inference map key mismatch with model path map - model_path_map["test-dummy"] = model_path_map.at("bmode_perspective"); + model_path_map["test-dummy"] = model_path_map.at("model_1"); status = create_specifications(); clear_specs(); model_path_map.erase("test-dummy"); @@ -181,7 +181,7 @@ void HoloInferTests::parameter_setup_test() { auto backup_infer_map = std::move(inference_map); auto backup_device_map = std::move(device_map); - model_path_map = {{"test_model", "../data/multiai_ultrasound/models/bmode_perspective.pt"}}; + model_path_map = {{"test_model", model_folder + "identity_model.pt"}}; pre_processor_map = {{"test_model", {"input_"}}}; inference_map = {{"test_model", {"output_"}}}; device_map = {}; @@ -193,15 +193,14 @@ void HoloInferTests::parameter_setup_test() { status, test_module, 25, test_identifier_params.at(25), HoloInfer::holoinfer_code::H_ERROR); // Test: Torch backend, Config file missing - std::filesystem::rename("../data/multiai_ultrasound/models/bmode_perspective.onnx", - "../data/multiai_ultrasound/models/bmode_perspective.pt"); + std::filesystem::rename(model_folder + "identity_model.onnx", model_folder + "identity_model.pt"); status = create_specifications(); clear_specs(); holoinfer_assert( status, test_module, 26, test_identifier_params.at(26), HoloInfer::holoinfer_code::H_ERROR); // Test: Torch backend, Inference node missing in Config file - std::ofstream torch_config_file("../data/multiai_ultrasound/models/bmode_perspective.yaml"); + std::ofstream torch_config_file(model_folder + "identity_model.yaml"); status = create_specifications(); clear_specs(); holoinfer_assert( @@ -220,8 +219,7 @@ void HoloInferTests::parameter_setup_test() { status, test_module, 28, test_identifier_params.at(28), HoloInfer::holoinfer_code::H_ERROR); // Test: Torch backend, dtype missing in input node in Config file - torch_config_file.open("../data/multiai_ultrasound/models/bmode_perspective.yaml", - std::ofstream::trunc); + torch_config_file.open(model_folder + "identity_model.yaml", std::ofstream::trunc); torch_inference["inference"]["input_nodes"]["input"]["id"] = "1"; std::cout << torch_inference << std::endl; torch_config_file << torch_inference; @@ -232,8 +230,7 @@ void HoloInferTests::parameter_setup_test() { status, test_module, 29, test_identifier_params.at(29), HoloInfer::holoinfer_code::H_ERROR); // Test: Torch backend, Incorrect dtype in config file - torch_config_file.open("../data/multiai_ultrasound/models/bmode_perspective.yaml", - std::ofstream::trunc); + torch_config_file.open(model_folder + "identity_model.yaml", std::ofstream::trunc); torch_inference["inference"]["input_nodes"]["input"]["dtype"] = "float"; torch_config_file << torch_inference; torch_config_file.close(); @@ -243,8 +240,7 @@ void HoloInferTests::parameter_setup_test() { status, test_module, 30, test_identifier_params.at(30), HoloInfer::holoinfer_code::H_ERROR); // Test: Torch backend, Output node missing in config file correct - torch_config_file.open("../data/multiai_ultrasound/models/bmode_perspective.yaml", - std::ofstream::trunc); + torch_config_file.open(model_folder + "identity_model.yaml", std::ofstream::trunc); torch_inference["inference"]["input_nodes"]["input"]["dtype"] = "kFloat32"; torch_config_file << torch_inference; torch_config_file.close(); @@ -254,9 +250,8 @@ void HoloInferTests::parameter_setup_test() { status, test_module, 31, test_identifier_params.at(31), HoloInfer::holoinfer_code::H_ERROR); // Restore all changes to previous state - std::filesystem::remove("../data/multiai_ultrasound/models/bmode_perspective.yaml"); - std::filesystem::rename("../data/multiai_ultrasound/models/bmode_perspective.pt", - "../data/multiai_ultrasound/models/bmode_perspective.onnx"); + std::filesystem::remove(model_folder + "identity_model.yaml"); + std::filesystem::rename(model_folder + "identity_model.pt", model_folder + "identity_model.onnx"); model_path_map = std::move(backup_path_map); pre_processor_map = std::move(backup_pre_map); inference_map = std::move(backup_infer_map); @@ -273,13 +268,13 @@ void HoloInferTests::parameter_setup_test() { // Test: ONNX backend, incorrect model file format backend = "onnxrt"; - auto pc_path = model_path_map.at("bmode_perspective"); - model_path_map.at("bmode_perspective") = "model.engine"; + auto pc_path = model_path_map.at("model_1"); + model_path_map.at("model_1") = "model.engine"; input_on_cuda = false; output_on_cuda = false; status = create_specifications(); clear_specs(); - model_path_map.at("bmode_perspective") = pc_path; + model_path_map.at("model_1") = pc_path; holoinfer_assert( status, test_module, 21, test_identifier_params.at(21), HoloInfer::holoinfer_code::H_ERROR); diff --git a/tests/system/demosaic_op_app.cpp b/tests/system/demosaic_op_app.cpp index 9a0c659a..48611a88 100644 --- a/tests/system/demosaic_op_app.cpp +++ b/tests/system/demosaic_op_app.cpp @@ -44,6 +44,7 @@ class DummyDemosaicApp : public holoscan::Application { Arg("columns", columns), Arg("channels", channels), Arg("tensor_name", tensor_name), + Arg("storage_type", std::string("device")), make_condition(3)); auto cuda_stream_pool = make_resource("cuda_stream", 0, 0, 0, 1, 5); @@ -66,15 +67,26 @@ class DummyDemosaicApp : public holoscan::Application { void set_explicit_stream_pool_init(bool value) { explicit_stream_pool_init_ = value; } + void set_storage_type(const std::string& storage_type) { storage_type_ = storage_type; } + private: bool explicit_stream_pool_init_ = false; + std::string storage_type_ = std::string("device"); }; -TEST(DemosaicOpApp, TestDummyDemosaicApp) { +class DemosaicStorageParameterizedTestFixture : public ::testing::TestWithParam {}; + +INSTANTIATE_TEST_CASE_P(DemosaicOpAppTests, DemosaicStorageParameterizedTestFixture, + ::testing::Values(std::string("device"), std::string("host"), + std::string("system"))); + +TEST_P(DemosaicStorageParameterizedTestFixture, TestDummyDemosaicApp) { // Test fix for issue 4313690 (failure to initialize graph when using BayerDemosaicOp) + std::string storage_type = GetParam(); using namespace holoscan; auto app = make_application(); + app->set_storage_type(storage_type); // capture output to check that the expected messages were logged testing::internal::CaptureStderr(); diff --git a/tests/system/distributed/standalone_fragments.cpp b/tests/system/distributed/standalone_fragments.cpp new file mode 100644 index 00000000..30409be8 --- /dev/null +++ b/tests/system/distributed/standalone_fragments.cpp @@ -0,0 +1,110 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include + +#include + +namespace holoscan { + +namespace { + +class DummyOp : public Operator { + public: + HOLOSCAN_OPERATOR_FORWARD_ARGS(DummyOp) + + DummyOp() = default; + + void setup(OperatorSpec& spec) override {} + + void compute(InputContext&, OutputContext& op_output, ExecutionContext&) override { + HOLOSCAN_LOG_INFO("Operator: {}, Index: {}", name(), index_); + // Sleep for 100ms to simulate some work + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + index_++; + } + + int index() const { return index_; } + + private: + int index_ = 1; +}; + +class Fragment1 : public holoscan::Fragment { + public: + Fragment1() = default; + + void compose() override { + using namespace holoscan; + auto tx = make_operator("tx", make_condition(10)); + add_operator(tx); + } +}; + +class Fragment2 : public holoscan::Fragment { + public: + Fragment2() = default; + + void compose() override { + using namespace holoscan; + auto rx = make_operator("rx", make_condition(5)); + add_operator(rx); + } +}; + +class StandaloneFragmentApp : public holoscan::Application { + public: + // Inherit the constructor + using Application::Application; + + void compose() override { + using namespace holoscan; + auto fragment1 = make_fragment("fragment1"); + auto fragment2 = make_fragment("fragment2"); + + add_fragment(fragment1); + add_fragment(fragment2); + } +}; + +} // namespace + +/////////////////////////////////////////////////////////////////////////////// +// Tests +/////////////////////////////////////////////////////////////////////////////// + +TEST(DistributedApp, TestStandaloneFragments) { + // Test that two fragments can be run independently in a distributed app (issue 4616519). + const std::vector args{"test_app", "--driver", "--worker", "--fragments", "all"}; + auto app = make_application(args); + + // capture output so that we can check that the expected value is present + testing::internal::CaptureStderr(); + + app->run(); + + std::string log_output = testing::internal::GetCapturedStderr(); + EXPECT_TRUE(log_output.find("Operator: tx, Index: 10") != std::string::npos); + EXPECT_TRUE(log_output.find("Operator: rx, Index: 5") != std::string::npos); +} + +} // namespace holoscan diff --git a/tests/system/format_converter_op_apps.cpp b/tests/system/format_converter_op_apps.cpp new file mode 100644 index 00000000..022a3d8f --- /dev/null +++ b/tests/system/format_converter_op_apps.cpp @@ -0,0 +1,104 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include + +#include "../config.hpp" +#include "ping_tensor_rx_op.hpp" +#include "ping_tensor_tx_op.hpp" +#include "tensor_compare_op.hpp" + +#include "holoscan/holoscan.hpp" +#include "holoscan/operators/format_converter/format_converter.hpp" + +using namespace holoscan; + +static HoloscanTestConfig test_config; + +using StringOrArg = std::variant; + +class FormatConverterApp : public holoscan::Application { + public: + explicit FormatConverterApp(const std::string& storage_type) + : Application(), storage_type_(storage_type) {} + + void compose() override { + const int32_t width = 32, height = 64; + auto count = make_condition(10); + auto shape_args = ArgList({Arg("rows", height), Arg("columns", width), Arg("channels", 4)}); + auto source = make_operator( + "ping_source", shape_args, Arg("storage_type", storage_type_), count); + auto pool = Arg("pool", make_resource("pool")); + auto in_dtype = Arg("in_dtype", std::string("rgba8888")); + auto out_dtype = Arg("out_dtype", std::string("rgb888")); + std::string in_tensor_name = "tensor"; + std::string out_tensor_name = "rgb"; + auto converter = make_operator("converter", + in_dtype, + out_dtype, + pool, + Arg("in_tensor_name", in_tensor_name), + Arg("out_tensor_name", out_tensor_name)); + auto rx = make_operator("rx", Arg("tensor_name", out_tensor_name)); + + add_flow(source, converter, {{"out", "source_video"}}); + add_flow(converter, rx, {{"tensor", "in"}}); + } + + void set_storage_type(const std::string& storage_type) { storage_type_ = storage_type; } + + private: + std::string storage_type_ = std::string("device"); + + FormatConverterApp() = delete; +}; + +void run_app(const std::string& failure_str = "", const std::string& storage_type = "device") { + auto app = make_application(storage_type); + + // capture output to check that the expected messages were logged + testing::internal::CaptureStderr(); + try { + app->run(); + } catch (const std::exception& ex) { + GTEST_FATAL_FAILURE_( + fmt::format("{}{}", testing::internal::GetCapturedStderr(), ex.what()).c_str()); + } + std::string log_output = testing::internal::GetCapturedStderr(); + if (failure_str.empty()) { + EXPECT_TRUE(log_output.find("error") == std::string::npos) << log_output; + } else { + EXPECT_TRUE(log_output.find(failure_str) != std::string::npos) << log_output; + } +} + +class FormatConverterStorageParameterizedTestFixture + : public ::testing::TestWithParam {}; + +INSTANTIATE_TEST_CASE_P(FormatConverterOpAppTests, FormatConverterStorageParameterizedTestFixture, + ::testing::Values(std::string("device"), std::string("host"), + std::string("system"))); + +// run this case with various tensor memory storage types +TEST_P(FormatConverterStorageParameterizedTestFixture, TestFormatConverterStorageTypes) { + std::string storage_type = GetParam(); + run_app("", storage_type); +} diff --git a/tests/system/holoviz_op_apps.cpp b/tests/system/holoviz_op_apps.cpp index a4c45677..1e389921 100644 --- a/tests/system/holoviz_op_apps.cpp +++ b/tests/system/holoviz_op_apps.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -42,7 +42,8 @@ class HolovizToHolovizApp : public holoscan::Application { ArgList get_enable_arg() { if (std::holds_alternative(enable_arg_)) { return ArgList({std::get(enable_arg_)}); } if (std::holds_alternative(enable_arg_)) { - return from_config(std::get(enable_arg_)); + std::string arg_name = std::get(enable_arg_); + if (arg_name != "") { return from_config(arg_name); } } return ArgList(); } @@ -52,8 +53,9 @@ class HolovizToHolovizApp : public holoscan::Application { auto allocator = Arg("allocator", make_resource("allocator")); auto count = make_condition(10); auto headless = Arg("headless", true); - auto shape = ArgList({Arg("rows", height), Arg("columns", width), Arg("channels", 3)}); - auto source = make_operator("ping_source", shape); + auto shape_args = ArgList({Arg("rows", height), Arg("columns", width), Arg("channels", 3)}); + auto source = make_operator( + "ping_source", shape_args, Arg("storage_type", storage_type_)); auto renderer = make_operator("renderer", count, headless, @@ -66,25 +68,34 @@ class HolovizToHolovizApp : public holoscan::Application { auto pool = Arg("pool", make_resource("pool")); auto in_dtype = Arg("in_dtype", std::string("rgba8888")); auto out_dtype = Arg("out_dtype", std::string("rgb888")); - auto video_to_tensor = - make_operator("converter", in_dtype, out_dtype, pool); - - auto comparator = make_operator("comparator"); add_flow(source, renderer, {{"out", "receivers"}}); - add_flow(source, comparator, {{"out", "input1"}}); - add_flow(renderer, video_to_tensor, {{"render_buffer_output", "source_video"}}); - add_flow(video_to_tensor, comparator, {{"tensor", "input2"}}); + + // TensorCompareOp only works for device tensors + if (storage_type_ == "device") { + auto comparator = make_operator("comparator"); + auto video_to_tensor = + make_operator("converter", in_dtype, out_dtype, pool); + + add_flow(renderer, video_to_tensor, {{"render_buffer_output", "source_video"}}); + add_flow(source, comparator, {{"out", "input1"}}); + add_flow(video_to_tensor, comparator, {{"tensor", "input2"}}); + } } + void set_storage_type(const std::string& storage_type) { storage_type_ = storage_type; } + private: StringOrArg enable_arg_; + std::string storage_type_ = std::string("device"); HolovizToHolovizApp() = delete; }; -void run_app(StringOrArg enable_arg, std::string failure_str = "") { +void run_app(StringOrArg enable_arg, const std::string& failure_str = "", + const std::string& storage_type = "device") { auto app = make_application(enable_arg); + app->set_storage_type(storage_type); const std::string config_file = test_config.get_test_data_file("app_config.yaml"); app->config(config_file); @@ -105,6 +116,20 @@ void run_app(StringOrArg enable_arg, std::string failure_str = "") { } } +class HolovizStorageParameterizedTestFixture : public ::testing::TestWithParam {}; + +INSTANTIATE_TEST_CASE_P(HolovizOpAppTests, HolovizStorageParameterizedTestFixture, + ::testing::Values(std::string("device"), std::string("host"), + std::string("system"))); + +// run this case with various tensor memory storage types +TEST_P(HolovizStorageParameterizedTestFixture, TestHolovizStorageTypes) { + std::string storage_type = GetParam(); + // run without any extra arguments, just to verify HolovizOp support various memory types + run_app("", "", storage_type); +} + +// run this case with various tensor memory storage types TEST(HolovizApps, TestEnableRenderBufferOutputYAML) { run_app("holoviz_enable_ports"); } diff --git a/tests/system/jobstatistics_app.cpp b/tests/system/jobstatistics_app.cpp new file mode 100644 index 00000000..58911abf --- /dev/null +++ b/tests/system/jobstatistics_app.cpp @@ -0,0 +1,147 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include + +#include +#include +#include "holoscan/holoscan.hpp" + +#include "env_wrapper.hpp" + +class SimplePingApp : public holoscan::Application { + public: + using Application::Application; + + void compose() override { + using namespace holoscan; + auto tx = make_operator("tx", make_resource(100)); + auto rx = make_operator("rx"); + add_flow(tx, rx, {{"out", "in"}}); + } +}; + +TEST(JobStatisticsApp, TestJobStatisticsDisabled) { + using namespace holoscan; + + EnvVarWrapper wrapper({ + std::make_pair("HOLOSCAN_ENABLE_GXF_JOB_STATISTICS", "0"), + }); + + auto app = make_application(); + + // capture output to check that the expected messages were logged + testing::internal::CaptureStdout(); + app->run(); + + std::string log_output = testing::internal::GetCapturedStdout(); + EXPECT_TRUE(log_output.find("Job Statistics Report") == std::string::npos); +} + +TEST(JobStatisticsApp, TestJobStatisticsEnabled) { + using namespace holoscan; + + EnvVarWrapper wrapper({ + std::make_pair("HOLOSCAN_ENABLE_GXF_JOB_STATISTICS", "TRUE"), + }); + + auto app = make_application(); + + // capture output to check that the expected messages were logged + testing::internal::CaptureStdout(); + app->run(); + + std::string console_output = testing::internal::GetCapturedStdout(); + EXPECT_TRUE(console_output.find("Job Statistics Report") != std::string::npos); + + // Codelet statistics report is disabled by default + EXPECT_TRUE(console_output.find("Codelet Statistics Report") == std::string::npos); +} + +TEST(JobStatisticsApp, TestJobStatisticsEnabledCountSet) { + using namespace holoscan; + + EnvVarWrapper wrapper({ + std::make_pair("HOLOSCAN_ENABLE_GXF_JOB_STATISTICS", "1"), + std::make_pair("HOLOSCAN_GXF_JOB_STATISTICS_COUNT", "35"), + std::make_pair("HOLOSCAN_LOG_LEVEL", "DEBUG"), + }); + + auto app = make_application(); + + // capture output to check that the expected messages were logged + testing::internal::CaptureStdout(); + testing::internal::CaptureStderr(); + + app->run(); + + std::string console_output = testing::internal::GetCapturedStdout(); + std::string log_output = testing::internal::GetCapturedStderr(); + EXPECT_TRUE(console_output.find("Job Statistics Report") != std::string::npos); + + // Rely on DEBUG level log output to detect the event_history_count that was set as the + // value is not shown in the report itself. + EXPECT_TRUE(log_output.find("event_history_count: 35") != std::string::npos); +} + +TEST(JobStatisticsApp, TestJobStatisticsCodeletReportEnabled) { + using namespace holoscan; + + EnvVarWrapper wrapper({ + std::make_pair("HOLOSCAN_ENABLE_GXF_JOB_STATISTICS", "true"), + std::make_pair("HOLOSCAN_GXF_JOB_STATISTICS_CODELET", "1"), + }); + + auto app = make_application(); + + // capture output to check that the expected messages were logged + testing::internal::CaptureStdout(); + app->run(); + + std::string console_output = testing::internal::GetCapturedStdout(); + EXPECT_TRUE(console_output.find("Job Statistics Report") != std::string::npos); + EXPECT_TRUE(console_output.find("Codelet Statistics Report") != std::string::npos); +} + +TEST(JobStatisticsApp, TestJobStatisticsFilePathSet) { + using namespace holoscan; + + std::string file_path = "temp_job_stats.json"; + EXPECT_FALSE(std::filesystem::exists(file_path)); + + EnvVarWrapper wrapper({ + std::make_pair("HOLOSCAN_ENABLE_GXF_JOB_STATISTICS", "true"), + std::make_pair("HOLOSCAN_GXF_JOB_STATISTICS_PATH", file_path), + }); + + auto app = make_application(); + + // capture output to check that the expected messages were logged + testing::internal::CaptureStdout(); + app->run(); + + std::string console_output = testing::internal::GetCapturedStdout(); + EXPECT_TRUE(console_output.find("Job Statistics Report") != std::string::npos); + + // check that the expected JSON file was created + EXPECT_TRUE(std::filesystem::exists(file_path)); + std::filesystem::remove(file_path); +} diff --git a/tests/system/multithreaded_app.cpp b/tests/system/multithreaded_app.cpp index 28bfbade..4df8fc32 100644 --- a/tests/system/multithreaded_app.cpp +++ b/tests/system/multithreaded_app.cpp @@ -34,8 +34,11 @@ class PingMultithreadApp : public holoscan::Application { public: void compose() override { using namespace holoscan; - auto tx = make_operator( - "tx", Arg("rows", 64), Arg("columns", 32), make_condition(NUM_ITER)); + auto tx = make_operator("tx", + Arg("rows", 64), + Arg("columns", 32), + Arg("storage_type", storage_type_), + make_condition(NUM_ITER)); for (int index = 1; index <= NUM_RX; ++index) { const auto rx_name = fmt::format("rx{}", index); @@ -43,6 +46,11 @@ class PingMultithreadApp : public holoscan::Application { add_flow(tx, rx); } } + + void set_storage_type(const std::string& storage_type) { storage_type_ = storage_type; } + + private: + std::string storage_type_ = std::string("device"); }; TEST(MultithreadedApp, TestSendingTensorToMultipleOperators) { @@ -70,13 +78,11 @@ TEST(MultithreadedApp, TestSendingTensorToMultipleOperators) { std::string log_output = testing::internal::GetCapturedStderr(); EXPECT_TRUE(log_output.find("null data") == std::string::npos); for (int i = 1; i < NUM_RX; ++i) { - EXPECT_TRUE(log_output.find(fmt::format( - "Rx message value - name:rx{}, data[0]:{}, nbytes:2048", i, NUM_ITER)) != + EXPECT_TRUE(log_output.find(fmt::format("Rx message value - name:rx{}, data[0]:", i)) != std::string::npos); } // Check that the last rx operator received the expected value and print the log if it didn't - EXPECT_TRUE(log_output.find(fmt::format( - "Rx message value - name:rx{}, data[0]:{}, nbytes:2048", NUM_RX, NUM_ITER)) != + EXPECT_TRUE(log_output.find(fmt::format("Rx message value - name:rx{}, data[0]:", NUM_RX)) != std::string::npos) << "=== LOG ===\n" << log_output << "\n===========\n"; diff --git a/tests/system/ping_tensor_tx_op.cpp b/tests/system/ping_tensor_tx_op.cpp index 55b09855..54daadb2 100644 --- a/tests/system/ping_tensor_tx_op.cpp +++ b/tests/system/ping_tensor_tx_op.cpp @@ -24,6 +24,8 @@ #include #include +#include + #define CUDA_TRY(stmt) \ ({ \ cudaError_t _holoscan_cuda_err = stmt; \ @@ -41,21 +43,60 @@ namespace holoscan { namespace ops { +void PingTensorTxOp::initialize() { + // Set up prerequisite parameters before calling Operator::initialize() + auto frag = fragment(); + + // Find if there is an argument for 'allocator' + auto has_allocator = std::find_if( + args().begin(), args().end(), [](const auto& arg) { return (arg.name() == "allocator"); }); + // Create the allocator if there is no argument provided. + if (has_allocator == args().end()) { + allocator_ = frag->make_resource("allocator"); + add_arg(allocator_.get()); + } + Operator::initialize(); +} + void PingTensorTxOp::setup(OperatorSpec& spec) { - spec.output("out"); + spec.output("out"); - spec.param( - rows_, "rows", "number of rows", "number of rows (default: 64)", static_cast(64)); + spec.param(allocator_, "allocator", "Allocator", "Allocator used to allocate tensor output."); + spec.param(storage_type_, + "storage_type", + "memory storage type", + "nvidia::gxf::MemoryStorageType enum indicating where the memory will be stored", + std::string("system")); + spec.param(batch_size_, + "batch_size", + "batch size", + "Size of the batch dimension (default: 0). The tensor shape will be " + "([batch], rows, [columns], [channels]) where [] around a dimension indicates that " + "it is only present if the corresponding parameter has a size > 0." + "If 0, no batch dimension will be present.", + static_cast(0)); + spec.param(rows_, + "rows", + "number of rows", + "Number of rows (default: 32), must be >= 1.", + static_cast(32)); spec.param(columns_, "columns", "number of columns", - "number of columns (default: 32)", - static_cast(32)); - spec.param(channels_, - "channels", - "channels", - "Number of channels. If 0, no channel dimension will be present. (default: 0)", - static_cast(0)); + "Number of columns (default: 64). If 0, no column dimension will be present.", + static_cast(64)); + spec.param( + channels_, + "channels", + "channels", + "Number of channels (default: 0). If 0, no channel dimension will be present. (default: 0)", + static_cast(0)); + spec.param(data_type_, + "data_type", + "data type for the tensor elements", + "must be one of {'int8_t', 'int16_t', 'int32_t', 'int64_t', 'uint8_t', 'uint16_t'," + "'uint32_t', 'uint64_t', 'float', 'double', 'complex', 'complex'}", + std::string{"uint8_t"}); spec.param(tensor_name_, "tensor_name", "output tensor name", @@ -63,59 +104,97 @@ void PingTensorTxOp::setup(OperatorSpec& spec) { std::string{"tensor"}); } +nvidia::gxf::PrimitiveType PingTensorTxOp::primitive_type(const std::string& data_type) { + HOLOSCAN_LOG_INFO("PingTensorTxOp data type = {}", data_type); + if (data_type == "int8_t") { + return nvidia::gxf::PrimitiveType::kInt8; + } else if (data_type == "int16_t") { + return nvidia::gxf::PrimitiveType::kInt16; + } else if (data_type == "int32_t") { + return nvidia::gxf::PrimitiveType::kInt32; + } else if (data_type == "int64_t") { + return nvidia::gxf::PrimitiveType::kInt64; + } else if (data_type == "uint8_t") { + return nvidia::gxf::PrimitiveType::kUnsigned8; + } else if (data_type == "uint16_t") { + return nvidia::gxf::PrimitiveType::kUnsigned16; + } else if (data_type == "uint32_t") { + return nvidia::gxf::PrimitiveType::kUnsigned32; + } else if (data_type == "uint64_t") { + return nvidia::gxf::PrimitiveType::kUnsigned64; + } else if (data_type == "float") { + return nvidia::gxf::PrimitiveType::kFloat32; + } else if (data_type == "double") { + return nvidia::gxf::PrimitiveType::kFloat64; + } else if (data_type == "complex") { + return nvidia::gxf::PrimitiveType::kComplex64; + } else if (data_type == "complex") { + return nvidia::gxf::PrimitiveType::kComplex128; + } + throw std::runtime_error(std::string("Unrecognized data_type: ") + data_type); +} + void PingTensorTxOp::compute(InputContext&, OutputContext& op_output, ExecutionContext& context) { + // the type of out_message is TensorMap + TensorMap out_message; + + auto gxf_context = context.context(); + auto frag = fragment(); + + // get Handle to underlying nvidia::gxf::Allocator from std::shared_ptr + auto allocator = + nvidia::gxf::Handle::Create(gxf_context, allocator_->gxf_cid()); + + auto gxf_tensor = std::make_shared(); + // Define the dimensions for the CUDA memory (64 x 32, uint8). + int batch_size = batch_size_.get(); int rows = rows_.get(); int columns = columns_.get(); int channels = channels_.get(); - // keep element type as kUnsigned8 for use with BayerDemosaicOp - nvidia::gxf::PrimitiveType element_type = nvidia::gxf::PrimitiveType::kUnsigned8; - int element_size = nvidia::gxf::PrimitiveTypeSize(element_type); - nvidia::gxf::Shape shape; - size_t nbytes; - if (channels == 0) { - shape = nvidia::gxf::Shape{rows, columns}; - nbytes = rows * columns * element_size; + auto dtype = element_type(); + + std::vector shape_vec; + if (batch_size > 0) { shape_vec.push_back(batch_size); } + shape_vec.push_back(rows); + if (columns > 0) { shape_vec.push_back(columns); } + if (channels > 0) { shape_vec.push_back(channels); } + auto tensor_shape = nvidia::gxf::Shape{shape_vec}; + + const uint64_t bytes_per_element = nvidia::gxf::PrimitiveTypeSize(dtype); + auto strides = nvidia::gxf::ComputeTrivialStrides(tensor_shape, bytes_per_element); + nvidia::gxf::MemoryStorageType storage_type; + auto storage_name = storage_type_.get(); + HOLOSCAN_LOG_DEBUG("storage_name = {}", storage_name); + if (storage_name == std::string("device")) { + storage_type = nvidia::gxf::MemoryStorageType::kDevice; + } else if (storage_name == std::string("host")) { + storage_type = nvidia::gxf::MemoryStorageType::kHost; + } else if (storage_name == std::string("system")) { + storage_type = nvidia::gxf::MemoryStorageType::kSystem; } else { - shape = nvidia::gxf::Shape{rows, columns, channels}; - nbytes = rows * columns * channels * element_size; + throw std::runtime_error(fmt::format( + "Unrecognized storage_device ({}), should be one of {'device', 'host', 'system'}", + storage_name)); + } + + // allocate a tensor of the specified shape and data type + auto result = gxf_tensor->reshapeCustom( + tensor_shape, dtype, bytes_per_element, strides, storage_type, allocator.value()); + if (!result) { HOLOSCAN_LOG_ERROR("failed to generate tensor"); } + + // Create Holoscan tensor + auto maybe_dl_ctx = (*gxf_tensor).toDLManagedTensorContext(); + if (!maybe_dl_ctx) { + HOLOSCAN_LOG_ERROR( + "failed to get std::shared_ptr from nvidia::gxf::Tensor"); } + std::shared_ptr holoscan_tensor = std::make_shared(maybe_dl_ctx.value()); + + // insert tensor into the TensorMap + out_message.insert({tensor_name_.get().c_str(), holoscan_tensor}); - // Create a shared pointer for the CUDA memory with a custom deleter. - auto pointer = std::shared_ptr(new void*, [](void** pointer) { - if (pointer != nullptr) { - if (*pointer != nullptr) { CUDA_TRY(cudaFree(*pointer)); } - delete pointer; - } - }); - - // Allocate and initialize the CUDA memory. - CUDA_TRY(cudaMalloc(pointer.get(), nbytes)); - std::vector data(nbytes); - for (size_t index = 0; index < data.size(); ++index) { data[index] = (index_ + index) % 256; } - CUDA_TRY(cudaMemcpy(*pointer, data.data(), nbytes, cudaMemcpyKind::cudaMemcpyHostToDevice)); - - // Holoscan Tensor doesn't support direct memory allocation. - // Thus, create an Entity and use GXF tensor to wrap the CUDA memory. - auto out_message = nvidia::gxf::Entity::New(context.context()); - auto gxf_tensor = out_message.value().add(tensor_name_.get().c_str()); - gxf_tensor.value()->wrapMemory(shape, - element_type, - element_size, - nvidia::gxf::ComputeTrivialStrides(shape, element_size), - nvidia::gxf::MemoryStorageType::kDevice, - *pointer, - [orig_pointer = pointer](void*) mutable { - orig_pointer.reset(); // decrement ref count - return nvidia::gxf::Success; - }); - HOLOSCAN_LOG_INFO("Tx message rows:{}, columns:{}, channels:{}", rows, columns, channels); - HOLOSCAN_LOG_INFO("Tx message value - index:{}, size:{} ", index_, gxf_tensor.value()->size()); - - // Emit the tensor. - op_output.emit(out_message.value(), "out"); - - index_++; + op_output.emit(out_message); } } // namespace ops diff --git a/tests/system/ping_tensor_tx_op.hpp b/tests/system/ping_tensor_tx_op.hpp index 740a0c82..eabbd69b 100644 --- a/tests/system/ping_tensor_tx_op.hpp +++ b/tests/system/ping_tensor_tx_op.hpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2023-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,6 +18,7 @@ #ifndef SYSTEM_PING_TENSOR_TX_OP_HPP #define SYSTEM_PING_TENSOR_TX_OP_HPP +#include #include #include @@ -25,21 +26,33 @@ namespace holoscan { namespace ops { -class PingTensorTxOp : public Operator { +class PingTensorTxOp : public holoscan::Operator { public: HOLOSCAN_OPERATOR_FORWARD_ARGS(PingTensorTxOp) PingTensorTxOp() = default; + void initialize() override; void setup(OperatorSpec& spec) override; + void compute(InputContext&, OutputContext& op_output, ExecutionContext& context) override; - void compute(InputContext&, OutputContext& op_output, ExecutionContext&) override; + nvidia::gxf::PrimitiveType element_type() { + if (element_type_.has_value()) { return element_type_.value(); } + element_type_ = primitive_type(data_type_.get()); + return element_type_.value(); + } private: - int index_ = 1; + nvidia::gxf::PrimitiveType primitive_type(const std::string& data_type); + std::optional element_type_; + + Parameter> allocator_; + Parameter storage_type_; + Parameter batch_size_; Parameter rows_; Parameter columns_; Parameter channels_; + Parameter data_type_; Parameter tensor_name_; };