diff --git a/.vscode/launch.json b/.vscode/launch.json index eada45bc..b50f43a6 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -915,6 +915,76 @@ }, //#endregion ucx_h264_endoscopy_tool_tracking + + //#region grpc_endoscopy_tool_tracking + { + "name": "(gdb) grpc_endoscopy_tool_tracking/cpp (edge)", + "type": "cppdbg", + "request": "launch", + "preLaunchTask": "Build grpc_endoscopy_tool_tracking (delay 3s)", + "program": "${workspaceFolder}/build/grpc_endoscopy_tool_tracking/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/grpc_endoscopy_tool_tracking_edge", + "environment": [ + { + "name": "HOLOSCAN_INPUT_PATH", + "value": "${env:HOLOHUB_DATA_DIR}/endoscopy" + }, + { + "name": "HOLOSCAN_LOG_LEVEL", + "value": "INFO" + } + ], + "stopAtEntry": false, + "cwd": "${workspaceFolder}/build/grpc_endoscopy_tool_tracking/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp", + "externalConsole": false, + "MIMode": "gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ], + "presentation": { + "hidden": false, + "group": "grpc_endoscopy_tool_tracking", + "order": 1 + } + }, + { + "name": "(gdb) grpc_endoscopy_tool_tracking/cpp (cloud)", + "type": "cppdbg", + "request": "launch", + "preLaunchTask": "Build grpc_endoscopy_tool_tracking", + "program": "${workspaceFolder}/build/grpc_endoscopy_tool_tracking/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/grpc_endoscopy_tool_tracking_cloud", + "environment": [ + { + "name": "HOLOSCAN_INPUT_PATH", + "value": "${env:HOLOHUB_DATA_DIR}/endoscopy" + }, + { + "name": "HOLOSCAN_LOG_LEVEL", + "value": "INFO" + } + ], + "stopAtEntry": false, + "cwd": "${workspaceFolder}/build/grpc_endoscopy_tool_tracking/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp", + "externalConsole": false, + "MIMode": "gdb", + "setupCommands": [ + { + "description": "Enable pretty-printing for gdb", + "text": "-enable-pretty-printing", + "ignoreFailures": true + } + ], + "presentation": { + "hidden": false, + "group": "grpc_endoscopy_tool_tracking", + "order": 1 + } + }, + //#endregion grpc_endoscopy_tool_tracking + //#region grpc_h264_endoscopy_tool_tracking { "name": "(gdb) grpc_h264_endoscopy_tool_tracking/cpp (edge)", @@ -1227,6 +1297,20 @@ "order": 11 } }, + { + "name": "(compound) grpc_endoscopy_tool_tracking/cpp (cloud & edge)", + "configurations": [ + "(gdb) grpc_endoscopy_tool_tracking/cpp (cloud)", + "(gdb) grpc_endoscopy_tool_tracking/cpp (edge)" + ], + "preLaunchTask": "Build grpc_endoscopy_tool_tracking", + "stopAll": true, + "presentation": { + "hidden": false, + "group": "grpc_endoscopy_tool_tracking", + "order": 11 + } + }, { "name": "(compound) grpc_h264_endoscopy_tool_tracking/cpp (cloud & edge)", "configurations": [ diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 50f78f55..06695da3 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -272,6 +272,53 @@ "group": "ucx_endoscopy_tool_tracking" } }, + { + "type": "shell", + "label": "Build grpc_endoscopy_tool_tracking", + "command": "./run", + "args": [ + "build", + "grpc_endoscopy_tool_tracking", + "--type", + "debug" + ], + "options": { + "cwd": "${env:WORKSPACE_DIR}" + }, + "group": "build", + "problemMatcher": [], + "detail": "CMake template build task", + "presentation": { + "echo": true, + "reveal": "silent", + "focus": true, + "panel": "dedicated", + "showReuseMessage": false, + "clear": true, + "group": "grpc_endoscopy_tool_tracking" + } + }, + { + "type": "shell", + "label": "Build grpc_endoscopy_tool_tracking (delay 3s)", + "command": "sleep 3", + "dependsOn": "Build grpc_endoscopy_tool_tracking", + "options": { + "cwd": "${env:WORKSPACE_DIR}" + }, + "group": "build", + "problemMatcher": [], + "detail": "CMake template build task", + "presentation": { + "echo": true, + "reveal": "silent", + "focus": true, + "panel": "dedicated", + "showReuseMessage": false, + "clear": true, + "group": "grpc_endoscopy_tool_tracking" + } + }, { "type": "shell", "label": "Build h264_endoscopy_tool_tracking", @@ -332,6 +379,53 @@ "problemMatcher": [], "detail": "CMake template build task" }, + { + "type": "shell", + "label": "Build grpc_endoscopy_tool_tracking", + "command": "./run", + "args": [ + "build", + "grpc_endoscopy_tool_tracking", + "--type", + "debug" + ], + "options": { + "cwd": "${env:WORKSPACE_DIR}" + }, + "group": "build", + "problemMatcher": [], + "detail": "CMake template build task", + "presentation": { + "echo": true, + "reveal": "silent", + "focus": true, + "panel": "dedicated", + "showReuseMessage": false, + "clear": true, + "group": "grpc_endoscopy_tool_tracking" + } + }, + { + "type": "shell", + "label": "Build grpc_endoscopy_tool_tracking (delay 3s)", + "command": "sleep 3", + "dependsOn": "Build grpc_endoscopy_tool_tracking", + "options": { + "cwd": "${env:WORKSPACE_DIR}" + }, + "group": "build", + "problemMatcher": [], + "detail": "CMake template build task", + "presentation": { + "echo": true, + "reveal": "silent", + "focus": true, + "panel": "dedicated", + "showReuseMessage": false, + "clear": true, + "group": "grpc_endoscopy_tool_tracking" + } + }, { "type": "shell", "label": "Build grpc_h264_endoscopy_tool_tracking", diff --git a/applications/distributed/grpc/CMakeLists.txt b/applications/distributed/grpc/CMakeLists.txt index 7ae8b35b..ac48b57a 100644 --- a/applications/distributed/grpc/CMakeLists.txt +++ b/applications/distributed/grpc/CMakeLists.txt @@ -14,6 +14,12 @@ # limitations under the License. + +add_holohub_application(grpc_endoscopy_tool_tracking DEPENDS + OPERATORS lstm_tensor_rt_inference + tool_tracking_postprocessor + grpc_operators) + add_holohub_application(grpc_h264_endoscopy_tool_tracking DEPENDS OPERATORS video_encoder tensor_to_video_buffer diff --git a/applications/distributed/grpc/grpc_endoscopy_tool_tracking/CMakeLists.txt b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/CMakeLists.txt new file mode 100644 index 00000000..33c69d8d --- /dev/null +++ b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/CMakeLists.txt @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.20) +project(grpc_endoscopy_tool_tracking LANGUAGES NONE) + +# Download the endoscopy sample data +if(HOLOHUB_DOWNLOAD_DATASETS) + include(holoscan_download_data) + holoscan_download_data(endoscopy + URL https://api.ngc.nvidia.com/v2/resources/nvidia/clara-holoscan/holoscan_endoscopy_sample_data/versions/20230222/zip + DOWNLOAD_NAME holoscan_endoscopy_sample_data_20230222.zip + URL_MD5 d54f84a562d29ed560a87d2607eba973 + DOWNLOAD_DIR ${HOLOHUB_DATA_DIR} + GENERATE_GXF_ENTITIES + GXF_ENTITIES_HEIGHT 480 + GXF_ENTITIES_WIDTH 854 + GXF_ENTITIES_CHANNELS 3 + GXF_ENTITIES_FRAMERATE 30 + ) +endif() + +add_subdirectory(cpp) diff --git a/applications/distributed/grpc/grpc_endoscopy_tool_tracking/README.md b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/README.md new file mode 100644 index 00000000..3093c860 --- /dev/null +++ b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/README.md @@ -0,0 +1,147 @@ +# Endoscopy Tool Tracking Application with gRPC + +This application demonstrates how to offload heavy workloads to a remote Holoscan application using gRPC. + +## Overview + +In this sample application, we divided the Endoscopy Tool Tracking application into a server and client application where the two communicate via gRPC. + +The client application inputs a video file and streams the video frames to the server application. The server application handles the heavy workloads of inferencing and post-processing of the video frames. It receives the video frames, processes each frame through the endoscopy tool tracking pipeline, and then streams the results to the client. + +![Overview](static/overview.png) +*Endoscopy Tool Tracking Application with gRPC* + +From the diagram above, we can see that both the App Cloud (the server) and the App Edge (the client) are very similar to the standalone [Endoscopy Tool Tracking](../../../endoscopy_tool_tracking/) application. This section will only describe the differences; for details on inference and post-processing, please refer to the link above. + +On the client side, we provided two examples, one using a single fragment and another one using two fragments. When comparing the client side to the standalone [Endoscopy Tool Tracking](../../../endoscopy_tool_tracking/) application, the differences are the queues and the gRPC client. We added the following: +- **Outgoing Requests** operator (`GrpcClientRequestOp`): It converts the video frames (GXF entities) received from the *Video Stream Replayer* operator into `EntityRequest` protobuf messages and queues each frame in the *Request Queue*. +- **gRPC Service & Client** (`EntityClientService` & `EntityClient`): The gRPC Service is responsible for controlling the life cycle of the gRPC client. The client connects to the remote gRPC server and then sends the requests found in the *Request Queue*. When it receives a response, it converts it into a GXF entity and queues it in the *Response Queue*. +- **Incoming Responses** operator (`GrpcClientResponseOp`): This operator is configured with an `AsynchronousCondition` condition to check the availability of the *Response Queue*. When notified of available responses in the queue, it dequeues each item and emits each to the output port. + +The App Cloud (the server) application consists of a gRPC server and a few components for managing Holoscan applications. When the server receives a new remote procedure call in this sample application, it launches a new instance of the Endoscopy Tool Tracking application. This is facilitated by the `ApplicationFactory` used for application registration. + +Under the hood, the Endoscopy Tool Tracking application here inherits a custom base class (`HoloscanGrpcApplication`) which manages the `Request Queue` and the `Response Queue` as well as the `GrpcServerRequestOp` and `GrpcServerResponseOp` operators for receiving requests and serving results, respectively. When the RPC is complete, the instance of the Endoscopy Tool Tracking application is destroyed and ready to serve the subsequent request. + + +## Requirements + +### Data + +[📦️ (NGC) Sample App Data for AI-based Endoscopy Tool Tracking](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/clara-holoscan/resources/holoscan_endoscopy_sample_data) + +The data is automatically downloaded when building the application. + +## Building and Running gRPC Endoscopy Tool Tracking Application + +* Building and running the application from the top level Holohub directory: + + +### Configuration + +The Edge application runs in a single-fragment mode by default. However, it can be configured to run in a mult-fragment mode, as in the picture above. + +To switch to multi-fragment mode, edit the [endoscopy_tool_tracking.yaml](./cpp/endoscopy_tool_tracking.yaml) YAML file and change `multifragment` to `true`. + +```yaml + +application: + multifragment: false + benchmarking: false +``` + +[Data Flow Tracking](https://docs.nvidia.com/holoscan/sdk-user-guide/flow_tracking.html) can also be enabled by editing the [endoscopy_tool_tracking.yaml](./cpp/endoscopy_tool_tracking.yaml) YAML file and change `benchmarking` to `true`. This enables the built-in mechanism to profile the application and analyze the fine-grained timing properties and data flow between operators. + +For example, on the server side, when a client disconnects, it will output the results for that session: + +```bash +Data Flow Tracking Results: +Total paths: 1 + +Path 1: grpc_request_op,format_converter,lstm_inferer,tool_tracking_postprocessor,grpc_response_op +Number of messages: 663 +Min Latency Message No: 249 +Min end-to-end Latency (ms): 1.868 +Avg end-to-end Latency (ms): 2.15161 +Max Latency Message No: 371 +Max end-to-end Latency (ms): 4.19 + +Number of source messages [format: source operator->transmitter name: number of messages]: +grpc_request_op->output: 683 +``` + +Similarly, on the client side, when it completes playing the video, it will print the results: + +```bash +Data Flow Tracking Results: +Total paths: 3 + +Path 1: incoming_responses,visualizer_op +Number of messages: 663 +Min Latency Message No: 249 +Min end-to-end Latency (ms): 0.214 +Avg end-to-end Latency (ms): 0.374005 +Max Latency Message No: 378 +Max end-to-end Latency (ms): 2.751 + +Path 2: replayer,outgoing_requests +Number of messages: 663 +Min Latency Message No: 379 +Min end-to-end Latency (ms): 24.854 +Avg end-to-end Latency (ms): 27.1886 +Max Latency Message No: 142 +Max end-to-end Latency (ms): 28.003 + +Path 3: replayer,visualizer_op +Number of messages: 663 +Min Latency Message No: 372 +Min end-to-end Latency (ms): 30.966 +Avg end-to-end Latency (ms): 33.325 +Max Latency Message No: 397 +Max end-to-end Latency (ms): 35.479 + +Number of source messages [format: source operator->transmitter name: number of messages]: +incoming_responses->output: 683 +replayer->output: 683 +``` + + +### C++ + +```bash +# Start the gRPC Server +./dev_container build_and_run grpc_endoscopy_tool_tracking --run_args cloud [--language cpp] + +# Start the gRPC Client +./dev_container build_and_run grpc_endoscopy_tool_tracking --run_args edge [--language cpp] +``` + + +## Dev Container + +To start the the Dev Container, run the following command from the root directory of Holohub: + +```bash +./dev_container vscode +``` + +### VS Code Launch Profiles + +#### C++ + +The following launch profiles are available: + +- **(compound) grpc_endoscopy_tool_tracking/cpp (cloud & edge)**: Launch both the gRPC server and the client. +- **(gdb) grpc_endoscopy_tool_tracking/cpp (cloud)**: Launch the gRPC server. +- **(gdb) grpc_endoscopy_tool_tracking/cpp (edge)**: Launch the gRPC client. + + +## Limitations & Known Issues + +- The connection between the server and the client is controlled by `rpc_timeout`. If no data is received or sent within the configured time, it assumes the call has been completed and hangs up. The `rpc_timeout` value can be configured in the [endoscopy_tool_tracking.yaml](./cpp/endoscopy_tool_tracking.yaml) file with a default of 5 seconds. Increasing this value may help on a slow network. +- The server can serve one request at any given time. Any subsequent call receives a `grpc::StatusCode::RESOURCE_EXHAUSTED` status. +- When debugging using the compound profile, the server may not be ready to serve, resulting in errors with the client application. When this happens, open [tasks.json](../../../.vscode/tasks.json), find `Build grpc_endoscopy_tool_tracking (delay 3s)`, and adjust the `command` field with a higher sleep value. +- The client is expected to exit with the following error. It is how the client application terminates when it completes streaming and displays the entire video. + ```bash + [error] [program.cpp:614] Event notification 2 for entity [video_in__outgoing_requests] with id [33] received in an unexpected state [Origin] + ``` + diff --git a/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/CMakeLists.txt b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/CMakeLists.txt new file mode 100644 index 00000000..d60975c5 --- /dev/null +++ b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/CMakeLists.txt @@ -0,0 +1,82 @@ +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.20) +project(grpc_endoscopy_tool_tracking CXX) + +find_package(holoscan 2.6 REQUIRED CONFIG + PATHS "/opt/nvidia/holoscan" "/workspace/holoscan-sdk/install") + +add_executable(grpc_endoscopy_tool_tracking_cloud + cloud/app_cloud_main.cpp + cloud/grpc_service.hpp + cloud/app_cloud_pipeline.hpp +) + +add_executable(grpc_endoscopy_tool_tracking_edge + edge/app_edge_main.cpp + edge/app_edge_single_fragment.hpp + edge/app_edge_multi_fragment.hpp + edge/video_input_fragment.hpp + edge/viz_fragment.hpp +) + +target_link_libraries(grpc_endoscopy_tool_tracking_cloud + PRIVATE + holoscan::core + holoscan::ops::format_converter + lstm_tensor_rt_inference + tool_tracking_postprocessor + grpc_operators +) + +target_link_libraries(grpc_endoscopy_tool_tracking_edge + PRIVATE + holoscan::core + holoscan::ops::holoviz + holoscan::ops::video_stream_replayer + grpc_operators +) + +# Copy the config to the binary directory +add_custom_target(grpc_endoscopy_tool_tracking_yaml + COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_CURRENT_SOURCE_DIR}/endoscopy_tool_tracking.yaml" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "endoscopy_tool_tracking.yaml" + BYPRODUCTS "endoscopy_tool_tracking.yaml" +) +add_dependencies(grpc_endoscopy_tool_tracking_edge grpc_endoscopy_tool_tracking_yaml) + +# Copy the launch script +add_custom_target(grpc_endoscopy_tool_tracking_launch_sh + COMMAND ${CMAKE_COMMAND} -E copy_if_different "${CMAKE_CURRENT_SOURCE_DIR}/launch.sh" ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS "launch.sh" + BYPRODUCTS "launch.sh" +) +add_dependencies(grpc_endoscopy_tool_tracking_edge grpc_endoscopy_tool_tracking_launch_sh) + +# Default to download datasets +option(HOLOHUB_DOWNLOAD_DATASETS "Download datasets" ON) + +# Download the endoscopy sample data +if(HOLOHUB_DOWNLOAD_DATASETS) + include(holoscan_download_data) + holoscan_download_data(endoscopy + URL https://api.ngc.nvidia.com/v2/resources/nvidia/clara-holoscan/holoscan_endoscopy_sample_data/versions/20230222/zip + DOWNLOAD_NAME holoscan_endoscopy_sample_data_20230222.zip + URL_MD5 d54f84a562d29ed560a87d2607eba973 + DOWNLOAD_DIR ${HOLOHUB_DATA_DIR} + ) + add_dependencies(grpc_endoscopy_tool_tracking_edge endoscopy_data) +endif() diff --git a/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/cloud/app_cloud_main.cpp b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/cloud/app_cloud_main.cpp new file mode 100644 index 00000000..f9cb5195 --- /dev/null +++ b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/cloud/app_cloud_main.cpp @@ -0,0 +1,186 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights + * reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include + +#include +#include + +#include "app_cloud_pipeline.hpp" +#include "grpc_service.hpp" + +using namespace holoscan; +using namespace holohub::grpc_h264_endoscopy_tool_tracking; + +/** Helper function to parse the command line arguments */ +bool parse_arguments(int argc, char** argv, uint32_t& port, std::string& data_path, + std::string& config_path) { + static struct option long_options[] = {{"port", required_argument, 0, 'p'}, + {"data", required_argument, 0, 'd'}, + {"config", required_argument, 0, 'c'}, + {0, 0, 0, 0}}; + + int c; + while (optind < argc) { + if ((c = getopt_long(argc, argv, "p:", long_options, NULL)) != -1) { + switch (c) { + case 'c': + config_path = optarg; + break; + case 'd': + data_path = optarg; + break; + case 'p': + try { + port = std::stoi(optarg); + if (port < 0 || port > 65535) { throw std::out_of_range("port number out of range"); } + } catch (const std::exception& e) { std::cerr << e.what() << ":" << optarg << '\n'; } + break; + default: + HOLOSCAN_LOG_ERROR("Unhandled option '{}'", static_cast(c)); + return false; + } + } + } + + return true; +} + +void signal_handler(int signum) { + HOLOSCAN_LOG_WARN("Caught signal {}. Stopping services...", signum); + std::thread myThread([] { GrpcService::get_instance(0, nullptr).stop(); }); + myThread.join(); +} + +/** Helper function to parse benchmarking setting from the configuration file */ +void parse_config(const std::string& config_path, bool& benchmarking) { + auto config = holoscan::Config(config_path); + auto& yaml_nodes = config.yaml_nodes(); + for (const auto& yaml_node : yaml_nodes) { + try { + auto application = yaml_node["application"]; + if (application.IsMap()) { benchmarking = application["benchmarking"].as(); } + } catch (std::exception& e) { + HOLOSCAN_LOG_ERROR("Error parsing configuration file: {}", e.what()); + benchmarking = false; + } + } +} + +/** Main function */ +/** + * @file app_cloud_main.cpp + * @brief Main entry point for the gRPC H264 Endoscopy Tool Tracking application. + * + * This application sets up and starts a gRPC server for endoscopy tool tracking using H264 video + * streams. + * + * The main function performs the following steps: + * 1. Parses command-line arguments to get the port number, data directory, and configuration file + * path. + * 2. If the configuration file path is not provided, it attempts to retrieve it from the + * environment variable `HOLOSCAN_CONFIG_PATH` or defaults to a file named + * `endoscopy_tool_tracking.yaml` in the executable's directory. + * 3. If the data directory is not provided, it attempts to retrieve it from the environment + * variable `HOLOSCAN_INPUT_PATH` or defaults to a directory named `data/endoscopy` in the current + * working directory. + * 4. Registers the Endoscopy Tool Tracking application with the `ApplicationFactory`. + * 5. Starts the gRPC service on the specified port. + * + * @param argc The number of command-line arguments. + * @param argv The array of command-line arguments. + * @return Returns 0 on successful execution, or 1 if argument parsing fails. + */ +int main(int argc, char** argv) { + // Parse the arguments + uint32_t port = 50051; + std::string config_path = ""; + std::string data_directory = ""; + + if (!parse_arguments(argc, argv, port, data_directory, config_path)) { return 1; } + + if (config_path.empty()) { + // Get the input data environment variable + auto config_file_path = std::getenv("HOLOSCAN_CONFIG_PATH"); + if (config_file_path == nullptr || config_file_path[0] == '\0') { + auto config_file = std::filesystem::canonical(argv[0]).parent_path(); + config_path = config_file / std::filesystem::path("endoscopy_tool_tracking.yaml"); + } else { + config_path = config_file_path; + } + } + + if (data_directory.empty()) { + // Get the input data environment variable + auto input_path = std::getenv("HOLOSCAN_INPUT_PATH"); + if (input_path != nullptr && input_path[0] != '\0') { + data_directory = std::string(input_path); + } else if (std::filesystem::is_directory(std::filesystem::current_path() / "data/endoscopy")) { + data_directory = std::string((std::filesystem::current_path() / "data/endoscopy").c_str()); + } else { + HOLOSCAN_LOG_ERROR( + "Input data not provided. Use --data or set HOLOSCAN_INPUT_PATH environment variable."); + exit(-1); + } + } + + bool benchmarking = false; + parse_config(config_path, benchmarking); + + // Register each gRPC service with a Holoscan application: + // - the callback function (create_application_instance_func) is used to create a new instance of + // the application when a new RPC call is received. + ApplicationFactory::get_instance()->register_application( + "EntityStream", + [config_path, data_directory, benchmarking]( + std::queue> incoming_request_queue, + std::queue> + outgoing_response_queue) { + ApplicationInstance application_instance; + application_instance.instance = holoscan::make_application( + incoming_request_queue, outgoing_response_queue); + application_instance.instance->config(config_path); + application_instance.instance->set_data_path(data_directory); + if (benchmarking) { + application_instance.tracker = &application_instance.instance->track(); + } + application_instance.future = application_instance.instance->run_async(); + return application_instance; + }); + + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = signal_handler; + sigaction(SIGINT, &sa, NULL); + sigaction(SIGHUP, &sa, NULL); + sigaction(SIGTERM, &sa, NULL); + + try { + GrpcService::get_instance(port, ApplicationFactory::get_instance()).start(); + } catch (std::exception& e) { + HOLOSCAN_LOG_ERROR("Error running gRPC service: {}", e.what()); + exit(-1); + } catch (...) { + HOLOSCAN_LOG_ERROR("Unknown error running gRPC service"); + exit(-2); + } + return 0; +} diff --git a/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/cloud/app_cloud_pipeline.hpp b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/cloud/app_cloud_pipeline.hpp new file mode 100644 index 00000000..c08026bb --- /dev/null +++ b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/cloud/app_cloud_pipeline.hpp @@ -0,0 +1,111 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights + * reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef GRPC_H264_ENDOSCOPY_TOOL_TRACKING_CPP_CLOUD_APP_CLOUD_PIPELINE_HPP +#define GRPC_H264_ENDOSCOPY_TOOL_TRACKING_CPP_CLOUD_APP_CLOUD_PIPELINE_HPP + +#include +#include +#include +#include + +#include +#include +#include + +namespace holohub::grpc_h264_endoscopy_tool_tracking { + +using namespace holoscan; +using namespace holoscan::ops; + +using holoscan::entity::EntityResponse; + +/** + * @class AppCloudPipeline + * @brief A class that represents the application pipeline for H264 endoscopy tool tracking. + * + * This class inherits from HoloscanGrpcApplication and is responsible for composing the pipeline + * for processing video frames and performing tool tracking using a series of operators. + * + * @note the `HoloscanGrpcApplication` base class composes the `GrpcServerRequestOp` and + * `GrpcServerResponseOp` operators to handle incoming requests and outgoing responses. + * It also configures the queues for handling requests and responses. + */ +class AppCloudPipeline : public HoloscanGrpcApplication { + public: + AppCloudPipeline(std::queue> incoming_request_queue, + std::queue> outgoing_response_queue) + : HoloscanGrpcApplication(incoming_request_queue, outgoing_response_queue) {} + + void compose() override { + // Call base class compose to initialize the queues. + HoloscanGrpcApplication::compose(); + + // Create the Endoscopy Tool Tracking (ETT) Pipeline similar to the regular ETT application. + uint32_t width = 854; + uint32_t height = 480; + int64_t source_block_size = width * height * 3 * 4; + int64_t source_num_blocks = 2; + + const std::shared_ptr cuda_stream_pool = + make_resource("cuda_stream", 0, 0, 0, 1, 5); + + auto format_converter = make_operator( + "format_converter", + from_config("format_converter"), + Arg("pool") = + make_resource("pool", 1, source_block_size, source_num_blocks), + Arg("cuda_stream_pool") = cuda_stream_pool); + + const std::string model_file_path = data_path + "/tool_loc_convlstm.onnx"; + const std::string engine_cache_dir = data_path + "/engines"; + + const uint64_t lstm_inferer_block_size = 107 * 60 * 7 * 4; + const uint64_t lstm_inferer_num_blocks = 2 + 5 * 2; + auto lstm_inferer = make_operator( + "lstm_inferer", + from_config("lstm_inference"), + Arg("model_file_path", model_file_path), + Arg("engine_cache_dir", engine_cache_dir), + Arg("pool") = make_resource( + "pool", 1, lstm_inferer_block_size, lstm_inferer_num_blocks), + Arg("cuda_stream_pool") = cuda_stream_pool); + + // Due to an underlying change in the GXF UCX extension in GXF 4.0 that results in a known issue + // where we have to allocate more blocks than expected when using a BlockMemoryPool, we need to + // use UnboundedAllocator for now. + auto tool_tracking_postprocessor = make_operator( + "tool_tracking_postprocessor", + streaming_enabled, + Arg("cuda_stream_pool") = cuda_stream_pool, + Arg("device_allocator") = make_resource( + "device_allocator", Arg("device_memory_max_size") = std::string("256MB"))); + + // Here we connect the GrpcServerRequestOp to the VideoDecoderRequestOp to send the received + // video frames for decoding. + add_flow(grpc_request_op, format_converter, {{"output", "source_video"}}); + + add_flow(format_converter, lstm_inferer); + add_flow(lstm_inferer, tool_tracking_postprocessor, {{"tensor", "in"}}); + + // Lastly, we connect the results from the tool tracking postprocessor to the + // GrpcServerResponseOp so the pipeline can return the results back to the client + add_flow(tool_tracking_postprocessor, grpc_response_op, {{"out", "input"}}); + } +}; +} // namespace holohub::grpc_h264_endoscopy_tool_tracking +#endif /* GRPC_H264_ENDOSCOPY_TOOL_TRACKING_CPP_CLOUD_APP_CLOUD_PIPELINE_HPP */ diff --git a/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/cloud/grpc_service.hpp b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/cloud/grpc_service.hpp new file mode 100644 index 00000000..55279752 --- /dev/null +++ b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/cloud/grpc_service.hpp @@ -0,0 +1,105 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights + * reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef GRPC_GRPC_ENDOSCOPY_TOOL_TRACKING_CPP_CLOUD_GRPC_SERVICE_HPP +#define GRPC_GRPC_ENDOSCOPY_TOOL_TRACKING_CPP_CLOUD_GRPC_SERVICE_HPP + +#include +#include + +#include +#include +#include +#include + +#include + +using grpc::Server; +using grpc::ServerBuilder; + +namespace holohub::grpc_h264_endoscopy_tool_tracking { + +using namespace holoscan::ops; + +/** + * @class GrpcService + * @brief A singleton class that manages a gRPC server for Holoscan applications. + * + * The GrpcService class is responsible for setting up and managing a gRPC server + * that listens for incoming requests and processes them based on user configured applications with + * the ApplicationFactory. It ensures that only one instance of the server is running at any given + * time. + * + * @note This class cannot be copied or assigned. + */ +class GrpcService { + public: + GrpcService(const GrpcService&) = delete; + GrpcService& operator=(const GrpcService&) = delete; + + static GrpcService& get_instance(uint32_t port, + std::shared_ptr application_factory) { + static GrpcService instance(fmt::format("0.0.0.0:{}", port), application_factory); + return instance; + } + + void start() { + grpc::EnableDefaultHealthCheckService(true); + grpc::reflection::InitProtoReflectionServerBuilderPlugin(); + + service_ = std::make_unique( + // Callback function to create a new instance of a Holoscan application when a new RPC call + // is received. + [this](const std::string& service_name, + std::queue>& incoming_request_queue, + std::queue>& outgoing_response_queue) { + return application_factory_->create_application_instance( + service_name, incoming_request_queue, outgoing_response_queue); + }, + // Callback function to handle the completion of an entity stream RPC. + [this](std::shared_ptr application_instance) { + application_factory_->destroy_application_instance(application_instance); + }); + + ServerBuilder builder; + builder.AddListeningPort(server_address_, grpc::InsecureServerCredentials()); + builder.RegisterService(service_.get()); + server_ = builder.BuildAndStart(); + HOLOSCAN_LOG_INFO("grpc: Server listening on {}", server_address_); + server_->Wait(); + } + + void stop() { + HOLOSCAN_LOG_INFO("grpc: Server shutting down"); + server_->Shutdown(); + } + + private: + std::string server_address_; + std::unique_ptr server_; + std::unique_ptr service_; + std::shared_ptr application_factory_; + + GrpcService(const std::string server_address, + std::shared_ptr application_factory) + : server_address_(server_address), application_factory_(application_factory) {} + + ~GrpcService() = default; +}; +} // namespace holohub::grpc_h264_endoscopy_tool_tracking + +#endif /* GRPC_GRPC_ENDOSCOPY_TOOL_TRACKING_CPP_CLOUD_GRPC_SERVICE_HPP */ diff --git a/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/edge/app_edge_main.cpp b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/edge/app_edge_main.cpp new file mode 100644 index 00000000..16fd04b2 --- /dev/null +++ b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/edge/app_edge_main.cpp @@ -0,0 +1,178 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights + * reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include + +// Run th edge app with a single fragment +#include "app_edge_single_fragment.hpp" + +// Run the edge app with two fragments +#include "app_edge_multi_fragment.hpp" + +using namespace holoscan; +using namespace holohub::grpc_h264_endoscopy_tool_tracking; + +/** Helper function to parse the command line arguments */ +bool parse_arguments(int argc, char** argv, std::string& data_path, std::string& config_path) { + static struct option long_options[] = { + {"data", required_argument, 0, 'd'}, {"config", required_argument, 0, 'c'}, {0, 0, 0, 0}}; + + int c; + while (optind < argc) { + if ((c = getopt_long(argc, argv, "d:c:", long_options, NULL)) != -1) { + switch (c) { + case 'c': + config_path = optarg; + break; + case 'd': + data_path = optarg; + break; + default: + holoscan::log_error("Unhandled option '{}'", static_cast(c)); + return false; + } + } + } + + return true; +} + +/** Helper function to parse fragment mode and benchmarking settings from the configuration file */ +void parse_config(const std::string& config_path, bool& multi_fragment_mode, bool& benchmarking) { + auto config = holoscan::Config(config_path); + auto& yaml_nodes = config.yaml_nodes(); + for (const auto& yaml_node : yaml_nodes) { + try { + auto application = yaml_node["application"]; + if (application.IsMap()) { + multi_fragment_mode = application["multifragment"].as(); + benchmarking = application["benchmarking"].as(); + } + } catch (std::exception& e) { + HOLOSCAN_LOG_ERROR("Error parsing configuration file: {}", e.what()); + multi_fragment_mode = false; + benchmarking = false; + } + } +} + +/** Main function */ +/** + * @file app_edge_main.cpp + * @brief Main entry point for the edge (client) side of the H.264 endoscopy tool tracking + * application. + * + * This file contains the main function which initializes and runs the application. + * It handles argument parsing, configuration, and data directory setup. + * + * @param argc Number of command-line arguments. + * @param argv Array of command-line arguments. + * @return int Exit status of the application. + * + * The main function performs the following steps: + * 1. Parses command-line arguments to obtain the data directory and configuration path. + * 2. If the data directory is not provided, it attempts to retrieve it from the environment + * variable `HOLOSCAN_INPUT_PATH` or defaults to a local "data/endoscopy" directory. + * 3. If the configuration path is not provided, it attempts to retrieve it from the environment + * variable `HOLOSCAN_CONFIG_PATH` or defaults to a local "endoscopy_tool_tracking.yaml" file. + * 4. Creates an instance of the application (`AppEdge`). + * 5. Configures the application with the provided configuration file. + * 6. Sets the data path for the application. + * 7. Runs the application. + */ +int main(int argc, char** argv) { + // Parse the arguments + std::string config_path = ""; + std::string data_directory = ""; + if (!parse_arguments(argc, argv, data_directory, config_path)) { return 1; } + + if (data_directory.empty()) { + // Get the input data environment variable + auto input_path = std::getenv("HOLOSCAN_INPUT_PATH"); + if (input_path != nullptr && input_path[0] != '\0') { + data_directory = std::string(input_path); + } else if (std::filesystem::is_directory(std::filesystem::current_path() / "data/endoscopy")) { + data_directory = std::string((std::filesystem::current_path() / "data/endoscopy").c_str()); + } else { + HOLOSCAN_LOG_ERROR( + "Input data not provided. Use --data or set HOLOSCAN_INPUT_PATH environment variable."); + exit(-1); + } + } + + if (config_path.empty()) { + // Get the input data environment variable + auto config_file_path = std::getenv("HOLOSCAN_CONFIG_PATH"); + if (config_file_path == nullptr || config_file_path[0] == '\0') { + auto config_file = std::filesystem::canonical(argv[0]).parent_path(); + config_path = config_file / std::filesystem::path("endoscopy_tool_tracking.yaml"); + } else { + config_path = config_file_path; + } + } + + bool multi_fragment_mode = false; + bool benchmarking = false; + parse_config(config_path, multi_fragment_mode, benchmarking); + if (multi_fragment_mode) { + HOLOSCAN_LOG_INFO("Running application in multi-fragment mode"); + auto app = holoscan::make_application(); + + HOLOSCAN_LOG_INFO("Using configuration file from {}", config_path); + app->config(config_path); + + HOLOSCAN_LOG_INFO("Using input data from {}", data_directory); + app->set_datapath(data_directory); + + std::unordered_map trackers; + if (benchmarking) { + HOLOSCAN_LOG_INFO("Benchmarking enabled"); + trackers = app->track_distributed(); + } + + app->run(); + + if (benchmarking) { + for (const auto& [name, tracker] : trackers) { + std::cout << "Fragment: " << name << std::endl; + tracker->print(); + } + } + } else { + HOLOSCAN_LOG_INFO("Running application in single fragment mode"); + auto app = holoscan::make_application(); + + HOLOSCAN_LOG_INFO("Using configuration file from {}", config_path); + app->config(config_path); + + HOLOSCAN_LOG_INFO("Using input data from {}", data_directory); + app->set_datapath(data_directory); + + DataFlowTracker* tracker = nullptr; + if (benchmarking) { + HOLOSCAN_LOG_INFO("Benchmarking enabled"); + tracker = &app->track(); + } + app->run(); + if (benchmarking) { tracker->print(); } + } + return 0; +} diff --git a/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/edge/app_edge_multi_fragment.hpp b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/edge/app_edge_multi_fragment.hpp new file mode 100644 index 00000000..f1b17568 --- /dev/null +++ b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/edge/app_edge_multi_fragment.hpp @@ -0,0 +1,67 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights + * reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef GRPC_H264_ENDOSCOPY_TOOL_TRACKING_CPP_EDGE_APP_EDGE_HPP +#define GRPC_H264_ENDOSCOPY_TOOL_TRACKING_CPP_EDGE_APP_EDGE_HPP + +#include +#include +#include + +#include "video_input_fragment.hpp" +#include "viz_fragment.hpp" + +namespace holohub::grpc_h264_endoscopy_tool_tracking { + +using namespace holoscan; + +/** + * @class AppEdgeMultiFragment + * @brief A two-fragment application for the H.264 endoscopy tool tracking application. + * + * This class inherits from the holoscan::Application and is a client application offloads the + * inference and process to a remote gRPC server. It is composed with two fragments, a video input + * fragment and a visualization fragment using Holoviz. This enables running the edge application on + * two systems, separating the input from the visualization. For example, a video surveillance + * camera capturing and streaming input to another system displaying the footage. + */ +class AppEdgeMultiFragment : public holoscan::Application { + public: + explicit AppEdgeMultiFragment(const std::vector& argv = {}) : Application(argv) {} + void set_datapath(const std::string& path) { datapath_ = path; } + + void compose() { + uint32_t width = 854; + uint32_t height = 480; + + auto video_in = make_fragment("video_in", datapath_, width, height); + auto viz = make_fragment("viz", width, height); + + // Connect the video input fragment to the visualization fragment. + // - Connect the decoded video frames to the visualizer. + // - Connect the inference & post-process results to the visualizer. + add_flow(video_in, + viz, + {{"replayer.output", "visualizer_op.receivers"}, + {"incoming_responses.output", "visualizer_op.receivers"}}); + } + + private: + std::string datapath_ = "data/endoscopy"; +}; +} // namespace holohub::grpc_h264_endoscopy_tool_tracking +#endif /* GRPC_H264_ENDOSCOPY_TOOL_TRACKING_CPP_EDGE_APP_EDGE_HPP */ diff --git a/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/edge/app_edge_single_fragment.hpp b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/edge/app_edge_single_fragment.hpp new file mode 100644 index 00000000..b354bd22 --- /dev/null +++ b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/edge/app_edge_single_fragment.hpp @@ -0,0 +1,123 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights + * reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef GRPC_GRPC_ENDOSCOPY_TOOL_TRACKING_CPP_EDGE_APP_EDGE_SINGLE_FRAGMENT_HPP +#define GRPC_GRPC_ENDOSCOPY_TOOL_TRACKING_CPP_EDGE_APP_EDGE_SINGLE_FRAGMENT_HPP + +#include +#include +#include +#include + +#include "video_input_fragment.hpp" +#include "viz_fragment.hpp" + +namespace holohub::grpc_h264_endoscopy_tool_tracking { + +using namespace holoscan; + +/** + * @class AppEdgeSingleFragment + * @brief A two-fragment application for the H.264 endoscopy tool tracking application. + * + * This class inherits from the holoscan::Application and is a client application offloads the + * inference and process to a remote gRPC server. It is composed with two fragments, a video input + * fragment and a visualization fragment using Holoviz. This enables running the edge application + * on two systems, separating the input from the visualization. For example, a video surveillance + * camera capturing and streaming input to another system displaying the footage. + */ +class AppEdgeSingleFragment : public holoscan::Application { + public: + explicit AppEdgeSingleFragment(const std::vector& argv = {}) : Application(argv) {} + ~AppEdgeSingleFragment() { + entity_client_service_->stop_entity_stream(); + } + + void set_datapath(const std::string& path) { datapath_ = path; } + + void compose() { + uint32_t width = 854; + uint32_t height = 480; + int64_t source_block_size = width * height * 3 * 4; + int64_t source_num_blocks = 2; + + condition_ = make_condition("response_available_condition"); + request_queue_ = + make_resource>>("request_queue"); + response_queue_ = + make_resource>>( + "response_queue", condition_); + + auto replayer = make_operator( + "replayer", + from_config("replayer"), + Arg("directory", datapath_), + Arg("allocator", make_resource("video_replayer_allocator"))); + + // The GrpcClientRequestOp is responsible for sending data to the gRPC server. + auto outgoing_requests = make_operator( + "outgoing_requests", + Arg("request_queue") = request_queue_, + Arg("allocator") = make_resource( + "pool", Arg("device_memory_max_size") = std::string("256MB"))); + + auto visualizer_op = make_operator( + "visualizer_op", + from_config("holoviz"), + Arg("width") = width, + Arg("height") = height, + Arg("allocator") = + make_resource("allocator", 1, source_block_size, source_num_blocks), + Arg("cuda_stream_pool") = make_resource("cuda_stream", 0, 0, 0, 1, 5)); + + // The GrpcClientResponseOp is responsible for handling incoming responses from the gRPC + // server. + auto incoming_responses = + make_operator("incoming_responses", + Arg("condition") = condition_, + Arg("response_queue") = response_queue_); + + // Send the frames to the gRPC server for processing. + add_flow(replayer, outgoing_requests, {{"output", "input"}}); + + // Here we add the operator to process the response queue with data received from the gRPC + // server. The operator will convert the data to a GXF Entity and send it to the Holoviz. + add_operator(incoming_responses); + + add_flow(replayer, visualizer_op, {{"output", "receivers"}}); + add_flow(incoming_responses, visualizer_op, {{"output", "receivers"}}); + + entity_client_service_ = std::make_shared( + from_config("grpc_client.server_address").as(), + from_config("grpc_client.rpc_timeout").as(), + from_config("grpc_client.interrupt").as(), + request_queue_, + response_queue_, + outgoing_requests); + entity_client_service_->start_entity_stream(); + } + + private: + std::shared_ptr>> request_queue_; + std::shared_ptr>> response_queue_; + std::shared_ptr condition_; + std::shared_ptr entity_client_service_; + std::string datapath_ = "data/endoscopy"; +}; +} // namespace holohub::grpc_h264_endoscopy_tool_tracking + +#endif /* GRPC_GRPC_ENDOSCOPY_TOOL_TRACKING_CPP_EDGE_APP_EDGE_SINGLE_FRAGMENT_HPP */ diff --git a/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/edge/video_input_fragment.hpp b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/edge/video_input_fragment.hpp new file mode 100644 index 00000000..e62612b5 --- /dev/null +++ b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/edge/video_input_fragment.hpp @@ -0,0 +1,106 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights + * reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef GRPC_H264_ENDOSCOPY_TOOL_TRACKING_CPP_EDGE_VIDEO_INPUT_FRAGMENT_HPP +#define GRPC_H264_ENDOSCOPY_TOOL_TRACKING_CPP_EDGE_VIDEO_INPUT_FRAGMENT_HPP + +#include +#include +#include +#include +#include + +#include + +#include "video_input_fragment.hpp" +#include "viz_fragment.hpp" + +namespace holohub::grpc_h264_endoscopy_tool_tracking { +using namespace holoscan; +using namespace holoscan::ops; + +/** + * @class VideoInputFragment + * @brief A fragment class for handling video input and processing. + * + * This class is responsible for reading video bitstreams, sending requests to a gRPC client for + * inference, and processing. The video bitstreams also decodes video frames for the Holoviz to + * display the frames. + */ +class VideoInputFragment : public holoscan::Fragment { + public: + explicit VideoInputFragment(const std::string& datapath, const uint32_t width, + const uint32_t height) + : datapath_(datapath), width_(width), height_(height) {} + + ~VideoInputFragment() { entity_client_service_->stop_entity_stream(); } + + void compose() override { + condition_ = make_condition("response_available_condition"); + request_queue_ = + make_resource>>("request_queue"); + response_queue_ = + make_resource>>( + "response_queue", condition_); + + auto replayer = make_operator( + "replayer", + from_config("replayer"), + Arg("directory", datapath_)); + + // The GrpcClientRequestOp is responsible for sending data to the gRPC server. + auto outgoing_requests = make_operator( + "outgoing_requests", + Arg("request_queue") = request_queue_, + Arg("allocator") = make_resource( + "pool", Arg("device_memory_max_size") = std::string("256MB"))); + + // The GrpcClientResponseOp is responsible for handling incoming responses from the gRPC server. + auto incoming_responses = + make_operator("incoming_responses", + Arg("condition") = condition_, + Arg("response_queue") = response_queue_); + + // Send the frames to the gRPC server for processing. + add_flow(replayer, outgoing_requests, {{"output", "input"}}); + + // Here we add the operator to process the response queue with data received from the gRPC + // server. The operator will convert the data to a GXF Entity and send it to the Holoviz. + add_operator(incoming_responses); + + entity_client_service_ = std::make_shared( + from_config("grpc_client.server_address").as(), + from_config("grpc_client.rpc_timeout").as(), + from_config("grpc_client.interrupt").as(), + request_queue_, + response_queue_, + outgoing_requests); + entity_client_service_->start_entity_stream(); + } + + private: + std::shared_ptr>> request_queue_; + std::shared_ptr>> response_queue_; + std::shared_ptr condition_; + std::shared_ptr entity_client_service_; + std::string datapath_; + uint32_t width_; + uint32_t height_; +}; + +} // namespace holohub::grpc_h264_endoscopy_tool_tracking +#endif /* GRPC_H264_ENDOSCOPY_TOOL_TRACKING_CPP_EDGE_VIDEO_INPUT_FRAGMENT_HPP */ diff --git a/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/edge/viz_fragment.hpp b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/edge/viz_fragment.hpp new file mode 100644 index 00000000..50374f06 --- /dev/null +++ b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/edge/viz_fragment.hpp @@ -0,0 +1,59 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights + * reserved. SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef GRPC_H264_ENDOSCOPY_TOOL_TRACKING_CPP_VIZ_FRAGMENT_HPP +#define GRPC_H264_ENDOSCOPY_TOOL_TRACKING_CPP_VIZ_FRAGMENT_HPP + +#include +#include + +namespace holohub::grpc_h264_endoscopy_tool_tracking { + +using namespace holoscan; +using namespace holoscan::ops; + +/** + * @class VizFragment + * @brief A fragment class for visualizing endoscopy tool tracking using Holoviz. + * + * This class inherits from holoscan::Fragment and is used to set up a visualization + * operator with specified width and height. + */ +class VizFragment : public holoscan::Fragment { + public: + VizFragment(const uint32_t width, const uint32_t height) + : width_(width), height_(height) {} + + void compose() override { + int64_t source_block_size = width_ * height_ * 3 * 4; + int64_t source_num_blocks = 2; + + auto visualizer_op = make_operator( + "visualizer_op", + from_config("holoviz"), + Arg("width") = width_, + Arg("height") = height_, + Arg("cuda_stream_pool") = make_resource("cuda_stream", 0, 0, 0, 1, 5)); + add_operator(visualizer_op); + } + + private: + uint32_t width_; + uint32_t height_; +}; +} // namespace holohub::grpc_h264_endoscopy_tool_tracking +#endif /* GRPC_H264_ENDOSCOPY_TOOL_TRACKING_CPP_VIZ_FRAGMENT_HPP */ diff --git a/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/endoscopy_tool_tracking.yaml b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/endoscopy_tool_tracking.yaml new file mode 100644 index 00000000..2909fd6c --- /dev/null +++ b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/endoscopy_tool_tracking.yaml @@ -0,0 +1,116 @@ +%YAML 1.2 +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +extensions: + - ../../../../../lib/gxf_extensions/libgxf_lstm_tensor_rt_inference.so + +application: + title: Endoscopy Tool Tracking - gRPC + version: 1.0 + inputFormats: [] + outputFormats: ["screen"] + multifragment: false # default: false, true to run in multi-fragment mode, false otherwise + benchmarking: false # default: false, true to enable Data Flow Benchmarking, false otherwise + +replayer: + basename: "surgical_video" + frame_rate: 0 # as specified in timestamps + repeat: false # default: false + realtime: true # default: true + count: 0 # default: 0 (no frame count restriction) + +format_converter: + out_tensor_name: source_video + out_dtype: "float32" + scale_min: 0.0 + scale_max: 255.0 + +grpc_server: + rpc_timeout: 5 + +grpc_client: + server_address: localhost:50051 + rpc_timeout: 5 + interrupt: true + +scheduler: + worker_thread_number: 8 + stop_on_deadlock: true + stop_on_deadlock_timeout: 2500 + +lstm_inference: + input_tensor_names: + - source_video + - cellstate_in + - hiddenstate_in + input_state_tensor_names: + - cellstate_in + - hiddenstate_in + input_binding_names: + - data_ph:0 # (shape=[1, 480, 854, 3], dtype=float32) <==> source_video + - cellstate_ph:0 # (shape=[1, 60, 107, 7], dtype=float32) == internal state + - hiddenstate_ph:0 # (shape=[1, 60, 107, 7], dtype=float32) == internal state + output_tensor_names: + - cellstate_out + - hiddenstate_out + - probs + - scaled_coords + - binary_masks + output_state_tensor_names: + - cellstate_out + - hiddenstate_out + output_binding_names: + - Model/net_states:0 # (shape=[ 1, 60, 107, 7], dtype=float32) + - Model/net_hidden:0 # (shape=[ 1, 60, 107, 7], dtype=float32) + - probs:0 # (shape=[1, 7], dtype=float32) + - Localize/scaled_coords:0 # (shape=[1, 7, 2], dtype=float32) + - Localize_1/binary_masks:0 # (shape=[1, 7, 60, 107], dtype=float32) + force_engine_update: false + verbose: true + max_workspace_size: 2147483648 + enable_fp16_: true + +tool_tracking_postprocessor: + +holoviz: + tensors: + - name: "" + type: color + opacity: 1.0 + priority: 0 + - name: mask + type: color + opacity: 1.0 + priority: 1 + - name: scaled_coords + type: crosses + opacity: 1.0 + line_width: 4 + color: [1.0, 0.0, 0.0, 1.0] + priority: 2 + - name: scaled_coords + type: text + opacity: 1.0 + priority: 3 + color: [1.0, 1.0, 1.0, 0.9] + text: + - Grasper + - Bipolar + - Hook + - Scissors + - Clipper + - Irrigator + - Spec.Bag diff --git a/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/launch.sh b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/launch.sh new file mode 100755 index 00000000..39d0f660 --- /dev/null +++ b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/launch.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +app="" +args=() + +run_command() { + local status=0 + local cmd="$*" + + echo -e "${YELLOW}[command]${NOCOLOR} ${cmd}" + + [ "$(echo -n "$@")" = "" ] && return 1 # return 1 if there is no command available + + if [ "${DO_DRY_RUN}" != "true" ]; then + eval "$@" + status=$? + fi + return $status +} + + +while [[ $# -gt 0 ]]; do + if [[ "$1" == "--"* ]]; then + args+=("$1" "$2") + shift 2 + elif [ -z "$app" ]; then + app=$1 + shift 1 + else + echo "Invalid argument: $1" + exit 1 + fi +done + +if [ "$app" == "cloud" ]; then + run_command ./grpc_endoscopy_tool_tracking_cloud "${args[@]}" +elif [ "$app" == "edge" ]; then + run_command ./grpc_endoscopy_tool_tracking_edge "${args[@]}" +else + echo "Invalid application: ${app:-}" + exit 1 +fi diff --git a/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/metadata.json b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/metadata.json new file mode 100644 index 00000000..6d9fd521 --- /dev/null +++ b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/cpp/metadata.json @@ -0,0 +1,47 @@ +{ + "application": { + "name": "gRPC-streaming Endoscopy Tool Tracking", + "authors": [ + { + "name": "Holoscan Team", + "affiliation": "NVIDIA" + } + ], + "language": "C++", + "version": "1.0", + "changelog": { + "1.0": "Initial Release" + }, + "dockerfile": "applications/h264/Dockerfile", + "holoscan_sdk": { + "minimum_required_version": "2.6.0", + "tested_versions": [ + "2.6.0" + ] + }, + "platforms": [ + "amd64", + "arm64" + ], + "tags": [ + "Endoscopy", + "Tracking", + "gRPC" + ], + "ranking": 0, + "dependencies": { + "data": [ + { + "name": "Holoscan Sample App Data for AI-based Endoscopy Tool Tracking", + "description": "This resource contains the convolutional LSTM model for tool tracking in laparoscopic videos by Nwoye et. al [1], and a sample surgical video.", + "url": "https://catalog.ngc.nvidia.com/orgs/nvidia/teams/clara-holoscan/resources/holoscan_endoscopy_sample_data", + "version": "20230222" + } + ] + }, + "run": { + "command": "./launch.sh --data /endoscopy", + "workdir": "holohub_app_bin" + } + } +} \ No newline at end of file diff --git a/applications/distributed/grpc/grpc_endoscopy_tool_tracking/static/overview.png b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/static/overview.png new file mode 100644 index 00000000..8d0b2a3d Binary files /dev/null and b/applications/distributed/grpc/grpc_endoscopy_tool_tracking/static/overview.png differ diff --git a/applications/distributed/grpc/grpc_h264_endoscopy_tool_tracking/cpp/edge/video_input_fragment.hpp b/applications/distributed/grpc/grpc_h264_endoscopy_tool_tracking/cpp/edge/video_input_fragment.hpp index a6be0322..f6cbac59 100644 --- a/applications/distributed/grpc/grpc_h264_endoscopy_tool_tracking/cpp/edge/video_input_fragment.hpp +++ b/applications/distributed/grpc/grpc_h264_endoscopy_tool_tracking/cpp/edge/video_input_fragment.hpp @@ -119,6 +119,7 @@ class VideoInputFragment : public holoscan::Fragment { entity_client_service_ = std::make_shared( from_config("grpc_client.server_address").as(), from_config("grpc_client.rpc_timeout").as(), + from_config("grpc_client.interrupt").as(), request_queue_, response_queue_, outgoing_requests); diff --git a/applications/distributed/grpc/grpc_h264_endoscopy_tool_tracking/cpp/endoscopy_tool_tracking.yaml b/applications/distributed/grpc/grpc_h264_endoscopy_tool_tracking/cpp/endoscopy_tool_tracking.yaml index 67ff5483..f04fcbc1 100644 --- a/applications/distributed/grpc/grpc_h264_endoscopy_tool_tracking/cpp/endoscopy_tool_tracking.yaml +++ b/applications/distributed/grpc/grpc_h264_endoscopy_tool_tracking/cpp/endoscopy_tool_tracking.yaml @@ -27,6 +27,7 @@ grpc_server: grpc_client: server_address: localhost:50051 rpc_timeout: 5 + interrupt: true scheduler: worker_thread_number: 8 diff --git a/applications/distributed/ucx/ucx_endoscopy_tool_tracking/cpp/metadata.json b/applications/distributed/ucx/ucx_endoscopy_tool_tracking/cpp/metadata.json index cc675163..0550769b 100644 --- a/applications/distributed/ucx/ucx_endoscopy_tool_tracking/cpp/metadata.json +++ b/applications/distributed/ucx/ucx_endoscopy_tool_tracking/cpp/metadata.json @@ -32,7 +32,7 @@ ] }, "run": { - "command": "/ucx_endoscopy_tool_tracking", + "command": "/ucx_endoscopy_tool_tracking --data /endoscopy", "workdir": "holohub_bin" } } diff --git a/operators/grpc_operators/client/entity_client_service.cpp b/operators/grpc_operators/client/entity_client_service.cpp index 361c621a..d4a80ea5 100644 --- a/operators/grpc_operators/client/entity_client_service.cpp +++ b/operators/grpc_operators/client/entity_client_service.cpp @@ -20,13 +20,14 @@ namespace holoscan::ops { EntityClientService::EntityClientService( - const std::string& server_address, const uint32_t rpc_timeout, + const std::string& server_address, const uint32_t rpc_timeout, const bool interrupt, std::shared_ptr>> request_queue, std::shared_ptr>> response_queue, std::shared_ptr grpc_request_operator) : server_address_(server_address), rpc_timeout_(rpc_timeout), + interrupt_(interrupt), request_queue_(request_queue), response_queue_(response_queue), grpc_request_operator_(grpc_request_operator) {} @@ -70,7 +71,7 @@ void EntityClientService::start_entity_stream_internal() { HOLOSCAN_LOG_ERROR("grpc client: EntityStream rpc failed: {}", e.what()); } entity_client_.reset(); - grpc_request_operator_->executor().interrupt(); + if (interrupt_) { grpc_request_operator_->executor().interrupt(); } } void EntityClientService::stop_entity_stream() { diff --git a/operators/grpc_operators/client/entity_client_service.hpp b/operators/grpc_operators/client/entity_client_service.hpp index 74f8ef95..6af48c91 100644 --- a/operators/grpc_operators/client/entity_client_service.hpp +++ b/operators/grpc_operators/client/entity_client_service.hpp @@ -45,7 +45,7 @@ namespace holoscan::ops { class EntityClientService { public: EntityClientService( - const std::string& server_address, const uint32_t rpc_timeout, + const std::string& server_address, const uint32_t rpc_timeout, const bool interrupt, std::shared_ptr>> request_queue, std::shared_ptr>> response_queue, @@ -85,6 +85,7 @@ class EntityClientService { const std::string server_address_; const uint32_t rpc_timeout_; + const bool interrupt_; std::shared_ptr>> request_queue_; std::shared_ptr>> response_queue_; std::shared_ptr grpc_request_operator_; diff --git a/operators/grpc_operators/server/application_factory.cpp b/operators/grpc_operators/server/application_factory.cpp index 8c78c8c1..618922c3 100644 --- a/operators/grpc_operators/server/application_factory.cpp +++ b/operators/grpc_operators/server/application_factory.cpp @@ -70,6 +70,7 @@ void ApplicationFactory::destroy_application_instance( if (instance.instance == application_instance) { instance.instance->stop_streaming(); instance.future.wait_for(std::chrono::seconds(1)); + if (instance.tracker != nullptr) { instance.tracker->print(); } application_instances_.erase(service_name); HOLOSCAN_LOG_INFO("Application instance deleted for {}", service_name); return; diff --git a/operators/grpc_operators/server/application_factory.hpp b/operators/grpc_operators/server/application_factory.hpp index d6efcd71..2f1fa49e 100644 --- a/operators/grpc_operators/server/application_factory.hpp +++ b/operators/grpc_operators/server/application_factory.hpp @@ -50,6 +50,7 @@ class HoloscanGrpcApplication; struct ApplicationInstance { std::shared_ptr instance; std::future future; + DataFlowTracker* tracker = nullptr; }; /**