Merge pull request #3 from NVIDIA-ISAAC-ROS/release-dp3.1

Isaac ROS 0.31.0 (DP3.1)

Author: jaiveersinghNV

README.md

@@ -66,14 +66,22 @@ Follow the steps below to run a sample benchmark for measuring performance of an
 
     ```bash
     cd ~/workspaces/isaac_ros-dev/src &&
-    git clone https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark && \
-    git clone https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_common && \
-    git clone https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_nitros && \
-    git clone https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_image_pipeline && \
-    git clone https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_apriltag
+      git clone https://github.com/NVIDIA-ISAAC-ROS/ros2_benchmark && \
+      git clone https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_benchmark && \
+      git clone https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_common && \
+      git clone https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_nitros && \
+      git clone https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_image_pipeline && \
+      git clone https://github.com/NVIDIA-ISAAC-ROS/isaac_ros_apriltag
     ```
 
-3. Pull down the r2b Dataset 2023 by following the instructions [here](https://github.com/NVIDIA-ISAAC-ROS/ros2_benchmark/blob/main/README.md#datasets).
+3. Pull down `r2b Dataset 2023` by following the instructions [here](https://github.com/NVIDIA-ISAAC-ROS/ros2_benchmark/blob/main/README.md#datasets) or fetch just the rosbag used in this Quickstart with the following command.
+
+    ```bash
+    mkdir -p ~/workspaces/isaac_ros-dev/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage && \
+      cd ~/workspaces/isaac_ros-dev/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage && \
+      wget --content-disposition 'https://api.ngc.nvidia.com/v2/resources/nvidia/isaac/r2bdataset2023/versions/1/files/r2b_storage/metadata.yaml' && \
+      wget --content-disposition 'https://api.ngc.nvidia.com/v2/resources/nvidia/isaac/r2bdataset2023/versions/1/files/r2b_storage/r2b_storage_0.db3'
+    ```
 
 4. Launch the Docker container using the `run_dev.sh` script:
 

isaac_ros_benchmark/package.xml

@@ -21,7 +21,7 @@
 <?xml-model href="http://download.ros.org/schema/package_format3.xsd" schematypens="http://www.w3.org/2001/XMLSchema"?>
 <package format="3">
   <name>isaac_ros_benchmark</name>
-  <version>0.30.0</version>
+  <version>0.31.0</version>
   <description>Isaac ROS benchmarking utilities</description>
 
   <maintainer email="[email protected]">Hemal Shah</maintainer>

results/apriltag_ros_apriltag_graph-agx_orin.json

@@ -0,0 +1 @@
+{"BasicPerformanceMetrics.RECEIVED_DURATION": 4973.735758463542, "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 56.29346542189134, "BasicPerformanceMetrics.MEAN_FRAME_RATE": 56.29534904224372, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0.0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 280.0, "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 21.679280598958332, "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 21.50146484375, "BasicPerformanceMetrics.MAX_JITTER": 4.03857421875, "BasicPerformanceMetrics.MIN_JITTER": 0.00732421875, "BasicPerformanceMetrics.MEAN_JITTER": 1.001255831272482, "BasicPerformanceMetrics.STD_DEV_JITTER": 0.5097108820386291, "CPUProfilingMetrics.MAX_CPU_UTIL": 26.166666666666668, "CPUProfilingMetrics.MIN_CPU_UTIL": 22.375, "CPUProfilingMetrics.MEAN_CPU_UTIL": 23.43166666666666, "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.6896382330957904, "CPUProfilingMetrics.BASELINE_CPU_UTIL": 23.647222222222222, "10.0fps": {"BasicPerformanceMetrics.RECEIVED_DURATION": 4899.74609375, "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203899623521941, "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.204610411094325, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 21.81689453125, "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 21.4755859375, "BasicPerformanceMetrics.MAX_JITTER": 2.171142578125, "BasicPerformanceMetrics.MIN_JITTER": 0.116455078125, "BasicPerformanceMetrics.MEAN_JITTER": 1.051605224609375, "BasicPerformanceMetrics.STD_DEV_JITTER": 0.5106522767872985, "CPUProfilingMetrics.MAX_CPU_UTIL": 5.691666666666666, "CPUProfilingMetrics.MIN_CPU_UTIL": 4.575, "CPUProfilingMetrics.MEAN_CPU_UTIL": 4.98, "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.3881079460379259, "CPUProfilingMetrics.BASELINE_CPU_UTIL": 5.691666666666666}, "30.0fps": {"BasicPerformanceMetrics.RECEIVED_DURATION": 4966.509765625, "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200823680277093, "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.202296396999746, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 21.622802734375, "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 21.380615234375, "BasicPerformanceMetrics.MAX_JITTER": 1.8173828125, "BasicPerformanceMetrics.MIN_JITTER": 0.025146484375, "BasicPerformanceMetrics.MEAN_JITTER": 0.9371766786317568, "BasicPerformanceMetrics.STD_DEV_JITTER": 0.4186160229648217, "CPUProfilingMetrics.MAX_CPU_UTIL": 13.775, "CPUProfilingMetrics.MIN_CPU_UTIL": 12.783333333333333, "CPUProfilingMetrics.MEAN_CPU_UTIL": 13.271666666666667, "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 0.4023748942908294, "CPUProfilingMetrics.BASELINE_CPU_UTIL": 13.408333333333333}, "custom": {"data_resolution": "HD (1280,720)"}, "metadata": {"BenchmarkMetadata.NAME": "apriltag_ros AprilTag Graph Benchmark", "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/apriltag_ros_apriltag_graph.py", "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T06:56:47Z", "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-5bj7cyeix5eolmrzzp4eu34h3y", "BenchmarkMetadata.DEVICE_ARCH": "aarch64", "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Sun Mar 19 07:55:28 PDT 2023", "BenchmarkMetadata.IDLE_CPU_UTIL": 0.08333333333333333, "BenchmarkMetadata.BENCHMARK_MODE": 1, "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 56.09375, "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", "BenchmarkMetadata.INPUT_DATA_SIZE": 3087908864, "BenchmarkMetadata.INPUT_DATA_HASH": "5e8f11201fe10dbac7307a8628553e94", "BenchmarkMetadata.INPUT_DATA_START_TIME": 3.0, "BenchmarkMetadata.INPUT_DATA_END_TIME": 3.5, "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n  additional_fixed_publisher_rate_tests:\n  - 10.0\n  - 30.0\n  - 60.0\n  assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n  benchmark_duration: 5.0\n  benchmark_mode: BenchmarkMode.LOOPING\n  benchmark_name: apriltag_ros AprilTag Graph Benchmark\n  benchmark_namespace: r2b\n  binary_search_acceptable_frame_loss_fraction: 0.05\n  binary_search_acceptable_frame_rate_drop: 10.0\n  binary_search_duration_fraction: 1.0\n  binary_search_terminal_interval_width: 10.0\n  cpu_profiling_interval_sec: 1.0\n  custom_report_info:\n    data_resolution: Resolution(1280, 720, HD)\n  default_config_version: 0.30.0\n  default_service_future_timeout_sec: 15.0\n  enable_cpu_profiler: true\n  enable_trial_buffer_preparation: false\n  enforce_publisher_rate: false\n  input_data_end_time: 3.5\n  input_data_path: datasets/r2b_dataset/r2b_storage\n  input_data_start_time: 3.0\n  linear_scan_acceptable_frame_loss_fraction: 0.05\n  linear_scan_acceptable_frame_rate_drop: 5.0\n  linear_scan_duration_fraction: 1.0\n  linear_scan_step_size: 5.0\n  load_data_in_real_time: false\n  log_file_name: kpi.json\n  log_folder: /osmo/data/output\n  monitor_info_list:\n  - calculators:\n    - class_name: BasicPerformanceCalculator\n      config:\n        report_prefix: ''\n      module_name: ros2_benchmark.basic_performance_calculator\n    service_name: start_monitoring0\n  play_messages_service_future_timeout_sec: 15.0\n  playback_message_buffer_size: 10\n  publish_tf_messages_in_set_data: false\n  publish_tf_static_messages_in_set_data: false\n  publisher_lower_frequency: 10.0\n  publisher_upper_frequency: 600.0\n  record_data_timeline: false\n  revise_timestamps_as_message_ids: false\n  set_data_service_future_timeout_sec: 60.0\n  setup_service_client_timeout_sec: 20.0\n  start_monitoring_service_timeout_sec: 10\n  start_recording_service_future_timeout_sec: 35.0\n  start_recording_service_timeout_sec: 30\n  test_iterations: 5\n"}}

results/apriltag_ros_apriltag_graph-agx_xavier.json

@@ -0,0 +1 @@
+{"BasicPerformanceMetrics.RECEIVED_DURATION": 4982.469156901042, "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 37.85641001734155, "BasicPerformanceMetrics.MEAN_FRAME_RATE": 37.302170642169436, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 2.3333333333333335, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 188.0, "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 32.547607421875, "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 48.551106770833336, "BasicPerformanceMetrics.MAX_JITTER": 6.840087890625, "BasicPerformanceMetrics.MIN_JITTER": 0.00341796875, "BasicPerformanceMetrics.MEAN_JITTER": 1.068137790845788, "BasicPerformanceMetrics.STD_DEV_JITTER": 0.8836308837055412, "CPUProfilingMetrics.MAX_CPU_UTIL": 29.8875, "CPUProfilingMetrics.MIN_CPU_UTIL": 0.5, "CPUProfilingMetrics.MEAN_CPU_UTIL": 14.451969696969698, "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 13.180950309858142, "CPUProfilingMetrics.BASELINE_CPU_UTIL": 27.900000000000002, "10.0fps": {"BasicPerformanceMetrics.RECEIVED_DURATION": 4899.93701171875, "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 10.203733889011675, "BasicPerformanceMetrics.MEAN_FRAME_RATE": 10.204212805270636, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 50, "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 32.474853515625, "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 32.244873046875, "BasicPerformanceMetrics.MAX_JITTER": 5.814453125, "BasicPerformanceMetrics.MIN_JITTER": 0.01904296875, "BasicPerformanceMetrics.MEAN_JITTER": 1.989654541015625, "BasicPerformanceMetrics.STD_DEV_JITTER": 1.3267283020236127, "CPUProfilingMetrics.MAX_CPU_UTIL": 12.1625, "CPUProfilingMetrics.MIN_CPU_UTIL": 7.9125, "CPUProfilingMetrics.MEAN_CPU_UTIL": 9.5675, "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.4882372122749785, "CPUProfilingMetrics.BASELINE_CPU_UTIL": 9.387500000000001}, "30.0fps": {"BasicPerformanceMetrics.RECEIVED_DURATION": 4965.954345703125, "BasicPerformanceMetrics.MEAN_PLAYBACK_FRAME_RATE": 30.200304106838292, "BasicPerformanceMetrics.MEAN_FRAME_RATE": 30.205674389614558, "BasicPerformanceMetrics.NUM_MISSED_FRAMES": 0, "BasicPerformanceMetrics.NUM_FRAMES_SENT": 150, "BasicPerformanceMetrics.FIRST_SENT_RECEIVED_LATENCY": 31.3466796875, "BasicPerformanceMetrics.LAST_SENT_RECEIVED_LATENCY": 30.463623046875, "BasicPerformanceMetrics.MAX_JITTER": 15.36767578125, "BasicPerformanceMetrics.MIN_JITTER": 0.029296875, "BasicPerformanceMetrics.MEAN_JITTER": 2.0854013803842903, "BasicPerformanceMetrics.STD_DEV_JITTER": 1.874367530717849, "CPUProfilingMetrics.MAX_CPU_UTIL": 26.012500000000003, "CPUProfilingMetrics.MIN_CPU_UTIL": 22.85, "CPUProfilingMetrics.MEAN_CPU_UTIL": 24.1475, "CPUProfilingMetrics.STD_DEV_CPU_UTIL": 1.1334515869678778, "CPUProfilingMetrics.BASELINE_CPU_UTIL": 22.85}, "custom": {"data_resolution": "HD (1280,720)"}, "metadata": {"BenchmarkMetadata.NAME": "apriltag_ros AprilTag Graph Benchmark", "BenchmarkMetadata.TEST_FILE_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/scripts/apriltag_ros_apriltag_graph.py", "BenchmarkMetadata.TEST_DATETIME": "2023-05-25T07:45:31Z", "BenchmarkMetadata.DEVICE_HOSTNAME": "ros-vmbbxwokrng7ferzoro546rhfm", "BenchmarkMetadata.DEVICE_ARCH": "aarch64", "BenchmarkMetadata.DEVICE_OS": "Linux 5.10.104-tegra #1 SMP PREEMPT Sun Mar 19 07:55:28 PDT 2023", "BenchmarkMetadata.IDLE_CPU_UTIL": 2.725, "BenchmarkMetadata.BENCHMARK_MODE": 1, "BenchmarkMetadata.PEAK_THROUGHPUT_PREDICTION": 37.65625, "BenchmarkMetadata.INPUT_DATA_PATH": "/workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets/datasets/r2b_dataset/r2b_storage", "BenchmarkMetadata.INPUT_DATA_SIZE": 3087908864, "BenchmarkMetadata.INPUT_DATA_HASH": "5e8f11201fe10dbac7307a8628553e94", "BenchmarkMetadata.INPUT_DATA_START_TIME": 3.0, "BenchmarkMetadata.INPUT_DATA_END_TIME": 3.5, "BenchmarkMetadata.CONFIG": "ros2_benchmark_config:\n  additional_fixed_publisher_rate_tests:\n  - 10.0\n  - 30.0\n  - 60.0\n  assets_root: /workspaces/isaac_ros-dev/ros_ws/src/ros2_benchmark/assets\n  benchmark_duration: 5.0\n  benchmark_mode: BenchmarkMode.LOOPING\n  benchmark_name: apriltag_ros AprilTag Graph Benchmark\n  benchmark_namespace: r2b\n  binary_search_acceptable_frame_loss_fraction: 0.05\n  binary_search_acceptable_frame_rate_drop: 10.0\n  binary_search_duration_fraction: 1.0\n  binary_search_terminal_interval_width: 10.0\n  cpu_profiling_interval_sec: 1.0\n  custom_report_info:\n    data_resolution: Resolution(1280, 720, HD)\n  default_config_version: 0.30.0\n  default_service_future_timeout_sec: 15.0\n  enable_cpu_profiler: true\n  enable_trial_buffer_preparation: false\n  enforce_publisher_rate: false\n  input_data_end_time: 3.5\n  input_data_path: datasets/r2b_dataset/r2b_storage\n  input_data_start_time: 3.0\n  linear_scan_acceptable_frame_loss_fraction: 0.05\n  linear_scan_acceptable_frame_rate_drop: 5.0\n  linear_scan_duration_fraction: 1.0\n  linear_scan_step_size: 5.0\n  load_data_in_real_time: false\n  log_file_name: kpi.json\n  log_folder: /osmo/data/output\n  monitor_info_list:\n  - calculators:\n    - class_name: BasicPerformanceCalculator\n      config:\n        report_prefix: ''\n      module_name: ros2_benchmark.basic_performance_calculator\n    service_name: start_monitoring0\n  play_messages_service_future_timeout_sec: 15.0\n  playback_message_buffer_size: 10\n  publish_tf_messages_in_set_data: false\n  publish_tf_static_messages_in_set_data: false\n  publisher_lower_frequency: 10.0\n  publisher_upper_frequency: 600.0\n  record_data_timeline: false\n  revise_timestamps_as_message_ids: false\n  set_data_service_future_timeout_sec: 60.0\n  setup_service_client_timeout_sec: 20.0\n  start_monitoring_service_timeout_sec: 10\n  start_recording_service_future_timeout_sec: 35.0\n  start_recording_service_timeout_sec: 30\n  test_iterations: 5\n"}}