Hdr with split as select

common/model-views.cpp

@@ -973,6 +973,9 @@ namespace rs2
             if (shared_filter->is<hole_filling_filter>())
                 model->enable(false);
 
+            if (shared_filter->is<sequence_id_filter>())
+                model->enable(false);
+
             if (shared_filter->is<decimation_filter>())
             {
                 if (is_rgb_camera)

examples/CMakeLists.txt

@@ -47,3 +47,4 @@ add_subdirectory(ar-basic)
 add_subdirectory(ar-advanced)
 add_subdirectory(pose-apriltag)
 add_subdirectory(tracking-and-depth)
+add_subdirectory(hdr)

examples/hdr/CMakeLists.txt

@@ -0,0 +1,16 @@
+# License: Apache 2.0. See LICENSE file in root directory.
+# Copyright(c) 2020 Intel Corporation. All Rights Reserved.
+#  minimum required cmake version: 3.1.0
+cmake_minimum_required(VERSION 3.1.0)
+
+project(RealsenseExamplesHdr)
+
+if(BUILD_GRAPHICAL_EXAMPLES)
+    add_executable(rs-hdr rs-hdr.cpp ../example.hpp)
+    set_property(TARGET rs-hdr PROPERTY CXX_STANDARD 11)
+    target_link_libraries(rs-hdr ${DEPENDENCIES})
+    include_directories(rs-hdr ../ ../../third-party/tclap/include)
+    set_target_properties (rs-hdr PROPERTIES FOLDER "Examples")
+
+    install(TARGETS rs-hdr RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
+endif()

examples/hdr/readme.md

@@ -0,0 +1,114 @@
+# rs-hdr Sample
+
+## Overview
+
+This sample demonstrates how to configure the camera for streaming and rendering Depth & Infrared data to the screen, using the High Dynamic Range (HDR) feature.  
+For explanations on the streaming and rendering, please refer to the rs-capture sample.
+
+## Code Overview 
+
+### Finidng device
+The HDR feature is available only for D400 Product line Devices. Finiding this device and getting its depth sensor:
+```cpp
+// finding a device of D400 product line for working with HDR feature
+if (dev.supports(RS2_CAMERA_INFO_PRODUCT_LINE) && 
+            std::string(dev.get_info(RS2_CAMERA_INFO_PRODUCT_LINE)) == "D400")
+{
+    device = dev;
+}
+rs2::depth_sensor depth_sensor = device.query_sensors().front();
+ ```
+
+
+### HDR Configuration
+Before starting the configuration, we need to disable the Auto Exposure option:
+```cpp
+// disable auto exposure before sending HDR configuration
+if (depth_sensor.get_option(RS2_OPTION_ENABLE_AUTO_EXPOSURE))
+    depth_sensor.set_option(RS2_OPTION_ENABLE_AUTO_EXPOSURE, 0);
+```
+
+Next, we start to configure the HDR, by setting the size of the HDR sequence:
+```cpp
+// setting the HDR sequence size to 2 frames
+depth_sensor.set_option(RS2_OPTION_SEQUENCE_SIZE, 2);
+```
+Choosing an ID for this HDR configuration. The configuration will not be saved in the firmware, but this ID can help users that configure another HDR configuration to know when the new HDR is streaming, by checking this value via the metadata:
+```cpp
+// configuring identifier for this hdr config (value must be in range [0,3])
+depth_sensor.set_option(RS2_OPTION_SEQUENCE_NAME, 1);
+```
+
+Configuring the first HDR sequence ID: 
+```cpp
+// configuration for the first HDR sequence ID
+depth_sensor.set_option(RS2_OPTION_SEQUENCE_ID, 1);
+depth_sensor.set_option(RS2_OPTION_EXPOSURE, 8500.f);
+depth_sensor.set_option(RS2_OPTION_GAIN, 16.f);
+```
+Configuring the second HDR sequence ID:
+```cpp
+// configuration for the second HDR sequence ID
+depth_sensor.set_option(RS2_OPTION_SEQUENCE_ID, 2);
+depth_sensor.set_option(RS2_OPTION_EXPOSURE, 150.f);
+depth_sensor.set_option(RS2_OPTION_GAIN, 16.f);
+```
+
+Resetting the sequence ID to 0. This action permits the next setting of the exposure and gain option to be targetted to the normal UVC option, instead of being targetted to the HDR configuration (this call could have been omitted in this specific example):
+```cpp
+// after setting the HDR sequence ID option to 0, setting exposure or gain
+// will be targetted to the normal (UVC) exposure and gain options (not HDR configuration)
+depth_sensor.set_option(RS2_OPTION_SEQUENCE_ID, 0);
+```
+
+Activating the HDR configuration in order to get the resulting streaming:
+```cpp
+// turning ON the HDR with the above configuration 
+depth_sensor.set_option(RS2_OPTION_HDR_ENABLED, 1);
+```
+
+Then, the pipe is configured with depth and infrared streams. 
+In HDR mode the Infrared stream is used as an auxiliary invalidation filter to handle outlier Depth pixels and therefore, to enhance the outcome. The merging algorithm can also work without the infrared stream (if it is not activated by the user), but it workds better with the infrared.
+```cpp
+// Start streaming with depth and infrared configuration
+// The HDR merging algorithm can work with both depth and infrared,or only with depth, 
+// but the resulting stream is better when both depth and infrared are used.
+rs2::config cfg;
+cfg.enable_stream(RS2_STREAM_DEPTH);
+cfg.enable_stream(RS2_STREAM_INFRARED, 1);
+pipe.start(cfg);
+```
+
+### Using Merging Filter
+Initializing the merging filter - will be further used in order to merge frames from both HDR sequence IDs that have been configured:
+```cpp
+// initializing the merging filter
+rs2::depth_merge merging_filter;
+```
+After getting the frames, by using the wait_for_frames method, the merging filter is used:
+```cpp
+// merging the frames from the different HDR sequence IDs
+auto merged_frameset = merging_filter.process(data). 
+        apply_filter(color_map);   // Find and colorize the depth data;
+app.show(merged_frameset);
+
+```
+
+### Using Spliting Filter 
+Initializing also the spliting filter, with the requested sequence ID as 2:
+```cpp
+// initializing the spliting filter
+rs2::depth_split spliting_filter;
+// setting the required sequence ID to be shown
+spliting_filter.set_option(RS2_OPTION_SELECT_ID, 2);
+```
+
+After getting the frames, by using the wait_for_frames method, the spliting filter is used:
+```cpp
+// getting frames only with the requested sequence ID
+auto split_frameset = spliting_filter.process(data). 
+apply_filter(color_map);   // Find and colorize the depth data;
+app.show(split_frameset);
+
+```
+

examples/hdr/rs-hdr.cpp

@@ -0,0 +1,166 @@
+// License: Apache 2.0. See LICENSE file in root directory.
+// Copyright(c) 2020 Intel Corporation. All Rights Reserved.
+
+#include <librealsense2/rs.hpp> // Include RealSense Cross Platform API
+#include "example.hpp"          // Include short list of convenience functions for rendering
+#include <iostream>
+
+// HDR Example demonstrates how to
+// use the HDR feature - only for D400 product line devices
+int main(int argc, char * argv[]) try
+{
+    rs2::context ctx;
+
+    rs2::device_list devices_list = ctx.query_devices();
+
+    size_t device_count = devices_list.size();
+    if (!device_count)
+    {
+        std::cout << "No device detected. Is it plugged in?\n";
+        return EXIT_SUCCESS;
+    }
+
+    rs2::device device;
+    bool device_found = false;
+    for (auto&& dev : devices_list)
+    {
+        // finding a device of D400 product line for working with HDR feature
+        if (dev.supports(RS2_CAMERA_INFO_PRODUCT_LINE) && 
+            std::string(dev.get_info(RS2_CAMERA_INFO_PRODUCT_LINE)) == "D400")
+        {
+            device = dev;
+            device_found = true;
+            break;
+        }
+    }
+
+    if (!device_found)
+    {
+        std::cout << "No device from D400 product line detected. Is it plugged in?\n";
+        return EXIT_SUCCESS;
+    }
+
+    rs2::depth_sensor depth_sensor = device.query_sensors().front();
+
+    // disable auto exposure before sending HDR configuration
+    if (depth_sensor.get_option(RS2_OPTION_ENABLE_AUTO_EXPOSURE))
+        depth_sensor.set_option(RS2_OPTION_ENABLE_AUTO_EXPOSURE, 0);
+
+    // setting the HDR sequence size to 2 frames
+    if (depth_sensor.supports(RS2_OPTION_SEQUENCE_SIZE))
+        depth_sensor.set_option(RS2_OPTION_SEQUENCE_SIZE, 2);
+    else
+    {
+        std::cout << "Firmware and/or SDK versions must be updated for the HDR feature to be supported.\n";
+        return EXIT_SUCCESS;
+    }
+
+    // configuring id for this hdr config (value must be in range [0,3])
+    depth_sensor.set_option(RS2_OPTION_SEQUENCE_NAME, 0);
+
+    // configuration for the first HDR sequence ID
+    depth_sensor.set_option(RS2_OPTION_SEQUENCE_ID, 1);
+    depth_sensor.set_option(RS2_OPTION_EXPOSURE, 8500);
+    depth_sensor.set_option(RS2_OPTION_GAIN, 16.f);
+
+    // configuration for the second HDR sequence ID
+    depth_sensor.set_option(RS2_OPTION_SEQUENCE_ID, 2);
+    depth_sensor.set_option(RS2_OPTION_EXPOSURE, 150);
+    depth_sensor.set_option(RS2_OPTION_GAIN, 16.f);
+
+    // after setting the HDR sequence ID opotion to 0, setting exposure or gain
+    // will be targetted to the normal (UVC) exposure and gain options (not HDR configuration)
+    depth_sensor.set_option(RS2_OPTION_SEQUENCE_ID, 0);
+
+    // turning ON the HDR with the above configuration 
+    depth_sensor.set_option(RS2_OPTION_HDR_ENABLED, 1);
+
+    // Create a simple OpenGL window for rendering:
+    window app(1280, 720, "RealSense Capture Example");
+
+    // Declare depth colorizer for pretty visualization of depth data
+    rs2::colorizer color_map;
+    // Declare rates printer for showing streaming rates of the enabled streams.
+    rs2::rates_printer printer;
+
+    // Declare RealSense pipeline, encapsulating the actual device and sensors
+    rs2::pipeline pipe;
+
+    // Start streaming with default recommended configuration
+    // The default video configuration contains Depth and Color streams
+    // If a device is capable to stream IMU data, both Gyro and Accelerometer are enabled by default
+    rs2::config cfg;
+    cfg.enable_stream(RS2_STREAM_DEPTH);
+    cfg.enable_stream(RS2_STREAM_INFRARED, 1);
+    pipe.start(cfg);
+
+    // initializing the merging filter
+    rs2::hdr_merge merging_filter;
+
+    // initializing the spliting filter
+    rs2::sequence_id_filter spliting_filter;
+
+    // setting the required sequence ID to be shown
+    spliting_filter.set_option(RS2_OPTION_SEQUENCE_ID, 2);
+
+    // flag used to see the original stream or the merged one
+    bool true_for_merge_false_for_split = true;
+    int frames_without_hdr_metadata_params = 0;
+    while (app) // Application still alive?
+    {
+        rs2::frameset data = pipe.wait_for_frames() .    // Wait for next set of frames from the camera
+            apply_filter(printer);     // Print each enabled stream frame rate
+
+        auto depth_frame = data.get_depth_frame();
+
+        if (!depth_frame.supports_frame_metadata(RS2_FRAME_METADATA_SUBPRESET_SEQUENCE_SIZE) ||
+            !depth_frame.supports_frame_metadata(RS2_FRAME_METADATA_SUBPRESET_SEQUENCE_ID))
+        {
+            ++frames_without_hdr_metadata_params;
+            if (frames_without_hdr_metadata_params > 20)
+            {
+                std::cout << "Firmware and/or SDK versions must be updated for the HDR feature to be supported.\n";
+                return EXIT_SUCCESS;
+            }
+            data = data.apply_filter(color_map);
+            app.show(data);
+            continue;
+        }
+
+        auto hdr_id = depth_frame.get_frame_metadata(RS2_FRAME_METADATA_SUBPRESET_ID);
+        auto hdr_seq_size = depth_frame.get_frame_metadata(RS2_FRAME_METADATA_SUBPRESET_SEQUENCE_SIZE);
+        auto hdr_seq_id = depth_frame.get_frame_metadata(RS2_FRAME_METADATA_SUBPRESET_SEQUENCE_ID);
+        auto exp = depth_frame.get_frame_metadata(RS2_FRAME_METADATA_ACTUAL_EXPOSURE);
+
+        std::cout << "frame hdr metadata: hdr_seq_id = "<< hdr_seq_id << ", exposure = " << exp << std::endl;
+        // The show method, when applied on frameset, break it to frames and upload each frame into a gl textures
+        // Each texture is displayed on different viewport according to it's stream unique id
+        if (true_for_merge_false_for_split)
+        {
+            // merging the frames from the different HDR sequence IDs 
+            auto merged_frameset = merging_filter.process(data). // merging frames with both hdr sequence IDs 
+                apply_filter(color_map);   // Find and colorize the depth data;
+            rs2_format format = merged_frameset.as<rs2::frameset>().get_depth_frame().get_profile().format();
+            std::cout << ", after merge format = " << rs2_format_to_string(format) << std::endl;
+            app.show(merged_frameset);
+        }
+        else
+        {
+            auto split_frameset = spliting_filter.process(data) . // getting frames only with the requested sequence ID
+                apply_filter(color_map);   // Find and colorize the depth data;
+            app.show(split_frameset);
+        }
+    }
+
+    return EXIT_SUCCESS;
+}
+catch (const rs2::error & e)
+{
+    std::cerr << "RealSense error calling " << e.get_failed_function() << "(" << e.get_failed_args() << "):\n    " << e.what() << std::endl;
+    return EXIT_FAILURE;
+}
+catch (const std::exception& e)
+{
+    std::cerr << e.what() << std::endl;
+    return EXIT_FAILURE;
+}

include/librealsense2/h/rs_context.h

@@ -54,7 +54,7 @@ void rs2_set_devices_changed_callback(const rs2_context* context, rs2_devices_ch
  * @return  A pointer to a device that plays data from the file, or null in case of failure
  */
 rs2_device* rs2_context_add_device(rs2_context* ctx, const char* file, rs2_error** error);
-    
+
 /**
  * Add an instance of software device to the context
  * \param ctx   The context to which the new device will be added
@@ -73,9 +73,9 @@ void rs2_context_remove_device(rs2_context* ctx, const char* file, rs2_error** e
 
 /**
  * Removes tracking module.
- * function query_devices() locks the tracking module in the tm_context object. 
+ * function query_devices() locks the tracking module in the tm_context object.
  * If the tracking module device is not used it should be removed using this function, so that other applications could find it.
- * This function can be used both before the call to query_device() to prevent enabling tracking modules or afterwards to 
+ * This function can be used both before the call to query_device() to prevent enabling tracking modules or afterwards to
  * release them.
  */
 void rs2_context_unload_tracking_module(rs2_context* ctx, rs2_error** error);

include/librealsense2/h/rs_device.h

@@ -147,7 +147,7 @@ void rs2_connect_tm2_controller(const rs2_device* device, const unsigned char* m
 void rs2_disconnect_tm2_controller(const rs2_device* device, int id, rs2_error** error);
 
 
-/** 
+/**
 * Reset device to factory calibration
 * \param[in] device       The RealSense device
 * \param[out] error       If non-null, receives any error that occurs during this call, otherwise, errors are ignored

include/librealsense2/h/rs_frame.h

@@ -62,6 +62,9 @@ typedef enum rs2_frame_metadata_value
     RS2_FRAME_METADATA_FRAME_LED_POWER                      , /**< Led power value 0-360. */
     RS2_FRAME_METADATA_RAW_FRAME_SIZE                       , /**< The number of transmitted payload bytes, not including metadata */
     RS2_FRAME_METADATA_GPIO_INPUT_DATA                      , /**< GPIO input data */
+    RS2_FRAME_METADATA_SUBPRESET_ID                         , /**< sub-preset id */
+    RS2_FRAME_METADATA_SUBPRESET_SEQUENCE_ID                , /**< sub-preset sequence id */
+    RS2_FRAME_METADATA_SUBPRESET_SEQUENCE_SIZE              , /**< sub-preset sequence size */
     RS2_FRAME_METADATA_COUNT
 } rs2_frame_metadata_value;
 const char* rs2_frame_metadata_to_string(rs2_frame_metadata_value metadata);

include/librealsense2/h/rs_option.h

@@ -97,6 +97,10 @@ extern "C" {
         RS2_OPTION_TRIGGER_CAMERA_ACCURACY_HEALTH, /**< Enable depth & color frame sync with periodic calibration for proper alignment */
         RS2_OPTION_RESET_CAMERA_ACCURACY_HEALTH,
         RS2_OPTION_HOST_PERFORMANCE, /**< Set host performance mode to optimize device settings so host can keep up with workload, for example, USB transaction granularity, setting option to low performance host leads to larger USB transaction size and reduced number of transactions which improves performance and stability if host is relatively weak as compared to workload */
+        RS2_OPTION_HDR_ENABLED,  /**< Enable / disable HDR */
+        RS2_OPTION_SEQUENCE_NAME, /**< HDR Sequence size */
+        RS2_OPTION_SEQUENCE_SIZE, /**< HDR Sequence size */
+        RS2_OPTION_SEQUENCE_ID, /**< HDR Sequence ID - 0 is not HDR; sequence ID for HDR configuartion starts from 1 */
         RS2_OPTION_COUNT /**< Number of enumeration values. Not a valid input: intended to be used in for-loops. */
     } rs2_option;
 

include/librealsense2/h/rs_pipeline.h

@@ -150,7 +150,7 @@ extern "C" {
     * Start the pipeline streaming with its default configuration.
     * The pipeline captures samples from the device, and delivers them to the through the provided frame callback.
     * Starting the pipeline is possible only when it is not started. If the pipeline was started, an exception is raised.
-    * When starting the pipeline with a callback both \c wait_for_frames() or \c poll_for_frames() will throw exception. 
+    * When starting the pipeline with a callback both \c wait_for_frames() or \c poll_for_frames() will throw exception.
     *
     * \param[in] pipe     A pointer to an instance of the pipeline
     * \param[in] callback callback object created from c++ application. ownership over the callback object is moved into the relevant streaming lock

include/librealsense2/h/rs_processing.h

@@ -251,6 +251,20 @@ rs2_processing_block* rs2_create_zero_order_invalidation_block(rs2_error** error
 */
 rs2_processing_block* rs2_create_huffman_depth_decompress_block(rs2_error** error);
 
+/**
+* Creates a hdr_merge processing block. 
+* The block merges between two depth frames with different exposure values
+* \param[out] error  if non-null, receives any error that occurs during this call, otherwise, errors are ignored
+*/
+rs2_processing_block* rs2_create_hdr_merge_processing_block(rs2_error** error);
+
+/**
+* Creates a sequence_id_filter processing block.
+* The block lets frames with the selected sequence id pass and blocks frames with other values
+* \param[out] error  if non-null, receives any error that occurs during this call, otherwise, errors are ignored
+*/
+rs2_processing_block* rs2_create_sequence_id_filter(rs2_error** error);
+
 /**
 * Retrieve processing block specific information, like name.
 * \param[in]  block     The processing block