Skip to content

Updated frame validator #4788

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 3, 2019
Merged

Updated frame validator #4788

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cd4076f
Updated frame validator for L500
aangerma Sep 2, 2019
53eff3a
Added try catch to open
aangerma Sep 3, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 35 additions & 31 deletions common/model-views.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -821,7 +821,8 @@ namespace rs2
bool* options_invalidated,
std::string& error_message)
{
for (auto i = 0; i < RS2_OPTION_COUNT; i++)

for (auto&& i:options->get_supported_options())
{
auto opt = static_cast<rs2_option>(i);

Expand Down Expand Up @@ -1579,43 +1580,46 @@ namespace rs2

for (auto&& pbm : post_processing) pbm->save_to_config_file();
}
if (next_option < RS2_OPTION_COUNT)
if (next_option < s->get_supported_options().size())
{
auto& opt_md = options_metadata[static_cast<rs2_option>(next_option)];
opt_md.update_all_fields(error_message, notifications);

if (next_option == RS2_OPTION_ENABLE_AUTO_EXPOSURE)
if (options_metadata.find(static_cast<rs2_option>(next_option)) != options_metadata.end())
{
auto old_ae_enabled = auto_exposure_enabled;
auto_exposure_enabled = opt_md.value > 0;
auto& opt_md = options_metadata[static_cast<rs2_option>(next_option)];
opt_md.update_all_fields(error_message, notifications);

if (!old_ae_enabled && auto_exposure_enabled)
if (next_option == RS2_OPTION_ENABLE_AUTO_EXPOSURE)
{
try
auto old_ae_enabled = auto_exposure_enabled;
auto_exposure_enabled = opt_md.value > 0;

if (!old_ae_enabled && auto_exposure_enabled)
{
if (s->is<roi_sensor>())
try
{
auto r = s->as<roi_sensor>().get_region_of_interest();
roi_rect.x = static_cast<float>(r.min_x);
roi_rect.y = static_cast<float>(r.min_y);
roi_rect.w = static_cast<float>(r.max_x - r.min_x);
roi_rect.h = static_cast<float>(r.max_y - r.min_y);
if (s->is<roi_sensor>())
{
auto r = s->as<roi_sensor>().get_region_of_interest();
roi_rect.x = static_cast<float>(r.min_x);
roi_rect.y = static_cast<float>(r.min_y);
roi_rect.w = static_cast<float>(r.max_x - r.min_x);
roi_rect.h = static_cast<float>(r.max_y - r.min_y);
}
}
catch (...)
{
auto_exposure_enabled = false;
}
}
catch (...)
{
auto_exposure_enabled = false;
}
}
}

if (next_option == RS2_OPTION_DEPTH_UNITS)
{
opt_md.dev->depth_units = opt_md.value;
}
if (next_option == RS2_OPTION_DEPTH_UNITS)
{
opt_md.dev->depth_units = opt_md.value;
}

if (next_option == RS2_OPTION_STEREO_BASELINE)
opt_md.dev->stereo_baseline = opt_md.value;
if (next_option == RS2_OPTION_STEREO_BASELINE)
opt_md.dev->stereo_baseline = opt_md.value;
}

next_option++;
}
Expand Down Expand Up @@ -3311,7 +3315,6 @@ namespace rs2
std::vector<frame> results;

auto res = handle_frame(f, source);

auto frame = source.allocate_composite_frame(res);

if(frame)
Expand Down Expand Up @@ -5309,8 +5312,9 @@ namespace rs2
dev_syncer = viewer.syncer->create_syncer();

std::string friendly_name = sub->s->get_info(RS2_CAMERA_INFO_NAME);
if ((friendly_name.find("Tracking") != std::string::npos) ||
(friendly_name.find("Motion") != std::string::npos))
if (!viewer.zo_sensors.load() &&
((friendly_name.find("Tracking") != std::string::npos) ||
(friendly_name.find("Motion") != std::string::npos)))
{
viewer.synchronization_enable = false;
}
Expand Down Expand Up @@ -5442,7 +5446,7 @@ namespace rs2
label = to_string() << "Controls ##" << sub->s->get_info(RS2_CAMERA_INFO_NAME) << "," << id;
if (ImGui::TreeNode(label.c_str()))
{
for (auto i = 0; i < RS2_OPTION_COUNT; i++)
for (auto&& i:sub->s->get_supported_options())
{
auto opt = static_cast<rs2_option>(i);
if (skip_option(opt)) continue;
Expand Down
4 changes: 3 additions & 1 deletion common/model-views.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -604,7 +604,9 @@ namespace rs2
bool draw_option(rs2_option opt, bool update_read_only_options,
std::string& error_message, notifications_model& model)
{
return options_metadata[opt].draw_option(update_read_only_options, streaming, error_message, model);
if(options_metadata.find(opt)!=options_metadata.end())
return options_metadata[opt].draw_option(update_read_only_options, streaming, error_message, model);
return false;
}

bool is_paused() const;
Expand Down
96 changes: 56 additions & 40 deletions src/frame-validator.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,90 +3,106 @@

#include "frame-validator.h"

#define INVALID_PIXELS_THRESHOLD 0.1

namespace librealsense
{
void frame_validator::on_frame(rs2_frame * f)
{
if (propagate((frame_interface*)f))
if (!_stopped && propagate((frame_interface*)f))
{
_user_callback->on_frame(f);
_sensor->set_frames_callback(_user_callback);
if(is_user_requseted_frame((frame_interface*)f))
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

requested

_user_callback->on_frame(f);
}
}

void frame_validator::release()
{

}
{}

frame_validator::frame_validator(std::shared_ptr<sensor_interface> sensor, frame_callback_ptr user_callback, stream_profiles current_requests) :
frame_validator::frame_validator(std::shared_ptr<sensor_interface> sensor, frame_callback_ptr user_callback, stream_profiles current_requests, stream_profiles validator_requests) :
_sensor(sensor),
_user_callback(user_callback),
_current_requests(current_requests)
{

}
_user_requests(current_requests),
_validator_requests(validator_requests),
_stopped(false),
_validated(false)
{}

frame_validator::~frame_validator()
{

}
{}

bool frame_validator::propagate(frame_interface* frame)
{
auto vf = dynamic_cast<video_frame*>(frame);
if (vf == nullptr) {
if(_validated)
return true;
}

auto vf = dynamic_cast<video_frame*>(frame);
if (vf == nullptr)
throw std::runtime_error(to_string() << "non video stream arrived to frame_validator");

auto stream = vf->get_stream();
if (stream->get_stream_type() != RS2_STREAM_DEPTH)

//all streams except ir will be droped untill the validation on ir,
//after the validation all streams will be passeded to user callback directly
if (stream->get_stream_type() != RS2_STREAM_INFRARED)
return false;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If for some reason Ir frames do not arrive for a considerable amount of time but others do - is it possible to generate a log, like after 100 consecutive "return false" ?


//start to validate only from the second frame
if(frame_num++ < 2)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Pls rename to _frame_num to differentiate from local vars

return false;

auto w = vf->get_width();
auto h = vf->get_height();
auto bpp = vf->get_bpp() / 8;
auto data = reinterpret_cast<const uint16_t*>(vf->get_frame_data());

bool non_empty_pixel_found = false;
auto data = static_cast<const void*>(vf->get_frame_data());

auto invalid_pixels = 0;

int y_begin = h * 0.5 - h * 0.05;
int y_end = h * 0.5 + h * 0.05;
int x_begin = w * 0.5 - w * 0.05;
int x_end = w * 0.5 + w * 0.05;
for (int y = y_begin; y < y_end; y++)
for (int y = 0; y < h*w; y++)
{
for (int x = x_begin; x < x_end; x++)
if (((byte*)data)[y] == 0)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is IR stream 8-bit only, no 12 bit formats?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes 8 bit

{
auto index = x + y * w;
if (data[index] != 0)
{
non_empty_pixel_found = true;
break;
}
invalid_pixels++;
}
}

if (non_empty_pixel_found)
if ((float)invalid_pixels/(w*h) < INVALID_PIXELS_THRESHOLD)
{
_validated = true;
return true;
}
else
{
LOG_ERROR("frame_source received an empty depth frame, restarting the sensor...");

LOG_ERROR("frame_source received corrupted depth frame ("<<(float)invalid_pixels/(w*h)<<"% invalid pixels), restarting the sensor...");
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

corrupted IR frame

auto s = _sensor;
auto cr = _current_requests;
auto vr = _validator_requests;
auto uc = _user_callback;
_reset_thread = std::thread([s, cr, uc]()
_stopped = true;
_reset_thread = std::thread([s, vr, uc]()
{
try {
std::this_thread::sleep_for(std::chrono::milliseconds(500));
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Add a comment for maintainers

s->stop();
s->close();
s->open(cr);
std::this_thread::sleep_for(std::chrono::milliseconds(500));
s->open(vr);
s->start(uc);
}
catch (...) {}
catch (...)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If an exception is raised during the stop->close->open->start then the system state is not defined.
the thread's shall complete with either streaming or a full stop. In both cases a log shall be provided

{}
});
_reset_thread.detach();
//std::this_thread::sleep_for(std::chrono::milliseconds(1000));
}
return false;

}

bool frame_validator::is_user_requseted_frame(frame_interface* frame)
{
return std::find_if(_user_requests.begin(), _user_requests.end(), [&](std::shared_ptr<stream_profile_interface> sp)
{
return is_stream_profiles_equals(frame->get_stream().get(), sp.get());
}) != _user_requests.end();
}
}
49 changes: 46 additions & 3 deletions src/frame-validator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,23 +4,66 @@
#pragma once

#include "sensor.h"

#include "stream.h"
#include "option.h"
namespace librealsense
{
const int RS2_OPTION_DEPTH_INVALIDATION_ENABLE = static_cast<rs2_option>(RS2_OPTION_COUNT + 9); /**< depth invalidation enabled*/

class depth_invalidation_option: public ptr_option<bool>
{
public:
depth_invalidation_option(bool min, bool max, bool step, bool def, bool* value, const std::string& desc):
ptr_option<bool>(min, max, step, def, value, desc),
_streaming(false){}

void set_streaming(bool streaming)
{
_streaming = streaming;
}

bool is_enabled() const override
{
return !_streaming;
}

private:
bool _streaming;
};

class frame_validator : public rs2_frame_callback
{
public:
explicit frame_validator(std::shared_ptr<sensor_interface> sensor, frame_callback_ptr user_callback, stream_profiles current_requests);
explicit frame_validator(std::shared_ptr<sensor_interface> sensor, frame_callback_ptr user_callback, stream_profiles user_requests, stream_profiles validator_requests);
virtual ~frame_validator();

void on_frame(rs2_frame * f) override;
void release() override;

static bool is_stream_profiles_equals(stream_profile_interface* l, stream_profile_interface* r)
{
auto vl = dynamic_cast<video_stream_profile_interface*>(l);
auto vr = dynamic_cast<video_stream_profile_interface*>(r);

if (!vl || !vr)
return false;

return l->get_framerate() == r->get_framerate() &&
vl->get_width() == vr->get_width() &&
vl->get_height() == vr->get_height();
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Stream format?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I try to find IR stream that corresponding to depth/confidence if the user asked depth/confidence and not ir ,so the format will not be the same only resolution and frame rate.

}

private:
bool propagate(frame_interface* frame);
bool is_user_requseted_frame(frame_interface* frame);

std::thread _reset_thread;
std::atomic<bool> _stopped;
std::atomic<bool> _validated;
int frame_num = 0;
frame_callback_ptr _user_callback;
stream_profiles _current_requests;
stream_profiles _user_requests;
stream_profiles _validator_requests;
std::shared_ptr<sensor_interface> _sensor;
};
}
Loading