File: tutorial-hsv-segmentation-pcl-viewer.cpp

package info (click to toggle)
visp 3.7.0-10
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 166,384 kB
  • sloc: cpp: 392,705; ansic: 224,448; xml: 23,444; python: 13,701; java: 4,792; sh: 207; objc: 145; makefile: 118
file content (349 lines) | stat: -rw-r--r-- 12,448 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
//! \example tutorial-hsv-segmentation-pcl-viewer.cpp

#include <iostream>
#include <visp3/core/vpConfig.h>

#if defined(VISP_HAVE_REALSENSE2) && defined(VISP_HAVE_PCL) && defined(VISP_HAVE_PCL_VISUALIZATION) && defined(VISP_HAVE_THREADS) && defined(VISP_HAVE_DISPLAY)
#include <visp3/core/vpCameraParameters.h>
#include <visp3/core/vpHSV.h>
#include <visp3/core/vpImageConvert.h>
#include <visp3/core/vpImageTools.h>
#include <visp3/core/vpPixelMeterConversion.h>
#include <visp3/core/vpColorDepthConversion.h>
#include <visp3/core/vpIoTools.h>
#include <visp3/gui/vpDisplayFactory.h>
//! [Include vpDisplayPCL header]
#include <visp3/gui/vpDisplayPCL.h>
//! [Include vpDisplayPCL header]
#include <visp3/sensor/vpRealSense2.h>

#ifdef ENABLE_VISP_NAMESPACE
using namespace VISP_NAMESPACE_NAME;
#endif

//! [Enum for mode choice]
/**
 * @brief Enumeration permitting to choose between running the blocking-mode display
 * example, the threaded-mode or both mode consecutively.
 */
typedef enum DisplayMode
{
  MONOTHREAD = 0, /*!< Only the monothread-mode display example will be run.*/
  THREADED = 1, /*!< Only the threaded-mode display example will be run.*/
  MODE_COUNT = 2
} DisplayMode;

/**
 * @brief Cast a \b DisplayMode enum value into a \b std::stirng.
 *
 * @param mode The display mode we want to cast into a string.
 * @return std::string The name of the \b DisplayMode enum value.
 */
std::string displayModeToString(const DisplayMode &mode)
{
  switch (mode) {
  case MONOTHREAD:
    return "monothread";
  case THREADED:
    return "threaded";
  default:
    break;
  }
  return "unknown";
}

/**
 * @brief Cast a string into a \b DisplayMode enum value.
 * If \b name is not found, return \b DisplayMode::MODE_COUNT .
 *
 * @param name The name of the display mode.
 * @return DisplayMode The corresponding \b DisplayMode enum value, or \b DisplayMode::MODE_COUNT if not found.
 */
DisplayMode displayModeFromString(const std::string &name)
{
  DisplayMode res = DisplayMode::MODE_COUNT;
  bool wasFound = false;
  std::string lowerCaseName = vpIoTools::toLowerCase(name);
  unsigned int i = 0;
  while ((i < DisplayMode::MODE_COUNT) && (!wasFound)) {
    DisplayMode candidate = (DisplayMode)i;
    if (lowerCaseName == displayModeToString(candidate)) {
      res = candidate;
      wasFound = true;
    }
    ++i;
  }
  return res;
}

/**
 * @brief Create a string that lists the different \b DisplayMode available.
 *
 * @param prefix The string that must prefix the list of modes.
 * @param sep The separator between the different modes.
 * @param suffix The string that must suffix the list of modes.
 * @return std::string The list containing the different modes.
 */
std::string getAvailableDisplayMode(const std::string &prefix = "< ", const std::string &sep = " , ", const std::string &suffix = " >")
{
  std::string modes(prefix);
  for (unsigned int i = 0; i < DisplayMode::MODE_COUNT - 1; ++i) {
    DisplayMode candidate = (DisplayMode)i;
    modes += displayModeToString(candidate) + sep;
  }
  DisplayMode candidate = (DisplayMode)(DisplayMode::MODE_COUNT - 1);
  modes += displayModeToString(candidate) + suffix;
  return modes;
}
//! [Enum for mode choice]

int main(int argc, const char *argv[])
{
  std::string opt_hsv_filename = "calib/hsv-thresholds.yml";
  bool opt_pcl_textured = false;
  bool opt_verbose = false;
  int opt_width = 848;
  int opt_height = 480;
  int opt_fps = 60;
  DisplayMode opt_mode = DisplayMode::MONOTHREAD;

  int i = 1;
  while (i < argc) {
    if (((std::string(argv[i]) == "--width") || (std::string(argv[i]) == "-v")) && ((i+1) < argc)) {
      opt_width = std::atoi(argv[++i]);
    }
    else if (((std::string(argv[i]) == "--height") || (std::string(argv[i]) == "-h")) && ((i+1) < argc)) {
      opt_height = std::atoi(argv[++i]);
    }
    else if (std::string(argv[i]) == "--display-mode" && i + 1 < argc) {
      opt_mode = displayModeFromString(std::string(argv[i + 1]));
      ++i;
    }
    else if ((std::string(argv[i]) == "--fps") && ((i+1) < argc)) {
      opt_fps = std::atoi(argv[++i]);
    }
    else if (std::string(argv[i]) == "--texture") {
      opt_pcl_textured = true;
    }
    else if ((std::string(argv[i]) == "--hsv-thresholds") && ((i+1) < argc)) {
      opt_hsv_filename = std::string(argv[++i]);
    }
    else if ((std::string(argv[i]) == "--verbose")  || (std::string(argv[i]) == "-v")) {
      opt_verbose = true;
    }
    else if ((std::string(argv[i]) == "--help") || (std::string(argv[i]) == "-h")) {
      std::cout << "\nSYNOPSIS " << std::endl
        << argv[0]
        << " [--width,-w <image width>]"
        << " [--height,-h <image height>]"
        << " [--fps <framerate>]"
        << " [--texture]"
        << " [--hsv-thresholds <filename.yml>]"
        << " [--verbose,-v]"
        << " [--help,-h]"
        << std::endl;
      std::cout << "\nOPTIONS " << std::endl
        << "  --width,-w <image width>" << std::endl
        << "    Realsense camera image width." << std::endl
        << "    Default: " << opt_width << std::endl
        << std::endl
        << "  --height,-h <image height>" << std::endl
        << "    Realsense camera image height." << std::endl
        << "    Default: " << opt_height << std::endl
        << std::endl
        << "  --fps <framerate>" << std::endl
        << "    Realsense camera framerate." << std::endl
        << "    Default: " << opt_fps << std::endl
        << std::endl
        << "  --texture" << std::endl
        << "    Enable textured point cloud adding RGB information to the 3D point." << std::endl
        << std::endl
        << "  --hsv-thresholds <filename.yaml>" << std::endl
        << "    Path to a yaml filename that contains H <min,max>, S <min,max>, V <min,max> threshold values." << std::endl
        << "    An Example of such a file could be:" << std::endl
        << "      rows: 6" << std::endl
        << "      cols: 1" << std::endl
        << "      data:" << std::endl
        << "        - [0]" << std::endl
        << "        - [42]" << std::endl
        << "        - [177]" << std::endl
        << "        - [237]" << std::endl
        << "        - [148]" << std::endl
        << "        - [208]" << std::endl
        << std::endl
        << "  --verbose, -v" << std::endl
        << "    Enable verbose mode." << std::endl
        << std::endl
        << "  --help, -h" << std::endl
        << "    Display this helper message." << std::endl
        << std::endl;
      return EXIT_SUCCESS;
    }
    ++i;
  }

  vpColVector hsv_values;
  if (vpColVector::loadYAML(opt_hsv_filename, hsv_values)) {
    std::cout << "Load HSV threshold values from " << opt_hsv_filename << std::endl;
    std::cout << "HSV low/high values: " << hsv_values.t() << std::endl;
  }
  else {
    std::cout << "Warning: unable to load HSV thresholds values from " << opt_hsv_filename << std::endl;
    return EXIT_FAILURE;
  }

  vpRealSense2 rs;
  rs2::config config;
  config.enable_stream(RS2_STREAM_COLOR, opt_width, opt_height, RS2_FORMAT_RGBA8, opt_fps);
  config.enable_stream(RS2_STREAM_DEPTH, opt_width, opt_height, RS2_FORMAT_Z16, opt_fps);
  config.disable_stream(RS2_STREAM_INFRARED, 1);
  config.disable_stream(RS2_STREAM_INFRARED, 2);
  rs2::align align_to(RS2_STREAM_COLOR);

  rs.open(config);

  float depth_scale = rs.getDepthScale();
  vpCameraParameters cam_depth = rs.getCameraParameters(RS2_STREAM_DEPTH,
                                                        vpCameraParameters::perspectiveProjWithoutDistortion);

  vpImage<vpRGBa> I(opt_height, opt_width);
  vpImage<unsigned char> mask(opt_height, opt_width);
  vpImage<uint16_t> depth_raw(opt_height, opt_width);
  vpImage<vpRGBa> I_segmented(opt_height, opt_width);

#if (VISP_CXX_STANDARD >= VISP_CXX_STANDARD_11)
  vpImage<vpHSV<unsigned char, true>> Ihsv;

  std::shared_ptr<vpDisplay> d_I = vpDisplayFactory::createDisplay(I, 0, 0, "Current frame");
  std::shared_ptr<vpDisplay> d_I_segmented = vpDisplayFactory::createDisplay(I_segmented, I.getWidth()+75, 0, "HSV segmented frame");
#else
  vpImage<unsigned char> H(opt_height, opt_width);
  vpImage<unsigned char> S(opt_height, opt_width);
  vpImage<unsigned char> V(opt_height, opt_width);

  vpDisplay *d_I = vpDisplayFactory::allocateDisplay(I, 0, 0, "Current frame");
  vpDisplay *d_I_segmented = vpDisplayFactory::allocateDisplay(I_segmented, I.getWidth()+75, 0, "HSV segmented frame");
#endif

  bool quit = false;
  double loop_time = 0., total_loop_time = 0.;
  long nb_iter = 0;
  float Z_min = 0.1;
  float Z_max = 2.5;

  //! [Create point cloud]
  pcl::PointCloud<pcl::PointXYZRGB>::Ptr pointcloud_color(new pcl::PointCloud<pcl::PointXYZRGB>());
  pcl::PointCloud<pcl::PointXYZ>::Ptr pointcloud(new pcl::PointCloud<pcl::PointXYZ>());
  //! [Create point cloud]

  //! [Create pcl viewer object]
  std::mutex pointcloud_mutex;
  vpDisplayPCL pcl_viewer(opt_width, opt_height);
  if (opt_mode == DisplayMode::THREADED) {
    if (opt_pcl_textured) {
      pcl_viewer.startThread(std::ref(pointcloud_mutex), pointcloud_color);
    }
    else {
      pcl_viewer.startThread(std::ref(pointcloud_mutex), pointcloud);
    }
  }
  else {
    if (opt_pcl_textured) {
      pcl_viewer.addPointCloud(std::ref(pointcloud_mutex), pointcloud_color);
    }
    else {
      pcl_viewer.addPointCloud(std::ref(pointcloud_mutex), pointcloud);
    }
  }
 //! [Create pcl viewer object]

  while (!quit) {
    double t = vpTime::measureTimeMs();
    rs.acquire((unsigned char *)I.bitmap, (unsigned char *)(depth_raw.bitmap), NULL, NULL, &align_to);

#if (VISP_CXX_STANDARD >= VISP_CXX_STANDARD_11)
    vpImageConvert::convert(I, Ihsv);
    vpImageTools::inRange(Ihsv, hsv_values, mask);
#else
    vpImageConvert::RGBaToHSV(reinterpret_cast<unsigned char *>(I.bitmap),
                              reinterpret_cast<unsigned char *>(H.bitmap),
                              reinterpret_cast<unsigned char *>(S.bitmap),
                              reinterpret_cast<unsigned char *>(V.bitmap), I.getSize());

    vpImageTools::inRange(reinterpret_cast<unsigned char *>(H.bitmap),
                          reinterpret_cast<unsigned char *>(S.bitmap),
                          reinterpret_cast<unsigned char *>(V.bitmap),
                          hsv_values,
                          reinterpret_cast<unsigned char *>(mask.bitmap),
                          mask.getSize());
#endif

    vpImageTools::inMask(I, mask, I_segmented);

    //! [Update point cloud with mutex protection]
    int pcl_size;
    if (opt_pcl_textured) {
      pcl_size = vpImageConvert::depthToPointCloud(I, depth_raw, depth_scale, cam_depth, pointcloud_color, &pointcloud_mutex, &mask, Z_min, Z_max);
    }
    else {
      pcl_size = vpImageConvert::depthToPointCloud(depth_raw, depth_scale, cam_depth, pointcloud, &pointcloud_mutex, &mask, Z_min, Z_max);
    }
    if (opt_verbose) {
      std::cout << "Segmented point cloud size: " << pcl_size << std::endl;
    }
    //! [Update point cloud with mutex protection]

    vpDisplay::display(I);
    vpDisplay::display(I_segmented);
    vpDisplay::displayText(I, 20, 20, "Click to quit...", vpColor::red);

    if (vpDisplay::getClick(I, false)) {
      quit = true;
    }

    //! [Refresh monothread viewer]
    if (opt_mode ==  DisplayMode::MONOTHREAD) {
      const bool blocking_mode = false;
      pcl_viewer.display(blocking_mode);
    }
    //! [Refresh monothread viewer]

    vpDisplay::flush(I);
    vpDisplay::flush(I_segmented);
    nb_iter++;
    loop_time = vpTime::measureTimeMs() - t;
    total_loop_time += loop_time;
  }

  std::cout << "Mean loop time: " << total_loop_time / nb_iter << std::endl;

#if (VISP_CXX_STANDARD < VISP_CXX_STANDARD_11)
  if (d_I != nullptr) {
    delete d_I;
  }

  if (d_I_segmented != nullptr) {
    delete d_I_segmented;
  }
#endif
  return EXIT_SUCCESS;
}
#else
int main()
{
#if !defined(VISP_HAVE_REALSENSE2)
  std::cout << "This tutorial needs librealsense as 3rd party." << std::endl;
#endif
#if !defined(VISP_HAVE_PCL)
  std::cout << "This tutorial needs pcl library as 3rd party." << std::endl;
#endif
#if !defined(VISP_HAVE_PCL_VISUALIZATION)
  std::cout << "This tutorial needs pcl visualization module." << std::endl;
#endif
#if !defined(VISP_HAVE_X11)
  std::cout << "This tutorial needs X11 3rd party enabled." << std::endl;
#endif
  std::cout << "Install missing 3rd party, configure and rebuild ViSP." << std::endl;
  return EXIT_SUCCESS;
}
#endif