Allow the Display node to display multiple video streams

examples/cpp/RVC4/Camera/camera_multiple_outputs.cpp

@@ -1,8 +1,11 @@
 // Includes common necessary includes for development using depthai library
 #include <stdexcept>
+#include <string>
 
 #include "depthai/capabilities/ImgFrameCapability.hpp"
 #include "depthai/depthai.hpp"
+#include "depthai/pipeline/node/Sync.hpp"
+#include "depthai/pipeline/node/host/Display.hpp"
 
 int main(int argc, char** argv) {
     if(argc < 4 || (argc - 1) % 3 != 0) {
@@ -16,13 +19,19 @@ int main(int argc, char** argv) {
     dai::Pipeline pipeline;
 
     auto camRgb = pipeline.create<dai::node::Camera>()->build();
+    auto sync = pipeline.create<dai::node::Sync>();
+    auto display = pipeline.create<dai::node::Display>();
+
+    sync->setSyncAttempts(0);
+    sync->setRunOnHost(true);
 
     if(sizes.empty()) {
         throw std::runtime_error("internal error to few sizes");
     }
 
-    std::vector<std::shared_ptr<dai::MessageQueue>> videos;
+    int index = 0;
     for(const auto& size : sizes) {
+        ++index;
         dai::ImgFrameCapability cap;
         cap.type = dai::ImgFrame::Type::NV12;  // Fastest
         cap.size.value = std::pair{std::get<0>(size), std::get<1>(size)};
@@ -40,28 +49,14 @@ int main(int argc, char** argv) {
                 throw std::runtime_error("Resize mode argument (every 3rd) must be 0, 1 or 2");
         }
         auto* output = camRgb->requestOutput(cap, true);
-        videos.push_back(output->createOutputQueue());
+        output->link(sync->inputs[std::to_string(index)]);
     }
 
+    sync->out.link(display->input);
+
     pipeline.start();
 
-    while(pipeline.isRunning()) {
-        size_t videoIndex = 0;
-        for(const auto& video : videos) {
-            auto videoIn = video->tryGet<dai::ImgFrame>();
-            // Get BGR frame from NV12 encoded video frame to show with opencv
-            // Visualizing the frame on slower hosts might have overhead
-            if(videoIn) {
-                cv::imshow("video_" + std::to_string(videoIndex), videoIn->getCvFrame());
-            }
-            ++videoIndex;
-        }
+    pipeline.wait();
 
-        int key = cv::waitKey(1);
-        if(key == 'q' || key == 'Q') {
-            pipeline.stop();
-            return 0;
-        }
-    }
     return 0;
 }

include/depthai/pipeline/node/host/Display.hpp

@@ -15,4 +15,4 @@ class Display : public dai::NodeCRTP<ThreadedHostNode, Display> {
     void run() override;
 };
 }  // namespace node
-}  // namespace dai
+}  // namespace dai

src/pipeline/node/host/Display.cpp

@@ -1,9 +1,12 @@
 #include "depthai/pipeline/node/host/Display.hpp"
 
 #include <chrono>
+#include <memory>
 #include <opencv2/opencv.hpp>
+#include <unordered_map>
 
 #include "depthai/pipeline/Pipeline.hpp"
+#include "pipeline/datatype/MessageGroup.hpp"
 namespace dai {
 namespace node {
 
@@ -34,12 +37,15 @@ class FPSCounter {
 Display::Display(std::string name) : name(std::move(name)) {}
 
 void Display::run() {
-    auto fpsCounter = FPSCounter();
+    std::unordered_map<std::string, FPSCounter> fpsCounters;
+    fpsCounters["default"] = FPSCounter();
     while(isRunning()) {
-        std::shared_ptr<dai::ImgFrame> imgFrame = input.get<dai::ImgFrame>();
+        auto msg = input.get<dai::Buffer>();
+        auto imgFrame = std::dynamic_pointer_cast<dai::ImgFrame>(msg);
+        auto msgGroup = std::dynamic_pointer_cast<dai::MessageGroup>(msg);
         if(imgFrame != nullptr) {
-            fpsCounter.update();
-            auto fps = fpsCounter.getFPS();
+            fpsCounters["default"].update();
+            auto fps = fpsCounters["default"].getFPS();
             using namespace std::chrono;
             auto latencyMs = duration_cast<milliseconds>(steady_clock::now() - imgFrame->getTimestamp());
             auto frame = imgFrame->getCvFrame();
@@ -53,6 +59,28 @@ void Display::run() {
                 auto parentPipeline = getParentPipeline();
                 parentPipeline.stop();
             }
+        } else if(msgGroup != nullptr) {
+            for(const auto& [k, v] : *msgGroup) {
+                auto imgFrame = std::dynamic_pointer_cast<dai::ImgFrame>(v);
+                if(imgFrame != nullptr) {
+                    fpsCounters[k].update();
+                    auto fps = fpsCounters[k].getFPS();
+                    using namespace std::chrono;
+                    auto latencyMs = duration_cast<milliseconds>(steady_clock::now() - imgFrame->getTimestamp());
+                    auto frame = imgFrame->getCvFrame();
+                    cv::putText(frame, fmt::format("FPS: {:.2f}", fps), cv::Point(10, 30), cv::FONT_HERSHEY_SIMPLEX, 1, cv::Scalar(0, 255, 0), 2);
+                    cv::putText(
+                        frame, fmt::format("Latency: {}ms", latencyMs.count()), cv::Point(10, 60), cv::FONT_HERSHEY_SIMPLEX, 1, cv::Scalar(0, 255, 0), 2);
+                    cv::imshow(name + " - " + k, frame);
+                }
+            }
+            auto key = cv::waitKey(1);
+            if(key == 'q') {
+                // Get the parent pipeline and stop it
+                // TODO(Morato) - add a convience stop method directly to the pipeline
+                auto parentPipeline = getParentPipeline();
+                parentPipeline.stop();
+            }
         }
     }
     fmt::print("Display node stopped\n");