From b2b49b49467818bd6ee5c8af876096f81afbc14b Mon Sep 17 00:00:00 2001 From: NicikD Date: Thu, 19 Sep 2024 10:01:04 +0200 Subject: [PATCH 1/6] Added keypoints parser device node. For now not working on device (only on host) and without python bindings. Also added keypoints parser example. --- CMakeLists.txt | 3 + examples/cpp/CMakeLists.txt | 4 + examples/cpp/parsers/keypoints_device.cpp | 50 ++++++++++ examples/cpp/parsers/keypoints_host.cpp | 51 ++++++++++ .../pipeline/datatype/DatatypeEnum.hpp | 1 + .../depthai/pipeline/datatype/Keypoints.hpp | 41 ++++++++ .../datatype/creators/KeypointsCreator.hpp | 42 ++++++++ include/depthai/pipeline/datatypes.hpp | 1 + .../depthai/pipeline/node/KeypointsParser.hpp | 97 +++++++++++++++++++ include/depthai/pipeline/nodes.hpp | 1 + .../properties/KeypointsParserProperties.hpp | 24 +++++ src/pipeline/datatype/DatatypeEnum.cpp | 3 + src/pipeline/datatype/Keypoints.cpp | 14 +++ .../datatype/creators/KeypointsCreator.cpp | 67 +++++++++++++ src/pipeline/node/KeypointsParser.cpp | 85 ++++++++++++++++ 15 files changed, 484 insertions(+) create mode 100644 examples/cpp/parsers/keypoints_device.cpp create mode 100644 examples/cpp/parsers/keypoints_host.cpp create mode 100644 include/depthai/pipeline/datatype/Keypoints.hpp create mode 100644 include/depthai/pipeline/datatype/creators/KeypointsCreator.hpp create mode 100644 include/depthai/pipeline/node/KeypointsParser.hpp create mode 100644 include/depthai/properties/KeypointsParserProperties.hpp create mode 100644 src/pipeline/datatype/Keypoints.cpp create mode 100644 src/pipeline/datatype/creators/KeypointsCreator.cpp create mode 100644 src/pipeline/node/KeypointsParser.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 772063235..760676cfb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -312,6 +312,7 @@ set(TARGET_CORE_SOURCES src/pipeline/node/AprilTag.cpp src/pipeline/node/ObjectTracker.cpp src/pipeline/node/IMU.cpp + src/pipeline/node/KeypointsParser.cpp src/pipeline/node/EdgeDetector.cpp src/pipeline/node/SPIIn.cpp src/pipeline/node/FeatureTracker.cpp @@ -325,9 +326,11 @@ set(TARGET_CORE_SOURCES src/pipeline/node/host/HostNode.cpp src/pipeline/datatype/DatatypeEnum.cpp src/pipeline/node/PointCloud.cpp + src/pipeline/datatype/creators/KeypointsCreator.cpp src/pipeline/datatype/Buffer.cpp src/pipeline/datatype/ImgFrame.cpp src/pipeline/datatype/ImgTransformations.cpp + src/pipeline/datatype/Keypoints.cpp src/pipeline/datatype/EncodedFrame.cpp src/pipeline/datatype/ImageManipConfig.cpp src/pipeline/datatype/ImageManipConfigV2.cpp diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index 7e07ca8cd..299e3a394 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -483,6 +483,10 @@ dai_add_example(threaded_host_node HostNodes/threaded_host_node.cpp ON OFF) # Model zoo dai_add_example(model_zoo RVC2/ModelZoo/model_zoo.cpp ON OFF) +# Keypoints parser +dai_add_example(keypoints_device parsers/keypoints_device.cpp ON OFF) +dai_add_example(keypoints_host parsers/keypoints_host.cpp ON OFF) + if(DEPTHAI_RTABMAP_SUPPORT) include(FetchContent) FetchContent_Declare(rerun_sdk URL https://github.com/rerun-io/rerun/releases/download/0.16.1/rerun_cpp_sdk.zip) diff --git a/examples/cpp/parsers/keypoints_device.cpp b/examples/cpp/parsers/keypoints_device.cpp new file mode 100644 index 000000000..d0ef22740 --- /dev/null +++ b/examples/cpp/parsers/keypoints_device.cpp @@ -0,0 +1,50 @@ +#include + +int main() { + dai::NNModelDescription modelDescription; + modelDescription.modelSlug = "mediapipe-face-landmarker"; + modelDescription.modelVersionSlug = "192x192"; + modelDescription.platform = "RVC2"; + std::string archivePath = dai::getModelFromZoo(modelDescription, true); + + dai::Pipeline pipeline; + + auto cam = pipeline.create()->build(); + auto largeOutput = cam->requestOutput(std::pair(720, 720), dai::ImgFrame::Type::BGR888p); + + auto manip = pipeline.create(); + manip->initialConfig.setResize(192, 192); + largeOutput->link(manip->inputImage); + + auto nn = pipeline.create()->build(manip->out, dai::NNArchive(archivePath)); + + auto parser = pipeline.create(); + parser->setNumKeypoints(468); + parser->setScaleFactor(192); + nn->out.link(parser->input); + + auto videoQ = largeOutput->createOutputQueue(); + auto keypointsQ = parser->out.createOutputQueue(); + + pipeline.start(); + + while(pipeline.isRunning()) { + auto frame = videoQ->get(); + auto cvFrame = frame->getCvFrame(); + auto keypoints = keypointsQ->get()->keypoints; + + for (auto keypoint : keypoints) { + int x = keypoint.x * frame->getWidth(); + int y = keypoint.y * frame->getHeight(); + cv::circle(cvFrame, cv::Point(x, y), 2, cv::Scalar(0, 255, 0), -1); + } + + cv::imshow("Display", cvFrame); + auto key = cv::waitKey(1); + if(key == 'q' || key == 'Q') { + break; + } + } + + return 0; +} diff --git a/examples/cpp/parsers/keypoints_host.cpp b/examples/cpp/parsers/keypoints_host.cpp new file mode 100644 index 000000000..79ade62ca --- /dev/null +++ b/examples/cpp/parsers/keypoints_host.cpp @@ -0,0 +1,51 @@ +#include + +int main() { + dai::NNModelDescription modelDescription; + modelDescription.modelSlug = "mediapipe-face-landmarker"; + modelDescription.modelVersionSlug = "192x192"; + modelDescription.platform = "RVC2"; + std::string archivePath = dai::getModelFromZoo(modelDescription, true); + + dai::Pipeline pipeline; + + auto cam = pipeline.create()->build(); + auto largeOutput = cam->requestOutput(std::pair(720, 720), dai::ImgFrame::Type::BGR888p); + + auto manip = pipeline.create(); + manip->initialConfig.setResize(192, 192); + largeOutput->link(manip->inputImage); + + auto nn = pipeline.create()->build(manip->out, dai::NNArchive(archivePath)); + + auto parser = pipeline.create(); + parser->setRunOnHost(true); + parser->setNumKeypoints(468); + parser->setScaleFactor(192); + nn->out.link(parser->input); + + auto videoQ = largeOutput->createOutputQueue(); + auto keypointsQ = parser->out.createOutputQueue(); + + pipeline.start(); + + while(pipeline.isRunning()) { + auto frame = videoQ->get(); + auto cvFrame = frame->getCvFrame(); + auto keypoints = keypointsQ->get()->keypoints; + + for (auto keypoint : keypoints) { + int x = keypoint.x * frame->getWidth(); + int y = keypoint.y * frame->getHeight(); + cv::circle(cvFrame, cv::Point(x, y), 2, cv::Scalar(0, 255, 0), -1); + } + + cv::imshow("Display", cvFrame); + auto key = cv::waitKey(1); + if(key == 'q' || key == 'Q') { + break; + } + } + + return 0; +} diff --git a/include/depthai/pipeline/datatype/DatatypeEnum.hpp b/include/depthai/pipeline/datatype/DatatypeEnum.hpp index d3456571b..7d2c8820c 100644 --- a/include/depthai/pipeline/datatype/DatatypeEnum.hpp +++ b/include/depthai/pipeline/datatype/DatatypeEnum.hpp @@ -24,6 +24,7 @@ enum class DatatypeEnum : std::int32_t { AprilTags, Tracklets, IMUData, + Keypoints, StereoDepthConfig, FeatureTrackerConfig, ToFConfig, diff --git a/include/depthai/pipeline/datatype/Keypoints.hpp b/include/depthai/pipeline/datatype/Keypoints.hpp new file mode 100644 index 000000000..099ab8c37 --- /dev/null +++ b/include/depthai/pipeline/datatype/Keypoints.hpp @@ -0,0 +1,41 @@ +#pragma once + +#include + +// project +#include "depthai/pipeline/datatype/Buffer.hpp" + +// shared +#include "depthai/common/Point3f.hpp" + +namespace dai { + +/** + * Keypoints message. Carries keypoints data. + */ + class Keypoints : public Buffer { + public: + /** + * Construct Keypoints message. + */ + Keypoints() = default; + virtual ~Keypoints() = default; + + /// Keypoints + std::vector keypoints; + + void serialize(std::vector& metadata, DatatypeEnum& datatype) const override { + metadata = utility::serialize(*this); + datatype = DatatypeEnum::Keypoints; + }; + + // getters + const std::vector& getKeypoints() const; + + // setters + Keypoints& setKeypoints(const std::vector& points); + + DEPTHAI_SERIALIZE(Keypoints, Buffer::sequenceNum, Buffer::ts, Buffer::tsDevice, keypoints); +}; + +} // namespace dai diff --git a/include/depthai/pipeline/datatype/creators/KeypointsCreator.hpp b/include/depthai/pipeline/datatype/creators/KeypointsCreator.hpp new file mode 100644 index 000000000..c6fb80142 --- /dev/null +++ b/include/depthai/pipeline/datatype/creators/KeypointsCreator.hpp @@ -0,0 +1,42 @@ +#pragma once + +#include + +// project +#include "depthai/pipeline/datatype/Keypoints.hpp" + +// shared +#include "depthai/common/Point2f.hpp" +#include "depthai/common/Point3f.hpp" + +namespace dai { + +/** + * Keypoints message creator + */ + +/** + * Create a DepthAI message for 3D keypoints + * + * @param keypoints detected 3D keypoints + * @param scores confidence scores for each keypoint + * @param confidence_threshold confidence threshold + * + * @returns keypoints message + */ +std::shared_ptr createKeypointsMessage(const std::vector keypoints); +std::shared_ptr createKeypointsMessage(const std::vector keypoints, const std::vector scores, float confidence_threshold); + +/** + * Create a DepthAI message for 2D keypoints + * + * @param keypoints detected 2D keypoints + * @param scores confidence scores for each keypoint + * @param confidence_threshold confidence threshold + * + * @returns keypoints message + */ +std::shared_ptr createKeypointsMessage(const std::vector keypoints); +std::shared_ptr createKeypointsMessage(const std::vector keypoints, const std::vector scores, float confidence_threshold); + +} // namespace dai diff --git a/include/depthai/pipeline/datatypes.hpp b/include/depthai/pipeline/datatypes.hpp index 35285d045..e8e2408c2 100644 --- a/include/depthai/pipeline/datatypes.hpp +++ b/include/depthai/pipeline/datatypes.hpp @@ -12,6 +12,7 @@ #include "datatype/EncodedFrame.hpp" #include "datatype/FeatureTrackerConfig.hpp" #include "datatype/IMUData.hpp" +#include "datatype/Keypoints.hpp" #include "datatype/ImageManipConfig.hpp" #include "datatype/ImageManipConfigV2.hpp" #include "datatype/ImgDetections.hpp" diff --git a/include/depthai/pipeline/node/KeypointsParser.hpp b/include/depthai/pipeline/node/KeypointsParser.hpp new file mode 100644 index 000000000..f68c2465e --- /dev/null +++ b/include/depthai/pipeline/node/KeypointsParser.hpp @@ -0,0 +1,97 @@ +#pragma once + +// project +#include "depthai/pipeline/DeviceNode.hpp" +#include "depthai/pipeline/datatype/NNData.hpp" +#include "depthai/properties/KeypointsParserProperties.hpp" +#include "depthai/pipeline/datatype/Keypoints.hpp" +#include "depthai/pipeline/datatype/creators/KeypointsCreator.hpp" + +// shared +#include "depthai/common/Point2f.hpp" +#include "depthai/common/Point3f.hpp" + +#if defined(__clang__) + #if __has_warning("-Wswitch-enum") + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wswitch-enum" + #endif +#elif defined(__GNUC__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wswitch-enum" +#elif defined(_MSC_VER) + #pragma warning(push) + #pragma warning(disable : 4061) +#endif +#ifdef DEPTHAI_XTENSOR_SUPPORT + #include "xtensor/xarray.hpp" + #include "xtensor/xmath.hpp" +#endif +#if defined(__clang__) + #if __has_warning("-Wswitch-enum") + #pragma clang diagnostic pop + #endif +#elif defined(__GNUC__) + #pragma GCC diagnostic pop +#elif defined(_MSC_VER) + #pragma warning(pop) +#endif + +namespace dai { +namespace node { + +/** + * @brief Keypoint detections parser node. + */ +class KeypointsParser : public DeviceNodeCRTP, public HostRunnable { + private: + bool runOnHostVar = false; + + public: + constexpr static const char* NAME = "KeypointsParser"; + using DeviceNodeCRTP::DeviceNodeCRTP; + + /** + * Input message with keypoint data from the neural network. + */ + Input input{*this, {"input", DEFAULT_GROUP, DEFAULT_BLOCKING, DEFAULT_QUEUE_SIZE, {{{DatatypeEnum::NNData, true}}}, DEFAULT_WAIT_FOR_MESSAGE}}; + + /** + * Outputs Keypoints message that carries the detected keypoints. + */ + Output out{*this, {"out", DEFAULT_GROUP, {{{DatatypeEnum::Keypoints, false}}}}}; + + /** + * Set the scale factor to divide the keypoints by. + */ + void setScaleFactor(float scaleFactor); + /** + * Get the scale factor to divide the keypoints by. + */ + float getScaleFactor() const; + + /** + * Set the number of keypoints. + */ + void setNumKeypoints(int numKeypoints); + /** + * Get the number of keypoints. + */ + int getNumKeypoints() const; + + /** + * Specify whether to run on host or device + * By default, the node will run on device. + */ + void setRunOnHost(bool runOnHost); + /** + * Check if the node is set to run on host + */ + bool runOnHost() const override; + + void run() override; + +}; + +} // namespace node +} // namespace dai diff --git a/include/depthai/pipeline/nodes.hpp b/include/depthai/pipeline/nodes.hpp index c499ff199..16747c230 100644 --- a/include/depthai/pipeline/nodes.hpp +++ b/include/depthai/pipeline/nodes.hpp @@ -13,6 +13,7 @@ #include "node/IMU.hpp" #include "node/ImageManip.hpp" #include "node/ImageManipV2.hpp" +#include "node/KeypointsParser.hpp" #include "node/MessageDemux.hpp" #include "node/MonoCamera.hpp" #include "node/NeuralNetwork.hpp" diff --git a/include/depthai/properties/KeypointsParserProperties.hpp b/include/depthai/properties/KeypointsParserProperties.hpp new file mode 100644 index 000000000..7a02bb2a3 --- /dev/null +++ b/include/depthai/properties/KeypointsParserProperties.hpp @@ -0,0 +1,24 @@ +#pragma once + +#include "depthai/properties/Properties.hpp" + +namespace dai { + +/** + * Specify properties for KeypointsParser. + */ +struct KeypointsParserProperties : PropertiesSerializable { + /** + * The scale factor to divide the keypoints by. + */ + float scaleFactor = 1; + + /** + * The number of keypoints. Default value is invalid, must be specified. + */ + int numKeypoints = -1; +}; + +DEPTHAI_SERIALIZE_EXT(KeypointsParserProperties, scaleFactor, numKeypoints); + +} // namespace dai diff --git a/src/pipeline/datatype/DatatypeEnum.cpp b/src/pipeline/datatype/DatatypeEnum.cpp index d31e46dc1..801923a57 100644 --- a/src/pipeline/datatype/DatatypeEnum.cpp +++ b/src/pipeline/datatype/DatatypeEnum.cpp @@ -27,6 +27,7 @@ const std::unordered_map> hierarchy = { DatatypeEnum::EdgeDetectorConfig, DatatypeEnum::Tracklets, DatatypeEnum::IMUData, + DatatypeEnum::Keypoints, DatatypeEnum::StereoDepthConfig, DatatypeEnum::FeatureTrackerConfig, DatatypeEnum::ToFConfig, @@ -56,6 +57,7 @@ const std::unordered_map> hierarchy = { DatatypeEnum::EdgeDetectorConfig, DatatypeEnum::Tracklets, DatatypeEnum::IMUData, + DatatypeEnum::Keypoints, DatatypeEnum::StereoDepthConfig, DatatypeEnum::FeatureTrackerConfig, DatatypeEnum::ToFConfig, @@ -83,6 +85,7 @@ const std::unordered_map> hierarchy = { {DatatypeEnum::EdgeDetectorConfig, {}}, {DatatypeEnum::Tracklets, {}}, {DatatypeEnum::IMUData, {}}, + {DatatypeEnum::Keypoints, {}}, {DatatypeEnum::StereoDepthConfig, {}}, {DatatypeEnum::FeatureTrackerConfig, {}}, {DatatypeEnum::ToFConfig, {}}, diff --git a/src/pipeline/datatype/Keypoints.cpp b/src/pipeline/datatype/Keypoints.cpp new file mode 100644 index 000000000..b5c2a7e5c --- /dev/null +++ b/src/pipeline/datatype/Keypoints.cpp @@ -0,0 +1,14 @@ +#include "depthai/pipeline/datatype/Keypoints.hpp" + +namespace dai { + +const std::vector& Keypoints::getKeypoints() const { + return keypoints; +} + +Keypoints& Keypoints::setKeypoints(const std::vector& points) { + keypoints = points; + return *this; +} + +} // namespace dai diff --git a/src/pipeline/datatype/creators/KeypointsCreator.cpp b/src/pipeline/datatype/creators/KeypointsCreator.cpp new file mode 100644 index 000000000..45be8dff4 --- /dev/null +++ b/src/pipeline/datatype/creators/KeypointsCreator.cpp @@ -0,0 +1,67 @@ +#include "depthai/pipeline/datatype/creators/KeypointsCreator.hpp" + +namespace dai { + +// 3D keypoints +std::shared_ptr createKeypointsMessage(const std::vector keypoints) { + std::shared_ptr keypointsMsg = std::make_shared(); + keypointsMsg->setKeypoints(keypoints); + + return keypointsMsg; +} + +std::shared_ptr createKeypointsMessage(const std::vector keypoints, const std::vector scores, float confidence_threshold) { + if (keypoints.size() != scores.size()) { + throw std::invalid_argument("Keypoints and scores should have the same length. Got " + + std::to_string(keypoints.size()) + + " keypoints and " + + std::to_string(scores.size()) + + " scores."); + } + + for (const auto& score: scores) { + if (0 > score || score > 1) { + throw std::invalid_argument("Scores should only contain values between 0 and 1."); + } + } + + if (0 > confidence_threshold || confidence_threshold > 1) { + throw std::invalid_argument("Confidence threshold should be between 0 and 1. Got " + std::to_string(confidence_threshold) + "."); + } + + std::vector filteredPoints = std::vector(); + + for (size_t i = 0; i < keypoints.size(); i++) { + if (scores[i] >= confidence_threshold) { + filteredPoints.push_back(keypoints[i]); + } + } + + return createKeypointsMessage(filteredPoints); +} + + +// 2D keypoints +std::shared_ptr createKeypointsMessage(const std::vector keypoints) { + std::vector points3d = std::vector(keypoints.size()); + for (size_t i = 0; i < keypoints.size(); i++) { + points3d[i].x = keypoints[i].x; + points3d[i].y = keypoints[i].y; + points3d[i].z = 0; + } + + return createKeypointsMessage(points3d); +} + +std::shared_ptr createKeypointsMessage(const std::vector keypoints, const std::vector scores, float confidence_threshold) { + std::vector points3d = std::vector(keypoints.size()); + for (size_t i = 0; i < keypoints.size(); i++) { + points3d[i].x = keypoints[i].x; + points3d[i].y = keypoints[i].y; + points3d[i].z = 0; + } + + return createKeypointsMessage(points3d, scores, confidence_threshold); +} + +} // namespace dai diff --git a/src/pipeline/node/KeypointsParser.cpp b/src/pipeline/node/KeypointsParser.cpp new file mode 100644 index 000000000..4d818aea4 --- /dev/null +++ b/src/pipeline/node/KeypointsParser.cpp @@ -0,0 +1,85 @@ +#include "depthai/pipeline/node/KeypointsParser.hpp" + +namespace dai { +namespace node { + +void KeypointsParser::setScaleFactor(float scaleFactor) { + properties.scaleFactor = scaleFactor; +} +float KeypointsParser::getScaleFactor() const { + return properties.scaleFactor; +} + +void KeypointsParser::setNumKeypoints(int numKeypoints) { + properties.numKeypoints = numKeypoints; +} +int KeypointsParser::getNumKeypoints() const { + return properties.numKeypoints; +} + +void KeypointsParser::setRunOnHost(bool runOnHost) { + runOnHostVar = runOnHost; +} +bool KeypointsParser::runOnHost() const { + return runOnHostVar; +} + +void KeypointsParser::run() { + auto numKeypoints = properties.numKeypoints; + auto scaleFactor = properties.scaleFactor; + + if(numKeypoints == -1) { + throw std::runtime_error("Number of keypoints must be specified!"); + } + + while(isRunning()) { + auto inputData = input.get(); + if(inputData == nullptr) { + throw std::invalid_argument("Received nullptr from input"); + } + + std::vector outputLayerNames = inputData->getAllLayerNames(); + if (outputLayerNames.size() != 1) { + throw std::invalid_argument("Expected 1 output layer, got " + std::to_string(outputLayerNames.size())); + } + + xt::xarray keypoints = inputData->getFirstTensor(true); + int totalCoords = std::accumulate(keypoints.shape().begin(), keypoints.shape().end(), 1, std::multiplies()); + + if (numKeypoints * 2 != totalCoords && numKeypoints * 3 != totalCoords) { + throw std::runtime_error("Expected 2 or 3 coordinates per keypoint, got " + std::to_string(float(totalCoords) / float(numKeypoints))); + } + int pointDimension = totalCoords / numKeypoints; + + keypoints = keypoints.reshape({numKeypoints, pointDimension}); + keypoints /= scaleFactor; + + std::shared_ptr msg = nullptr; + + if (pointDimension == 2) { + std::vector keypointsVector = std::vector(numKeypoints); + for (int i = 0; i < numKeypoints; i++) { + keypointsVector[i].x = keypoints(i, 0); + keypointsVector[i].y = keypoints(i, 1); + } + msg = createKeypointsMessage(keypointsVector); + } + else { + std::vector keypointsVector = std::vector(numKeypoints); + for (int i = 0; i < numKeypoints; i++) { + keypointsVector[i].x = keypoints(i, 0); + keypointsVector[i].y = keypoints(i, 1); + keypointsVector[i].z = keypoints(i, 2); + } + msg = createKeypointsMessage(keypointsVector); + } + + msg->setTimestamp(inputData->getTimestamp()); + msg->setTimestampDevice(inputData->getTimestampDevice()); + msg->setSequenceNum(inputData->getSequenceNum()); + out.send(msg); + } +} + +} // namespace node +} // namespace dai From 5a538e7b049324b01f322be361e85eee304ed785 Mon Sep 17 00:00:00 2001 From: NicikD Date: Thu, 19 Sep 2024 16:34:40 +0200 Subject: [PATCH 2/6] Added python bindings to KeypointsParser node and a python example. --- bindings/python/CMakeLists.txt | 2 + bindings/python/src/DatatypeBindings.cpp | 3 ++ .../pipeline/datatype/KeypointsBindings.cpp | 47 ++++++++++++++++ .../pipeline/node/KeypointsParserBindings.cpp | 54 +++++++++++++++++++ .../python/src/pipeline/node/NodeBindings.cpp | 2 + examples/python/RVC2/parsers/keypoints.py | 48 +++++++++++++++++ 6 files changed, 156 insertions(+) create mode 100644 bindings/python/src/pipeline/datatype/KeypointsBindings.cpp create mode 100644 bindings/python/src/pipeline/node/KeypointsParserBindings.cpp create mode 100644 examples/python/RVC2/parsers/keypoints.py diff --git a/bindings/python/CMakeLists.txt b/bindings/python/CMakeLists.txt index 17cdbb3f4..324604f76 100644 --- a/bindings/python/CMakeLists.txt +++ b/bindings/python/CMakeLists.txt @@ -95,6 +95,7 @@ set(SOURCE_LIST src/pipeline/node/SpatialDetectionNetworkBindings.cpp src/pipeline/node/ObjectTrackerBindings.cpp src/pipeline/node/IMUBindings.cpp + src/pipeline/node/KeypointsParserBindings.cpp src/pipeline/node/EdgeDetectorBindings.cpp src/pipeline/node/FeatureTrackerBindings.cpp src/pipeline/node/ToFBindings.cpp @@ -126,6 +127,7 @@ set(SOURCE_LIST src/pipeline/datatype/ImgFrameBindings.cpp src/pipeline/datatype/EncodedFrameBindings.cpp src/pipeline/datatype/IMUDataBindings.cpp + src/pipeline/datatype/KeypointsBindings.cpp src/pipeline/datatype/MessageGroupBindings.cpp src/pipeline/datatype/NNDataBindings.cpp src/pipeline/datatype/SpatialImgDetectionsBindings.cpp diff --git a/bindings/python/src/DatatypeBindings.cpp b/bindings/python/src/DatatypeBindings.cpp index 0e9e8aa1e..7625b3191 100644 --- a/bindings/python/src/DatatypeBindings.cpp +++ b/bindings/python/src/DatatypeBindings.cpp @@ -17,6 +17,7 @@ void bind_imgdetections(pybind11::module& m, void* pCallstack); void bind_imgframe(pybind11::module& m, void* pCallstack); void bind_encodedframe(pybind11::module& m, void* pCallstack); void bind_imudata(pybind11::module& m, void* pCallstack); +void bind_keypoints(pybind11::module& m, void* pCallstack); void bind_message_group(pybind11::module& m, void* pCallstack); void bind_nndata(pybind11::module& m, void* pCallstack); void bind_spatialimgdetections(pybind11::module& m, void* pCallstack); @@ -51,6 +52,7 @@ void DatatypeBindings::addToCallstack(std::deque& callstack) { callstack.push_front(bind_imgframe); callstack.push_front(bind_encodedframe); callstack.push_front(bind_imudata); + callstack.push_front(bind_keypoints); callstack.push_front(bind_message_group); callstack.push_front(bind_nndata); callstack.push_front(bind_spatialimgdetections); @@ -96,6 +98,7 @@ void DatatypeBindings::bind(pybind11::module& m, void* pCallstack){ .value("ImageManipConfigV2", DatatypeEnum::ImageManipConfigV2) .value("CameraControl", DatatypeEnum::CameraControl) .value("ImgDetections", DatatypeEnum::ImgDetections) + .value("Keypoints", DatatypeEnum::Keypoints) .value("SpatialImgDetections", DatatypeEnum::SpatialImgDetections) .value("SystemInformation", DatatypeEnum::SystemInformation) .value("SpatialLocationCalculatorConfig", DatatypeEnum::SpatialLocationCalculatorConfig) diff --git a/bindings/python/src/pipeline/datatype/KeypointsBindings.cpp b/bindings/python/src/pipeline/datatype/KeypointsBindings.cpp new file mode 100644 index 000000000..bc9cf4049 --- /dev/null +++ b/bindings/python/src/pipeline/datatype/KeypointsBindings.cpp @@ -0,0 +1,47 @@ +#include "DatatypeBindings.hpp" +#include "pipeline/CommonBindings.hpp" + +// depthai +#include "depthai/pipeline/datatype/Keypoints.hpp" +//pybind +#include +#include + + +void bind_keypoints(pybind11::module& m, void* pCallstack){ + + using namespace dai; + + py::class_, Buffer, std::shared_ptr> keypoints(m, "Keypoints", DOC(dai, Keypoints)); + + /////////////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////// + // Call the rest of the type defines, then perform the actual bindings + Callstack* callstack = (Callstack*) pCallstack; + auto cb = callstack->top(); + callstack->pop(); + cb(m, pCallstack); + // Actual bindings + /////////////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////// + + // Message + keypoints + .def(py::init<>()) + + // getters + .def("getTimestamp", &Keypoints::Buffer::getTimestamp, DOC(dai, Buffer, getTimestamp)) + .def("getTimestampDevice", &Keypoints::Buffer::getTimestampDevice, DOC(dai, Buffer, getTimestampDevice)) + .def("getSequenceNum", &Keypoints::Buffer::getSequenceNum, DOC(dai, Buffer, getSequenceNum)) + .def("getKeypoints", &Keypoints::getKeypoints, DOC(dai, Keypoints, getKeypoints)) + + // setters + .def("setTimestamp", &Keypoints::Buffer::setTimestamp, py::arg("timestamp"), DOC(dai, Buffer, setTimestamp)) + .def("setTimestampDevice", &Keypoints::Buffer::setTimestampDevice, DOC(dai, Buffer, setTimestampDevice)) + .def("setSequenceNum", &Keypoints::Buffer::setSequenceNum, DOC(dai, Buffer, setSequenceNum)) + .def("setKeypoints", &Keypoints::setKeypoints, py::arg("points"), DOC(dai, Keypoints, setKeypoints)) + ; + +} diff --git a/bindings/python/src/pipeline/node/KeypointsParserBindings.cpp b/bindings/python/src/pipeline/node/KeypointsParserBindings.cpp new file mode 100644 index 000000000..c5746a16a --- /dev/null +++ b/bindings/python/src/pipeline/node/KeypointsParserBindings.cpp @@ -0,0 +1,54 @@ +#include "Common.hpp" +#include "NodeBindings.hpp" + +#include "depthai/pipeline/Node.hpp" +#include "depthai/pipeline/Pipeline.hpp" +#include "depthai/properties/KeypointsParserProperties.hpp" +#include "depthai/pipeline/node/KeypointsParser.hpp" + +void bind_keypointsparser(pybind11::module& m, void* pCallstack){ + + using namespace dai; + using namespace dai::node; + + // Node and Properties declare upfront + py::class_ keypointsParserProperties(m, "keypointsParserProperties", DOC(dai, KeypointsParserProperties)); + auto keypointsParser = ADD_NODE(KeypointsParser); + + /////////////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////// + // Call the rest of the type defines, then perform the actual bindings + Callstack* callstack = (Callstack*) pCallstack; + auto cb = callstack->top(); + callstack->pop(); + cb(m, pCallstack); + // Actual bindings + /////////////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////// + + // Properties + keypointsParserProperties + .def_readwrite("scaleFactor", &KeypointsParserProperties::scaleFactor) + .def_readwrite("numKeypoints", &KeypointsParserProperties::numKeypoints) + ; + + // Node + keypointsParser + .def_readonly("input", &KeypointsParser::input, DOC(dai, node, KeypointsParser, input)) + .def_readonly("out", &KeypointsParser::out, DOC(dai, node, KeypointsParser, out)) + + // getters + .def("runOnHost", &KeypointsParser::runOnHost, DOC(dai, node, KeypointsParser, runOnHost)) + .def("getScaleFactor", &KeypointsParser::getScaleFactor, DOC(dai, node, KeypointsParser, getScaleFactor)) + .def("getNumKeypoints", &KeypointsParser::getNumKeypoints, DOC(dai, node, KeypointsParser, getNumKeypoints)) + + // setters + .def("setRunOnHost", &KeypointsParser::setRunOnHost, DOC(dai, node, KeypointsParser, setRunOnHost)) + .def("setScaleFactor", &KeypointsParser::setScaleFactor, DOC(dai, node, KeypointsParser, setScaleFactor)) + .def("setNumKeypoints", &KeypointsParser::setNumKeypoints, DOC(dai, node, KeypointsParser, setNumKeypoints)) + ; + daiNodeModule.attr("KeypointsParser").attr("Properties") = keypointsParserProperties; + +} diff --git a/bindings/python/src/pipeline/node/NodeBindings.cpp b/bindings/python/src/pipeline/node/NodeBindings.cpp index 63e0bd18e..29922e4ce 100644 --- a/bindings/python/src/pipeline/node/NodeBindings.cpp +++ b/bindings/python/src/pipeline/node/NodeBindings.cpp @@ -147,6 +147,7 @@ void bind_spatiallocationcalculator(pybind11::module& m, void* pCallstack); void bind_spatialdetectionnetwork(pybind11::module& m, void* pCallstack); void bind_objecttracker(pybind11::module& m, void* pCallstack); void bind_imu(pybind11::module& m, void* pCallstack); +void bind_keypointsparser(pybind11::module& m, void* pCallstack); void bind_edgedetector(pybind11::module& m, void* pCallstack); void bind_featuretracker(pybind11::module& m, void* pCallstack); void bind_apriltag(pybind11::module& m, void* pCallstack); @@ -192,6 +193,7 @@ void NodeBindings::addToCallstack(std::deque& callstack) { callstack.push_front(bind_spatialdetectionnetwork); callstack.push_front(bind_objecttracker); callstack.push_front(bind_imu); + callstack.push_front(bind_keypointsparser); callstack.push_front(bind_edgedetector); callstack.push_front(bind_featuretracker); callstack.push_front(bind_apriltag); diff --git a/examples/python/RVC2/parsers/keypoints.py b/examples/python/RVC2/parsers/keypoints.py new file mode 100644 index 000000000..9d4eab55f --- /dev/null +++ b/examples/python/RVC2/parsers/keypoints.py @@ -0,0 +1,48 @@ +import depthai as dai + +import cv2 + +RUN_ON_HOST = True + +modelDescription = dai.NNModelDescription(modelSlug="mediapipe-face-landmarker", modelVersionSlug="192x192", platform="RVC2") +archivePath = dai.getModelFromZoo(modelDescription, useCached=True) +nnArchive = dai.NNArchive(archivePath) + +with dai.Pipeline() as pipeline: + + print("Creating pipeline...") + cam = pipeline.create(dai.node.Camera).build(dai.CameraBoardSocket.CAM_A) + full = cam.requestOutput((720, 720), dai.ImgFrame.Type.BGR888p) + + manip = pipeline.create(dai.node.ImageManip) + manip.initialConfig.setResize(192, 192) + full.link(manip.inputImage) + + nn = pipeline.create(dai.node.NeuralNetwork).build( + input=manip.out, + nnArchive=nnArchive + ) + + parser = pipeline.create(dai.node.KeypointsParser) + parser.setNumKeypoints(468) + parser.setScaleFactor(192) + parser.setRunOnHost(RUN_ON_HOST) + nn.out.link(parser.input) + + video_q = full.createOutputQueue() + keypoints_q = parser.out.createOutputQueue() + + pipeline.start() + + while pipeline.isRunning(): + frame = video_q.get().getCvFrame() + keypoints: dai.Keypoints = keypoints_q.get() + + for keypoint in keypoints.getKeypoints(): + x, y = keypoint.x, keypoint.y + x, y = int(x * frame.shape[1]), int(y * frame.shape[0]) + frame = cv2.circle(frame, (int(x), int(y)), 2, (0, 255, 0), -1) + + cv2.imshow("Frame", frame) + if cv2.waitKey(1) == ord("q"): + break From b9a67cbd3997fbaefe387817093e90db830192da Mon Sep 17 00:00:00 2001 From: NicikD Date: Thu, 19 Sep 2024 18:55:04 +0200 Subject: [PATCH 3/6] Moved RVC2 examples to RVC2 folder, refactored creation of keypoints message to only use Keypoints class, moving into new branch so "v3_develop_" is not in the name of the branch --- CMakeLists.txt | 1 - .../pipeline/datatype/KeypointsBindings.cpp | 3 +- examples/cpp/CMakeLists.txt | 4 +- .../{ => RVC2}/parsers/keypoints_device.cpp | 0 .../cpp/{ => RVC2}/parsers/keypoints_host.cpp | 0 .../depthai/pipeline/datatype/Keypoints.hpp | 28 +++++++- .../datatype/creators/KeypointsCreator.hpp | 42 ------------ .../depthai/pipeline/node/KeypointsParser.hpp | 1 - src/pipeline/datatype/Keypoints.cpp | 57 +++++++++++++++- .../datatype/creators/KeypointsCreator.cpp | 67 ------------------- src/pipeline/node/KeypointsParser.cpp | 8 +-- 11 files changed, 89 insertions(+), 122 deletions(-) rename examples/cpp/{ => RVC2}/parsers/keypoints_device.cpp (100%) rename examples/cpp/{ => RVC2}/parsers/keypoints_host.cpp (100%) delete mode 100644 include/depthai/pipeline/datatype/creators/KeypointsCreator.hpp delete mode 100644 src/pipeline/datatype/creators/KeypointsCreator.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 760676cfb..a3b4487d6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -326,7 +326,6 @@ set(TARGET_CORE_SOURCES src/pipeline/node/host/HostNode.cpp src/pipeline/datatype/DatatypeEnum.cpp src/pipeline/node/PointCloud.cpp - src/pipeline/datatype/creators/KeypointsCreator.cpp src/pipeline/datatype/Buffer.cpp src/pipeline/datatype/ImgFrame.cpp src/pipeline/datatype/ImgTransformations.cpp diff --git a/bindings/python/src/pipeline/datatype/KeypointsBindings.cpp b/bindings/python/src/pipeline/datatype/KeypointsBindings.cpp index bc9cf4049..1d3bd3136 100644 --- a/bindings/python/src/pipeline/datatype/KeypointsBindings.cpp +++ b/bindings/python/src/pipeline/datatype/KeypointsBindings.cpp @@ -41,7 +41,8 @@ void bind_keypoints(pybind11::module& m, void* pCallstack){ .def("setTimestamp", &Keypoints::Buffer::setTimestamp, py::arg("timestamp"), DOC(dai, Buffer, setTimestamp)) .def("setTimestampDevice", &Keypoints::Buffer::setTimestampDevice, DOC(dai, Buffer, setTimestampDevice)) .def("setSequenceNum", &Keypoints::Buffer::setSequenceNum, DOC(dai, Buffer, setSequenceNum)) - .def("setKeypoints", &Keypoints::setKeypoints, py::arg("points"), DOC(dai, Keypoints, setKeypoints)) + // Binds only the overload that takes 3D points without scores and confidence threshold + .def("setKeypoints", py::overload_cast&>(&Keypoints::setKeypoints), DOC(dai, Keypoints, setKeypoints)) ; } diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index 299e3a394..f5392c5eb 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -484,8 +484,8 @@ dai_add_example(threaded_host_node HostNodes/threaded_host_node.cpp ON OFF) dai_add_example(model_zoo RVC2/ModelZoo/model_zoo.cpp ON OFF) # Keypoints parser -dai_add_example(keypoints_device parsers/keypoints_device.cpp ON OFF) -dai_add_example(keypoints_host parsers/keypoints_host.cpp ON OFF) +dai_add_example(keypoints_device RVC2/parsers/keypoints_device.cpp ON OFF) +dai_add_example(keypoints_host RVC2/parsers/keypoints_host.cpp ON OFF) if(DEPTHAI_RTABMAP_SUPPORT) include(FetchContent) diff --git a/examples/cpp/parsers/keypoints_device.cpp b/examples/cpp/RVC2/parsers/keypoints_device.cpp similarity index 100% rename from examples/cpp/parsers/keypoints_device.cpp rename to examples/cpp/RVC2/parsers/keypoints_device.cpp diff --git a/examples/cpp/parsers/keypoints_host.cpp b/examples/cpp/RVC2/parsers/keypoints_host.cpp similarity index 100% rename from examples/cpp/parsers/keypoints_host.cpp rename to examples/cpp/RVC2/parsers/keypoints_host.cpp diff --git a/include/depthai/pipeline/datatype/Keypoints.hpp b/include/depthai/pipeline/datatype/Keypoints.hpp index 099ab8c37..c616302c4 100644 --- a/include/depthai/pipeline/datatype/Keypoints.hpp +++ b/include/depthai/pipeline/datatype/Keypoints.hpp @@ -6,6 +6,7 @@ #include "depthai/pipeline/datatype/Buffer.hpp" // shared +#include "depthai/common/Point2f.hpp" #include "depthai/common/Point3f.hpp" namespace dai { @@ -19,7 +20,7 @@ namespace dai { * Construct Keypoints message. */ Keypoints() = default; - virtual ~Keypoints() = default; + ~Keypoints() override = default; /// Keypoints std::vector keypoints; @@ -33,7 +34,30 @@ namespace dai { const std::vector& getKeypoints() const; // setters - Keypoints& setKeypoints(const std::vector& points); + + /** + * Set 3D keypoints + * + * @param keypoints detected 3D keypoints + * @param scores confidence scores for each keypoint + * @param confidence_threshold confidence threshold + * + * @returns keypoints message + */ + Keypoints& setKeypoints(const std::vector& keypoints); + Keypoints& setKeypoints(const std::vector& keypoints, const std::vector& scores, float confidenceThreshold); + + /** + * Set 2D keypoints + * + * @param keypoints detected 2D keypoints + * @param scores confidence scores for each keypoint + * @param confidence_threshold confidence threshold + * + * @returns keypoints message + */ + Keypoints& setKeypoints(const std::vector& keypoints); + Keypoints& setKeypoints(const std::vector& keypoints, const std::vector& scores, float confidenceThreshold); DEPTHAI_SERIALIZE(Keypoints, Buffer::sequenceNum, Buffer::ts, Buffer::tsDevice, keypoints); }; diff --git a/include/depthai/pipeline/datatype/creators/KeypointsCreator.hpp b/include/depthai/pipeline/datatype/creators/KeypointsCreator.hpp deleted file mode 100644 index c6fb80142..000000000 --- a/include/depthai/pipeline/datatype/creators/KeypointsCreator.hpp +++ /dev/null @@ -1,42 +0,0 @@ -#pragma once - -#include - -// project -#include "depthai/pipeline/datatype/Keypoints.hpp" - -// shared -#include "depthai/common/Point2f.hpp" -#include "depthai/common/Point3f.hpp" - -namespace dai { - -/** - * Keypoints message creator - */ - -/** - * Create a DepthAI message for 3D keypoints - * - * @param keypoints detected 3D keypoints - * @param scores confidence scores for each keypoint - * @param confidence_threshold confidence threshold - * - * @returns keypoints message - */ -std::shared_ptr createKeypointsMessage(const std::vector keypoints); -std::shared_ptr createKeypointsMessage(const std::vector keypoints, const std::vector scores, float confidence_threshold); - -/** - * Create a DepthAI message for 2D keypoints - * - * @param keypoints detected 2D keypoints - * @param scores confidence scores for each keypoint - * @param confidence_threshold confidence threshold - * - * @returns keypoints message - */ -std::shared_ptr createKeypointsMessage(const std::vector keypoints); -std::shared_ptr createKeypointsMessage(const std::vector keypoints, const std::vector scores, float confidence_threshold); - -} // namespace dai diff --git a/include/depthai/pipeline/node/KeypointsParser.hpp b/include/depthai/pipeline/node/KeypointsParser.hpp index f68c2465e..90a8c9e5b 100644 --- a/include/depthai/pipeline/node/KeypointsParser.hpp +++ b/include/depthai/pipeline/node/KeypointsParser.hpp @@ -5,7 +5,6 @@ #include "depthai/pipeline/datatype/NNData.hpp" #include "depthai/properties/KeypointsParserProperties.hpp" #include "depthai/pipeline/datatype/Keypoints.hpp" -#include "depthai/pipeline/datatype/creators/KeypointsCreator.hpp" // shared #include "depthai/common/Point2f.hpp" diff --git a/src/pipeline/datatype/Keypoints.cpp b/src/pipeline/datatype/Keypoints.cpp index b5c2a7e5c..4ba639a01 100644 --- a/src/pipeline/datatype/Keypoints.cpp +++ b/src/pipeline/datatype/Keypoints.cpp @@ -6,9 +6,62 @@ const std::vector& Keypoints::getKeypoints() const { return keypoints; } -Keypoints& Keypoints::setKeypoints(const std::vector& points) { - keypoints = points; +// 3D keypoints +Keypoints& Keypoints::setKeypoints(const std::vector& keypoints) { + this->keypoints = keypoints; + return *this; } +Keypoints& Keypoints::setKeypoints(const std::vector& keypoints, const std::vector& scores, float confidenceThreshold) { + if (keypoints.size() != scores.size()) { + throw std::invalid_argument("Keypoints and scores should have the same length. Got " + + std::to_string(keypoints.size()) + + " keypoints and " + + std::to_string(scores.size()) + + " scores."); + } + + for (const auto& score: scores) { + if (0 > score || score > 1) { + throw std::invalid_argument("Scores should only contain values between 0 and 1."); + } + } + + if (0 > confidenceThreshold || confidenceThreshold > 1) { + throw std::invalid_argument("Confidence threshold should be between 0 and 1. Got " + std::to_string(confidenceThreshold) + "."); + } + + std::vector filteredPoints = std::vector(); + + for (size_t i = 0; i < keypoints.size(); i++) { + if (scores[i] >= confidenceThreshold) { + filteredPoints.push_back(keypoints[i]); + } + } + + return this->setKeypoints(filteredPoints); +} + +// 2D keypoints +Keypoints& Keypoints::setKeypoints(const std::vector& keypoints) { + std::vector points3d = std::vector(keypoints.size()); + for (size_t i = 0; i < keypoints.size(); i++) { + points3d[i].x = keypoints[i].x; + points3d[i].y = keypoints[i].y; + points3d[i].z = 0; + } + + return this->setKeypoints(points3d); +} +Keypoints& Keypoints::setKeypoints(const std::vector& keypoints, const std::vector& scores, float confidenceThreshold) { + std::vector points3d = std::vector(keypoints.size()); + for (size_t i = 0; i < keypoints.size(); i++) { + points3d[i].x = keypoints[i].x; + points3d[i].y = keypoints[i].y; + points3d[i].z = 0; + } + + return this->setKeypoints(points3d, scores, confidenceThreshold); +} } // namespace dai diff --git a/src/pipeline/datatype/creators/KeypointsCreator.cpp b/src/pipeline/datatype/creators/KeypointsCreator.cpp deleted file mode 100644 index 45be8dff4..000000000 --- a/src/pipeline/datatype/creators/KeypointsCreator.cpp +++ /dev/null @@ -1,67 +0,0 @@ -#include "depthai/pipeline/datatype/creators/KeypointsCreator.hpp" - -namespace dai { - -// 3D keypoints -std::shared_ptr createKeypointsMessage(const std::vector keypoints) { - std::shared_ptr keypointsMsg = std::make_shared(); - keypointsMsg->setKeypoints(keypoints); - - return keypointsMsg; -} - -std::shared_ptr createKeypointsMessage(const std::vector keypoints, const std::vector scores, float confidence_threshold) { - if (keypoints.size() != scores.size()) { - throw std::invalid_argument("Keypoints and scores should have the same length. Got " - + std::to_string(keypoints.size()) - + " keypoints and " - + std::to_string(scores.size()) - + " scores."); - } - - for (const auto& score: scores) { - if (0 > score || score > 1) { - throw std::invalid_argument("Scores should only contain values between 0 and 1."); - } - } - - if (0 > confidence_threshold || confidence_threshold > 1) { - throw std::invalid_argument("Confidence threshold should be between 0 and 1. Got " + std::to_string(confidence_threshold) + "."); - } - - std::vector filteredPoints = std::vector(); - - for (size_t i = 0; i < keypoints.size(); i++) { - if (scores[i] >= confidence_threshold) { - filteredPoints.push_back(keypoints[i]); - } - } - - return createKeypointsMessage(filteredPoints); -} - - -// 2D keypoints -std::shared_ptr createKeypointsMessage(const std::vector keypoints) { - std::vector points3d = std::vector(keypoints.size()); - for (size_t i = 0; i < keypoints.size(); i++) { - points3d[i].x = keypoints[i].x; - points3d[i].y = keypoints[i].y; - points3d[i].z = 0; - } - - return createKeypointsMessage(points3d); -} - -std::shared_ptr createKeypointsMessage(const std::vector keypoints, const std::vector scores, float confidence_threshold) { - std::vector points3d = std::vector(keypoints.size()); - for (size_t i = 0; i < keypoints.size(); i++) { - points3d[i].x = keypoints[i].x; - points3d[i].y = keypoints[i].y; - points3d[i].z = 0; - } - - return createKeypointsMessage(points3d, scores, confidence_threshold); -} - -} // namespace dai diff --git a/src/pipeline/node/KeypointsParser.cpp b/src/pipeline/node/KeypointsParser.cpp index 4d818aea4..117e81fc1 100644 --- a/src/pipeline/node/KeypointsParser.cpp +++ b/src/pipeline/node/KeypointsParser.cpp @@ -47,14 +47,14 @@ void KeypointsParser::run() { int totalCoords = std::accumulate(keypoints.shape().begin(), keypoints.shape().end(), 1, std::multiplies()); if (numKeypoints * 2 != totalCoords && numKeypoints * 3 != totalCoords) { - throw std::runtime_error("Expected 2 or 3 coordinates per keypoint, got " + std::to_string(float(totalCoords) / float(numKeypoints))); + throw std::runtime_error("Expected 2 or 3 coordinates per keypoint, got " + std::to_string(static_cast(totalCoords) / static_cast(numKeypoints))); } int pointDimension = totalCoords / numKeypoints; keypoints = keypoints.reshape({numKeypoints, pointDimension}); keypoints /= scaleFactor; - std::shared_ptr msg = nullptr; + std::shared_ptr msg = std::make_shared(); if (pointDimension == 2) { std::vector keypointsVector = std::vector(numKeypoints); @@ -62,7 +62,7 @@ void KeypointsParser::run() { keypointsVector[i].x = keypoints(i, 0); keypointsVector[i].y = keypoints(i, 1); } - msg = createKeypointsMessage(keypointsVector); + msg->setKeypoints(keypointsVector); } else { std::vector keypointsVector = std::vector(numKeypoints); @@ -71,7 +71,7 @@ void KeypointsParser::run() { keypointsVector[i].y = keypoints(i, 1); keypointsVector[i].z = keypoints(i, 2); } - msg = createKeypointsMessage(keypointsVector); + msg->setKeypoints(keypointsVector); } msg->setTimestamp(inputData->getTimestamp()); From 3581cd50ff62bf2e92a584a7588439e170f7d173 Mon Sep 17 00:00:00 2001 From: NicikD Date: Wed, 25 Sep 2024 16:54:07 +0200 Subject: [PATCH 4/6] Moved function definitions out of #ifdef DEPTHAI_XTENSOR_SUPPORT, wouldnt compile otherwise --- include/depthai/pipeline/datatype/NNData.hpp | 24 ++++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/include/depthai/pipeline/datatype/NNData.hpp b/include/depthai/pipeline/datatype/NNData.hpp index 4a7aecfed..b9f0961ad 100644 --- a/include/depthai/pipeline/datatype/NNData.hpp +++ b/include/depthai/pipeline/datatype/NNData.hpp @@ -494,18 +494,6 @@ class NNData : public Buffer { } } - /** - * Get the datatype of a given tensor - * @returns TensorInfo::DataType tensor datatype - */ - TensorInfo::DataType getTensorDatatype(const std::string& name); - - /** - * Get the datatype of the first tensor - * @returns TensorInfo::DataType tensor datatype - */ - TensorInfo::DataType getFirstTensorDatatype(); - /** * Convenience function to retrieve values from the first tensor * @returns xt::xarray<_Ty> tensor @@ -519,6 +507,18 @@ class NNData : public Buffer { return {}; } #endif + /** + * Get the datatype of a given tensor + * @returns TensorInfo::DataType tensor datatype + */ + TensorInfo::DataType getTensorDatatype(const std::string& name); + + /** + * Get the datatype of the first tensor + * @returns TensorInfo::DataType tensor datatype + */ + TensorInfo::DataType getFirstTensorDatatype(); + void serialize(std::vector& metadata, DatatypeEnum& datatype) const override { metadata = utility::serialize(*this); datatype = DatatypeEnum::NNData; From b47bf3770cb69028a15a89027e57d6a8f8ee6c7d Mon Sep 17 00:00:00 2001 From: NicikD Date: Wed, 25 Sep 2024 18:35:13 +0200 Subject: [PATCH 5/6] Created Keypoint object to add per point confidence to KeypointsParser output. Made KeypointsParser to compile when building without xtensor support. Added implicit metadata setting to KeypointsParser from NN archive. --- .../pipeline/datatype/KeypointsBindings.cpp | 14 ++- .../pipeline/node/KeypointsParserBindings.cpp | 2 + .../cpp/RVC2/parsers/keypoints_device.cpp | 7 +- examples/cpp/RVC2/parsers/keypoints_host.cpp | 7 +- examples/python/RVC2/parsers/keypoints.py | 4 +- .../depthai/pipeline/datatype/Keypoints.hpp | 38 ++++-- .../depthai/pipeline/node/KeypointsParser.hpp | 7 ++ src/pipeline/datatype/Keypoints.cpp | 116 ++++++++++++++---- src/pipeline/node/KeypointsParser.cpp | 54 ++++++-- 9 files changed, 190 insertions(+), 59 deletions(-) diff --git a/bindings/python/src/pipeline/datatype/KeypointsBindings.cpp b/bindings/python/src/pipeline/datatype/KeypointsBindings.cpp index 1d3bd3136..4265050ec 100644 --- a/bindings/python/src/pipeline/datatype/KeypointsBindings.cpp +++ b/bindings/python/src/pipeline/datatype/KeypointsBindings.cpp @@ -13,6 +13,7 @@ void bind_keypoints(pybind11::module& m, void* pCallstack){ using namespace dai; py::class_, Buffer, std::shared_ptr> keypoints(m, "Keypoints", DOC(dai, Keypoints)); + py::class_ keypoint(m, "Keypoint", DOC(dai, Keypoint)); /////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////// @@ -27,6 +28,15 @@ void bind_keypoints(pybind11::module& m, void* pCallstack){ /////////////////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////////////////// + // Single point struct + keypoint + .def(py::init<>()) + .def_readwrite("x", &Keypoint::x) + .def_readwrite("y", &Keypoint::y) + .def_readwrite("z", &Keypoint::z) + .def_readwrite("confidence", &Keypoint::confidence) + ; + // Message keypoints .def(py::init<>()) @@ -41,8 +51,8 @@ void bind_keypoints(pybind11::module& m, void* pCallstack){ .def("setTimestamp", &Keypoints::Buffer::setTimestamp, py::arg("timestamp"), DOC(dai, Buffer, setTimestamp)) .def("setTimestampDevice", &Keypoints::Buffer::setTimestampDevice, DOC(dai, Buffer, setTimestampDevice)) .def("setSequenceNum", &Keypoints::Buffer::setSequenceNum, DOC(dai, Buffer, setSequenceNum)) - // Binds only the overload that takes 3D points without scores and confidence threshold - .def("setKeypoints", py::overload_cast&>(&Keypoints::setKeypoints), DOC(dai, Keypoints, setKeypoints)) + // Binds only the overload that takes Keypoint objects + .def("setKeypoints", py::overload_cast&>(&Keypoints::setKeypoints), DOC(dai, Keypoints, setKeypoints)) ; } diff --git a/bindings/python/src/pipeline/node/KeypointsParserBindings.cpp b/bindings/python/src/pipeline/node/KeypointsParserBindings.cpp index c5746a16a..52890a1aa 100644 --- a/bindings/python/src/pipeline/node/KeypointsParserBindings.cpp +++ b/bindings/python/src/pipeline/node/KeypointsParserBindings.cpp @@ -39,6 +39,8 @@ void bind_keypointsparser(pybind11::module& m, void* pCallstack){ .def_readonly("input", &KeypointsParser::input, DOC(dai, node, KeypointsParser, input)) .def_readonly("out", &KeypointsParser::out, DOC(dai, node, KeypointsParser, out)) + .def("build", &KeypointsParser::build, DOC(dai, node, KeypointsParser, build)) + // getters .def("runOnHost", &KeypointsParser::runOnHost, DOC(dai, node, KeypointsParser, runOnHost)) .def("getScaleFactor", &KeypointsParser::getScaleFactor, DOC(dai, node, KeypointsParser, getScaleFactor)) diff --git a/examples/cpp/RVC2/parsers/keypoints_device.cpp b/examples/cpp/RVC2/parsers/keypoints_device.cpp index d0ef22740..993778216 100644 --- a/examples/cpp/RVC2/parsers/keypoints_device.cpp +++ b/examples/cpp/RVC2/parsers/keypoints_device.cpp @@ -6,6 +6,7 @@ int main() { modelDescription.modelVersionSlug = "192x192"; modelDescription.platform = "RVC2"; std::string archivePath = dai::getModelFromZoo(modelDescription, true); + dai::NNArchive nnArchive(archivePath); dai::Pipeline pipeline; @@ -16,11 +17,9 @@ int main() { manip->initialConfig.setResize(192, 192); largeOutput->link(manip->inputImage); - auto nn = pipeline.create()->build(manip->out, dai::NNArchive(archivePath)); + auto nn = pipeline.create()->build(manip->out, nnArchive); - auto parser = pipeline.create(); - parser->setNumKeypoints(468); - parser->setScaleFactor(192); + auto parser = pipeline.create()->build(nnArchive); nn->out.link(parser->input); auto videoQ = largeOutput->createOutputQueue(); diff --git a/examples/cpp/RVC2/parsers/keypoints_host.cpp b/examples/cpp/RVC2/parsers/keypoints_host.cpp index 79ade62ca..c44f6ec3a 100644 --- a/examples/cpp/RVC2/parsers/keypoints_host.cpp +++ b/examples/cpp/RVC2/parsers/keypoints_host.cpp @@ -6,6 +6,7 @@ int main() { modelDescription.modelVersionSlug = "192x192"; modelDescription.platform = "RVC2"; std::string archivePath = dai::getModelFromZoo(modelDescription, true); + dai::NNArchive nnArchive(archivePath); dai::Pipeline pipeline; @@ -16,12 +17,10 @@ int main() { manip->initialConfig.setResize(192, 192); largeOutput->link(manip->inputImage); - auto nn = pipeline.create()->build(manip->out, dai::NNArchive(archivePath)); + auto nn = pipeline.create()->build(manip->out, nnArchive); - auto parser = pipeline.create(); + auto parser = pipeline.create()->build(nnArchive); parser->setRunOnHost(true); - parser->setNumKeypoints(468); - parser->setScaleFactor(192); nn->out.link(parser->input); auto videoQ = largeOutput->createOutputQueue(); diff --git a/examples/python/RVC2/parsers/keypoints.py b/examples/python/RVC2/parsers/keypoints.py index 9d4eab55f..57da3babc 100644 --- a/examples/python/RVC2/parsers/keypoints.py +++ b/examples/python/RVC2/parsers/keypoints.py @@ -23,9 +23,7 @@ nnArchive=nnArchive ) - parser = pipeline.create(dai.node.KeypointsParser) - parser.setNumKeypoints(468) - parser.setScaleFactor(192) + parser = pipeline.create(dai.node.KeypointsParser).build(nnArchive) parser.setRunOnHost(RUN_ON_HOST) nn.out.link(parser.input) diff --git a/include/depthai/pipeline/datatype/Keypoints.hpp b/include/depthai/pipeline/datatype/Keypoints.hpp index c616302c4..59523f7ba 100644 --- a/include/depthai/pipeline/datatype/Keypoints.hpp +++ b/include/depthai/pipeline/datatype/Keypoints.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include // project #include "depthai/pipeline/datatype/Buffer.hpp" @@ -11,6 +12,20 @@ namespace dai { +struct Keypoint { + float x = 0.f; + float y = 0.f; + // TODO(NicikD): + // should be + // std::optional z; + // std::optional confidence; + // but DEPTHAI_SERIALIZE_EXT doesn't support std::optional + float z = 0.f; + float confidence = -1.f; +}; + +DEPTHAI_SERIALIZE_EXT(Keypoint, x, y, z, confidence); + /** * Keypoints message. Carries keypoints data. */ @@ -23,7 +38,7 @@ namespace dai { ~Keypoints() override = default; /// Keypoints - std::vector keypoints; + std::vector keypoints; void serialize(std::vector& metadata, DatatypeEnum& datatype) const override { metadata = utility::serialize(*this); @@ -31,33 +46,36 @@ namespace dai { }; // getters - const std::vector& getKeypoints() const; + const std::vector& getKeypoints() const; // setters + Keypoints& setKeypoints(const std::vector& keypoints); /** - * Set 3D keypoints + * From 3D points * * @param keypoints detected 3D keypoints * @param scores confidence scores for each keypoint - * @param confidence_threshold confidence threshold + * @param confidence_threshold confidence threshold, filters out keypoints with confidence below threshold * * @returns keypoints message */ - Keypoints& setKeypoints(const std::vector& keypoints); - Keypoints& setKeypoints(const std::vector& keypoints, const std::vector& scores, float confidenceThreshold); + Keypoints& setKeypoints(const std::vector& points); + Keypoints& setKeypoints(const std::vector& points, const std::vector& scores); + Keypoints& setKeypoints(const std::vector& points, const std::vector& scores, float confidenceThreshold); /** - * Set 2D keypoints + * From 2D points * * @param keypoints detected 2D keypoints * @param scores confidence scores for each keypoint - * @param confidence_threshold confidence threshold + * @param confidence_threshold confidence threshold, filters out keypoints with confidence below threshold * * @returns keypoints message */ - Keypoints& setKeypoints(const std::vector& keypoints); - Keypoints& setKeypoints(const std::vector& keypoints, const std::vector& scores, float confidenceThreshold); + Keypoints& setKeypoints(const std::vector& points); + Keypoints& setKeypoints(const std::vector& points, const std::vector& scores); + Keypoints& setKeypoints(const std::vector& points, const std::vector& scores, float confidenceThreshold); DEPTHAI_SERIALIZE(Keypoints, Buffer::sequenceNum, Buffer::ts, Buffer::tsDevice, keypoints); }; diff --git a/include/depthai/pipeline/node/KeypointsParser.hpp b/include/depthai/pipeline/node/KeypointsParser.hpp index 90a8c9e5b..d46af0564 100644 --- a/include/depthai/pipeline/node/KeypointsParser.hpp +++ b/include/depthai/pipeline/node/KeypointsParser.hpp @@ -9,6 +9,7 @@ // shared #include "depthai/common/Point2f.hpp" #include "depthai/common/Point3f.hpp" +#include "depthai/nn_archive/NNArchive.hpp" #if defined(__clang__) #if __has_warning("-Wswitch-enum") @@ -44,6 +45,7 @@ namespace node { */ class KeypointsParser : public DeviceNodeCRTP, public HostRunnable { private: + bool isBuilt = false; bool runOnHostVar = false; public: @@ -60,6 +62,11 @@ class KeypointsParser : public DeviceNodeCRTP build(const NNArchive& nnArchive); + /** * Set the scale factor to divide the keypoints by. */ diff --git a/src/pipeline/datatype/Keypoints.cpp b/src/pipeline/datatype/Keypoints.cpp index 4ba639a01..623188133 100644 --- a/src/pipeline/datatype/Keypoints.cpp +++ b/src/pipeline/datatype/Keypoints.cpp @@ -2,20 +2,31 @@ namespace dai { -const std::vector& Keypoints::getKeypoints() const { +const std::vector& Keypoints::getKeypoints() const { return keypoints; } -// 3D keypoints -Keypoints& Keypoints::setKeypoints(const std::vector& keypoints) { +Keypoints& Keypoints::setKeypoints(const std::vector& keypoints) { this->keypoints = keypoints; return *this; } -Keypoints& Keypoints::setKeypoints(const std::vector& keypoints, const std::vector& scores, float confidenceThreshold) { - if (keypoints.size() != scores.size()) { + +// 3D keypoints +Keypoints& Keypoints::setKeypoints(const std::vector& points) { + std::vector keypoints = std::vector(points.size()); + for (size_t i = 0; i < points.size(); i++) { + keypoints[i].x = points[i].x; + keypoints[i].y = points[i].y; + keypoints[i].z = points[i].z; + }; + + return this->setKeypoints(keypoints); +} +Keypoints& Keypoints::setKeypoints(const std::vector& points, const std::vector& scores) { + if (points.size() != scores.size()) { throw std::invalid_argument("Keypoints and scores should have the same length. Got " - + std::to_string(keypoints.size()) + + std::to_string(points.size()) + " keypoints and " + std::to_string(scores.size()) + " scores."); @@ -27,41 +38,100 @@ Keypoints& Keypoints::setKeypoints(const std::vector& keypoints, const } } + std::vector keypoints = std::vector(points.size()); + for (size_t i = 0; i < points.size(); i++) { + keypoints[i].x = points[i].x; + keypoints[i].y = points[i].y; + keypoints[i].z = points[i].z; + keypoints[i].confidence = scores[i]; + }; + + return this->setKeypoints(keypoints); +} +Keypoints& Keypoints::setKeypoints(const std::vector& points, const std::vector& scores, float confidenceThreshold) { + if (points.size() != scores.size()) { + throw std::invalid_argument("Keypoints and scores should have the same length. Got " + + std::to_string(points.size()) + + " keypoints and " + + std::to_string(scores.size()) + + " scores."); + } + if (0 > confidenceThreshold || confidenceThreshold > 1) { throw std::invalid_argument("Confidence threshold should be between 0 and 1. Got " + std::to_string(confidenceThreshold) + "."); } std::vector filteredPoints = std::vector(); + std::vector filteredScores = std::vector(); - for (size_t i = 0; i < keypoints.size(); i++) { + for (size_t i = 0; i < points.size(); i++) { if (scores[i] >= confidenceThreshold) { - filteredPoints.push_back(keypoints[i]); + filteredPoints.push_back(points[i]); + filteredScores.push_back(scores[i]); } } - return this->setKeypoints(filteredPoints); + return this->setKeypoints(filteredPoints, filteredScores); } // 2D keypoints -Keypoints& Keypoints::setKeypoints(const std::vector& keypoints) { - std::vector points3d = std::vector(keypoints.size()); - for (size_t i = 0; i < keypoints.size(); i++) { - points3d[i].x = keypoints[i].x; - points3d[i].y = keypoints[i].y; - points3d[i].z = 0; +Keypoints& Keypoints::setKeypoints(const std::vector& points) { + std::vector keypoints = std::vector(points.size()); + for (size_t i = 0; i < points.size(); i++) { + keypoints[i].x = points[i].x; + keypoints[i].y = points[i].y; + }; + + return this->setKeypoints(keypoints); +} +Keypoints& Keypoints::setKeypoints(const std::vector& points, const std::vector& scores) { + if (points.size() != scores.size()) { + throw std::invalid_argument("Keypoints and scores should have the same length. Got " + + std::to_string(points.size()) + + " keypoints and " + + std::to_string(scores.size()) + + " scores."); } - return this->setKeypoints(points3d); + for (const auto& score: scores) { + if (0 > score || score > 1) { + throw std::invalid_argument("Scores should only contain values between 0 and 1."); + } + } + + std::vector keypoints = std::vector(points.size()); + for (size_t i = 0; i < points.size(); i++) { + keypoints[i].x = points[i].x; + keypoints[i].y = points[i].y; + keypoints[i].confidence = scores[i]; + }; + + return this->setKeypoints(keypoints); } -Keypoints& Keypoints::setKeypoints(const std::vector& keypoints, const std::vector& scores, float confidenceThreshold) { - std::vector points3d = std::vector(keypoints.size()); - for (size_t i = 0; i < keypoints.size(); i++) { - points3d[i].x = keypoints[i].x; - points3d[i].y = keypoints[i].y; - points3d[i].z = 0; +Keypoints& Keypoints::setKeypoints(const std::vector& points, const std::vector& scores, float confidenceThreshold) { + if (points.size() != scores.size()) { + throw std::invalid_argument("Keypoints and scores should have the same length. Got " + + std::to_string(points.size()) + + " keypoints and " + + std::to_string(scores.size()) + + " scores."); + } + + if (0 > confidenceThreshold || confidenceThreshold > 1) { + throw std::invalid_argument("Confidence threshold should be between 0 and 1. Got " + std::to_string(confidenceThreshold) + "."); + } + + std::vector filteredPoints = std::vector(); + std::vector filteredScores = std::vector(); + + for (size_t i = 0; i < points.size(); i++) { + if (scores[i] >= confidenceThreshold) { + filteredPoints.push_back(points[i]); + filteredScores.push_back(scores[i]); + } } - return this->setKeypoints(points3d, scores, confidenceThreshold); + return this->setKeypoints(filteredPoints, filteredScores); } } // namespace dai diff --git a/src/pipeline/node/KeypointsParser.cpp b/src/pipeline/node/KeypointsParser.cpp index 117e81fc1..5dbbe10ba 100644 --- a/src/pipeline/node/KeypointsParser.cpp +++ b/src/pipeline/node/KeypointsParser.cpp @@ -3,6 +3,28 @@ namespace dai { namespace node { +std::shared_ptr KeypointsParser::build(const NNArchive& nnArchive) { + if(isBuilt) { + throw std::runtime_error("KeypointsParser node is already built"); + } + + if (nnArchive.getConfig().getConfigV1().has_value() + && nnArchive.getConfig().getConfigV1().value().model.heads.has_value() + && !nnArchive.getConfig().getConfigV1().value().model.heads.value().empty()) { + nlohmann::json metadata = nnArchive.getConfig().getConfigV1().value().model.heads.value()[0].metadata.extraParams; + + if (metadata.contains("n_keypoints")) { + setNumKeypoints(metadata["n_keypoints"]); + } + if (metadata.contains("scale_factor")) { + setScaleFactor(metadata["scale_factor"]); + } + } + + isBuilt = true; + return std::static_pointer_cast(shared_from_this()); +} + void KeypointsParser::setScaleFactor(float scaleFactor) { properties.scaleFactor = scaleFactor; } @@ -24,6 +46,7 @@ bool KeypointsParser::runOnHost() const { return runOnHostVar; } +#ifdef DEPTHAI_XTENSOR_SUPPORT void KeypointsParser::run() { auto numKeypoints = properties.numKeypoints; auto scaleFactor = properties.scaleFactor; @@ -43,35 +66,35 @@ void KeypointsParser::run() { throw std::invalid_argument("Expected 1 output layer, got " + std::to_string(outputLayerNames.size())); } - xt::xarray keypoints = inputData->getFirstTensor(true); - int totalCoords = std::accumulate(keypoints.shape().begin(), keypoints.shape().end(), 1, std::multiplies()); + xt::xarray keypointsData = inputData->getFirstTensor(true); + int totalCoords = std::accumulate(keypointsData.shape().begin(), keypointsData.shape().end(), 1, std::multiplies()); if (numKeypoints * 2 != totalCoords && numKeypoints * 3 != totalCoords) { throw std::runtime_error("Expected 2 or 3 coordinates per keypoint, got " + std::to_string(static_cast(totalCoords) / static_cast(numKeypoints))); } int pointDimension = totalCoords / numKeypoints; - keypoints = keypoints.reshape({numKeypoints, pointDimension}); - keypoints /= scaleFactor; + keypointsData = keypointsData.reshape({numKeypoints, pointDimension}); + keypointsData /= scaleFactor; std::shared_ptr msg = std::make_shared(); if (pointDimension == 2) { - std::vector keypointsVector = std::vector(numKeypoints); + std::vector keypoints = std::vector(numKeypoints); for (int i = 0; i < numKeypoints; i++) { - keypointsVector[i].x = keypoints(i, 0); - keypointsVector[i].y = keypoints(i, 1); + keypoints[i].x = keypointsData(i, 0); + keypoints[i].y = keypointsData(i, 1); } - msg->setKeypoints(keypointsVector); + msg->setKeypoints(keypoints); } else { - std::vector keypointsVector = std::vector(numKeypoints); + std::vector keypoints = std::vector(numKeypoints); for (int i = 0; i < numKeypoints; i++) { - keypointsVector[i].x = keypoints(i, 0); - keypointsVector[i].y = keypoints(i, 1); - keypointsVector[i].z = keypoints(i, 2); + keypoints[i].x = keypointsData(i, 0); + keypoints[i].y = keypointsData(i, 1); + keypoints[i].z = keypointsData(i, 2); } - msg->setKeypoints(keypointsVector); + msg->setKeypoints(keypoints); } msg->setTimestamp(inputData->getTimestamp()); @@ -80,6 +103,11 @@ void KeypointsParser::run() { out.send(msg); } } +#else +void KeypointsParser::run() { + throw std::runtime_error("KeypointsParser node requires xtensor support"); +} +#endif } // namespace node } // namespace dai From 876ead8256ca7d435714000c0d4d7b461e07c5d5 Mon Sep 17 00:00:00 2001 From: NicikD Date: Wed, 25 Sep 2024 19:06:45 +0200 Subject: [PATCH 6/6] Changed Keypoint z and confidence member values to be std::optional instead of having a default value --- include/depthai/pipeline/datatype/Keypoints.hpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/include/depthai/pipeline/datatype/Keypoints.hpp b/include/depthai/pipeline/datatype/Keypoints.hpp index 59523f7ba..9ce227f25 100644 --- a/include/depthai/pipeline/datatype/Keypoints.hpp +++ b/include/depthai/pipeline/datatype/Keypoints.hpp @@ -9,19 +9,15 @@ // shared #include "depthai/common/Point2f.hpp" #include "depthai/common/Point3f.hpp" +#include "depthai/common/optional.hpp" namespace dai { struct Keypoint { float x = 0.f; float y = 0.f; - // TODO(NicikD): - // should be - // std::optional z; - // std::optional confidence; - // but DEPTHAI_SERIALIZE_EXT doesn't support std::optional - float z = 0.f; - float confidence = -1.f; + std::optional z; + std::optional confidence;; }; DEPTHAI_SERIALIZE_EXT(Keypoint, x, y, z, confidence);