C++ Demo - Object Detection (NanoDet) (#232)

* Functional and Refactored demo.cpp for MobileNet

* Fix inference timer text, added timer reset.

* Updated README.md, remove FPS text for single image processing

* Add matching saved image message

* Removing inference time printout for video inputs

* Update FPS text to 2 decimal places

* Update coding style for braces

* Address PR comments. Adjusted to C++11 standard.

* Addressed PR comments. Added C++11 cmake configuration, extracted classID and confidence function, and use NMSBoxesBatched now.

models/object_detection_nanodet/CMakeLists.txt

@@ -0,0 +1,32 @@
+cmake_minimum_required(VERSION 3.24)
+set(project_name "opencv_zoo_object_detection_nanodet")
+
+PROJECT (${project_name})
+
+set(OPENCV_VERSION "4.9.0")
+set(OPENCV_INSTALLATION_PATH "" CACHE PATH "Where to look for OpenCV installation")
+find_package(OpenCV ${OPENCV_VERSION} REQUIRED HINTS ${OPENCV_INSTALLATION_PATH})
+# Find OpenCV, you may need to set OpenCV_DIR variable
+# to the absolute path to the directory containing OpenCVConfig.cmake file
+# via the command line or GUI
+
+file(GLOB SourceFile
+    "demo.cpp")
+# If the package has been found, several variables will
+# be set, you can find the full list with descriptions
+# in the OpenCVConfig.cmake file.
+# Print some message showing some of them
+message(STATUS "OpenCV library status:")
+message(STATUS "    config: ${OpenCV_DIR}")
+message(STATUS "    version: ${OpenCV_VERSION}")
+message(STATUS "    libraries: ${OpenCV_LIBS}")
+message(STATUS "    include path: ${OpenCV_INCLUDE_DIRS}")
+
+# Declare the executable target built from your sources
+add_executable(${project_name} ${SourceFile})
+
+# Set C++ compilation standard to C++11
+set(CMAKE_CXX_STANDARD 11)
+
+# Link your application with OpenCV libraries
+target_link_libraries(${project_name} PRIVATE ${OpenCV_LIBS})

models/object_detection_nanodet/README.md

@@ -7,6 +7,8 @@ Note:
 
 ## Demo
 
+### Python
+
 Run the following command to try the demo: 
 ```shell
 # detect on camera input
@@ -17,6 +19,23 @@ python demo.py --input /path/to/image -v
 Note: 
 - image result saved as "result.jpg"
 
+### C++
+
+Install latest OpenCV and CMake >= 3.24.0 to get started with:
+
+```shell
+# A typical and default installation path of OpenCV is /usr/local
+cmake -B build -D OPENCV_INSTALLATION_PATH=/path/to/opencv/installation .
+cmake --build build
+
+# detect on camera input
+./build/opencv_zoo_object_detection_nanodet
+# detect on an image
+./build/opencv_zoo_object_detection_nanodet -i=/path/to/image
+# get help messages
+./build/opencv_zoo_object_detection_nanodet -h
+```
+
 
 ## Results
 

models/object_detection_nanodet/demo.cpp

@@ -0,0 +1,508 @@
+#include <vector>
+#include <string>
+#include <iostream>
+
+#include <opencv2/opencv.hpp>
+
+using namespace std;
+using namespace cv;
+using namespace dnn;
+
+const auto backendTargetPairs = vector<pair<Backend, Target>>
+{
+    {DNN_BACKEND_OPENCV, DNN_TARGET_CPU},
+    {DNN_BACKEND_CUDA, DNN_TARGET_CUDA},
+    {DNN_BACKEND_CUDA, DNN_TARGET_CUDA_FP16},
+    {DNN_BACKEND_TIMVX, DNN_TARGET_NPU},
+    {DNN_BACKEND_CANN, DNN_TARGET_NPU}
+};
+
+const vector<string> nanodetClassLabels = 
+{
+    "person", "bicycle", "car", "motorcycle", "airplane", "bus",
+    "train", "truck", "boat", "traffic light", "fire hydrant",
+    "stop sign", "parking meter", "bench", "bird", "cat", "dog",
+    "horse", "sheep", "cow", "elephant", "bear", "zebra", "giraffe",
+    "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee",
+    "skis", "snowboard", "sports ball", "kite", "baseball bat",
+    "baseball glove", "skateboard", "surfboard", "tennis racket",
+    "bottle", "wine glass", "cup", "fork", "knife", "spoon", "bowl",
+    "banana", "apple", "sandwich", "orange", "broccoli", "carrot",
+    "hot dog", "pizza", "donut", "cake", "chair", "couch",
+    "potted plant", "bed", "dining table", "toilet", "tv", "laptop",
+    "mouse", "remote", "keyboard", "cell phone", "microwave",
+    "oven", "toaster", "sink", "refrigerator", "book", "clock",
+    "vase", "scissors", "teddy bear", "hair drier", "toothbrush" 
+};
+
+class NanoDet 
+{
+public:
+    NanoDet(const String& modelPath, const float probThresh = 0.35, const float iouThresh = 0.6, 
+            const Backend bId = DNN_BACKEND_DEFAULT, const Target tId = DNN_TARGET_CPU) :
+        modelPath(modelPath), probThreshold(probThresh),
+        iouThreshold(iouThresh), backendId(bId), targetId(tId), 
+        imageShape(416, 416), regMax(7)
+    {
+        this->strides = { 8, 16, 32, 64 };
+        this->net = readNet(modelPath);
+        setBackendAndTarget(bId, tId);
+        this->project = Mat::zeros(1, this->regMax + 1, CV_32F);
+        for (size_t i = 0; i <= this->regMax; ++i) 
+        {
+            this->project.at<float>(0, i) = static_cast<float>(i);
+        }
+        this->mean = Scalar(103.53, 116.28, 123.675);
+        this->std = Scalar(1.0 / 57.375, 1.0 / 57.12, 1.0 / 58.395);
+        this->generateAnchors();
+    }
+
+    void setBackendAndTarget(Backend bId, Target tId)
+    {
+        this->net.setPreferableBackend(bId);
+        this->net.setPreferableTarget(tId);
+    }
+
+    Mat preProcess(const Mat& inputImage)
+    {
+        Image2BlobParams paramNanodet;
+        paramNanodet.datalayout = DNN_LAYOUT_NCHW;
+        paramNanodet.ddepth = CV_32F;
+        paramNanodet.mean = this->mean;
+        paramNanodet.scalefactor = this->std;
+        paramNanodet.size = this->imageShape;
+        Mat blob;
+        blobFromImageWithParams(inputImage, blob, paramNanodet);
+        return blob;
+    }
+
+    Mat infer(const Mat& sourceImage)
+    {
+        Mat blob = this->preProcess(sourceImage);
+        this->net.setInput(blob);
+        vector<Mat> modelOutput;
+        this->net.forward(modelOutput, this->net.getUnconnectedOutLayersNames());
+        Mat preds = this->postProcess(modelOutput);
+        return preds;
+    }
+
+    Mat reshapeIfNeeded(const Mat& input) 
+    {
+        if (input.dims == 3)
+        {
+            return input.reshape(0, input.size[1]);
+        }
+        return input;
+    }
+
+    Mat softmaxActivation(const Mat& input) 
+    {
+        Mat x_exp, x_sum, x_repeat_sum, result;
+        exp(input.reshape(0, input.total() / (this->regMax + 1)), x_exp);
+        reduce(x_exp, x_sum, 1, REDUCE_SUM, CV_32F);
+        repeat(x_sum, 1, this->regMax + 1, x_repeat_sum);
+        divide(x_exp, x_repeat_sum, result);
+        return result;
+    }
+
+    Mat applyProjection(Mat& input) 
+    {
+        Mat repeat_project;
+        repeat(this->project, input.rows, 1, repeat_project);
+        multiply(input, repeat_project, input);
+        reduce(input, input, 1, REDUCE_SUM, CV_32F);
+        Mat projection = input.col(0).clone();
+        return projection.reshape(0, projection.total() / 4);
+    }
+
+    void preNMS(Mat& anchors, Mat& bbox_pred, Mat& cls_score, const int nms_pre = 1000)
+    {
+        Mat max_scores;
+        reduce(cls_score, max_scores, 1, REDUCE_MAX);
+
+        Mat indices;
+        sortIdx(max_scores.t(), indices, SORT_DESCENDING);
+
+        Mat indices_float = indices.colRange(0, nms_pre);
+        Mat selected_anchors, selected_bbox_pred, selected_cls_score;
+        for (int j = 0; j < indices_float.cols; ++j) 
+        {
+            selected_anchors.push_back(anchors.row(indices_float.at<int>(j)));
+            selected_bbox_pred.push_back(bbox_pred.row(indices_float.at<int>(j)));
+            selected_cls_score.push_back(cls_score.row(indices_float.at<int>(j)));
+        }
+
+        anchors = selected_anchors;
+        bbox_pred = selected_bbox_pred;
+        cls_score = selected_cls_score;
+    }
+
+    void clipBoundingBoxes(Mat& x1, Mat& y1, Mat& x2, Mat& y2) 
+    {
+        Mat zeros = Mat::zeros(x1.size(), x1.type());
+        x1 = min(max(x1, zeros), Scalar(this->imageShape.width - 1));
+        y1 = min(max(y1, zeros), Scalar(this->imageShape.height - 1));
+        x2 = min(max(x2, zeros), Scalar(this->imageShape.width - 1));
+        y2 = min(max(y2, zeros), Scalar(this->imageShape.height - 1));
+    }
+
+    Mat calculateBoundingBoxes(const Mat& anchors, const Mat& bbox_pred) 
+    {
+        Mat x1 = anchors.col(0) - bbox_pred.col(0);
+        Mat y1 = anchors.col(1) - bbox_pred.col(1);
+        Mat x2 = anchors.col(0) + bbox_pred.col(2);
+        Mat y2 = anchors.col(1) + bbox_pred.col(3);
+
+        clipBoundingBoxes(x1, y1, x2, y2);
+
+        Mat bboxes;
+        hconcat(vector<Mat>{x1, y1, x2, y2}, bboxes);
+
+        return bboxes;
+    }
+
+    vector<Rect2d> bboxMatToRect2d(const Mat& bboxes)
+    {
+        Mat bboxes_wh(bboxes.clone());
+        bboxes_wh.colRange(2, 4) = bboxes_wh.colRange(2, 4) -= bboxes_wh.colRange(0, 2);
+        vector<Rect2d> boxesXYXY;
+        for (size_t i = 0; i < bboxes_wh.rows; i++) 
+        {
+            boxesXYXY.emplace_back(bboxes.at<float>(i, 0),
+                                   bboxes.at<float>(i, 1),
+                                   bboxes.at<float>(i, 2),
+                                   bboxes.at<float>(i, 3));
+        }
+        return boxesXYXY;
+    }
+
+    Mat postProcess(const vector<Mat>& preds)
+    {
+        vector<Mat> cls_scores, bbox_preds;
+        for (size_t i = 0; i < preds.size(); i += 2) 
+        {
+            cls_scores.push_back(preds[i]);
+            bbox_preds.push_back(preds[i + 1]);
+        }
+
+        vector<Mat> bboxes_mlvl;
+        vector<Mat> scores_mlvl;
+
+        for (size_t i = 0; i < strides.size(); ++i) 
+        {
+            if (i >= cls_scores.size() || i >= bbox_preds.size()) continue;
+            // Extract necessary data
+            int stride = strides[i];
+            Mat cls_score = reshapeIfNeeded(cls_scores[i]);
+            Mat bbox_pred = reshapeIfNeeded(bbox_preds[i]);
+            Mat anchors = anchorsMlvl[i].t();
+
+            // Softmax activation, projection, and calculate bounding boxes
+            bbox_pred = softmaxActivation(bbox_pred);
+            bbox_pred = applyProjection(bbox_pred);
+            bbox_pred = stride * bbox_pred;
+
+            const int nms_pre = 1000;
+            if (nms_pre > 0 && cls_score.rows > nms_pre) 
+            {
+                preNMS(anchors, bbox_pred, cls_score, nms_pre);
+            }
+            
+            Mat bboxes = calculateBoundingBoxes(anchors, bbox_pred);
+
+            
+            bboxes_mlvl.push_back(bboxes);
+            scores_mlvl.push_back(cls_score);
+        }
+        Mat bboxes;
+        Mat scores;
+        vconcat(bboxes_mlvl, bboxes);
+        vconcat(scores_mlvl, scores);
+
+        vector<Rect2d> boxesXYXY = bboxMatToRect2d(bboxes);
+        vector<int> classIds;
+        vector<float> confidences;
+        for (size_t i = 0; i < scores.rows; ++i) 
+        {
+            Point maxLoc;
+            minMaxLoc(scores.row(i), nullptr, nullptr, nullptr, &maxLoc);
+            classIds.push_back(maxLoc.x);
+            confidences.push_back(scores.at<float>(i, maxLoc.x));
+        }
+
+        vector<int> indices;
+        NMSBoxesBatched(boxesXYXY, confidences, classIds, probThreshold, iouThreshold, indices);
+        Mat result(int(indices.size()), 6, CV_32FC1);
+        int row = 0;
+        for (auto idx : indices)
+        {
+            bboxes.rowRange(idx, idx + 1).copyTo(result(Rect(0, row, 4, 1)));
+            result.at<float>(row, 4) = confidences[idx];
+            result.at<float>(row, 5) = static_cast<float>(classIds[idx]);
+            row++;
+        }
+        if (indices.size() == 0)
+        {
+            return Mat();
+        }
+        return result;
+    }
+
+    void generateAnchors()
+    {
+        for (const int stride : strides) {
+            int feat_h = this->imageShape.height / stride;
+            int feat_w = this->imageShape.width / stride;
+            
+            vector<Mat> anchors;
+            
+            for (int y = 0; y < feat_h; ++y) 
+            {
+                for (int x = 0; x < feat_w; ++x) 
+                {
+                    float shift_x = x * stride;
+                    float shift_y = y * stride;
+                    float cx = shift_x + 0.5 * (stride - 1);
+                    float cy = shift_y + 0.5 * (stride - 1);
+                    Mat anchor_point = (Mat_<float>(2, 1) << cx, cy);
+                    anchors.push_back(anchor_point);
+                }
+            }
+            Mat anchors_mat;
+            hconcat(anchors, anchors_mat);
+            this->anchorsMlvl.push_back(anchors_mat);
+        }
+    }
+private:
+    Net net;
+    String modelPath;
+    vector<int> strides;
+    Size imageShape;
+    int regMax;
+    float probThreshold;
+    float iouThreshold;
+    Backend backendId;
+    Target targetId;
+    Mat project;
+    Scalar mean;
+    Scalar std;
+    vector<Mat> anchorsMlvl;
+};
+
+// Function to resize and pad an image and return both the image and scale information
+tuple<Mat, vector<double>> letterbox(const Mat& sourceImage, const Size& target_size = Size(416, 416)) 
+{
+    Mat img = sourceImage.clone();
+
+    double top = 0, left = 0, newh = target_size.height, neww = target_size.width;
+
+    if (img.rows != img.cols) 
+    {
+        double hw_scale = static_cast<double>(img.rows) / img.cols;
+        if (hw_scale > 1) 
+        {
+            newh = target_size.height;
+            neww = static_cast<int>(target_size.width / hw_scale);
+            resize(img, img, Size(neww, newh), 0, 0, INTER_AREA);
+            left = static_cast<int>((target_size.width - neww) * 0.5);
+            copyMakeBorder(img, img, 0, 0, left, target_size.width - neww - left, BORDER_CONSTANT, Scalar(0));
+        } 
+        else 
+        {
+            newh = static_cast<int>(target_size.height * hw_scale);
+            neww = target_size.width;
+            resize(img, img, Size(neww, newh), 0, 0, INTER_AREA);
+            top = static_cast<int>((target_size.height - newh) * 0.5);
+            copyMakeBorder(img, img, top, target_size.height - newh - top, 0, 0, BORDER_CONSTANT, Scalar(0));
+        }
+    } 
+    else 
+    {
+        resize(img, img, target_size, 0, 0, INTER_AREA);
+    }
+    vector<double> letterbox_scale = {top, left, newh, neww};
+
+    return make_tuple(img, letterbox_scale);
+}
+
+// Function to scale bounding boxes back to original image coordinates
+vector<int> unletterbox(const Mat& bbox, const Size& original_image_shape, const vector<double>& letterbox_scale) 
+{
+    vector<int> ret(bbox.cols);
+
+    int h = original_image_shape.height;
+    int w = original_image_shape.width;
+    double top = letterbox_scale[0];
+    double left = letterbox_scale[1];
+    double newh = letterbox_scale[2];
+    double neww = letterbox_scale[3];
+
+    if (h == w) 
+    {
+        double ratio = static_cast<double>(h) / newh;
+        for (int& val : ret) 
+        {
+            val = static_cast<int>(val * ratio);
+        }
+        return ret;
+    }
+
+    double ratioh = static_cast<double>(h) / newh;
+    double ratiow = static_cast<double>(w) / neww;
+    ret[0] = max(static_cast<int>((bbox.at<float>(0) - left) * ratiow), 0);
+    ret[1] = max(static_cast<int>((bbox.at<float>(1) - top) * ratioh), 0);
+    ret[2] = min(static_cast<int>((bbox.at<float>(2) - left) * ratiow), w);
+    ret[3] = min(static_cast<int>((bbox.at<float>(3) - top) * ratioh), h);
+
+    return ret;
+}
+
+// Function to visualize predictions on an image
+Mat visualize(const Mat& preds, const Mat& result_image, const vector<double>& letterbox_scale, bool video, double fps = 0.0) 
+{
+    Mat visualized_image = result_image.clone();
+
+    // Draw FPS if provided
+    if (fps > 0.0 && video) 
+    {
+        std::ostringstream fps_stream;
+        fps_stream << "FPS: " << std::fixed << std::setprecision(2) << fps;
+        putText(visualized_image, fps_stream.str(), Point(10, 25), FONT_HERSHEY_SIMPLEX, 1, Scalar(0, 0, 255), 2);
+    }
+
+    // Draw bounding boxes and labels for each prediction
+    for (size_t i = 0; i < preds.rows; i++) 
+    {
+        Mat pred = preds.row(i);
+        Mat bbox = pred.colRange(0, 4);
+        double conf = pred.at<float>(4);
+        int classid = static_cast<int>(pred.at<float>(5));
+
+        // Convert bbox coordinates back to original image space
+        vector<int> unnormalized_bbox = unletterbox(bbox, visualized_image.size(), letterbox_scale);
+
+        // Draw bounding box
+        rectangle(visualized_image, Point(unnormalized_bbox[0], unnormalized_bbox[1]),
+                      Point(unnormalized_bbox[2], unnormalized_bbox[3]), Scalar(0, 255, 0), 2);
+
+        // Draw label
+        stringstream label;
+        label << nanodetClassLabels[classid] << ": " << fixed << setprecision(2) << conf;
+        putText(visualized_image, label.str(), Point(unnormalized_bbox[0], unnormalized_bbox[1] - 10),
+                  FONT_HERSHEY_SIMPLEX, 1, Scalar(0, 255, 0), 2);
+    }
+
+    return visualized_image;
+}
+
+void processImage(Mat& inputImage, NanoDet& nanodet, TickMeter& tm, bool save, bool vis, bool video)
+{
+    cvtColor(inputImage, inputImage, COLOR_BGR2RGB);
+    tuple<Mat, vector<double>> w = letterbox(inputImage);
+    Mat inputBlob = get<0>(w);
+    vector<double> letterboxScale  = get<1>(w);
+    
+    tm.start();
+    Mat predictions = nanodet.infer(inputBlob);
+    tm.stop();
+    if (!video)
+    {
+        cout << "Inference time: " << tm.getTimeMilli() << " ms\n";
+    }
+
+    Mat img = visualize(predictions, inputImage, letterboxScale, video, tm.getFPS());
+    cvtColor(img, img, COLOR_BGR2RGB);
+    if (save)
+    {
+        static const string kOutputName = "result.jpg";
+        imwrite(kOutputName, img);
+        if (!video)
+        {
+            cout << "Results saved to " + kOutputName << endl;
+        }
+    }
+    if (vis)
+    {
+        static const string kWinName = "model";
+        imshow(kWinName, img);
+    }
+}
+
+
+const String keys =
+        "{ help  h          |                                               | Print help message. }"
+        "{ model m          | object_detection_nanodet_2022nov.onnx         | Usage: Path to the model, defaults to object_detection_nanodet_2022nov.onnx  }"
+        "{ input i          |                                               | Path to the input image. Omit for using the default camera.}"
+        "{ confidence       | 0.35                                          | Class confidence }"
+        "{ nms              | 0.6                                           | Enter nms IOU threshold }"
+        "{ save s           | true                                          | Specify to save results. This flag is invalid when using the camera. }"
+        "{ vis v            | true                                          | Specify to open a window for result visualization. This flag is invalid when using the camera. }"
+        "{ backend bt       | 0                                             | Choose one of computation backends: "
+        "0: (default) OpenCV implementation + CPU, "
+        "1: CUDA + GPU (CUDA), "
+        "2: CUDA + GPU (CUDA FP16), "
+        "3: TIM-VX + NPU, "
+        "4: CANN + NPU}";
+
+int main(int argc, char** argv)
+{
+    CommandLineParser parser(argc, argv, keys);
+
+    parser.about("Use this script to run Nanodet inference using OpenCV, a contribution by Sri Siddarth Chakaravarthy as part of GSOC_2022.");
+    if (parser.has("help"))
+    {
+        parser.printMessage();
+        return 0;
+    }
+
+    string model = parser.get<String>("model");
+    string inputPath = parser.get<String>("input");
+    float confThreshold = parser.get<float>("confidence");
+    float nmsThreshold = parser.get<float>("nms");
+    bool save = parser.get<bool>("save");
+    bool vis = parser.get<bool>("vis");
+    int backendTargetid = parser.get<int>("backend");
+
+    if (model.empty())
+    {
+        CV_Error(Error::StsError, "Model file " + model + " not found");
+    }
+
+    NanoDet nanodet(model, confThreshold, nmsThreshold,
+                    backendTargetPairs[backendTargetid].first, backendTargetPairs[backendTargetid].second);
+
+    TickMeter tm;
+    if (parser.has("input"))
+    {
+        Mat inputImage = imread(samples::findFile(inputPath));
+        static const bool kNotVideo = false;
+        processImage(inputImage, nanodet, tm, save, vis, kNotVideo);
+        waitKey(0);
+    }
+    else
+    {
+        VideoCapture cap;
+        cap.open(0);
+        if (!cap.isOpened())
+        {
+            CV_Error(Error::StsError, "Cannot open video or file");
+        }
+
+        Mat frame;
+        while (waitKey(1) < 0)
+        {
+            cap >> frame;
+            if (frame.empty())
+            {
+                cout << "Frame is empty" << endl;
+                waitKey();
+                break;
+            }
+            tm.reset();
+            static const bool kIsVideo = true;
+            processImage(frame, nanodet, tm, save, vis, kIsVideo);
+        }
+        cap.release();
+    }
+    return 0;
+}