Upload 27 files

model_farm_yolov8s_qcs6490_qnn2.16_int8_aidlite/README.md

@@ -0,0 +1,59 @@
+## Model Information
+### Source model
+- Input shape: 640x640
+- Number of parameters: 10.65M
+- Model size: 42.7M
+- Output shape: 1x84x8400
+
+Source model repository: [yolov8](https://github.com/ultralytics/ultralytics)
+
+### Converted model
+
+- Precision: INT8
+- Backend: QNN2.16
+- Target Device: FV01 QCS6490
+
+## Model Conversion Reference
+User can find model conversion reference at [aimo.aidlux.com](https://aimo.aidlux.com/#/public/04384e46-ee9f-4c6d-875b-daa4702aa6d0)
+
+## Inference with AidLite SDK
+
+### SDK installation
+Model Farm uses AidLite SDK as the model inference SDK. For details, please refer to the [AidLite Developer Documentation](https://v2.docs.aidlux.com/en/sdk-api/aidlite-sdk/)
+
+- install AidLite SDK
+
+```bash
+# Install the appropriate version of the aidlite sdk
+sudo aid-pkg update
+sudo aid-pkg install aidlite-sdk
+# Download the qnn version that matches the above backend. Eg Install QNN2.23 Aidlite: sudo aid-pkg install aidlite-qnn223
+sudo aid-pkg install aidlite-{QNN VERSION}
+# eg: Install QNN 2.23 Aidlite: sudo aid-pkg install aidlite-qnn223
+```
+
+- Verify AidLite SDK
+
+```bash
+# aidlite sdk c++ check
+python3 -c "import aidlite ; print(aidlite.get_library_version())"
+
+# aidlite sdk python check
+python3 -c "import aidlite ; print(aidlite.get_py_library_version())"
+```
+
+### Run Demo
+
+#### python
+```bash
+cd model_farm_yolov8s_qcs6490_qnn2.16_int8_aidlite
+python3  python/run_test.py --target_model ./models/cutoff_yolov8s_w8a8.qnn216.ctx.bin --imgs ./python/bus.jpg  --invoke_nums 10
+```
+
+#### cpp
+```bash
+cd model_farm_yolov8s_qcs6490_qnn2.16_int8_aidlite/cpp
+mkdir -p build && cd build
+cmake .. && make
+./run_test --target_model ../../models/cutoff_yolov8s_w8a8.qnn216.ctx.bin --imgs ../bus.jpg  --invoke_nums 10
+```

model_farm_yolov8s_qcs6490_qnn2.16_int8_aidlite/cpp/CMakeLists.txt

@@ -0,0 +1,30 @@
+cmake_minimum_required (VERSION 3.5)
+project("run_test")
+
+find_package(OpenCV REQUIRED)
+
+message(STATUS "oPENCV Library status:")
+message(STATUS ">version:${OpenCV_VERSION}")
+message(STATUS "Include:${OpenCV_INCLUDE_DIRS}")
+
+set(CMAKE_CXX_FLAGS "-Wno-error=deprecated-declarations -Wno-deprecated-declarations")
+
+include_directories(
+    /usr/local/include
+    /usr/include/opencv4
+)
+
+link_directories(
+    /usr/local/lib/
+)
+
+file(GLOB SRC_LISTS 
+    ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp  
+)
+
+add_executable(run_test ${SRC_LISTS})
+
+target_link_libraries(run_test
+    aidlite
+	${OpenCV_LIBS}
+)

model_farm_yolov8s_qcs6490_qnn2.16_int8_aidlite/cpp/main.cpp

@@ -0,0 +1,349 @@
+#include <iostream>
+#include <string>
+#include <algorithm>
+#include <cctype>
+#include <opencv2/opencv.hpp>
+#include <aidlux/aidlite/aidlite.hpp>
+#include <vector>
+#include <numeric>
+
+const float INPUT_WIDTH = 640.0;
+const float INPUT_HEIGHT = 640.0;
+const float SCORE_THRESHOLD = 0.25;
+const float NMS_THRESHOLD = 0.45;
+const float CONFIDENCE_THRESHOLD = 0.25;
+const uint32_t size = 640;
+const uint32_t out_size = 8400;
+
+const int FONT_FACE = cv::FONT_HERSHEY_SIMPLEX;
+cv::Scalar WHITE = cv::Scalar(255,255,255);
+
+const float FONT_SCALE = 0.75;
+const int THICKNESS = 1;
+
+const std::vector<std::string> class_list = {
+    "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train",
+    "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter",
+    "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear",
+    "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase",
+    "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat",
+    "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle",
+    "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple",
+    "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut",
+    "cake", "chair", "couch", "potted plant", "bed", "dining table", "toilet",
+    "TV", "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave",
+    "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase",
+    "scissors", "teddy bear", "hair drier", "toothbrush"
+};
+
+using namespace Aidlux::Aidlite;
+
+struct Args {
+    std::string target_model = "../../models/cutoff_yolov8s_w8a8.qnn216.ctx.bin.aidem";
+    std::string imgs = "../bus.jpg";
+    int invoke_nums = 10;
+    std::string model_type = "QNN";
+};
+
+Args parse_args(int argc, char* argv[]) {
+    Args args;
+    for (int i = 1; i < argc; ++i) {
+        std::string arg = argv[i];
+        if (arg == "--target_model" && i + 1 < argc) {
+            args.target_model = argv[++i];
+        } else if (arg == "--imgs" && i + 1 < argc) {
+            args.imgs = argv[++i];
+        } else if (arg == "--invoke_nums" && i + 1 < argc) {
+            args.invoke_nums = std::stoi(argv[++i]);
+        } else if (arg == "--model_type" && i + 1 < argc) {
+            args.model_type = argv[++i];
+        }
+    }
+    return args;
+}
+
+std::string to_lower(const std::string& str) {
+    std::string lower_str = str;
+    std::transform(lower_str.begin(), lower_str.end(), lower_str.begin(), [](unsigned char c) {
+        return std::tolower(c);
+    });
+    return lower_str;
+}
+
+
+void concatenate(float* qnn_trans_data, float* qnn_mul_data, int batch, int num_elements, int trans_dim, int mul_dim, std::vector<float>& output) {
+    int out_dim = trans_dim + mul_dim + 1; 
+    output.resize(batch * num_elements * out_dim); 
+    for (int i = 0; i < batch * num_elements; ++i) {
+        std::memcpy(&output[i * out_dim], &qnn_mul_data[i * mul_dim], mul_dim * sizeof(float));
+        float max_val = *std::max_element(&qnn_trans_data[i * trans_dim], &qnn_trans_data[i * trans_dim + trans_dim]);
+        output[i * out_dim + 4] = max_val;
+        std::memcpy(&output[i * out_dim + 5], &qnn_trans_data[i * trans_dim], trans_dim * sizeof(float));
+    }
+}
+
+void transformData(const float* input, float* output, int C, int N) {
+    for (int c = 0; c < C; ++c) {
+        for (int n = 0; n < N; ++n) {
+            output[n * C + c] = input[c * N + n];
+        }
+    }
+}
+
+double img_process(cv::Mat frame, cv::Mat &img_input, int size) {
+    cv::Mat img_processed = frame.clone();
+    int height = img_processed.rows;
+    int width = img_processed.cols;
+    int length = std::max(height, width);
+    double scala = static_cast<double>(length) / size;
+    
+    cv::Mat image = cv::Mat::zeros(cv::Size(length, length), CV_8UC3);
+    img_processed.copyTo(image(cv::Rect(0, 0, width, height)));
+    
+    cv::cvtColor(image, img_input, cv::COLOR_BGR2RGB);
+    cv::resize(img_input, img_input, cv::Size(size, size));
+    
+    cv::Mat mean_data = cv::Mat::zeros(img_input.size(), CV_32FC3);
+    cv::Mat std_data(img_input.size(), CV_32FC3, cv::Scalar(255, 255, 255));
+    img_input.convertTo(img_input, CV_32FC3);
+    img_input = (img_input - mean_data) / std_data;
+    return scala;
+}
+
+
+cv::Scalar generate_colors(int i, bool bgr = false) {
+    static const std::vector<std::string> hex_colors = {
+        "FF3838", "FF9D97", "FF701F", "FFB21D", "CFD231", "48F90A",
+        "92CC17", "3DDB86", "1A9334", "00D4BB", "2C99A8", "00C2FF",
+        "344593", "6473FF", "0018EC", "8438FF", "520085", "CB38FF",
+        "FF95C8", "FF37C7"
+    };
+
+    int num = hex_colors.size();
+    std::string hex = hex_colors[i % num];
+
+    int r = std::stoi(hex.substr(0, 2), nullptr, 16);
+    int g = std::stoi(hex.substr(2, 2), nullptr, 16);
+    int b = std::stoi(hex.substr(4, 2), nullptr, 16);
+
+    if (bgr)
+        return cv::Scalar(b, g, r);
+    else
+        return cv::Scalar(r, g, b);
+}
+
+void draw_label(cv::Mat& input_image, std::string label, int left, int top, cv::Scalar color)
+{
+    int baseLine;
+    cv::Size label_size = cv::getTextSize(label, FONT_FACE, FONT_SCALE, THICKNESS, &baseLine);
+    int y = top - label_size.height - baseLine;
+    if (y < 0) {
+        y = top ;
+    }
+    cv::Point tlc(left, y);
+    cv::Point brc(left + label_size.width, y + label_size.height + baseLine);
+    rectangle(input_image, tlc, brc, color, cv::FILLED);
+    putText(input_image, label, cv::Point(left, y + label_size.height), FONT_FACE, FONT_SCALE, WHITE, THICKNESS);
+}
+
+
+cv::Mat post_process(cv::Mat &input_image, std::vector<float> &outputs, const std::vector<std::string> &class_name, const double ratio)
+{
+    // Initialize vectors to hold respective outputs while unwrapping detections.
+    std::vector<int> class_ids;
+    std::vector<float> confidences;
+    std::vector<cv::Rect> boxes;
+    
+    // Iterate through outputs for each box prediction
+    for (int i = 0; i < outputs.size(); i+=85)
+    {
+        float confidence = outputs[i+4];
+        if (confidence >= CONFIDENCE_THRESHOLD)
+        {
+            // Create a 1x80 Mat and store class scores of 80 classes.
+            cv::Mat scores(1, class_name.size(), CV_32FC1, outputs.data() + i + 5);
+            cv::Point class_id;
+            double max_class_score;
+            
+            // For multi-label, check each class score
+            for (int c = 0; c < class_name.size(); c++) {
+                float class_score = scores.at<float>(0, c);
+                
+                // If class score is above threshold, consider this class for the box
+                if (class_score > SCORE_THRESHOLD) {
+                    // Store class ID and confidence in the pre-defined respective vectors.
+                    confidences.push_back(confidence * class_score);  // Multiply with confidence
+                    class_ids.push_back(c);  // class index
+                    // Center and box dimension.
+                    float cx = outputs[i];
+                    float cy = outputs[i+1];
+                    float w = outputs[i+2];
+                    float h = outputs[i+3];
+
+                    int left = int((cx - 0.5 * w) * ratio);
+                    int top = int((cy - 0.5 * h) * ratio);
+                    int width = int(w * ratio);
+                    int height = int(h * ratio);
+
+                    // Store good detections in the boxes vector.
+                    boxes.push_back(cv::Rect(left, top, width, height));
+                }
+            }
+        }
+    }
+
+    // Perform Non Maximum Suppression and draw predictions.
+    std::vector<int> indices;
+    cv::dnn::NMSBoxes(boxes, confidences, SCORE_THRESHOLD, NMS_THRESHOLD, indices);
+    printf("Detected {%ld} targets.\n", indices.size());
+    
+    // Loop over NMS results and draw bounding boxes
+    for (int i = 0; i < indices.size(); i++)
+    {
+        int idx = indices[i];
+        cv::Rect box = boxes[idx];
+
+        int left = box.x;
+        int top = box.y;
+        int width = box.width;
+        int height = box.height;
+        cv::Scalar color = generate_colors(class_ids[idx]);
+        // Draw bounding box.
+        rectangle(input_image, cv::Point(left, top), cv::Point(left + width, top + height), color, 2*THICKNESS);
+
+        // Get the label for the class name and its confidence.
+        std::string label = cv::format("%.2f", confidences[idx]);
+        label = class_name[class_ids[idx]] + ":" + label;
+        // Draw class labels.
+        draw_label(input_image, label, left, top, color);
+    }
+    printf("Processing finished.\n");
+    return input_image;
+}
+
+int invoke(const Args& args) {
+    std::cout << "Start main ... ... Model Path: " << args.target_model << "\n"
+              << "Image Path: " << args.imgs << "\n"
+              << "Inference Nums: " << args.invoke_nums << "\n"
+              << "Model Type: " << args.model_type << "\n";
+    Model* model = Model::create_instance(args.target_model);
+    if(model == nullptr){
+        printf("Create model failed !\n");
+        return EXIT_FAILURE;
+    }
+    Config* config = Config::create_instance();
+    if(config == nullptr){
+        printf("Create config failed !\n");
+        return EXIT_FAILURE;
+    }
+    config->implement_type = ImplementType::TYPE_LOCAL;
+    std::string model_type_lower = to_lower(args.model_type);
+    if (model_type_lower == "qnn"){
+        config->framework_type = FrameworkType::TYPE_QNN;
+    } else if (model_type_lower == "snpe2" || model_type_lower == "snpe") {
+        config->framework_type = FrameworkType::TYPE_SNPE2;
+    }
+    config->accelerate_type = AccelerateType::TYPE_DSP;
+    config->is_quantify_model = 1;
+
+    std::vector<std::vector<uint32_t>> input_shapes = {{1, size, size, 3}};
+    std::vector<std::vector<uint32_t>> output_shapes = {{1, 4, out_size}, {1, 80, out_size}};
+    model->set_model_properties(input_shapes, DataType::TYPE_FLOAT32, output_shapes, DataType::TYPE_FLOAT32);
+    std::unique_ptr<Interpreter> fast_interpreter = InterpreterBuilder::build_interpretper_from_model_and_config(model, config);
+    if(fast_interpreter == nullptr){
+        printf("build_interpretper_from_model_and_config failed !\n");
+        return EXIT_FAILURE;
+    }
+    int result = fast_interpreter->init();
+    if(result != EXIT_SUCCESS){
+        printf("interpreter->init() failed !\n");
+        return EXIT_FAILURE;
+    }
+    // load model
+    fast_interpreter->load_model();
+    if(result != EXIT_SUCCESS){
+        printf("interpreter->load_model() failed !\n");
+        return EXIT_FAILURE;
+    }
+    printf("detect model load success!\n");
+
+    cv::Mat frame = cv::imread(args.imgs);
+    if (frame.empty()) {
+        printf("detect image load failed!\n");
+        return 1;
+    }
+    printf("img_src cols: %d, img_src rows: %d\n", frame.cols, frame.rows);
+    cv::Mat input_img;
+    double scale = img_process(frame, input_img, size);
+    if (input_img.empty()) {
+        printf("detect input_img load failed!\n");
+        return 1;
+    }
+
+    float *qnn_trans_data = nullptr;
+    float *qnn_mul_data = nullptr;
+
+    std::vector<float> invoke_time;
+    for (int i = 0; i < args.invoke_nums; ++i) {
+        result = fast_interpreter->set_input_tensor(0, input_img.data);
+        if(result != EXIT_SUCCESS){
+            printf("interpreter->set_input_tensor() failed !\n");
+            return EXIT_FAILURE;
+        }
+          // 开始计时
+        auto t1 = std::chrono::high_resolution_clock::now();
+        result = fast_interpreter->invoke();
+        auto t2 = std::chrono::high_resolution_clock::now();
+        std::chrono::duration<double> cost_time = t2 - t1;
+        invoke_time.push_back(cost_time.count() * 1000);
+        if(result != EXIT_SUCCESS){
+            printf("interpreter->invoke() failed !\n");
+            return EXIT_FAILURE;
+        }
+        uint32_t out_data_1 = 0;
+        result = fast_interpreter->get_output_tensor(0, (void**)&qnn_trans_data, &out_data_1);
+        if(result != EXIT_SUCCESS){
+            printf("interpreter->get_output_tensor() 1 failed !\n");
+            return EXIT_FAILURE;
+        }
+        uint32_t out_data_2 = 0;
+        result = fast_interpreter->get_output_tensor(1, (void**)&qnn_mul_data, &out_data_2);
+        if(result != EXIT_SUCCESS){
+            printf("interpreter->get_output_tensor() 2 failed !\n");
+            return EXIT_FAILURE;
+        }
+    }
+
+    float max_invoke_time = *std::max_element(invoke_time.begin(), invoke_time.end());
+    float min_invoke_time = *std::min_element(invoke_time.begin(), invoke_time.end());
+    float mean_invoke_time = std::accumulate(invoke_time.begin(), invoke_time.end(), 0.0f) / args.invoke_nums;
+    float var_invoketime = 0.0f;
+    for (auto time : invoke_time) {
+        var_invoketime += (time - mean_invoke_time) * (time - mean_invoke_time);
+    }
+    var_invoketime /= args.invoke_nums;
+    printf("=======================================\n");
+    printf("QNN inference %d times :\n --mean_invoke_time is %f \n --max_invoke_time is %f \n --min_invoke_time is %f \n --var_invoketime is %f\n", 
+        args.invoke_nums, mean_invoke_time, max_invoke_time, min_invoke_time, var_invoketime);
+    printf("=======================================\n");
+
+    // std::vector<std::vector<uint32_t>> output_shapes = {{1, 4, out_size}, {1, 80, out_size}};
+    float* pos_data = new float[4 * out_size];
+    float* class_data = new float[80 * out_size];
+    transformData(qnn_trans_data, pos_data, 4, out_size);
+    transformData(qnn_mul_data, class_data, 80, out_size);
+    
+    // post process
+    std::vector<float> qnn_concat;
+    concatenate(class_data, pos_data , 1, out_size, 80, 4, qnn_concat);
+    cv::Mat img = post_process(frame, qnn_concat, class_list, scale);
+    cv::imwrite("./results.png", img);
+    fast_interpreter->destory();
+    return 0;
+}
+
+
+int main(int argc, char* argv[]) {
+    Args args = parse_args(argc, argv);
+    return invoke(args);
+}

model_farm_yolov8s_qcs6490_qnn2.16_int8_aidlite/models/cutoff_yolov8s_w8a8.qnn216.ctx.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60eef403584262893959e5f133f4b523205a71d166750bc1e0853303536cb5ed
+size 11804696

model_farm_yolov8s_qcs6490_qnn2.16_int8_aidlite/python/run_test.py

@@ -0,0 +1,30 @@
+import cv2
+import argparse
+from yolov8s import Yolov8S
+from utils import draw_detect_res
+
+def parser_args():
+    parser = argparse.ArgumentParser(description="Run model benchmarks")
+    parser.add_argument('--target_model',type=str,default='./models/cutoff_yolov8s_w8a8.qnn231.ctx.bin.aidem',help="inference model path")
+    parser.add_argument('--imgs',type=str,default='./python/bus.jpg',help="Predict images path")
+    parser.add_argument('--height',type=int,default=640,help="run backend")
+    parser.add_argument('--weight',type=int,default=640,help="run backend")
+    parser.add_argument('--cls_num',type=int,default=80,help="run backend")
+    parser.add_argument('--invoke_nums',type=int,default=10,help="Inference nums")
+    parser.add_argument('--model_type',type=str,default='QNN',help="run backend")
+    args = parser.parse_args()
+    return args
+
+if __name__ == "__main__":
+    args = parser_args()
+    height = args.height
+    weight = args.weight
+
+    
+    model = Yolov8S(args.target_model, args.weight, args.height, args.cls_num)
+    frame = cv2.imread(args.imgs)
+    
+    out_boxes= model(frame, args.invoke_nums)
+    print(f"=================== \n Detect {len(out_boxes)} targets.")
+    result = draw_detect_res(frame, out_boxes)
+    cv2.imwrite("./python/result.jpg", result)

model_farm_yolov8s_qcs6490_qnn2.16_int8_aidlite/python/utils.py

@@ -0,0 +1,123 @@
+import numpy as np
+import cv2
+
+CLASSES = ("person", "bicycle", "car", "motorbike ", "aeroplane ", "bus ", "train", "truck ", "boat", "traffic light",
+           "fire hydrant", "stop sign ", "parking meter", "bench", "bird", "cat", "dog ", "horse ", "sheep", "cow", "elephant",
+           "bear", "zebra ", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite",
+           "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", "knife ",
+           "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza ", "donut", "cake", "chair", "sofa",
+           "pottedplant", "bed", "diningtable", "toilet ", "tvmonitor", "laptop	", "mouse	", "remote ", "keyboard ", "cell phone", "microwave ",
+           "oven ", "toaster", "sink", "refrigerator ", "book", "clock", "vase", "scissors ", "teddy bear ", "hair drier", "toothbrush ")
+           
+def eqprocess(image, size1, size2):
+    h,w,_ = image.shape
+    mask = np.zeros((size1,size2,3),dtype=np.float32)
+    scale1 = h /size1
+    scale2 = w / size2
+    if scale1 > scale2:
+        scale = scale1
+    else:
+        scale = scale2
+    img = cv2.resize(image,(int(w / scale),int(h / scale)))
+    mask[:int(h / scale),:int(w / scale),:] = img
+    return mask, scale
+
+def xywh2xyxy(x):
+    '''
+    Box (center x, center y, width, height) to (x1, y1, x2, y2)
+    '''
+    y = np.copy(x)
+    y[:, 0] = x[:, 0] - x[:, 2] / 2  # top left x
+    y[:, 1] = x[:, 1] - x[:, 3] / 2  # top left y
+    y[:, 2] = x[:, 0] + x[:, 2] / 2  # bottom right x
+    y[:, 3] = x[:, 1] + x[:, 3] / 2  # bottom right y
+    return y
+
+def xyxy2xywh(box):
+    '''
+    Box (left_top x, left_top y, right_bottom x, right_bottom y) to (left_top x, left_top y, width, height)
+    '''
+    box[:, 2:] = box[:, 2:] - box[:, :2]
+    return box
+
+def NMS(dets, scores, thresh):
+    '''
+    单类NMS算法
+    dets.shape = (N, 5), (left_top x, left_top y, right_bottom x, right_bottom y, Scores)
+    '''
+    x1 = dets[:,0]
+    y1 = dets[:,1]
+    x2 = dets[:,2]
+    y2 = dets[:,3]
+    areas = (y2-y1+1) * (x2-x1+1)
+    keep = []
+    index = scores.argsort()[::-1]
+    while index.size >0:
+        i = index[0]       # every time the first is the biggst, and add it directly
+        keep.append(i)
+        x11 = np.maximum(x1[i], x1[index[1:]])    # calculate the points of overlap 
+        y11 = np.maximum(y1[i], y1[index[1:]])
+        x22 = np.minimum(x2[i], x2[index[1:]])
+        y22 = np.minimum(y2[i], y2[index[1:]])
+        w = np.maximum(0, x22-x11+1)    # the weights of overlap
+        h = np.maximum(0, y22-y11+1)    # the height of overlap
+        overlaps = w*h
+        ious = overlaps / (areas[i]+areas[index[1:]] - overlaps)
+        idx = np.where(ious<=thresh)[0]
+        index = index[idx+1]   # because index start from 1
+ 
+    return keep
+
+
+def draw_detect_res(img, det_pred):
+    '''
+    检测结果绘制
+    '''
+    if det_pred is None:
+        return img
+
+    img = img.astype(np.uint8)
+    im_canvas = img.copy()
+    color_step = int(255/len(CLASSES))
+    for i in range(len(det_pred)):
+        x1, y1, x2, y2 = [int(t) for t in det_pred[i][:4]]
+        cls_id = int(det_pred[i][5])
+        print(i+1,[x1,y1,x2,y2],det_pred[i][4], f'{CLASSES[cls_id]}')
+        cv2.putText(img, f'{CLASSES[cls_id]}', (x1, y1-6), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1)
+        cv2.rectangle(img, (x1, y1), (x2, y2), (0, int(cls_id*color_step), int(255-cls_id*color_step)),thickness = 2)
+    img = cv2.addWeighted(im_canvas, 0.3, img, 0.7, 0)
+    return img
+
+def scale_mask(masks, im0_shape):
+    masks = cv2.resize(masks, (im0_shape[1], im0_shape[0]),
+                        interpolation=cv2.INTER_LINEAR)
+    if len(masks.shape) == 2:
+        masks = masks[:, :, None]
+    return masks
+
+def crop_mask(masks, boxes):
+    n, h, w = masks.shape
+    x1, y1, x2, y2 = np.split(boxes[:, :, None], 4, 1)
+    r = np.arange(w, dtype=x1.dtype)[None, None, :]
+    c = np.arange(h, dtype=x1.dtype)[None, :, None]
+    return masks * ((r >= x1) * (r < x2) * (c >= y1) * (c < y2))
+
+def process_mask(protos, masks_in, bboxes, im0_shape):
+    c, mh, mw = protos.shape
+    masks = np.matmul(masks_in, protos.reshape((c, -1))).reshape((-1, mh, mw)).transpose(1, 2, 0)  # HWN
+    masks = np.ascontiguousarray(masks)
+    masks = scale_mask(masks, im0_shape)  # re-scale mask from P3 shape to original input image shape
+    masks = np.einsum('HWN -> NHW', masks)  # HWN -> NHW
+    masks = crop_mask(masks, bboxes)
+    return np.greater(masks, 0.5)
+
+def masks2segments(masks):
+    segments = []
+    for x in masks.astype('uint8'):
+        c = cv2.findContours(x, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)[0]  # CHAIN_APPROX_SIMPLE
+        if c:
+            c = np.array(c[np.array([len(x) for x in c]).argmax()]).reshape(-1, 2)
+        else:
+            c = np.zeros((0, 2))  # no segments found
+        segments.append(c.astype('float32'))
+    return segments

model_farm_yolov8s_qcs6490_qnn2.16_int8_aidlite/python/yolov8s.py

@@ -0,0 +1,86 @@
+import numpy as np
+import cv2
+import aidlite
+from utils import eqprocess, xywh2xyxy, NMS
+import time
+
+OBJ_THRESH = 0.45
+NMS_THRESH = 0.45
+
+class Yolov8S(object):
+    def __init__(self, model_path, width, height, class_num):
+        self.class_num = class_num
+        self.width = width
+        self.height = height
+        input_shape = [[1,height,width,3]]
+        self.blocks = int(height * width * ( 1 / 64 + 1 / 256 + 1 / 1024))
+        self.maskw = int(width / 4)
+        self.maskh = int(height / 4)
+        self.output_shape = [[1,4,self.blocks],[1,class_num,self.blocks]]
+    
+        self.model = aidlite.Model.create_instance(model_path)
+        if self.model is None:
+            print("Create model failed !")
+            return
+        self.model.set_model_properties(input_shape, aidlite.DataType.TYPE_FLOAT32, self.output_shape, aidlite.DataType.TYPE_FLOAT32)
+        
+        self.config = aidlite.Config.create_instance()
+        if self.config is None:
+            print("build_interpretper_from_model_and_config failed !")
+            return
+        
+        self.config.framework_type = aidlite.FrameworkType.TYPE_QNN216
+        self.config.accelerate_type = aidlite.AccelerateType.TYPE_DSP
+        self.config.is_quantify_model = 1
+        
+        self.interpreter = aidlite.InterpreterBuilder.build_interpretper_from_model_and_config(self.model, self.config)
+        if self.interpreter is None:
+            print("build_interpretper_from_model_and_config failed !")
+            return
+        
+        self.interpreter.init()
+        self.interpreter.load_model()
+    
+    def __call__(self, frame,invoke_nums):
+        img, scale = eqprocess(frame, self.height, self.width)
+        img = img / 255
+        img = img.astype(np.float32)
+        self.interpreter.set_input_tensor(0,img.data)
+        
+        invoke_time=[]
+        for i  in range(invoke_nums):
+            t1=time.time()
+            self.interpreter.invoke()
+            cost_time = (time.time()-t1)*1000
+            invoke_time.append(cost_time)
+
+        max_invoke_time = max(invoke_time)
+        min_invoke_time = min(invoke_time)
+        mean_invoke_time = sum(invoke_time)/invoke_nums
+        var_invoketime=np.var(invoke_time)
+        print("====================================")
+        print(f"QNN invoke {invoke_nums} times:\n --mean_invoke_time is {mean_invoke_time} \n --max_invoke_time is {max_invoke_time} \n --min_invoke_time is {min_invoke_time} \n --var_invoketime is {var_invoketime}")
+        print("====================================")
+
+        qnn_1 = self.interpreter.get_output_tensor(0)
+        qnn_2 = self.interpreter.get_output_tensor(1)
+        qnn_out = sorted([qnn_1,qnn_2], key=len)
+
+        qnn_local = qnn_out[0].reshape(*self.output_shape[0])
+        qnn_conf = qnn_out[1].reshape(*self.output_shape[1])
+
+        
+        x = np.concatenate([qnn_local, qnn_conf], axis = 1).transpose(0,2,1)
+        x = x[np.amax(x[..., 4:], axis=-1) > OBJ_THRESH]
+        if len(x) < 1:
+            return None, None
+    
+        x = np.c_[x[..., :4], np.amax(x[..., 4:], axis=-1), np.argmax(x[..., 4:], axis=-1)]
+        
+        x[:, :4] = xywh2xyxy(x[:, :4])
+        index = NMS(x[:, :4], x[:, 4], NMS_THRESH)
+        out_boxes = x[index]
+        out_boxes[..., :4] = out_boxes[..., :4] * scale
+        
+        return out_boxes
+

model_farm_yolov8s_qcs8550_qnn2.16_fp16_aidlite/README.md

@@ -0,0 +1,59 @@
+## Model Information
+### Source model
+- Input shape: 640x640
+- Number of parameters: 10.65M
+- Model size: 42.7M
+- Output shape: 1x84x8400
+
+Source model repository: [yolov8](https://github.com/ultralytics/ultralytics)
+
+### Converted model
+
+- Precision: FP16
+- Backend: QNN2.16
+- Target Device: SNM972 QCS8550
+
+## Model Conversion Reference
+User can find model conversion reference at [aimo.aidlux.com](https://aimo.aidlux.com/#/public/8204cbbb-e7dc-4c2b-8eb8-06e11fa82e29)
+
+## Inference with AidLite SDK
+
+### SDK installation
+Model Farm uses AidLite SDK as the model inference SDK. For details, please refer to the [AidLite Developer Documentation](https://v2.docs.aidlux.com/en/sdk-api/aidlite-sdk/)
+
+- install AidLite SDK
+
+```bash
+# Install the appropriate version of the aidlite sdk
+sudo aid-pkg update
+sudo aid-pkg install aidlite-sdk
+# Download the qnn version that matches the above backend. Eg Install QNN2.23 Aidlite: sudo aid-pkg install aidlite-qnn223
+sudo aid-pkg install aidlite-{QNN VERSION}
+```
+
+- Verify AidLite SDK
+
+```bash
+# Install the appropriate version of the aidlite sdk
+sudo aid-pkg update
+sudo aid-pkg install aidlite-sdk
+# Download the qnn version that matches the above backend. Eg Install QNN2.23 Aidlite: sudo aid-pkg install aidlite-qnn223
+sudo aid-pkg install aidlite-{QNN VERSION}
+# eg: Install QNN 2.23 Aidlite: sudo aid-pkg install aidlite-qnn223
+```
+
+### Run Demo
+
+#### python
+```bash
+cd model_farm_yolov8s_qcs8550_qnn2.16_fp16_aidlite
+python3  python/run_test.py --target_model ./models/cutoff_yolov8s_fp16.qnn216.ctx.bin --imgs ./python/bus.jpg  --invoke_nums 10
+```
+
+#### cpp
+```bash
+cd model_farm_yolov8s_qcs8550_qnn2.16_fp16_aidlite/cpp
+mkdir -p build && cd build
+cmake .. && make
+./run_test --target_model ../../models/cutoff_yolov8s_fp16.qnn216.ctx.bin --imgs ../bus.jpg  --invoke_nums 10
+```

model_farm_yolov8s_qcs8550_qnn2.16_fp16_aidlite/cpp/CMakeLists.txt

@@ -0,0 +1,30 @@
+cmake_minimum_required (VERSION 3.5)
+project("run_test")
+
+find_package(OpenCV REQUIRED)
+
+message(STATUS "oPENCV Library status:")
+message(STATUS ">version:${OpenCV_VERSION}")
+message(STATUS "Include:${OpenCV_INCLUDE_DIRS}")
+
+set(CMAKE_CXX_FLAGS "-Wno-error=deprecated-declarations -Wno-deprecated-declarations")
+
+include_directories(
+    /usr/local/include
+    /usr/include/opencv4
+)
+
+link_directories(
+    /usr/local/lib/
+)
+
+file(GLOB SRC_LISTS 
+    ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp  
+)
+
+add_executable(run_test ${SRC_LISTS})
+
+target_link_libraries(run_test
+    aidlite
+	${OpenCV_LIBS}
+)

model_farm_yolov8s_qcs8550_qnn2.16_fp16_aidlite/cpp/main.cpp

@@ -0,0 +1,349 @@
+#include <iostream>
+#include <string>
+#include <algorithm>
+#include <cctype>
+#include <opencv2/opencv.hpp>
+#include <aidlux/aidlite/aidlite.hpp>
+#include <vector>
+#include <numeric>
+
+const float INPUT_WIDTH = 640.0;
+const float INPUT_HEIGHT = 640.0;
+const float SCORE_THRESHOLD = 0.25;
+const float NMS_THRESHOLD = 0.45;
+const float CONFIDENCE_THRESHOLD = 0.25;
+const uint32_t size = 640;
+const uint32_t out_size = 8400;
+
+const int FONT_FACE = cv::FONT_HERSHEY_SIMPLEX;
+cv::Scalar WHITE = cv::Scalar(255,255,255);
+
+const float FONT_SCALE = 0.75;
+const int THICKNESS = 1;
+
+const std::vector<std::string> class_list = {
+    "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train",
+    "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter",
+    "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear",
+    "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase",
+    "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat",
+    "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle",
+    "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple",
+    "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut",
+    "cake", "chair", "couch", "potted plant", "bed", "dining table", "toilet",
+    "TV", "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave",
+    "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase",
+    "scissors", "teddy bear", "hair drier", "toothbrush"
+};
+
+using namespace Aidlux::Aidlite;
+
+struct Args {
+    std::string target_model = "../../models/cutoff_yolov8s_fp16.qnn216.ctx.bin.aidem";
+    std::string imgs = "../bus.jpg";
+    int invoke_nums = 10;
+    std::string model_type = "QNN";
+};
+
+Args parse_args(int argc, char* argv[]) {
+    Args args;
+    for (int i = 1; i < argc; ++i) {
+        std::string arg = argv[i];
+        if (arg == "--target_model" && i + 1 < argc) {
+            args.target_model = argv[++i];
+        } else if (arg == "--imgs" && i + 1 < argc) {
+            args.imgs = argv[++i];
+        } else if (arg == "--invoke_nums" && i + 1 < argc) {
+            args.invoke_nums = std::stoi(argv[++i]);
+        } else if (arg == "--model_type" && i + 1 < argc) {
+            args.model_type = argv[++i];
+        }
+    }
+    return args;
+}
+
+std::string to_lower(const std::string& str) {
+    std::string lower_str = str;
+    std::transform(lower_str.begin(), lower_str.end(), lower_str.begin(), [](unsigned char c) {
+        return std::tolower(c);
+    });
+    return lower_str;
+}
+
+
+void concatenate(float* qnn_trans_data, float* qnn_mul_data, int batch, int num_elements, int trans_dim, int mul_dim, std::vector<float>& output) {
+    int out_dim = trans_dim + mul_dim + 1; 
+    output.resize(batch * num_elements * out_dim); 
+    for (int i = 0; i < batch * num_elements; ++i) {
+        std::memcpy(&output[i * out_dim], &qnn_mul_data[i * mul_dim], mul_dim * sizeof(float));
+        float max_val = *std::max_element(&qnn_trans_data[i * trans_dim], &qnn_trans_data[i * trans_dim + trans_dim]);
+        output[i * out_dim + 4] = max_val;
+        std::memcpy(&output[i * out_dim + 5], &qnn_trans_data[i * trans_dim], trans_dim * sizeof(float));
+    }
+}
+
+void transformData(const float* input, float* output, int C, int N) {
+    for (int c = 0; c < C; ++c) {
+        for (int n = 0; n < N; ++n) {
+            output[n * C + c] = input[c * N + n];
+        }
+    }
+}
+
+double img_process(cv::Mat frame, cv::Mat &img_input, int size) {
+    cv::Mat img_processed = frame.clone();
+    int height = img_processed.rows;
+    int width = img_processed.cols;
+    int length = std::max(height, width);
+    double scala = static_cast<double>(length) / size;
+    
+    cv::Mat image = cv::Mat::zeros(cv::Size(length, length), CV_8UC3);
+    img_processed.copyTo(image(cv::Rect(0, 0, width, height)));
+    
+    cv::cvtColor(image, img_input, cv::COLOR_BGR2RGB);
+    cv::resize(img_input, img_input, cv::Size(size, size));
+    
+    cv::Mat mean_data = cv::Mat::zeros(img_input.size(), CV_32FC3);
+    cv::Mat std_data(img_input.size(), CV_32FC3, cv::Scalar(255, 255, 255));
+    img_input.convertTo(img_input, CV_32FC3);
+    img_input = (img_input - mean_data) / std_data;
+    return scala;
+}
+
+
+cv::Scalar generate_colors(int i, bool bgr = false) {
+    static const std::vector<std::string> hex_colors = {
+        "FF3838", "FF9D97", "FF701F", "FFB21D", "CFD231", "48F90A",
+        "92CC17", "3DDB86", "1A9334", "00D4BB", "2C99A8", "00C2FF",
+        "344593", "6473FF", "0018EC", "8438FF", "520085", "CB38FF",
+        "FF95C8", "FF37C7"
+    };
+
+    int num = hex_colors.size();
+    std::string hex = hex_colors[i % num];
+
+    int r = std::stoi(hex.substr(0, 2), nullptr, 16);
+    int g = std::stoi(hex.substr(2, 2), nullptr, 16);
+    int b = std::stoi(hex.substr(4, 2), nullptr, 16);
+
+    if (bgr)
+        return cv::Scalar(b, g, r);
+    else
+        return cv::Scalar(r, g, b);
+}
+
+void draw_label(cv::Mat& input_image, std::string label, int left, int top, cv::Scalar color)
+{
+    int baseLine;
+    cv::Size label_size = cv::getTextSize(label, FONT_FACE, FONT_SCALE, THICKNESS, &baseLine);
+    int y = top - label_size.height - baseLine;
+    if (y < 0) {
+        y = top ;
+    }
+    cv::Point tlc(left, y);
+    cv::Point brc(left + label_size.width, y + label_size.height + baseLine);
+    rectangle(input_image, tlc, brc, color, cv::FILLED);
+    putText(input_image, label, cv::Point(left, y + label_size.height), FONT_FACE, FONT_SCALE, WHITE, THICKNESS);
+}
+
+
+cv::Mat post_process(cv::Mat &input_image, std::vector<float> &outputs, const std::vector<std::string> &class_name, const double ratio)
+{
+    // Initialize vectors to hold respective outputs while unwrapping detections.
+    std::vector<int> class_ids;
+    std::vector<float> confidences;
+    std::vector<cv::Rect> boxes;
+    
+    // Iterate through outputs for each box prediction
+    for (int i = 0; i < outputs.size(); i+=85)
+    {
+        float confidence = outputs[i+4];
+        if (confidence >= CONFIDENCE_THRESHOLD)
+        {
+            // Create a 1x80 Mat and store class scores of 80 classes.
+            cv::Mat scores(1, class_name.size(), CV_32FC1, outputs.data() + i + 5);
+            cv::Point class_id;
+            double max_class_score;
+            
+            // For multi-label, check each class score
+            for (int c = 0; c < class_name.size(); c++) {
+                float class_score = scores.at<float>(0, c);
+                
+                // If class score is above threshold, consider this class for the box
+                if (class_score > SCORE_THRESHOLD) {
+                    // Store class ID and confidence in the pre-defined respective vectors.
+                    confidences.push_back(confidence * class_score);  // Multiply with confidence
+                    class_ids.push_back(c);  // class index
+                    // Center and box dimension.
+                    float cx = outputs[i];
+                    float cy = outputs[i+1];
+                    float w = outputs[i+2];
+                    float h = outputs[i+3];
+
+                    int left = int((cx - 0.5 * w) * ratio);
+                    int top = int((cy - 0.5 * h) * ratio);
+                    int width = int(w * ratio);
+                    int height = int(h * ratio);
+
+                    // Store good detections in the boxes vector.
+                    boxes.push_back(cv::Rect(left, top, width, height));
+                }
+            }
+        }
+    }
+
+    // Perform Non Maximum Suppression and draw predictions.
+    std::vector<int> indices;
+    cv::dnn::NMSBoxes(boxes, confidences, SCORE_THRESHOLD, NMS_THRESHOLD, indices);
+    printf("Detected {%ld} targets.\n", indices.size());
+    
+    // Loop over NMS results and draw bounding boxes
+    for (int i = 0; i < indices.size(); i++)
+    {
+        int idx = indices[i];
+        cv::Rect box = boxes[idx];
+
+        int left = box.x;
+        int top = box.y;
+        int width = box.width;
+        int height = box.height;
+        cv::Scalar color = generate_colors(class_ids[idx]);
+        // Draw bounding box.
+        rectangle(input_image, cv::Point(left, top), cv::Point(left + width, top + height), color, 2*THICKNESS);
+
+        // Get the label for the class name and its confidence.
+        std::string label = cv::format("%.2f", confidences[idx]);
+        label = class_name[class_ids[idx]] + ":" + label;
+        // Draw class labels.
+        draw_label(input_image, label, left, top, color);
+    }
+    printf("Processing finished.\n");
+    return input_image;
+}
+
+int invoke(const Args& args) {
+    std::cout << "Start main ... ... Model Path: " << args.target_model << "\n"
+              << "Image Path: " << args.imgs << "\n"
+              << "Inference Nums: " << args.invoke_nums << "\n"
+              << "Model Type: " << args.model_type << "\n";
+    Model* model = Model::create_instance(args.target_model);
+    if(model == nullptr){
+        printf("Create model failed !\n");
+        return EXIT_FAILURE;
+    }
+    Config* config = Config::create_instance();
+    if(config == nullptr){
+        printf("Create config failed !\n");
+        return EXIT_FAILURE;
+    }
+    config->implement_type = ImplementType::TYPE_LOCAL;
+    std::string model_type_lower = to_lower(args.model_type);
+    if (model_type_lower == "qnn"){
+        config->framework_type = FrameworkType::TYPE_QNN;
+    } else if (model_type_lower == "snpe2" || model_type_lower == "snpe") {
+        config->framework_type = FrameworkType::TYPE_SNPE2;
+    }
+    config->accelerate_type = AccelerateType::TYPE_DSP;
+    config->is_quantify_model = 1;
+
+    std::vector<std::vector<uint32_t>> input_shapes = {{1, size, size, 3}};
+    std::vector<std::vector<uint32_t>> output_shapes = {{1, 4, out_size}, {1, 80, out_size}};
+    model->set_model_properties(input_shapes, DataType::TYPE_FLOAT32, output_shapes, DataType::TYPE_FLOAT32);
+    std::unique_ptr<Interpreter> fast_interpreter = InterpreterBuilder::build_interpretper_from_model_and_config(model, config);
+    if(fast_interpreter == nullptr){
+        printf("build_interpretper_from_model_and_config failed !\n");
+        return EXIT_FAILURE;
+    }
+    int result = fast_interpreter->init();
+    if(result != EXIT_SUCCESS){
+        printf("interpreter->init() failed !\n");
+        return EXIT_FAILURE;
+    }
+    // load model
+    fast_interpreter->load_model();
+    if(result != EXIT_SUCCESS){
+        printf("interpreter->load_model() failed !\n");
+        return EXIT_FAILURE;
+    }
+    printf("detect model load success!\n");
+
+    cv::Mat frame = cv::imread(args.imgs);
+    if (frame.empty()) {
+        printf("detect image load failed!\n");
+        return 1;
+    }
+    printf("img_src cols: %d, img_src rows: %d\n", frame.cols, frame.rows);
+    cv::Mat input_img;
+    double scale = img_process(frame, input_img, size);
+    if (input_img.empty()) {
+        printf("detect input_img load failed!\n");
+        return 1;
+    }
+
+    float *qnn_trans_data = nullptr;
+    float *qnn_mul_data = nullptr;
+
+    std::vector<float> invoke_time;
+    for (int i = 0; i < args.invoke_nums; ++i) {
+        result = fast_interpreter->set_input_tensor(0, input_img.data);
+        if(result != EXIT_SUCCESS){
+            printf("interpreter->set_input_tensor() failed !\n");
+            return EXIT_FAILURE;
+        }
+          // 开始计时
+        auto t1 = std::chrono::high_resolution_clock::now();
+        result = fast_interpreter->invoke();
+        auto t2 = std::chrono::high_resolution_clock::now();
+        std::chrono::duration<double> cost_time = t2 - t1;
+        invoke_time.push_back(cost_time.count() * 1000);
+        if(result != EXIT_SUCCESS){
+            printf("interpreter->invoke() failed !\n");
+            return EXIT_FAILURE;
+        }
+        uint32_t out_data_1 = 0;
+        result = fast_interpreter->get_output_tensor(0, (void**)&qnn_trans_data, &out_data_1);
+        if(result != EXIT_SUCCESS){
+            printf("interpreter->get_output_tensor() 1 failed !\n");
+            return EXIT_FAILURE;
+        }
+        uint32_t out_data_2 = 0;
+        result = fast_interpreter->get_output_tensor(1, (void**)&qnn_mul_data, &out_data_2);
+        if(result != EXIT_SUCCESS){
+            printf("interpreter->get_output_tensor() 2 failed !\n");
+            return EXIT_FAILURE;
+        }
+    }
+
+    float max_invoke_time = *std::max_element(invoke_time.begin(), invoke_time.end());
+    float min_invoke_time = *std::min_element(invoke_time.begin(), invoke_time.end());
+    float mean_invoke_time = std::accumulate(invoke_time.begin(), invoke_time.end(), 0.0f) / args.invoke_nums;
+    float var_invoketime = 0.0f;
+    for (auto time : invoke_time) {
+        var_invoketime += (time - mean_invoke_time) * (time - mean_invoke_time);
+    }
+    var_invoketime /= args.invoke_nums;
+    printf("=======================================\n");
+    printf("QNN inference %d times :\n --mean_invoke_time is %f \n --max_invoke_time is %f \n --min_invoke_time is %f \n --var_invoketime is %f\n", 
+        args.invoke_nums, mean_invoke_time, max_invoke_time, min_invoke_time, var_invoketime);
+    printf("=======================================\n");
+
+    // std::vector<std::vector<uint32_t>> output_shapes = {{1, 4, out_size}, {1, 80, out_size}};
+    float* pos_data = new float[4 * out_size];
+    float* class_data = new float[80 * out_size];
+    transformData(qnn_trans_data, pos_data, 4, out_size);
+    transformData(qnn_mul_data, class_data, 80, out_size);
+    
+    // post process
+    std::vector<float> qnn_concat;
+    concatenate(class_data, pos_data , 1, out_size, 80, 4, qnn_concat);
+    cv::Mat img = post_process(frame, qnn_concat, class_list, scale);
+    cv::imwrite("./results.png", img);
+    fast_interpreter->destory();
+    return 0;
+}
+
+
+int main(int argc, char* argv[]) {
+    Args args = parse_args(argc, argv);
+    return invoke(args);
+}

model_farm_yolov8s_qcs8550_qnn2.16_fp16_aidlite/models/cutoff_yolov8s_fp16.qnn216.ctx.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68c451e6059b8cbbfcc732cfe83571223a7e0c4120c1437d78d32d5e4f828d8e
+size 23003136

model_farm_yolov8s_qcs8550_qnn2.16_fp16_aidlite/python/run_test.py

@@ -0,0 +1,30 @@
+import cv2
+import argparse
+from yolov8s import Yolov8S
+from utils import draw_detect_res
+
+def parser_args():
+    parser = argparse.ArgumentParser(description="Run model benchmarks")
+    parser.add_argument('--target_model',type=str,default='./models/cutoff_yolov8s_w8a8.qnn231.ctx.bin.aidem',help="inference model path")
+    parser.add_argument('--imgs',type=str,default='./python/bus.jpg',help="Predict images path")
+    parser.add_argument('--height',type=int,default=640,help="run backend")
+    parser.add_argument('--weight',type=int,default=640,help="run backend")
+    parser.add_argument('--cls_num',type=int,default=80,help="run backend")
+    parser.add_argument('--invoke_nums',type=int,default=10,help="Inference nums")
+    parser.add_argument('--model_type',type=str,default='QNN',help="run backend")
+    args = parser.parse_args()
+    return args
+
+if __name__ == "__main__":
+    args = parser_args()
+    height = args.height
+    weight = args.weight
+
+    
+    model = Yolov8S(args.target_model, args.weight, args.height, args.cls_num)
+    frame = cv2.imread(args.imgs)
+    
+    out_boxes= model(frame,args.invoke_nums)
+    print(f"=================== \n Detect {len(out_boxes)} targets.")
+    result = draw_detect_res(frame, out_boxes)
+    cv2.imwrite("./python/result.jpg", result)

model_farm_yolov8s_qcs8550_qnn2.16_fp16_aidlite/python/utils.py

@@ -0,0 +1,123 @@
+import numpy as np
+import cv2
+
+CLASSES = ("person", "bicycle", "car", "motorbike ", "aeroplane ", "bus ", "train", "truck ", "boat", "traffic light",
+           "fire hydrant", "stop sign ", "parking meter", "bench", "bird", "cat", "dog ", "horse ", "sheep", "cow", "elephant",
+           "bear", "zebra ", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite",
+           "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", "knife ",
+           "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza ", "donut", "cake", "chair", "sofa",
+           "pottedplant", "bed", "diningtable", "toilet ", "tvmonitor", "laptop	", "mouse	", "remote ", "keyboard ", "cell phone", "microwave ",
+           "oven ", "toaster", "sink", "refrigerator ", "book", "clock", "vase", "scissors ", "teddy bear ", "hair drier", "toothbrush ")
+           
+def eqprocess(image, size1, size2):
+    h,w,_ = image.shape
+    mask = np.zeros((size1,size2,3),dtype=np.float32)
+    scale1 = h /size1
+    scale2 = w / size2
+    if scale1 > scale2:
+        scale = scale1
+    else:
+        scale = scale2
+    img = cv2.resize(image,(int(w / scale),int(h / scale)))
+    mask[:int(h / scale),:int(w / scale),:] = img
+    return mask, scale
+
+def xywh2xyxy(x):
+    '''
+    Box (center x, center y, width, height) to (x1, y1, x2, y2)
+    '''
+    y = np.copy(x)
+    y[:, 0] = x[:, 0] - x[:, 2] / 2  # top left x
+    y[:, 1] = x[:, 1] - x[:, 3] / 2  # top left y
+    y[:, 2] = x[:, 0] + x[:, 2] / 2  # bottom right x
+    y[:, 3] = x[:, 1] + x[:, 3] / 2  # bottom right y
+    return y
+
+def xyxy2xywh(box):
+    '''
+    Box (left_top x, left_top y, right_bottom x, right_bottom y) to (left_top x, left_top y, width, height)
+    '''
+    box[:, 2:] = box[:, 2:] - box[:, :2]
+    return box
+
+def NMS(dets, scores, thresh):
+    '''
+    单类NMS算法
+    dets.shape = (N, 5), (left_top x, left_top y, right_bottom x, right_bottom y, Scores)
+    '''
+    x1 = dets[:,0]
+    y1 = dets[:,1]
+    x2 = dets[:,2]
+    y2 = dets[:,3]
+    areas = (y2-y1+1) * (x2-x1+1)
+    keep = []
+    index = scores.argsort()[::-1]
+    while index.size >0:
+        i = index[0]       # every time the first is the biggst, and add it directly
+        keep.append(i)
+        x11 = np.maximum(x1[i], x1[index[1:]])    # calculate the points of overlap 
+        y11 = np.maximum(y1[i], y1[index[1:]])
+        x22 = np.minimum(x2[i], x2[index[1:]])
+        y22 = np.minimum(y2[i], y2[index[1:]])
+        w = np.maximum(0, x22-x11+1)    # the weights of overlap
+        h = np.maximum(0, y22-y11+1)    # the height of overlap
+        overlaps = w*h
+        ious = overlaps / (areas[i]+areas[index[1:]] - overlaps)
+        idx = np.where(ious<=thresh)[0]
+        index = index[idx+1]   # because index start from 1
+ 
+    return keep
+
+
+def draw_detect_res(img, det_pred):
+    '''
+    检测结果绘制
+    '''
+    if det_pred is None:
+        return img
+
+    img = img.astype(np.uint8)
+    im_canvas = img.copy()
+    color_step = int(255/len(CLASSES))
+    for i in range(len(det_pred)):
+        x1, y1, x2, y2 = [int(t) for t in det_pred[i][:4]]
+        cls_id = int(det_pred[i][5])
+        print(i+1,[x1,y1,x2,y2],det_pred[i][4], f'{CLASSES[cls_id]}')
+        cv2.putText(img, f'{CLASSES[cls_id]}', (x1, y1-6), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1)
+        cv2.rectangle(img, (x1, y1), (x2, y2), (0, int(cls_id*color_step), int(255-cls_id*color_step)),thickness = 2)
+    img = cv2.addWeighted(im_canvas, 0.3, img, 0.7, 0)
+    return img
+
+def scale_mask(masks, im0_shape):
+    masks = cv2.resize(masks, (im0_shape[1], im0_shape[0]),
+                        interpolation=cv2.INTER_LINEAR)
+    if len(masks.shape) == 2:
+        masks = masks[:, :, None]
+    return masks
+
+def crop_mask(masks, boxes):
+    n, h, w = masks.shape
+    x1, y1, x2, y2 = np.split(boxes[:, :, None], 4, 1)
+    r = np.arange(w, dtype=x1.dtype)[None, None, :]
+    c = np.arange(h, dtype=x1.dtype)[None, :, None]
+    return masks * ((r >= x1) * (r < x2) * (c >= y1) * (c < y2))
+
+def process_mask(protos, masks_in, bboxes, im0_shape):
+    c, mh, mw = protos.shape
+    masks = np.matmul(masks_in, protos.reshape((c, -1))).reshape((-1, mh, mw)).transpose(1, 2, 0)  # HWN
+    masks = np.ascontiguousarray(masks)
+    masks = scale_mask(masks, im0_shape)  # re-scale mask from P3 shape to original input image shape
+    masks = np.einsum('HWN -> NHW', masks)  # HWN -> NHW
+    masks = crop_mask(masks, bboxes)
+    return np.greater(masks, 0.5)
+
+def masks2segments(masks):
+    segments = []
+    for x in masks.astype('uint8'):
+        c = cv2.findContours(x, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)[0]  # CHAIN_APPROX_SIMPLE
+        if c:
+            c = np.array(c[np.array([len(x) for x in c]).argmax()]).reshape(-1, 2)
+        else:
+            c = np.zeros((0, 2))  # no segments found
+        segments.append(c.astype('float32'))
+    return segments

model_farm_yolov8s_qcs8550_qnn2.16_fp16_aidlite/python/yolov8s.py

@@ -0,0 +1,86 @@
+import numpy as np
+import cv2
+import aidlite
+from utils import eqprocess, xywh2xyxy, NMS
+import time
+
+OBJ_THRESH = 0.45
+NMS_THRESH = 0.45
+
+class Yolov8S(object):
+    def __init__(self, model_path, width, height, class_num):
+        self.class_num = class_num
+        self.width = width
+        self.height = height
+        input_shape = [[1,height,width,3]]
+        self.blocks = int(height * width * ( 1 / 64 + 1 / 256 + 1 / 1024))
+        self.maskw = int(width / 4)
+        self.maskh = int(height / 4)
+        self.output_shape = [[1,4,self.blocks],[1,class_num,self.blocks]]
+    
+        self.model = aidlite.Model.create_instance(model_path)
+        if self.model is None:
+            print("Create model failed !")
+            return
+        self.model.set_model_properties(input_shape, aidlite.DataType.TYPE_FLOAT32, self.output_shape, aidlite.DataType.TYPE_FLOAT32)
+        
+        self.config = aidlite.Config.create_instance()
+        if self.config is None:
+            print("build_interpretper_from_model_and_config failed !")
+            return
+        
+        self.config.framework_type = aidlite.FrameworkType.TYPE_QNN
+        self.config.accelerate_type = aidlite.AccelerateType.TYPE_DSP
+        self.config.is_quantify_model = 1
+        
+        self.interpreter = aidlite.InterpreterBuilder.build_interpretper_from_model_and_config(self.model, self.config)
+        if self.interpreter is None:
+            print("build_interpretper_from_model_and_config failed !")
+            return
+        
+        self.interpreter.init()
+        self.interpreter.load_model()
+    
+    def __call__(self, frame,invoke_nums):
+        img, scale = eqprocess(frame, self.height, self.width)
+        img = img / 255
+        img = img.astype(np.float32)
+        self.interpreter.set_input_tensor(0,img.data)
+        
+        invoke_time=[]
+        for i  in range(invoke_nums):
+            t1=time.time()
+            self.interpreter.invoke()
+            cost_time = (time.time()-t1)*1000
+            invoke_time.append(cost_time)
+
+        max_invoke_time = max(invoke_time)
+        min_invoke_time = min(invoke_time)
+        mean_invoke_time = sum(invoke_time)/invoke_nums
+        var_invoketime=np.var(invoke_time)
+        print("====================================")
+        print(f"QNN invoke {invoke_nums} times:\n --mean_invoke_time is {mean_invoke_time} \n --max_invoke_time is {max_invoke_time} \n --min_invoke_time is {min_invoke_time} \n --var_invoketime is {var_invoketime}")
+        print("====================================")
+
+        qnn_1 = self.interpreter.get_output_tensor(0)
+        qnn_2 = self.interpreter.get_output_tensor(1)
+        qnn_out = sorted([qnn_1,qnn_2], key=len)
+
+        qnn_local = qnn_out[0].reshape(*self.output_shape[0])
+        qnn_conf = qnn_out[1].reshape(*self.output_shape[1])
+
+        
+        x = np.concatenate([qnn_local, qnn_conf], axis = 1).transpose(0,2,1)
+        x = x[np.amax(x[..., 4:], axis=-1) > OBJ_THRESH]
+        if len(x) < 1:
+            return None, None
+    
+        x = np.c_[x[..., :4], np.amax(x[..., 4:], axis=-1), np.argmax(x[..., 4:], axis=-1)]
+        
+        x[:, :4] = xywh2xyxy(x[:, :4])
+        index = NMS(x[:, :4], x[:, 4], NMS_THRESH)
+        out_boxes = x[index]
+        out_boxes[..., :4] = out_boxes[..., :4] * scale
+        
+        return out_boxes
+

model_farm_yolov8s_qcs8550_qnn2.16_int8_aidlite/README.md

@@ -0,0 +1,59 @@
+## Model Information
+### Source model
+- Input shape: 640x640
+- Number of parameters: 10.65M
+- Model size: 42.7M
+- Output shape: 1x84x8400
+
+Source model repository: [yolov8](https://github.com/ultralytics/ultralytics)
+
+### Converted model
+
+- Precision: INT8
+- Backend: QNN2.16
+- Target Device: SNM972 QCS8550
+
+## Model Conversion Reference
+User can find model conversion reference at [aimo.aidlux.com](https://aimo.aidlux.com/#/public/34252a53-87e0-4fe2-b1bb-87c153fc21b7)
+
+## Inference with AidLite SDK
+
+### SDK installation
+Model Farm uses AidLite SDK as the model inference SDK. For details, please refer to the [AidLite Developer Documentation](https://v2.docs.aidlux.com/en/sdk-api/aidlite-sdk/)
+
+- install AidLite SDK
+
+```bash
+# Install the appropriate version of the aidlite sdk
+sudo aid-pkg update
+sudo aid-pkg install aidlite-sdk
+# Download the qnn version that matches the above backend. Eg Install QNN2.23 Aidlite: sudo aid-pkg install aidlite-qnn223
+sudo aid-pkg install aidlite-{QNN VERSION}
+# eg: Install QNN 2.23 Aidlite: sudo aid-pkg install aidlite-qnn223
+```
+
+- Verify AidLite SDK
+
+```bash
+# aidlite sdk c++ check
+python3 -c "import aidlite ; print(aidlite.get_library_version())"
+
+# aidlite sdk python check
+python3 -c "import aidlite ; print(aidlite.get_py_library_version())"
+```
+
+### Run Demo
+
+#### python
+```bash
+cd model_farm_yolov8s_qcs8550_qnn2.16_int8_aidlite
+python3  python/run_test.py --target_model ./models/cutoff_yolov8s_w8a8.qnn216.ctx.bin --imgs ./python/bus.jpg  --invoke_nums 10
+```
+
+#### cpp
+```bash
+cd model_farm_yolov8s_qcs8550_qnn2.16_int8_aidlite/cpp
+mkdir -p build && cd build
+cmake .. && make
+./run_test --target_model ../../models/cutoff_yolov8s_w8a8.qnn216.ctx.bin --imgs ../bus.jpg  --invoke_nums 10
+```

model_farm_yolov8s_qcs8550_qnn2.16_int8_aidlite/cpp/CMakeLists.txt

@@ -0,0 +1,30 @@
+cmake_minimum_required (VERSION 3.5)
+project("run_test")
+
+find_package(OpenCV REQUIRED)
+
+message(STATUS "oPENCV Library status:")
+message(STATUS ">version:${OpenCV_VERSION}")
+message(STATUS "Include:${OpenCV_INCLUDE_DIRS}")
+
+set(CMAKE_CXX_FLAGS "-Wno-error=deprecated-declarations -Wno-deprecated-declarations")
+
+include_directories(
+    /usr/local/include
+    /usr/include/opencv4
+)
+
+link_directories(
+    /usr/local/lib/
+)
+
+file(GLOB SRC_LISTS 
+    ${CMAKE_CURRENT_SOURCE_DIR}/*.cpp  
+)
+
+add_executable(run_test ${SRC_LISTS})
+
+target_link_libraries(run_test
+    aidlite
+	${OpenCV_LIBS}
+)

model_farm_yolov8s_qcs8550_qnn2.16_int8_aidlite/cpp/main.cpp

@@ -0,0 +1,349 @@
+#include <iostream>
+#include <string>
+#include <algorithm>
+#include <cctype>
+#include <opencv2/opencv.hpp>
+#include <aidlux/aidlite/aidlite.hpp>
+#include <vector>
+#include <numeric>
+
+const float INPUT_WIDTH = 640.0;
+const float INPUT_HEIGHT = 640.0;
+const float SCORE_THRESHOLD = 0.25;
+const float NMS_THRESHOLD = 0.45;
+const float CONFIDENCE_THRESHOLD = 0.25;
+const uint32_t size = 640;
+const uint32_t out_size = 8400;
+
+const int FONT_FACE = cv::FONT_HERSHEY_SIMPLEX;
+cv::Scalar WHITE = cv::Scalar(255,255,255);
+
+const float FONT_SCALE = 0.75;
+const int THICKNESS = 1;
+
+const std::vector<std::string> class_list = {
+    "person", "bicycle", "car", "motorcycle", "airplane", "bus", "train",
+    "truck", "boat", "traffic light", "fire hydrant", "stop sign", "parking meter",
+    "bench", "bird", "cat", "dog", "horse", "sheep", "cow", "elephant", "bear",
+    "zebra", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase",
+    "frisbee", "skis", "snowboard", "sports ball", "kite", "baseball bat",
+    "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle",
+    "wine glass", "cup", "fork", "knife", "spoon", "bowl", "banana", "apple",
+    "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza", "donut",
+    "cake", "chair", "couch", "potted plant", "bed", "dining table", "toilet",
+    "TV", "laptop", "mouse", "remote", "keyboard", "cell phone", "microwave",
+    "oven", "toaster", "sink", "refrigerator", "book", "clock", "vase",
+    "scissors", "teddy bear", "hair drier", "toothbrush"
+};
+
+using namespace Aidlux::Aidlite;
+
+struct Args {
+    std::string target_model = "../../models/cutoff_yolov8s_w8a8.qnn216.ctx.bin.aidem";
+    std::string imgs = "../bus.jpg";
+    int invoke_nums = 10;
+    std::string model_type = "QNN";
+};
+
+Args parse_args(int argc, char* argv[]) {
+    Args args;
+    for (int i = 1; i < argc; ++i) {
+        std::string arg = argv[i];
+        if (arg == "--target_model" && i + 1 < argc) {
+            args.target_model = argv[++i];
+        } else if (arg == "--imgs" && i + 1 < argc) {
+            args.imgs = argv[++i];
+        } else if (arg == "--invoke_nums" && i + 1 < argc) {
+            args.invoke_nums = std::stoi(argv[++i]);
+        } else if (arg == "--model_type" && i + 1 < argc) {
+            args.model_type = argv[++i];
+        }
+    }
+    return args;
+}
+
+std::string to_lower(const std::string& str) {
+    std::string lower_str = str;
+    std::transform(lower_str.begin(), lower_str.end(), lower_str.begin(), [](unsigned char c) {
+        return std::tolower(c);
+    });
+    return lower_str;
+}
+
+
+void concatenate(float* qnn_trans_data, float* qnn_mul_data, int batch, int num_elements, int trans_dim, int mul_dim, std::vector<float>& output) {
+    int out_dim = trans_dim + mul_dim + 1; 
+    output.resize(batch * num_elements * out_dim); 
+    for (int i = 0; i < batch * num_elements; ++i) {
+        std::memcpy(&output[i * out_dim], &qnn_mul_data[i * mul_dim], mul_dim * sizeof(float));
+        float max_val = *std::max_element(&qnn_trans_data[i * trans_dim], &qnn_trans_data[i * trans_dim + trans_dim]);
+        output[i * out_dim + 4] = max_val;
+        std::memcpy(&output[i * out_dim + 5], &qnn_trans_data[i * trans_dim], trans_dim * sizeof(float));
+    }
+}
+
+void transformData(const float* input, float* output, int C, int N) {
+    for (int c = 0; c < C; ++c) {
+        for (int n = 0; n < N; ++n) {
+            output[n * C + c] = input[c * N + n];
+        }
+    }
+}
+
+double img_process(cv::Mat frame, cv::Mat &img_input, int size) {
+    cv::Mat img_processed = frame.clone();
+    int height = img_processed.rows;
+    int width = img_processed.cols;
+    int length = std::max(height, width);
+    double scala = static_cast<double>(length) / size;
+    
+    cv::Mat image = cv::Mat::zeros(cv::Size(length, length), CV_8UC3);
+    img_processed.copyTo(image(cv::Rect(0, 0, width, height)));
+    
+    cv::cvtColor(image, img_input, cv::COLOR_BGR2RGB);
+    cv::resize(img_input, img_input, cv::Size(size, size));
+    
+    cv::Mat mean_data = cv::Mat::zeros(img_input.size(), CV_32FC3);
+    cv::Mat std_data(img_input.size(), CV_32FC3, cv::Scalar(255, 255, 255));
+    img_input.convertTo(img_input, CV_32FC3);
+    img_input = (img_input - mean_data) / std_data;
+    return scala;
+}
+
+
+cv::Scalar generate_colors(int i, bool bgr = false) {
+    static const std::vector<std::string> hex_colors = {
+        "FF3838", "FF9D97", "FF701F", "FFB21D", "CFD231", "48F90A",
+        "92CC17", "3DDB86", "1A9334", "00D4BB", "2C99A8", "00C2FF",
+        "344593", "6473FF", "0018EC", "8438FF", "520085", "CB38FF",
+        "FF95C8", "FF37C7"
+    };
+
+    int num = hex_colors.size();
+    std::string hex = hex_colors[i % num];
+
+    int r = std::stoi(hex.substr(0, 2), nullptr, 16);
+    int g = std::stoi(hex.substr(2, 2), nullptr, 16);
+    int b = std::stoi(hex.substr(4, 2), nullptr, 16);
+
+    if (bgr)
+        return cv::Scalar(b, g, r);
+    else
+        return cv::Scalar(r, g, b);
+}
+
+void draw_label(cv::Mat& input_image, std::string label, int left, int top, cv::Scalar color)
+{
+    int baseLine;
+    cv::Size label_size = cv::getTextSize(label, FONT_FACE, FONT_SCALE, THICKNESS, &baseLine);
+    int y = top - label_size.height - baseLine;
+    if (y < 0) {
+        y = top ;
+    }
+    cv::Point tlc(left, y);
+    cv::Point brc(left + label_size.width, y + label_size.height + baseLine);
+    rectangle(input_image, tlc, brc, color, cv::FILLED);
+    putText(input_image, label, cv::Point(left, y + label_size.height), FONT_FACE, FONT_SCALE, WHITE, THICKNESS);
+}
+
+
+cv::Mat post_process(cv::Mat &input_image, std::vector<float> &outputs, const std::vector<std::string> &class_name, const double ratio)
+{
+    // Initialize vectors to hold respective outputs while unwrapping detections.
+    std::vector<int> class_ids;
+    std::vector<float> confidences;
+    std::vector<cv::Rect> boxes;
+    
+    // Iterate through outputs for each box prediction
+    for (int i = 0; i < outputs.size(); i+=85)
+    {
+        float confidence = outputs[i+4];
+        if (confidence >= CONFIDENCE_THRESHOLD)
+        {
+            // Create a 1x80 Mat and store class scores of 80 classes.
+            cv::Mat scores(1, class_name.size(), CV_32FC1, outputs.data() + i + 5);
+            cv::Point class_id;
+            double max_class_score;
+            
+            // For multi-label, check each class score
+            for (int c = 0; c < class_name.size(); c++) {
+                float class_score = scores.at<float>(0, c);
+                
+                // If class score is above threshold, consider this class for the box
+                if (class_score > SCORE_THRESHOLD) {
+                    // Store class ID and confidence in the pre-defined respective vectors.
+                    confidences.push_back(confidence * class_score);  // Multiply with confidence
+                    class_ids.push_back(c);  // class index
+                    // Center and box dimension.
+                    float cx = outputs[i];
+                    float cy = outputs[i+1];
+                    float w = outputs[i+2];
+                    float h = outputs[i+3];
+
+                    int left = int((cx - 0.5 * w) * ratio);
+                    int top = int((cy - 0.5 * h) * ratio);
+                    int width = int(w * ratio);
+                    int height = int(h * ratio);
+
+                    // Store good detections in the boxes vector.
+                    boxes.push_back(cv::Rect(left, top, width, height));
+                }
+            }
+        }
+    }
+
+    // Perform Non Maximum Suppression and draw predictions.
+    std::vector<int> indices;
+    cv::dnn::NMSBoxes(boxes, confidences, SCORE_THRESHOLD, NMS_THRESHOLD, indices);
+    printf("Detected {%ld} targets.\n", indices.size());
+    
+    // Loop over NMS results and draw bounding boxes
+    for (int i = 0; i < indices.size(); i++)
+    {
+        int idx = indices[i];
+        cv::Rect box = boxes[idx];
+
+        int left = box.x;
+        int top = box.y;
+        int width = box.width;
+        int height = box.height;
+        cv::Scalar color = generate_colors(class_ids[idx]);
+        // Draw bounding box.
+        rectangle(input_image, cv::Point(left, top), cv::Point(left + width, top + height), color, 2*THICKNESS);
+
+        // Get the label for the class name and its confidence.
+        std::string label = cv::format("%.2f", confidences[idx]);
+        label = class_name[class_ids[idx]] + ":" + label;
+        // Draw class labels.
+        draw_label(input_image, label, left, top, color);
+    }
+    printf("Processing finished.\n");
+    return input_image;
+}
+
+int invoke(const Args& args) {
+    std::cout << "Start main ... ... Model Path: " << args.target_model << "\n"
+              << "Image Path: " << args.imgs << "\n"
+              << "Inference Nums: " << args.invoke_nums << "\n"
+              << "Model Type: " << args.model_type << "\n";
+    Model* model = Model::create_instance(args.target_model);
+    if(model == nullptr){
+        printf("Create model failed !\n");
+        return EXIT_FAILURE;
+    }
+    Config* config = Config::create_instance();
+    if(config == nullptr){
+        printf("Create config failed !\n");
+        return EXIT_FAILURE;
+    }
+    config->implement_type = ImplementType::TYPE_LOCAL;
+    std::string model_type_lower = to_lower(args.model_type);
+    if (model_type_lower == "qnn"){
+        config->framework_type = FrameworkType::TYPE_QNN;
+    } else if (model_type_lower == "snpe2" || model_type_lower == "snpe") {
+        config->framework_type = FrameworkType::TYPE_SNPE2;
+    }
+    config->accelerate_type = AccelerateType::TYPE_DSP;
+    config->is_quantify_model = 1;
+
+    std::vector<std::vector<uint32_t>> input_shapes = {{1, size, size, 3}};
+    std::vector<std::vector<uint32_t>> output_shapes = {{1, 4, out_size}, {1, 80, out_size}};
+    model->set_model_properties(input_shapes, DataType::TYPE_FLOAT32, output_shapes, DataType::TYPE_FLOAT32);
+    std::unique_ptr<Interpreter> fast_interpreter = InterpreterBuilder::build_interpretper_from_model_and_config(model, config);
+    if(fast_interpreter == nullptr){
+        printf("build_interpretper_from_model_and_config failed !\n");
+        return EXIT_FAILURE;
+    }
+    int result = fast_interpreter->init();
+    if(result != EXIT_SUCCESS){
+        printf("interpreter->init() failed !\n");
+        return EXIT_FAILURE;
+    }
+    // load model
+    fast_interpreter->load_model();
+    if(result != EXIT_SUCCESS){
+        printf("interpreter->load_model() failed !\n");
+        return EXIT_FAILURE;
+    }
+    printf("detect model load success!\n");
+
+    cv::Mat frame = cv::imread(args.imgs);
+    if (frame.empty()) {
+        printf("detect image load failed!\n");
+        return 1;
+    }
+    printf("img_src cols: %d, img_src rows: %d\n", frame.cols, frame.rows);
+    cv::Mat input_img;
+    double scale = img_process(frame, input_img, size);
+    if (input_img.empty()) {
+        printf("detect input_img load failed!\n");
+        return 1;
+    }
+
+    float *qnn_trans_data = nullptr;
+    float *qnn_mul_data = nullptr;
+
+    std::vector<float> invoke_time;
+    for (int i = 0; i < args.invoke_nums; ++i) {
+        result = fast_interpreter->set_input_tensor(0, input_img.data);
+        if(result != EXIT_SUCCESS){
+            printf("interpreter->set_input_tensor() failed !\n");
+            return EXIT_FAILURE;
+        }
+          // 开始计时
+        auto t1 = std::chrono::high_resolution_clock::now();
+        result = fast_interpreter->invoke();
+        auto t2 = std::chrono::high_resolution_clock::now();
+        std::chrono::duration<double> cost_time = t2 - t1;
+        invoke_time.push_back(cost_time.count() * 1000);
+        if(result != EXIT_SUCCESS){
+            printf("interpreter->invoke() failed !\n");
+            return EXIT_FAILURE;
+        }
+        uint32_t out_data_1 = 0;
+        result = fast_interpreter->get_output_tensor(0, (void**)&qnn_trans_data, &out_data_1);
+        if(result != EXIT_SUCCESS){
+            printf("interpreter->get_output_tensor() 1 failed !\n");
+            return EXIT_FAILURE;
+        }
+        uint32_t out_data_2 = 0;
+        result = fast_interpreter->get_output_tensor(1, (void**)&qnn_mul_data, &out_data_2);
+        if(result != EXIT_SUCCESS){
+            printf("interpreter->get_output_tensor() 2 failed !\n");
+            return EXIT_FAILURE;
+        }
+    }
+
+    float max_invoke_time = *std::max_element(invoke_time.begin(), invoke_time.end());
+    float min_invoke_time = *std::min_element(invoke_time.begin(), invoke_time.end());
+    float mean_invoke_time = std::accumulate(invoke_time.begin(), invoke_time.end(), 0.0f) / args.invoke_nums;
+    float var_invoketime = 0.0f;
+    for (auto time : invoke_time) {
+        var_invoketime += (time - mean_invoke_time) * (time - mean_invoke_time);
+    }
+    var_invoketime /= args.invoke_nums;
+    printf("=======================================\n");
+    printf("QNN inference %d times :\n --mean_invoke_time is %f \n --max_invoke_time is %f \n --min_invoke_time is %f \n --var_invoketime is %f\n", 
+        args.invoke_nums, mean_invoke_time, max_invoke_time, min_invoke_time, var_invoketime);
+    printf("=======================================\n");
+
+    // std::vector<std::vector<uint32_t>> output_shapes = {{1, 4, out_size}, {1, 80, out_size}};
+    float* pos_data = new float[4 * out_size];
+    float* class_data = new float[80 * out_size];
+    transformData(qnn_trans_data, pos_data, 4, out_size);
+    transformData(qnn_mul_data, class_data, 80, out_size);
+    
+    // post process
+    std::vector<float> qnn_concat;
+    concatenate(class_data, pos_data , 1, out_size, 80, 4, qnn_concat);
+    cv::Mat img = post_process(frame, qnn_concat, class_list, scale);
+    cv::imwrite("./results.png", img);
+    fast_interpreter->destory();
+    return 0;
+}
+
+
+int main(int argc, char* argv[]) {
+    Args args = parse_args(argc, argv);
+    return invoke(args);
+}

model_farm_yolov8s_qcs8550_qnn2.16_int8_aidlite/models/cutoff_yolov8s_w8a8.qnn216.ctx.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25255e85b913a3d079c9a975049b192ae05b77232cb78c0b5077f339c38129e6
+size 11690008

model_farm_yolov8s_qcs8550_qnn2.16_int8_aidlite/python/run_test.py

@@ -0,0 +1,30 @@
+import cv2
+import argparse
+from yolov8s import Yolov8S
+from utils import draw_detect_res
+
+def parser_args():
+    parser = argparse.ArgumentParser(description="Run model benchmarks")
+    parser.add_argument('--target_model',type=str,default='./models/cutoff_yolov8s_w8a8.qnn231.ctx.bin.aidem',help="inference model path")
+    parser.add_argument('--imgs',type=str,default='./python/bus.jpg',help="Predict images path")
+    parser.add_argument('--height',type=int,default=640,help="run backend")
+    parser.add_argument('--weight',type=int,default=640,help="run backend")
+    parser.add_argument('--cls_num',type=int,default=80,help="run backend")
+    parser.add_argument('--invoke_nums',type=int,default=10,help="Inference nums")
+    parser.add_argument('--model_type',type=str,default='QNN',help="run backend")
+    args = parser.parse_args()
+    return args
+
+if __name__ == "__main__":
+    args = parser_args()
+    height = args.height
+    weight = args.weight
+
+    
+    model = Yolov8S(args.target_model, args.weight, args.height, args.cls_num)
+    frame = cv2.imread(args.imgs)
+    
+    out_boxes= model(frame, args.invoke_nums)
+    print(f"=================== \n Detect {len(out_boxes)} targets.")
+    result = draw_detect_res(frame, out_boxes)
+    cv2.imwrite("./python/result.jpg", result)

model_farm_yolov8s_qcs8550_qnn2.16_int8_aidlite/python/utils.py

@@ -0,0 +1,123 @@
+import numpy as np
+import cv2
+
+CLASSES = ("person", "bicycle", "car", "motorbike ", "aeroplane ", "bus ", "train", "truck ", "boat", "traffic light",
+           "fire hydrant", "stop sign ", "parking meter", "bench", "bird", "cat", "dog ", "horse ", "sheep", "cow", "elephant",
+           "bear", "zebra ", "giraffe", "backpack", "umbrella", "handbag", "tie", "suitcase", "frisbee", "skis", "snowboard", "sports ball", "kite",
+           "baseball bat", "baseball glove", "skateboard", "surfboard", "tennis racket", "bottle", "wine glass", "cup", "fork", "knife ",
+           "spoon", "bowl", "banana", "apple", "sandwich", "orange", "broccoli", "carrot", "hot dog", "pizza ", "donut", "cake", "chair", "sofa",
+           "pottedplant", "bed", "diningtable", "toilet ", "tvmonitor", "laptop	", "mouse	", "remote ", "keyboard ", "cell phone", "microwave ",
+           "oven ", "toaster", "sink", "refrigerator ", "book", "clock", "vase", "scissors ", "teddy bear ", "hair drier", "toothbrush ")
+           
+def eqprocess(image, size1, size2):
+    h,w,_ = image.shape
+    mask = np.zeros((size1,size2,3),dtype=np.float32)
+    scale1 = h /size1
+    scale2 = w / size2
+    if scale1 > scale2:
+        scale = scale1
+    else:
+        scale = scale2
+    img = cv2.resize(image,(int(w / scale),int(h / scale)))
+    mask[:int(h / scale),:int(w / scale),:] = img
+    return mask, scale
+
+def xywh2xyxy(x):
+    '''
+    Box (center x, center y, width, height) to (x1, y1, x2, y2)
+    '''
+    y = np.copy(x)
+    y[:, 0] = x[:, 0] - x[:, 2] / 2  # top left x
+    y[:, 1] = x[:, 1] - x[:, 3] / 2  # top left y
+    y[:, 2] = x[:, 0] + x[:, 2] / 2  # bottom right x
+    y[:, 3] = x[:, 1] + x[:, 3] / 2  # bottom right y
+    return y
+
+def xyxy2xywh(box):
+    '''
+    Box (left_top x, left_top y, right_bottom x, right_bottom y) to (left_top x, left_top y, width, height)
+    '''
+    box[:, 2:] = box[:, 2:] - box[:, :2]
+    return box
+
+def NMS(dets, scores, thresh):
+    '''
+    单类NMS算法
+    dets.shape = (N, 5), (left_top x, left_top y, right_bottom x, right_bottom y, Scores)
+    '''
+    x1 = dets[:,0]
+    y1 = dets[:,1]
+    x2 = dets[:,2]
+    y2 = dets[:,3]
+    areas = (y2-y1+1) * (x2-x1+1)
+    keep = []
+    index = scores.argsort()[::-1]
+    while index.size >0:
+        i = index[0]       # every time the first is the biggst, and add it directly
+        keep.append(i)
+        x11 = np.maximum(x1[i], x1[index[1:]])    # calculate the points of overlap 
+        y11 = np.maximum(y1[i], y1[index[1:]])
+        x22 = np.minimum(x2[i], x2[index[1:]])
+        y22 = np.minimum(y2[i], y2[index[1:]])
+        w = np.maximum(0, x22-x11+1)    # the weights of overlap
+        h = np.maximum(0, y22-y11+1)    # the height of overlap
+        overlaps = w*h
+        ious = overlaps / (areas[i]+areas[index[1:]] - overlaps)
+        idx = np.where(ious<=thresh)[0]
+        index = index[idx+1]   # because index start from 1
+ 
+    return keep
+
+
+def draw_detect_res(img, det_pred):
+    '''
+    检测结果绘制
+    '''
+    if det_pred is None:
+        return img
+
+    img = img.astype(np.uint8)
+    im_canvas = img.copy()
+    color_step = int(255/len(CLASSES))
+    for i in range(len(det_pred)):
+        x1, y1, x2, y2 = [int(t) for t in det_pred[i][:4]]
+        cls_id = int(det_pred[i][5])
+        print(i+1,[x1,y1,x2,y2],det_pred[i][4], f'{CLASSES[cls_id]}')
+        cv2.putText(img, f'{CLASSES[cls_id]}', (x1, y1-6), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1)
+        cv2.rectangle(img, (x1, y1), (x2, y2), (0, int(cls_id*color_step), int(255-cls_id*color_step)),thickness = 2)
+    img = cv2.addWeighted(im_canvas, 0.3, img, 0.7, 0)
+    return img
+
+def scale_mask(masks, im0_shape):
+    masks = cv2.resize(masks, (im0_shape[1], im0_shape[0]),
+                        interpolation=cv2.INTER_LINEAR)
+    if len(masks.shape) == 2:
+        masks = masks[:, :, None]
+    return masks
+
+def crop_mask(masks, boxes):
+    n, h, w = masks.shape
+    x1, y1, x2, y2 = np.split(boxes[:, :, None], 4, 1)
+    r = np.arange(w, dtype=x1.dtype)[None, None, :]
+    c = np.arange(h, dtype=x1.dtype)[None, :, None]
+    return masks * ((r >= x1) * (r < x2) * (c >= y1) * (c < y2))
+
+def process_mask(protos, masks_in, bboxes, im0_shape):
+    c, mh, mw = protos.shape
+    masks = np.matmul(masks_in, protos.reshape((c, -1))).reshape((-1, mh, mw)).transpose(1, 2, 0)  # HWN
+    masks = np.ascontiguousarray(masks)
+    masks = scale_mask(masks, im0_shape)  # re-scale mask from P3 shape to original input image shape
+    masks = np.einsum('HWN -> NHW', masks)  # HWN -> NHW
+    masks = crop_mask(masks, bboxes)
+    return np.greater(masks, 0.5)
+
+def masks2segments(masks):
+    segments = []
+    for x in masks.astype('uint8'):
+        c = cv2.findContours(x, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)[0]  # CHAIN_APPROX_SIMPLE
+        if c:
+            c = np.array(c[np.array([len(x) for x in c]).argmax()]).reshape(-1, 2)
+        else:
+            c = np.zeros((0, 2))  # no segments found
+        segments.append(c.astype('float32'))
+    return segments

model_farm_yolov8s_qcs8550_qnn2.16_int8_aidlite/python/yolov8s.py

@@ -0,0 +1,86 @@
+import numpy as np
+import cv2
+import aidlite
+from utils import eqprocess, xywh2xyxy, NMS
+import time
+
+OBJ_THRESH = 0.45
+NMS_THRESH = 0.45
+
+class Yolov8S(object):
+    def __init__(self, model_path, width, height, class_num):
+        self.class_num = class_num
+        self.width = width
+        self.height = height
+        input_shape = [[1,height,width,3]]
+        self.blocks = int(height * width * ( 1 / 64 + 1 / 256 + 1 / 1024))
+        self.maskw = int(width / 4)
+        self.maskh = int(height / 4)
+        self.output_shape = [[1,4,self.blocks],[1,class_num,self.blocks]]
+    
+        self.model = aidlite.Model.create_instance(model_path)
+        if self.model is None:
+            print("Create model failed !")
+            return
+        self.model.set_model_properties(input_shape, aidlite.DataType.TYPE_FLOAT32, self.output_shape, aidlite.DataType.TYPE_FLOAT32)
+        
+        self.config = aidlite.Config.create_instance()
+        if self.config is None:
+            print("build_interpretper_from_model_and_config failed !")
+            return
+        
+        self.config.framework_type = aidlite.FrameworkType.TYPE_QNN216
+        self.config.accelerate_type = aidlite.AccelerateType.TYPE_DSP
+        self.config.is_quantify_model = 1
+        
+        self.interpreter = aidlite.InterpreterBuilder.build_interpretper_from_model_and_config(self.model, self.config)
+        if self.interpreter is None:
+            print("build_interpretper_from_model_and_config failed !")
+            return
+        
+        self.interpreter.init()
+        self.interpreter.load_model()
+    
+    def __call__(self, frame,invoke_nums):
+        img, scale = eqprocess(frame, self.height, self.width)
+        img = img / 255
+        img = img.astype(np.float32)
+        self.interpreter.set_input_tensor(0,img.data)
+        
+        invoke_time=[]
+        for i  in range(invoke_nums):
+            t1=time.time()
+            self.interpreter.invoke()
+            cost_time = (time.time()-t1)*1000
+            invoke_time.append(cost_time)
+
+        max_invoke_time = max(invoke_time)
+        min_invoke_time = min(invoke_time)
+        mean_invoke_time = sum(invoke_time)/invoke_nums
+        var_invoketime=np.var(invoke_time)
+        print("====================================")
+        print(f"QNN invoke {invoke_nums} times:\n --mean_invoke_time is {mean_invoke_time} \n --max_invoke_time is {max_invoke_time} \n --min_invoke_time is {min_invoke_time} \n --var_invoketime is {var_invoketime}")
+        print("====================================")
+
+        qnn_1 = self.interpreter.get_output_tensor(0)
+        qnn_2 = self.interpreter.get_output_tensor(1)
+        qnn_out = sorted([qnn_1,qnn_2], key=len)
+
+        qnn_local = qnn_out[0].reshape(*self.output_shape[0])
+        qnn_conf = qnn_out[1].reshape(*self.output_shape[1])
+
+        
+        x = np.concatenate([qnn_local, qnn_conf], axis = 1).transpose(0,2,1)
+        x = x[np.amax(x[..., 4:], axis=-1) > OBJ_THRESH]
+        if len(x) < 1:
+            return None, None
+    
+        x = np.c_[x[..., :4], np.amax(x[..., 4:], axis=-1), np.argmax(x[..., 4:], axis=-1)]
+        
+        x[:, :4] = xywh2xyxy(x[:, :4])
+        index = NMS(x[:, :4], x[:, 4], NMS_THRESH)
+        out_boxes = x[index]
+        out_boxes[..., :4] = out_boxes[..., :4] * scale
+        
+        return out_boxes
+