// Ultralytics 🚀 AGPL-3.0 License - https://ultralytics.com/license #include "inference.hpp" #include #include #include #include #include #include #define IOU_THRESHOLD 0.45 uint16_t float32_to_float16(float value) { __m128 input = _mm_set_ss(value); return _mm_cvtsi128_si32(_mm_cvtps_ph(input, 0)); } float float16_to_float32(uint16_t half) { uint16_t sign = (half & 0x8000) >> 15; uint16_t exponent = (half & 0x7C00) >> 10; uint16_t mantissa = (half & 0x03FF); if (exponent == 0) { if (mantissa == 0) { return sign ? -0.0f : 0.0f; } return std::ldexp(mantissa / 1024.0f, -14) * (sign ? -1.0f : 1.0f); } else if (exponent == 31) { return mantissa ? NAN : (sign ? -INFINITY : INFINITY); } float real_value = std::ldexp(1.0f + mantissa / 1024.0f, exponent - 15); return sign ? -real_value : real_value; } void Image::preprocess(cv::Mat* img, std::vector& triton_data, int input_w, int input_h) { int w, h, x, y; float r_w = input_w / (img->cols*1.0); float r_h = input_h / (img->rows*1.0); if (r_h > r_w) { w = input_w; h = r_w * img->rows; x = 0; y = (input_h - h) / 2; } else { w = r_h * img->cols; h = input_h; x = (input_w - w) / 2; y = 0; } cv::Mat re(h, w, CV_8UC3); cv::resize(*img, re, re.size(), 0, 0, cv::INTER_LINEAR); cv::Mat out(input_h, input_w, CV_8UC3, cv::Scalar(114, 114, 114)); re.copyTo(out(cv::Rect(x, y, re.cols, re.rows))); cv::cvtColor(out, out, cv::COLOR_BGR2RGB); unsigned char* data = (unsigned char*)out.data; int step = out.step; for (int yy = 0; yy < input_h; ++yy) { for (int kk = 0; kk < 3; ++kk) { for (int xx = 0; xx < input_w; ++xx) { float temp_f = data[yy * step + xx * 3 + kk] / 255.0f; triton_data[kk * input_w * input_h + yy * input_w + xx] = float32_to_float16(temp_f); } } } } int getDetectionsFromTritonRawData(std::vector& detection_results, std::vector &detections, std::vector& object_class_list, float confidence_threshold, int image_width, int image_height) { const size_t shape[3] = {1, object_class_list.size()+4, 8400}; std::vector boxes; for (size_t i = 0; i < shape[2]; i++) { int x = int(detection_results[0 * shape[2] + i]); int y = int(detection_results[1 * shape[2] + i]); int w = int(detection_results[2 * shape[2] + i]); int h = int(detection_results[3 * shape[2] + i]); for(size_t j =0 ; j < object_class_list.size(); j++) { if (detection_results[(4+j) * shape[2] + i] > 0.01) { BoundingBox box; box.x = static_cast(x); box.y = static_cast(y); box.w = static_cast(w); box.h = static_cast(h); box.score = detection_results[(4 + j) * shape[2] + i]; box.class_id = j; boxes.push_back(box); } } } auto nms_boxes = NMS(boxes, IOU_THRESHOLD); detections.clear(); if(nms_boxes.size()==0) { return 0; } float scale_x = 0.0; float scale_y = 0.0; int x1,y1 = 0; int x2,y2 = 0; float shift_factor_x = 0.6; float shift_factor_y = 0.5; int offset_shift = (image_width/640.0f)*10; if (image_width<=640) { scale_x = static_cast(image_width - 640.0f ) * 0.5 ; scale_y = static_cast(image_height - 640.0f) * 0.5 ; } for (size_t i = 0; i < nms_boxes.size(); ++i) { if (nms_boxes[i].score< confidence_threshold) { continue; } struct detection_struct tespit_yapi ; detections.push_back(tespit_yapi); detections[detections.size() - 1].confidence_score = nms_boxes[i].score; if (image_width==640) { scale_x = static_cast(image_width - 640.0f ) * 0.5 ; scale_y = static_cast(image_height - 640.0f) * 0.5 ; x1 = static_cast((nms_boxes[i].x - nms_boxes[i].w/2) + scale_x); y1 = static_cast((nms_boxes[i].y - nms_boxes[i].h/2) + scale_y) ; x2 = static_cast((nms_boxes[i].x + nms_boxes[i].w/2) + scale_x); y2 = static_cast((nms_boxes[i].y + nms_boxes[i].h/2) + scale_y); } else if(image_width>=1080) { x1 = static_cast((nms_boxes[i].x - nms_boxes[i].w/2) * (image_width/640) ); y1 = static_cast((nms_boxes[i].y - nms_boxes[i].h/2) * (image_width/640) - ((image_width - image_height) / 2.0)) ; x2 = static_cast((nms_boxes[i].x + nms_boxes[i].w/2) * (image_width/640) ); y2 = static_cast((nms_boxes[i].y + nms_boxes[i].h/2) * (image_width/640)- ((image_width - image_height) / 2.0)); } float x_center, y_center, width, height; x_center = (x1 + x2) / 2.0f; y_center = (y1 + y2) / 2.0f; width = x2 - x1; height = y2 - y1; detections[detections.size() - 1].bbox.x = x_center - width/2 ; detections[detections.size() - 1].bbox.y = y_center - height/2; detections[detections.size() - 1].bbox.width = width ; detections[detections.size() - 1].bbox.height = height; if (detections[detections.size() - 1].bbox.x <= 0) detections[detections.size() - 1].bbox.x = offset_shift; if (detections[detections.size() - 1].bbox.y <= 0) detections[detections.size() - 1].bbox.y = offset_shift; if (detections[detections.size() - 1].bbox.x + detections[detections.size() - 1].bbox.width >= image_width) { detections[detections.size() - 1].bbox.width -= detections[detections.size() - 1].bbox.x + detections[detections.size() - 1].bbox.width - image_width + offset_shift ; } if (detections[detections.size() - 1].bbox.y + detections[detections.size() - 1].bbox.height >= image_height) { detections[detections.size() - 1].bbox.height -= detections[detections.size() - 1].bbox.y + detections[detections.size() - 1].bbox.height - image_height + offset_shift ; } detections[detections.size() - 1].name = object_class_list[nms_boxes[i].class_id]; detections[detections.size() - 1].class_id = nms_boxes[i].class_id; } return 0; } float IoU(const BoundingBox& box1, const BoundingBox& box2) { float x1_min = box1.x - box1.w / 2.0f; float y1_min = box1.y - box1.h / 2.0f; float x1_max = box1.x + box1.w / 2.0f; float y1_max = box1.y + box1.h / 2.0f; float x2_min = box2.x - box2.w / 2.0f; float y2_min = box2.y - box2.h / 2.0f; float x2_max = box2.x + box2.w / 2.0f; float y2_max = box2.y + box2.h / 2.0f; float inter_x_min = std::max(x1_min, x2_min); float inter_y_min = std::max(y1_min, y2_min); float inter_x_max = std::min(x1_max, x2_max); float inter_y_max = std::min(y1_max, y2_max); float inter_width = inter_x_max - inter_x_min; float inter_height = inter_y_max - inter_y_min; if (inter_width <= 0 || inter_height <= 0) return 0.0f; float inter_area = inter_width * inter_height; float area1 = (x1_max - x1_min) * (y1_max - y1_min); float area2 = (x2_max - x2_min) * (y2_max - y2_min); float union_area = area1 + area2 - inter_area; return inter_area / union_area; } std::vector NMS(const std::vector& boxes, float iou_threshold) { std::vector result; std::vector sorted_boxes = boxes; std::sort(sorted_boxes.begin(), sorted_boxes.end(), [](const BoundingBox& a, const BoundingBox& b) { return a.score > b.score; }); std::vector suppressed(sorted_boxes.size(), false); for (size_t i = 0; i < sorted_boxes.size(); ++i) { if (suppressed[i]) continue; result.push_back(sorted_boxes[i]); for (size_t j = i + 1; j < sorted_boxes.size(); ++j) { if (suppressed[j]) continue; if (IoU(sorted_boxes[i], sorted_boxes[j]) > iou_threshold) { suppressed[j] = true; } } } return result; } TritonCommunication::TritonCommunication(std::string triton_address, std::string model_name, std::string model_version, int image_channel, int image_width, int image_height, int class_count) : options(model_name) { triton::client::Error err; this->triton_url = triton_address; this->options.model_version_ = model_version; this->shape = {1, image_channel, image_width, image_height}; this->input_byte_size = image_channel * image_width * image_height * sizeof(uint16_t) ; this->output_byte_size = (class_count + 4) * 8400 * sizeof(uint16_t); err = tc::InferenceServerGrpcClient::Create(&(this->client), this->triton_url); if (!err.IsOk()) { std::cout<< "Create grpc client error:"<IsServerReady(&live); if (!err.IsOk() || !live) { std::cout<< "Triton server is not live !"<IsModelReady(&model_ready,model_name,model_version); if (!err.IsOk() || !model_ready) { std::cerr << "Model:[" << model_name << "] has not been deployed on Triton Server. Triton Server Address:["< input0_ptr; input0_ptr.reset(input0); err = input0_ptr->AppendRaw((const uint8_t*)image_data, this->input_byte_size); std::vector inputs = {input0_ptr.get()}; tc::InferResult* results; err = client->Infer(&results, options, inputs); results_ptr.reset(results); float *output0_data; size_t output0_byte_size; std::vector result_fp16_raw_data; results->RawData("output0", (const uint8_t**)&output0_data, &output0_byte_size); // output0 is a specific name for the output tensor. result_fp16_raw_data.resize(output0_byte_size/sizeof(uint16_t)); std::memcpy(result_fp16_raw_data.data(), output0_data, output0_byte_size); std::vector float32_data(num_elements); for (size_t i = 0; i < num_elements; i++) { float32_data[i] = float16_to_float32(result_fp16_raw_data[i]); } output_raw_data = float32_data; }