From b85d79e673ac28d0f1bcf65a1773990bff6cd0b4 Mon Sep 17 00:00:00 2001 From: Emmanuel Benazera Date: Sat, 16 Jan 2021 15:11:51 +0100 Subject: [PATCH] feat: support for batches for NCNN image models --- src/backends/ncnn/ncnninputconns.h | 95 ++++++----- src/backends/ncnn/ncnnlib.cc | 255 +++++++++++++++-------------- tests/ut-ncnnapi.cc | 26 ++- 3 files changed, 211 insertions(+), 165 deletions(-) diff --git a/src/backends/ncnn/ncnninputconns.h b/src/backends/ncnn/ncnninputconns.h index a9bc56acb..8246dd002 100644 --- a/src/backends/ncnn/ncnninputconns.h +++ b/src/backends/ncnn/ncnninputconns.h @@ -98,40 +98,48 @@ namespace dd { throw; } - cv::Mat bgr = this->_images.at(0); - _height = bgr.rows; - _width = bgr.cols; - _in = ncnn::Mat::from_pixels(bgr.data, ncnn::Mat::PIXEL_BGR, bgr.cols, - bgr.rows); - if (_has_mean_scalar) + for (size_t i = 0; i < _images.size(); ++i) { - if (_std.empty()) - { - _in.substract_mean_normalize(_mean.data(), 0); - } - else - { - _in.substract_mean_normalize(_mean.data(), _std.data()); - } - } - else if (_scale != 1.0) - { - float norm = 1.0 / _scale; - if (_bw) - { - std::vector vscale = { norm }; - _in.substract_mean_normalize(0, vscale.data()); - } - else - { - std::vector vscale = { norm, norm, norm }; - _in.substract_mean_normalize(0, vscale.data()); - } + cv::Mat bgr = this->_images.at(i); + _height = bgr.rows; + _width = bgr.cols; + + _in.push_back(ncnn::Mat::from_pixels(bgr.data, ncnn::Mat::PIXEL_BGR, + bgr.cols, bgr.rows)); + { + if (_has_mean_scalar) + { + if (_std.empty()) + { + _in.at(i).substract_mean_normalize(_mean.data(), 0); + } + else + { + _in.at(i).substract_mean_normalize(_mean.data(), + _std.data()); + } + } + else if (_scale != 1.0) + { + float norm = 1.0 / _scale; + if (_bw) + { + std::vector vscale = { norm }; + _in.at(i).substract_mean_normalize(0, vscale.data()); + } + else + { + std::vector vscale = { norm, norm, norm }; + _in.at(i).substract_mean_normalize(0, vscale.data()); + } + } + _ids.push_back(this->_uris.at(i)); + _imgs_size.insert(std::pair>( + this->_ids.at(i), this->_images_size.at(i))); + } + _out = std::vector(_ids.size(), ncnn::Mat()); } - _ids.push_back(this->_uris.at(0)); - _imgs_size.insert(std::pair>( - this->_ids.at(0), this->_images_size.at(0))); } double unscale_res(double res, int nout) @@ -140,8 +148,8 @@ namespace dd } public: - ncnn::Mat _in; - ncnn::Mat _out; + std::vector _in; + std::vector _out; std::vector _ids; /**< input ids (e.g. image ids) */ }; @@ -207,7 +215,8 @@ namespace dd _height += l; } // Mat(w,h) - _in.create(_width, _height); + _in.emplace_back(_width, _height); + _out.emplace_back(); int mati = 0; @@ -222,20 +231,20 @@ namespace dd for (unsigned int si = 0; si < this->_csvtsdata.size(); ++si) { if (_continuation) - _in[mati++] = 1.0; + _in.at(0)[mati++] = 1.0; else - _in[mati++] = 0.0; + _in.at(0)[mati++] = 0.0; for (int di = 0; di < _ntargets; ++di) - _in[mati++] = 0.0; + _in.at(0)[mati++] = 0.0; for (unsigned int di : input_pos) - _in[mati++] = this->_csvtsdata[si][0]._v[di]; + _in.at(0)[mati++] = this->_csvtsdata[si][0]._v[di]; for (unsigned int ti = 1; ti < this->_csvtsdata[si].size(); ++ti) { - _in[mati++] = 1.0; + _in.at(0)[mati++] = 1.0; for (int di = 0; di < _ntargets; ++di) - _in[mati++] = 0.0; + _in.at(0)[mati++] = 0.0; for (unsigned int di : input_pos) - _in[mati++] = this->_csvtsdata[si][ti]._v[di]; + _in.at(0)[mati++] = this->_csvtsdata[si][ti]._v[di]; } } _ids.push_back(this->_uris.at(0)); @@ -255,8 +264,8 @@ namespace dd } public: - ncnn::Mat _in; - ncnn::Mat _out; + std::vector _in; + std::vector _out; int _height; int _width; std::vector _ids; /**< input ids (e.g. image ids) */ diff --git a/src/backends/ncnn/ncnnlib.cc b/src/backends/ncnn/ncnnlib.cc index b6732b58e..7bb8b291d 100644 --- a/src/backends/ncnn/ncnnlib.cc +++ b/src/backends/ncnn/ncnnlib.cc @@ -211,11 +211,6 @@ namespace dd _net->set_input_h(_old_height); } - ncnn::Extractor ex = _net->create_extractor(); - - ex.set_num_threads(_threads); - ex.input(_inputBlob.c_str(), inputc._in); - APIData ad_output = ad.getobj("parameters").getobj("output"); // Get bbox @@ -250,18 +245,8 @@ namespace dd else out_blob = "prob"; } - ret = ex.extract(out_blob.c_str(), inputc._out); - if (ret == -1) - { - throw MLLibInternalException("NCNN internal error"); - } std::vector vrad; - std::vector probs; - std::vector cats; - std::vector bboxes; - std::vector series; - APIData rad; // Get confidence_threshold float confidence_threshold = 0.0; @@ -280,133 +265,163 @@ namespace dd if (best == -1 || best > _nclasses) best = _nclasses; - if (bbox == true) + // for loop around batch size +#pragma omp parallel for num_threads(_threads) + for (size_t b = 0; b < inputc._ids.size(); b++) { - std::string uri = inputc._ids.at(0); - auto bit = inputc._imgs_size.find(uri); - int rows = 1; - int cols = 1; - if (bit != inputc._imgs_size.end()) - { - // original image size - rows = (*bit).second.first; - cols = (*bit).second.second; - } - else + std::vector probs; + std::vector cats; + std::vector bboxes; + std::vector series; + APIData rad; + + ncnn::Extractor ex = _net->create_extractor(); + ex.set_num_threads(_threads); + ex.input(_inputBlob.c_str(), inputc._in.at(b)); + + ret = ex.extract(out_blob.c_str(), inputc._out.at(b)); + if (ret == -1) { - throw MLLibInternalException( - "Couldn't find original image size for " + uri); + throw MLLibInternalException("NCNN internal error"); } - for (int i = 0; i < inputc._out.h; i++) + + if (bbox == true) { - const float *values = inputc._out.row(i); - if (values[1] < confidence_threshold) - break; // output is sorted by confidence - - cats.push_back(this->_mlmodel.get_hcorresp(values[0])); - probs.push_back(values[1]); - - APIData ad_bbox; - ad_bbox.add("xmin", static_cast(values[2] * (cols - 1))); - ad_bbox.add("ymin", static_cast(values[3] * (rows - 1))); - ad_bbox.add("xmax", static_cast(values[4] * (cols - 1))); - ad_bbox.add("ymax", static_cast(values[5] * (rows - 1))); - bboxes.push_back(ad_bbox); + std::string uri = inputc._ids.at(b); + auto bit = inputc._imgs_size.find(uri); + int rows = 1; + int cols = 1; + if (bit != inputc._imgs_size.end()) + { + // original image size + rows = (*bit).second.first; + cols = (*bit).second.second; + } + else + { + throw MLLibInternalException( + "Couldn't find original image size for " + uri); + } + for (int i = 0; i < inputc._out.at(b).h; i++) + { + const float *values = inputc._out.at(b).row(i); + if (values[1] < confidence_threshold) + break; // output is sorted by confidence + + cats.push_back(this->_mlmodel.get_hcorresp(values[0])); + probs.push_back(values[1]); + + APIData ad_bbox; + ad_bbox.add("xmin", + static_cast(values[2] * (cols - 1))); + ad_bbox.add("ymin", + static_cast(values[3] * (rows - 1))); + ad_bbox.add("xmax", + static_cast(values[4] * (cols - 1))); + ad_bbox.add("ymax", + static_cast(values[5] * (rows - 1))); + bboxes.push_back(ad_bbox); + } } - } - else if (ctc == true) - { - int alphabet = inputc._out.w; - int time_step = inputc._out.h; - std::vector pred_label_seq_with_blank(time_step); - for (int t = 0; t < time_step; ++t) + else if (ctc == true) { - const float *values = inputc._out.row(t); - pred_label_seq_with_blank[t] = std::distance( - values, std::max_element(values, values + alphabet)); - } + int alphabet = inputc._out.at(b).w; + int time_step = inputc._out.at(b).h; + std::vector pred_label_seq_with_blank(time_step); + for (int t = 0; t < time_step; ++t) + { + const float *values = inputc._out.at(b).row(t); + pred_label_seq_with_blank[t] = std::distance( + values, std::max_element(values, values + alphabet)); + } - std::vector pred_label_seq; - int prev = blank_label; - for (int t = 0; t < time_step; ++t) - { - int cur = pred_label_seq_with_blank[t]; - if (cur != prev && cur != blank_label) - pred_label_seq.push_back(cur); - prev = cur; + std::vector pred_label_seq; + int prev = blank_label; + for (int t = 0; t < time_step; ++t) + { + int cur = pred_label_seq_with_blank[t]; + if (cur != prev && cur != blank_label) + pred_label_seq.push_back(cur); + prev = cur; + } + std::string outstr; + std::ostringstream oss; + for (auto l : pred_label_seq) + outstr + += char(std::atoi(this->_mlmodel.get_hcorresp(l).c_str())); + cats.push_back(outstr); + probs.push_back(1.0); } - std::string outstr; - std::ostringstream oss; - for (auto l : pred_label_seq) - outstr += char(std::atoi(this->_mlmodel.get_hcorresp(l).c_str())); - cats.push_back(outstr); - probs.push_back(1.0); - } - else if (_timeserie) - { - std::vector tsl = inputc._timeseries_lengths; - for (unsigned int tsi = 0; tsi < tsl.size(); ++tsi) + else if (_timeserie) { - for (int ti = 0; ti < tsl[tsi]; ++ti) + std::vector tsl = inputc._timeseries_lengths; + for (unsigned int tsi = 0; tsi < tsl.size(); ++tsi) { - std::vector predictions; - for (int k = 0; k < inputc._ntargets; ++k) + for (int ti = 0; ti < tsl[tsi]; ++ti) { - double res = inputc._out.row(ti)[k]; - predictions.push_back(inputc.unscale_res(res, k)); + std::vector predictions; + for (int k = 0; k < inputc._ntargets; ++k) + { + double res = inputc._out.at(b).row(ti)[k]; + predictions.push_back(inputc.unscale_res(res, k)); + } + APIData ts; + ts.add("out", predictions); + series.push_back(ts); } - APIData ts; - ts.add("out", predictions); - series.push_back(ts); } } - } - else - { - std::vector cls_scores; - - cls_scores.resize(inputc._out.w); - for (int j = 0; j < inputc._out.w; j++) - { - cls_scores[j] = inputc._out[j]; - } - int size = cls_scores.size(); - std::vector> vec; - vec.resize(size); - for (int i = 0; i < size; i++) + else { - vec[i] = std::make_pair(cls_scores[i], i); - } + std::vector cls_scores; + + cls_scores.resize(inputc._out.at(b).w); + for (int j = 0; j < inputc._out.at(b).w; j++) + { + cls_scores[j] = inputc._out.at(b)[j]; + } + int size = cls_scores.size(); + std::vector> vec; + vec.resize(size); + for (int i = 0; i < size; i++) + { + vec[i] = std::make_pair(cls_scores[i], i); + } - std::partial_sort(vec.begin(), vec.begin() + best, vec.end(), - std::greater>()); + std::partial_sort(vec.begin(), vec.begin() + best, vec.end(), + std::greater>()); - for (int i = 0; i < best; i++) + for (int i = 0; i < best; i++) + { + if (vec[i].first < confidence_threshold) + continue; + cats.push_back(this->_mlmodel.get_hcorresp(vec[i].second)); + probs.push_back(vec[i].first); + } + } + + rad.add("uri", inputc._ids.at(b)); + rad.add("loss", 0.0); + rad.add("cats", cats); + if (bbox == true) + rad.add("bboxes", bboxes); + if (_timeserie) { - if (vec[i].first < confidence_threshold) - continue; - cats.push_back(this->_mlmodel.get_hcorresp(vec[i].second)); - probs.push_back(vec[i].first); + rad.add("series", series); + rad.add("probs", std::vector(series.size(), 1.0)); } - } + else + rad.add("probs", probs); - rad.add("uri", inputc._ids.at(0)); - rad.add("loss", 0.0); - rad.add("cats", cats); - if (bbox == true) - rad.add("bboxes", bboxes); - if (_timeserie) - { - rad.add("series", series); - rad.add("probs", std::vector(series.size(), 1.0)); - } - else - rad.add("probs", probs); + if (_timeserie) + out.add("timeseries", true); - if (_timeserie) - out.add("timeseries", true); +#pragma omp critical + { + vrad.push_back(rad); + } + } // end for batch_size - vrad.push_back(rad); tout.add_results(vrad); out.add("nclasses", this->_nclasses); if (bbox == true) diff --git a/tests/ut-ncnnapi.cc b/tests/ut-ncnnapi.cc index 214d846f0..53c0ae1dd 100644 --- a/tests/ut-ncnnapi.cc +++ b/tests/ut-ncnnapi.cc @@ -94,7 +94,8 @@ TEST(ncnnapi, service_predict_bbox) jpredictstr = "{\"service\":\"imgserv\",\"parameters\":{\"input\":{\"height\":300," "\"width\":300,\"mean\":[128,128,128],\"std\":[255,255,255]}," - "\"output\":{\"bbox\":true}},\"data\":[\"" + "\"output\":{\"bbox\":true,\"confidence_threshold\":0.25}},\"data\":[" + "\"" + squeezenet_ssd_repo + "face.jpg\"]}"; joutstr = japi.jrender(japi.service_predict(jpredictstr)); std::cout << "joutstr=" << joutstr << std::endl; @@ -110,7 +111,8 @@ TEST(ncnnapi, service_predict_bbox) jpredictstr = "{\"service\":\"imgserv\",\"parameters\":{\"input\":{\"height\":300," "\"width\":300,\"scale\":0.0039}," - "\"output\":{\"bbox\":true}},\"data\":[\"" + "\"output\":{\"bbox\":true,\"confidence_threshold\":0.25}},\"data\":[" + "\"" + squeezenet_ssd_repo + "face.jpg\"]}"; joutstr = japi.jrender(japi.service_predict(jpredictstr)); std::cout << "joutstr=" << joutstr << std::endl; @@ -121,6 +123,26 @@ TEST(ncnnapi, service_predict_bbox) cl1 = jd["body"]["predictions"][0]["classes"][0]["cat"].GetString(); ASSERT_TRUE(jd["body"]["predictions"][0]["classes"][0]["prob"].GetDouble() > 0.4); + + // predict with batch_size > 1 + jpredictstr + = "{\"service\":\"imgserv\",\"parameters\":{\"input\":{\"height\":300," + "\"width\":300},\"output\":{\"bbox\":true,\"confidence_threshold\":0." + "25}},\"data\":[\"" + + squeezenet_ssd_repo + "face.jpg\",\"" + squeezenet_ssd_repo + + "cat.jpg\"]}"; + // std::cerr << "predict=" << jpredictstr << std::endl; + joutstr = japi.jrender(japi.service_predict(jpredictstr)); + std::cout << "joutstr=" << joutstr << std::endl; + jd.Parse(joutstr.c_str()); + ASSERT_TRUE(!jd.HasParseError()); + ASSERT_EQ(200, jd["status"]["code"]); + ASSERT_TRUE(jd["body"]["predictions"].IsArray()); + cl1 = jd["body"]["predictions"][0]["classes"][0]["cat"].GetString(); + ASSERT_TRUE(jd["body"]["predictions"][0]["classes"][0]["prob"].GetDouble() + > 0.4); + ASSERT_TRUE(jd["body"]["predictions"][1]["classes"][0]["prob"].GetDouble() + > 0.4); } TEST(ncnnapi, service_predict_classification)