#include "precomp.hpp" #include <map> #include <iostream> #include <fstream> #if defined WIN32 || defined _WIN32 || defined WIN64 || defined _WIN64 #ifndef NOMINMAX #define NOMINMAX #endif #include <windows.h> #endif #ifdef HAVE_CUDA #include "opencv2/core/cuda.hpp" #endif #ifdef ANDROID # include <sys/time.h> #endif using namespace perf; int64 TestBase::timeLimitDefault = 0; unsigned int TestBase::iterationsLimitDefault = (unsigned int)(-1); int64 TestBase::_timeadjustment = 0; // Item [0] will be considered the default implementation. static std::vector<std::string> available_impls; static std::string param_impl; static enum PERF_STRATEGY strategyForce = PERF_STRATEGY_DEFAULT; static enum PERF_STRATEGY strategyModule = PERF_STRATEGY_SIMPLE; static double param_max_outliers; static double param_max_deviation; static unsigned int param_min_samples; static unsigned int param_force_samples; static uint64 param_seed; static double param_time_limit; static int param_threads; static bool param_write_sanity; static bool param_verify_sanity; #ifdef CV_COLLECT_IMPL_DATA static bool param_collect_impl; #endif extern bool test_ipp_check; #ifdef HAVE_CUDA static int param_cuda_device; #endif #ifdef ANDROID static int param_affinity_mask; static bool log_power_checkpoints; #include <sys/syscall.h> #include <pthread.h> static void setCurrentThreadAffinityMask(int mask) { pid_t pid=gettid(); int syscallres=syscall(__NR_sched_setaffinity, pid, sizeof(mask), &mask); if (syscallres) { int err=errno; err=err;//to avoid warnings about unused variables LOGE("Error in the syscall setaffinity: mask=%d=0x%x err=%d=0x%x", mask, mask, err, err); } } #endif static double perf_stability_criteria = 0.03; // 3% namespace { class PerfEnvironment: public ::testing::Environment { public: void TearDown() { cv::setNumThreads(-1); } }; } // namespace static void randu(cv::Mat& m) { const int bigValue = 0x00000FFF; if (m.depth() < CV_32F) { int minmax[] = {0, 256}; cv::Mat mr = cv::Mat(m.rows, (int)(m.cols * m.elemSize()), CV_8U, m.ptr(), m.step[0]); cv::randu(mr, cv::Mat(1, 1, CV_32S, minmax), cv::Mat(1, 1, CV_32S, minmax + 1)); } else if (m.depth() == CV_32F) { //float minmax[] = {-FLT_MAX, FLT_MAX}; float minmax[] = {-bigValue, bigValue}; cv::Mat mr = m.reshape(1); cv::randu(mr, cv::Mat(1, 1, CV_32F, minmax), cv::Mat(1, 1, CV_32F, minmax + 1)); } else { //double minmax[] = {-DBL_MAX, DBL_MAX}; double minmax[] = {-bigValue, bigValue}; cv::Mat mr = m.reshape(1); cv::randu(mr, cv::Mat(1, 1, CV_64F, minmax), cv::Mat(1, 1, CV_64F, minmax + 1)); } } /*****************************************************************************************\ * inner exception class for early termination \*****************************************************************************************/ class PerfEarlyExitException: public cv::Exception {}; /*****************************************************************************************\ * ::perf::Regression \*****************************************************************************************/ Regression& Regression::instance() { static Regression single; return single; } Regression& Regression::add(TestBase* test, const std::string& name, cv::InputArray array, double eps, ERROR_TYPE err) { if(test) test->setVerified(); return instance()(name, array, eps, err); } Regression& Regression::addMoments(TestBase* test, const std::string& name, const cv::Moments& array, double eps, ERROR_TYPE err) { int len = (int)sizeof(cv::Moments) / sizeof(double); cv::Mat m(1, len, CV_64F, (void*)&array); return Regression::add(test, name, m, eps, err); } Regression& Regression::addKeypoints(TestBase* test, const std::string& name, const std::vector<cv::KeyPoint>& array, double eps, ERROR_TYPE err) { int len = (int)array.size(); cv::Mat pt (len, 1, CV_32FC2, len ? (void*)&array[0].pt : 0, sizeof(cv::KeyPoint)); cv::Mat size (len, 1, CV_32FC1, len ? (void*)&array[0].size : 0, sizeof(cv::KeyPoint)); cv::Mat angle (len, 1, CV_32FC1, len ? (void*)&array[0].angle : 0, sizeof(cv::KeyPoint)); cv::Mat response(len, 1, CV_32FC1, len ? (void*)&array[0].response : 0, sizeof(cv::KeyPoint)); cv::Mat octave (len, 1, CV_32SC1, len ? (void*)&array[0].octave : 0, sizeof(cv::KeyPoint)); cv::Mat class_id(len, 1, CV_32SC1, len ? (void*)&array[0].class_id : 0, sizeof(cv::KeyPoint)); return Regression::add(test, name + "-pt", pt, eps, ERROR_ABSOLUTE) (name + "-size", size, eps, ERROR_ABSOLUTE) (name + "-angle", angle, eps, ERROR_ABSOLUTE) (name + "-response", response, eps, err) (name + "-octave", octave, eps, ERROR_ABSOLUTE) (name + "-class_id", class_id, eps, ERROR_ABSOLUTE); } Regression& Regression::addMatches(TestBase* test, const std::string& name, const std::vector<cv::DMatch>& array, double eps, ERROR_TYPE err) { int len = (int)array.size(); cv::Mat queryIdx(len, 1, CV_32SC1, len ? (void*)&array[0].queryIdx : 0, sizeof(cv::DMatch)); cv::Mat trainIdx(len, 1, CV_32SC1, len ? (void*)&array[0].trainIdx : 0, sizeof(cv::DMatch)); cv::Mat imgIdx (len, 1, CV_32SC1, len ? (void*)&array[0].imgIdx : 0, sizeof(cv::DMatch)); cv::Mat distance(len, 1, CV_32FC1, len ? (void*)&array[0].distance : 0, sizeof(cv::DMatch)); return Regression::add(test, name + "-queryIdx", queryIdx, DBL_EPSILON, ERROR_ABSOLUTE) (name + "-trainIdx", trainIdx, DBL_EPSILON, ERROR_ABSOLUTE) (name + "-imgIdx", imgIdx, DBL_EPSILON, ERROR_ABSOLUTE) (name + "-distance", distance, eps, err); } void Regression::Init(const std::string& testSuitName, const std::string& ext) { instance().init(testSuitName, ext); } void Regression::init(const std::string& testSuitName, const std::string& ext) { if (!storageInPath.empty()) { LOGE("Subsequent initialization of Regression utility is not allowed."); return; } #ifndef WINRT const char *data_path_dir = getenv("OPENCV_TEST_DATA_PATH"); #else const char *data_path_dir = OPENCV_TEST_DATA_PATH; #endif const char *path_separator = "/"; if (data_path_dir) { int len = (int)strlen(data_path_dir)-1; if (len < 0) len = 0; std::string path_base = (data_path_dir[0] == 0 ? std::string(".") : std::string(data_path_dir)) + (data_path_dir[len] == '/' || data_path_dir[len] == '\\' ? "" : path_separator) + "perf" + path_separator; storageInPath = path_base + testSuitName + ext; storageOutPath = path_base + testSuitName; } else { storageInPath = testSuitName + ext; storageOutPath = testSuitName; } suiteName = testSuitName; try { if (storageIn.open(storageInPath, cv::FileStorage::READ)) { rootIn = storageIn.root(); if (storageInPath.length() > 3 && storageInPath.substr(storageInPath.length()-3) == ".gz") storageOutPath += "_new"; storageOutPath += ext; } } catch(cv::Exception&) { LOGE("Failed to open sanity data for reading: %s", storageInPath.c_str()); } if(!storageIn.isOpened()) storageOutPath = storageInPath; } Regression::Regression() : regRNG(cv::getTickCount())//this rng should be really random { } Regression::~Regression() { if (storageIn.isOpened()) storageIn.release(); if (storageOut.isOpened()) { if (!currentTestNodeName.empty()) storageOut << "}"; storageOut.release(); } } cv::FileStorage& Regression::write() { if (!storageOut.isOpened() && !storageOutPath.empty()) { int mode = (storageIn.isOpened() && storageInPath == storageOutPath) ? cv::FileStorage::APPEND : cv::FileStorage::WRITE; storageOut.open(storageOutPath, mode); if (!storageOut.isOpened()) { LOGE("Could not open \"%s\" file for writing", storageOutPath.c_str()); storageOutPath.clear(); } else if (mode == cv::FileStorage::WRITE && !rootIn.empty()) { //TODO: write content of rootIn node into the storageOut } } return storageOut; } std::string Regression::getCurrentTestNodeName() { const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info(); if (test_info == 0) return "undefined"; std::string nodename = std::string(test_info->test_case_name()) + "--" + test_info->name(); size_t idx = nodename.find_first_of('/'); if (idx != std::string::npos) nodename.erase(idx); const char* type_param = test_info->type_param(); if (type_param != 0) (nodename += "--") += type_param; const char* value_param = test_info->value_param(); if (value_param != 0) (nodename += "--") += value_param; for(size_t i = 0; i < nodename.length(); ++i) if (!isalnum(nodename[i]) && '_' != nodename[i]) nodename[i] = '-'; return nodename; } bool Regression::isVector(cv::InputArray a) { return a.kind() == cv::_InputArray::STD_VECTOR_MAT || a.kind() == cv::_InputArray::STD_VECTOR_VECTOR || a.kind() == cv::_InputArray::STD_VECTOR_UMAT; } double Regression::getElem(cv::Mat& m, int y, int x, int cn) { switch (m.depth()) { case CV_8U: return *(m.ptr<unsigned char>(y, x) + cn); case CV_8S: return *(m.ptr<signed char>(y, x) + cn); case CV_16U: return *(m.ptr<unsigned short>(y, x) + cn); case CV_16S: return *(m.ptr<signed short>(y, x) + cn); case CV_32S: return *(m.ptr<signed int>(y, x) + cn); case CV_32F: return *(m.ptr<float>(y, x) + cn); case CV_64F: return *(m.ptr<double>(y, x) + cn); default: return 0; } } void Regression::write(cv::Mat m) { if (!m.empty() && m.dims < 2) return; double min, max; cv::minMaxIdx(m, &min, &max); write() << "min" << min << "max" << max; write() << "last" << "{" << "x" << m.size.p[1] - 1 << "y" << m.size.p[0] - 1 << "val" << getElem(m, m.size.p[0] - 1, m.size.p[1] - 1, m.channels() - 1) << "}"; int x, y, cn; x = regRNG.uniform(0, m.size.p[1]); y = regRNG.uniform(0, m.size.p[0]); cn = regRNG.uniform(0, m.channels()); write() << "rng1" << "{" << "x" << x << "y" << y; if(cn > 0) write() << "cn" << cn; write() << "val" << getElem(m, y, x, cn) << "}"; x = regRNG.uniform(0, m.size.p[1]); y = regRNG.uniform(0, m.size.p[0]); cn = regRNG.uniform(0, m.channels()); write() << "rng2" << "{" << "x" << x << "y" << y; if (cn > 0) write() << "cn" << cn; write() << "val" << getElem(m, y, x, cn) << "}"; } void Regression::verify(cv::FileNode node, cv::Mat actual, double eps, std::string argname, ERROR_TYPE err) { if (!actual.empty() && actual.dims < 2) return; double expect_min = (double)node["min"]; double expect_max = (double)node["max"]; if (err == ERROR_RELATIVE) eps *= std::max(std::abs(expect_min), std::abs(expect_max)); double actual_min, actual_max; cv::minMaxIdx(actual, &actual_min, &actual_max); ASSERT_NEAR(expect_min, actual_min, eps) << argname << " has unexpected minimal value" << std::endl; ASSERT_NEAR(expect_max, actual_max, eps) << argname << " has unexpected maximal value" << std::endl; cv::FileNode last = node["last"]; double actual_last = getElem(actual, actual.size.p[0] - 1, actual.size.p[1] - 1, actual.channels() - 1); int expect_cols = (int)last["x"] + 1; int expect_rows = (int)last["y"] + 1; ASSERT_EQ(expect_cols, actual.size.p[1]) << argname << " has unexpected number of columns" << std::endl; ASSERT_EQ(expect_rows, actual.size.p[0]) << argname << " has unexpected number of rows" << std::endl; double expect_last = (double)last["val"]; ASSERT_NEAR(expect_last, actual_last, eps) << argname << " has unexpected value of the last element" << std::endl; cv::FileNode rng1 = node["rng1"]; int x1 = rng1["x"]; int y1 = rng1["y"]; int cn1 = rng1["cn"]; double expect_rng1 = (double)rng1["val"]; // it is safe to use x1 and y1 without checks here because we have already // verified that mat size is the same as recorded double actual_rng1 = getElem(actual, y1, x1, cn1); ASSERT_NEAR(expect_rng1, actual_rng1, eps) << argname << " has unexpected value of the ["<< x1 << ":" << y1 << ":" << cn1 <<"] element" << std::endl; cv::FileNode rng2 = node["rng2"]; int x2 = rng2["x"]; int y2 = rng2["y"]; int cn2 = rng2["cn"]; double expect_rng2 = (double)rng2["val"]; double actual_rng2 = getElem(actual, y2, x2, cn2); ASSERT_NEAR(expect_rng2, actual_rng2, eps) << argname << " has unexpected value of the ["<< x2 << ":" << y2 << ":" << cn2 <<"] element" << std::endl; } void Regression::write(cv::InputArray array) { write() << "kind" << array.kind(); write() << "type" << array.type(); if (isVector(array)) { int total = (int)array.total(); int idx = regRNG.uniform(0, total); write() << "len" << total; write() << "idx" << idx; cv::Mat m = array.getMat(idx); if (m.total() * m.channels() < 26) //5x5 or smaller write() << "val" << m; else write(m); } else { if (array.total() * array.channels() < 26) //5x5 or smaller write() << "val" << array.getMat(); else write(array.getMat()); } } static int countViolations(const cv::Mat& expected, const cv::Mat& actual, const cv::Mat& diff, double eps, double* max_violation = 0, double* max_allowed = 0) { cv::Mat diff64f; diff.reshape(1).convertTo(diff64f, CV_64F); cv::Mat expected_abs = cv::abs(expected.reshape(1)); cv::Mat actual_abs = cv::abs(actual.reshape(1)); cv::Mat maximum, mask; cv::max(expected_abs, actual_abs, maximum); cv::multiply(maximum, cv::Vec<double, 1>(eps), maximum, CV_64F); cv::compare(diff64f, maximum, mask, cv::CMP_GT); int v = cv::countNonZero(mask); if (v > 0 && max_violation != 0 && max_allowed != 0) { int loc[10] = {0}; cv::minMaxIdx(maximum, 0, max_allowed, 0, loc, mask); *max_violation = diff64f.at<double>(loc[0], loc[1]); } return v; } void Regression::verify(cv::FileNode node, cv::InputArray array, double eps, ERROR_TYPE err) { int expected_kind = (int)node["kind"]; int expected_type = (int)node["type"]; ASSERT_EQ(expected_kind, array.kind()) << " Argument \"" << node.name() << "\" has unexpected kind"; ASSERT_EQ(expected_type, array.type()) << " Argument \"" << node.name() << "\" has unexpected type"; cv::FileNode valnode = node["val"]; if (isVector(array)) { int expected_length = (int)node["len"]; ASSERT_EQ(expected_length, (int)array.total()) << " Vector \"" << node.name() << "\" has unexpected length"; int idx = node["idx"]; cv::Mat actual = array.getMat(idx); if (valnode.isNone()) { ASSERT_LE((size_t)26, actual.total() * (size_t)actual.channels()) << " \"" << node.name() << "[" << idx << "]\" has unexpected number of elements"; verify(node, actual, eps, cv::format("%s[%d]", node.name().c_str(), idx), err); } else { cv::Mat expected; valnode >> expected; if(expected.empty()) { ASSERT_TRUE(actual.empty()) << " expected empty " << node.name() << "[" << idx<< "]"; } else { ASSERT_EQ(expected.size(), actual.size()) << " " << node.name() << "[" << idx<< "] has unexpected size"; cv::Mat diff; cv::absdiff(expected, actual, diff); if (err == ERROR_ABSOLUTE) { if (!cv::checkRange(diff, true, 0, 0, eps)) { if(expected.total() * expected.channels() < 12) std::cout << " Expected: " << std::endl << expected << std::endl << " Actual:" << std::endl << actual << std::endl; double max; cv::minMaxIdx(diff.reshape(1), 0, &max); FAIL() << " Absolute difference (=" << max << ") between argument \"" << node.name() << "[" << idx << "]\" and expected value is greater than " << eps; } } else if (err == ERROR_RELATIVE) { double maxv, maxa; int violations = countViolations(expected, actual, diff, eps, &maxv, &maxa); if (violations > 0) { if(expected.total() * expected.channels() < 12) std::cout << " Expected: " << std::endl << expected << std::endl << " Actual:" << std::endl << actual << std::endl; FAIL() << " Relative difference (" << maxv << " of " << maxa << " allowed) between argument \"" << node.name() << "[" << idx << "]\" and expected value is greater than " << eps << " in " << violations << " points"; } } } } } else { if (valnode.isNone()) { ASSERT_LE((size_t)26, array.total() * (size_t)array.channels()) << " Argument \"" << node.name() << "\" has unexpected number of elements"; verify(node, array.getMat(), eps, "Argument \"" + node.name() + "\"", err); } else { cv::Mat expected; valnode >> expected; cv::Mat actual = array.getMat(); if(expected.empty()) { ASSERT_TRUE(actual.empty()) << " expected empty " << node.name(); } else { ASSERT_EQ(expected.size(), actual.size()) << " Argument \"" << node.name() << "\" has unexpected size"; cv::Mat diff; cv::absdiff(expected, actual, diff); if (err == ERROR_ABSOLUTE) { if (!cv::checkRange(diff, true, 0, 0, eps)) { if(expected.total() * expected.channels() < 12) std::cout << " Expected: " << std::endl << expected << std::endl << " Actual:" << std::endl << actual << std::endl; double max; cv::minMaxIdx(diff.reshape(1), 0, &max); FAIL() << " Difference (=" << max << ") between argument1 \"" << node.name() << "\" and expected value is greater than " << eps; } } else if (err == ERROR_RELATIVE) { double maxv, maxa; int violations = countViolations(expected, actual, diff, eps, &maxv, &maxa); if (violations > 0) { if(expected.total() * expected.channels() < 12) std::cout << " Expected: " << std::endl << expected << std::endl << " Actual:" << std::endl << actual << std::endl; FAIL() << " Relative difference (" << maxv << " of " << maxa << " allowed) between argument \"" << node.name() << "\" and expected value is greater than " << eps << " in " << violations << " points"; } } } } } } Regression& Regression::operator() (const std::string& name, cv::InputArray array, double eps, ERROR_TYPE err) { // exit if current test is already failed if(::testing::UnitTest::GetInstance()->current_test_info()->result()->Failed()) return *this; if(!array.empty() && array.depth() == CV_USRTYPE1) { ADD_FAILURE() << " Can not check regression for CV_USRTYPE1 data type for " << name; return *this; } std::string nodename = getCurrentTestNodeName(); cv::FileNode n = rootIn[nodename]; if(n.isNone()) { if(param_write_sanity) { if (nodename != currentTestNodeName) { if (!currentTestNodeName.empty()) write() << "}"; currentTestNodeName = nodename; write() << nodename << "{"; } // TODO: verify that name is alphanumeric, current error message is useless write() << name << "{"; write(array); write() << "}"; } else if(param_verify_sanity) { ADD_FAILURE() << " No regression data for " << name << " argument"; } } else { cv::FileNode this_arg = n[name]; if (!this_arg.isMap()) ADD_FAILURE() << " No regression data for " << name << " argument"; else verify(this_arg, array, eps, err); } return *this; } /*****************************************************************************************\ * ::perf::performance_metrics \*****************************************************************************************/ performance_metrics::performance_metrics() { clear(); } void performance_metrics::clear() { bytesIn = 0; bytesOut = 0; samples = 0; outliers = 0; gmean = 0; gstddev = 0; mean = 0; stddev = 0; median = 0; min = 0; frequency = 0; terminationReason = TERM_UNKNOWN; } /*****************************************************************************************\ * Performance validation results \*****************************************************************************************/ static bool perf_validation_enabled = false; static std::string perf_validation_results_directory; static std::map<std::string, float> perf_validation_results; static std::string perf_validation_results_outfile; static double perf_validation_criteria = 0.03; // 3 % static double perf_validation_time_threshold_ms = 0.1; static int perf_validation_idle_delay_ms = 3000; // 3 sec static void loadPerfValidationResults(const std::string& fileName) { perf_validation_results.clear(); std::ifstream infile(fileName.c_str()); while (!infile.eof()) { std::string name; float value = 0; if (!(infile >> value)) { if (infile.eof()) break; // it is OK std::cout << "ERROR: Can't load performance validation results from " << fileName << "!" << std::endl; return; } infile.ignore(1); if (!(std::getline(infile, name))) { std::cout << "ERROR: Can't load performance validation results from " << fileName << "!" << std::endl; return; } if (!name.empty() && name[name.size() - 1] == '\r') // CRLF processing on Linux name.resize(name.size() - 1); perf_validation_results[name] = value; } std::cout << "Performance validation results loaded from " << fileName << " (" << perf_validation_results.size() << " entries)" << std::endl; } static void savePerfValidationResult(const std::string& name, float value) { perf_validation_results[name] = value; } static void savePerfValidationResults() { if (!perf_validation_results_outfile.empty()) { std::ofstream outfile((perf_validation_results_directory + perf_validation_results_outfile).c_str()); std::map<std::string, float>::const_iterator i; for (i = perf_validation_results.begin(); i != perf_validation_results.end(); ++i) { outfile << i->second << ';'; outfile << i->first << std::endl; } outfile.close(); std::cout << "Performance validation results saved (" << perf_validation_results.size() << " entries)" << std::endl; } } class PerfValidationEnvironment : public ::testing::Environment { public: virtual ~PerfValidationEnvironment() {} virtual void SetUp() {} virtual void TearDown() { savePerfValidationResults(); } }; /*****************************************************************************************\ * ::perf::TestBase \*****************************************************************************************/ void TestBase::Init(int argc, const char* const argv[]) { std::vector<std::string> plain_only; plain_only.push_back("plain"); TestBase::Init(plain_only, argc, argv); } void TestBase::Init(const std::vector<std::string> & availableImpls, int argc, const char* const argv[]) { available_impls = availableImpls; const std::string command_line_keys = "{ perf_max_outliers |8 |percent of allowed outliers}" "{ perf_min_samples |10 |minimal required numer of samples}" "{ perf_force_samples |100 |force set maximum number of samples for all tests}" "{ perf_seed |809564 |seed for random numbers generator}" "{ perf_threads |-1 |the number of worker threads, if parallel execution is enabled}" "{ perf_write_sanity |false |create new records for sanity checks}" "{ perf_verify_sanity |false |fail tests having no regression data for sanity checks}" "{ perf_impl |" + available_impls[0] + "|the implementation variant of functions under test}" "{ perf_list_impls |false |list available implementation variants and exit}" "{ perf_run_cpu |false |deprecated, equivalent to --perf_impl=plain}" "{ perf_strategy |default |specifies performance measuring strategy: default, base or simple (weak restrictions)}" "{ perf_read_validation_results | |specifies file name with performance results from previous run}" "{ perf_write_validation_results | |specifies file name to write performance validation results}" #ifdef ANDROID "{ perf_time_limit |6.0 |default time limit for a single test (in seconds)}" "{ perf_affinity_mask |0 |set affinity mask for the main thread}" "{ perf_log_power_checkpoints | |additional xml logging for power measurement}" #else "{ perf_time_limit |3.0 |default time limit for a single test (in seconds)}" #endif "{ perf_max_deviation |1.0 |}" #ifdef HAVE_IPP "{ perf_ipp_check |false |check whether IPP works without failures}" #endif #ifdef CV_COLLECT_IMPL_DATA "{ perf_collect_impl |false |collect info about executed implementations}" #endif "{ help h |false |print help info}" #ifdef HAVE_CUDA "{ perf_cuda_device |0 |run CUDA test suite onto specific CUDA capable device}" "{ perf_cuda_info_only |false |print an information about system and an available CUDA devices and then exit.}" #endif ; cv::CommandLineParser args(argc, argv, command_line_keys); if (args.get<bool>("help")) { args.printMessage(); return; } ::testing::AddGlobalTestEnvironment(new PerfEnvironment); param_impl = args.get<bool>("perf_run_cpu") ? "plain" : args.get<std::string>("perf_impl"); std::string perf_strategy = args.get<std::string>("perf_strategy"); if (perf_strategy == "default") { // nothing } else if (perf_strategy == "base") { strategyForce = PERF_STRATEGY_BASE; } else if (perf_strategy == "simple") { strategyForce = PERF_STRATEGY_SIMPLE; } else { printf("No such strategy: %s\n", perf_strategy.c_str()); exit(1); } param_max_outliers = std::min(100., std::max(0., args.get<double>("perf_max_outliers"))); param_min_samples = std::max(1u, args.get<unsigned int>("perf_min_samples")); param_max_deviation = std::max(0., args.get<double>("perf_max_deviation")); param_seed = args.get<unsigned int>("perf_seed"); param_time_limit = std::max(0., args.get<double>("perf_time_limit")); param_force_samples = args.get<unsigned int>("perf_force_samples"); param_write_sanity = args.get<bool>("perf_write_sanity"); param_verify_sanity = args.get<bool>("perf_verify_sanity"); #ifdef HAVE_IPP test_ipp_check = !args.get<bool>("perf_ipp_check") ? getenv("OPENCV_IPP_CHECK") != NULL : true; #endif param_threads = args.get<int>("perf_threads"); #ifdef CV_COLLECT_IMPL_DATA param_collect_impl = args.get<bool>("perf_collect_impl"); #endif #ifdef ANDROID param_affinity_mask = args.get<int>("perf_affinity_mask"); log_power_checkpoints = args.has("perf_log_power_checkpoints"); #endif bool param_list_impls = args.get<bool>("perf_list_impls"); if (param_list_impls) { fputs("Available implementation variants:", stdout); for (size_t i = 0; i < available_impls.size(); ++i) { putchar(' '); fputs(available_impls[i].c_str(), stdout); } putchar('\n'); exit(0); } if (std::find(available_impls.begin(), available_impls.end(), param_impl) == available_impls.end()) { printf("No such implementation: %s\n", param_impl.c_str()); exit(1); } #ifdef CV_COLLECT_IMPL_DATA if(param_collect_impl) cv::setUseCollection(1); else cv::setUseCollection(0); #endif #ifdef HAVE_CUDA bool printOnly = args.get<bool>("perf_cuda_info_only"); if (printOnly) exit(0); #endif if (available_impls.size() > 1) printf("[----------]\n[ INFO ] \tImplementation variant: %s.\n[----------]\n", param_impl.c_str()), fflush(stdout); #ifdef HAVE_CUDA param_cuda_device = std::max(0, std::min(cv::cuda::getCudaEnabledDeviceCount(), args.get<int>("perf_cuda_device"))); if (param_impl == "cuda") { cv::cuda::DeviceInfo info(param_cuda_device); if (!info.isCompatible()) { printf("[----------]\n[ FAILURE ] \tDevice %s is NOT compatible with current CUDA module build.\n[----------]\n", info.name()), fflush(stdout); exit(-1); } cv::cuda::setDevice(param_cuda_device); printf("[----------]\n[ GPU INFO ] \tRun test suite on %s GPU.\n[----------]\n", info.name()), fflush(stdout); } #endif { #ifndef WINRT const char* path = getenv("OPENCV_PERF_VALIDATION_DIR"); #else const char* path = OPENCV_PERF_VALIDATION_DIR; #endif if (path) perf_validation_results_directory = path; } std::string fileName_perf_validation_results_src = args.get<std::string>("perf_read_validation_results"); if (!fileName_perf_validation_results_src.empty()) { perf_validation_enabled = true; loadPerfValidationResults(perf_validation_results_directory + fileName_perf_validation_results_src); } perf_validation_results_outfile = args.get<std::string>("perf_write_validation_results"); if (!perf_validation_results_outfile.empty()) { perf_validation_enabled = true; ::testing::AddGlobalTestEnvironment(new PerfValidationEnvironment()); } if (!args.check()) { args.printErrors(); exit(1); } timeLimitDefault = param_time_limit == 0.0 ? 1 : (int64)(param_time_limit * cv::getTickFrequency()); iterationsLimitDefault = param_force_samples == 0 ? (unsigned)(-1) : param_force_samples; _timeadjustment = _calibrate(); } void TestBase::RecordRunParameters() { ::testing::Test::RecordProperty("cv_implementation", param_impl); ::testing::Test::RecordProperty("cv_num_threads", param_threads); #ifdef HAVE_CUDA if (param_impl == "cuda") { cv::cuda::DeviceInfo info(param_cuda_device); ::testing::Test::RecordProperty("cv_cuda_gpu", info.name()); } #endif } std::string TestBase::getSelectedImpl() { return param_impl; } enum PERF_STRATEGY TestBase::setModulePerformanceStrategy(enum PERF_STRATEGY strategy) { enum PERF_STRATEGY ret = strategyModule; strategyModule = strategy; return ret; } enum PERF_STRATEGY TestBase::getCurrentModulePerformanceStrategy() { return strategyForce == PERF_STRATEGY_DEFAULT ? strategyModule : strategyForce; } int64 TestBase::_calibrate() { class _helper : public ::perf::TestBase { public: performance_metrics& getMetrics() { return calcMetrics(); } virtual void TestBody() {} virtual void PerfTestBody() { //the whole system warmup SetUp(); cv::Mat a(2048, 2048, CV_32S, cv::Scalar(1)); cv::Mat b(2048, 2048, CV_32S, cv::Scalar(2)); declare.time(30); double s = 0; for(declare.iterations(20); startTimer(), next(); stopTimer()) s+=a.dot(b); declare.time(s); //self calibration SetUp(); for(declare.iterations(1000); startTimer(), next(); stopTimer()){} } }; _timeadjustment = 0; _helper h; h.PerfTestBody(); double compensation = h.getMetrics().min; if (getCurrentModulePerformanceStrategy() == PERF_STRATEGY_SIMPLE) { CV_Assert(compensation < 0.01 * cv::getTickFrequency()); compensation = 0.0f; // simple strategy doesn't require any compensation } LOGD("Time compensation is %.0f", compensation); return (int64)compensation; } #ifdef _MSC_VER # pragma warning(push) # pragma warning(disable:4355) // 'this' : used in base member initializer list #endif TestBase::TestBase(): testStrategy(PERF_STRATEGY_DEFAULT), declare(this) { lastTime = totalTime = timeLimit = 0; nIters = currentIter = runsPerIteration = 0; minIters = param_min_samples; verified = false; perfValidationStage = 0; } #ifdef _MSC_VER # pragma warning(pop) #endif void TestBase::declareArray(SizeVector& sizes, cv::InputOutputArray a, WarmUpType wtype) { if (!a.empty()) { sizes.push_back(std::pair<int, cv::Size>(getSizeInBytes(a), getSize(a))); warmup(a, wtype); } else if (a.kind() != cv::_InputArray::NONE) ADD_FAILURE() << " Uninitialized input/output parameters are not allowed for performance tests"; } void TestBase::warmup(cv::InputOutputArray a, WarmUpType wtype) { if (a.empty()) return; else if (a.isUMat()) { if (wtype == WARMUP_RNG || wtype == WARMUP_WRITE) { int depth = a.depth(); if (depth == CV_8U) cv::randu(a, 0, 256); else if (depth == CV_8S) cv::randu(a, -128, 128); else if (depth == CV_16U) cv::randu(a, 0, 1024); else if (depth == CV_32F || depth == CV_64F) cv::randu(a, -1.0, 1.0); else if (depth == CV_16S || depth == CV_32S) cv::randu(a, -4096, 4096); else CV_Error(cv::Error::StsUnsupportedFormat, "Unsupported format"); } return; } else if (a.kind() != cv::_InputArray::STD_VECTOR_MAT && a.kind() != cv::_InputArray::STD_VECTOR_VECTOR) warmup_impl(a.getMat(), wtype); else { size_t total = a.total(); for (size_t i = 0; i < total; ++i) warmup_impl(a.getMat((int)i), wtype); } } int TestBase::getSizeInBytes(cv::InputArray a) { if (a.empty()) return 0; int total = (int)a.total(); if (a.kind() != cv::_InputArray::STD_VECTOR_MAT && a.kind() != cv::_InputArray::STD_VECTOR_VECTOR) return total * CV_ELEM_SIZE(a.type()); int size = 0; for (int i = 0; i < total; ++i) size += (int)a.total(i) * CV_ELEM_SIZE(a.type(i)); return size; } cv::Size TestBase::getSize(cv::InputArray a) { if (a.kind() != cv::_InputArray::STD_VECTOR_MAT && a.kind() != cv::_InputArray::STD_VECTOR_VECTOR) return a.size(); return cv::Size(); } PERF_STRATEGY TestBase::getCurrentPerformanceStrategy() const { if (strategyForce == PERF_STRATEGY_DEFAULT) return (testStrategy == PERF_STRATEGY_DEFAULT) ? strategyModule : testStrategy; else return strategyForce; } bool TestBase::next() { static int64 lastActivityPrintTime = 0; if (currentIter != (unsigned int)-1) { if (currentIter + 1 != times.size()) ADD_FAILURE() << " next() is called before stopTimer()"; } else { lastActivityPrintTime = 0; metrics.clear(); } cv::theRNG().state = param_seed; //this rng should generate same numbers for each run ++currentIter; bool has_next = false; do { assert(currentIter == times.size()); if (currentIter == 0) { has_next = true; break; } if (getCurrentPerformanceStrategy() == PERF_STRATEGY_BASE) { has_next = currentIter < nIters && totalTime < timeLimit; } else { assert(getCurrentPerformanceStrategy() == PERF_STRATEGY_SIMPLE); if (totalTime - lastActivityPrintTime >= cv::getTickFrequency() * 10) { std::cout << '.' << std::endl; lastActivityPrintTime = totalTime; } if (currentIter >= nIters) { has_next = false; break; } if (currentIter < minIters) { has_next = true; break; } calcMetrics(); if (fabs(metrics.mean) > 1e-6) has_next = metrics.stddev > perf_stability_criteria * fabs(metrics.mean); else has_next = true; } } while (false); if (perf_validation_enabled && !has_next) { calcMetrics(); double median_ms = metrics.median * 1000.0f / metrics.frequency; const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info(); std::string name = (test_info == 0) ? "" : std::string(test_info->test_case_name()) + "--" + test_info->name(); if (!perf_validation_results.empty() && !name.empty()) { std::map<std::string, float>::iterator i = perf_validation_results.find(name); bool isSame = false; bool found = false; bool grow = false; if (i != perf_validation_results.end()) { found = true; double prev_result = i->second; grow = median_ms > prev_result; isSame = fabs(median_ms - prev_result) <= perf_validation_criteria * fabs(median_ms); if (!isSame) { if (perfValidationStage == 0) { printf("Performance is changed (samples = %d, median):\n %.2f ms (current)\n %.2f ms (previous)\n", (int)times.size(), median_ms, prev_result); } } } else { if (perfValidationStage == 0) printf("New performance result is detected\n"); } if (!isSame) { if (perfValidationStage < 2) { if (perfValidationStage == 0 && currentIter <= minIters * 3 && currentIter < nIters) { unsigned int new_minIters = std::max(minIters * 5, currentIter * 3); printf("Increase minIters from %u to %u\n", minIters, new_minIters); minIters = new_minIters; has_next = true; perfValidationStage++; } else if (found && currentIter >= nIters && median_ms > perf_validation_time_threshold_ms && (grow || metrics.stddev > perf_stability_criteria * fabs(metrics.mean))) { printf("Performance is unstable, it may be a result of overheat problems\n"); printf("Idle delay for %d ms... \n", perf_validation_idle_delay_ms); #if defined WIN32 || defined _WIN32 || defined WIN64 || defined _WIN64 #ifndef WINRT_8_0 Sleep(perf_validation_idle_delay_ms); #else WaitForSingleObjectEx(GetCurrentThread(), perf_validation_idle_delay_ms, FALSE); #endif #else usleep(perf_validation_idle_delay_ms * 1000); #endif has_next = true; minIters = std::min(minIters * 5, nIters); // reset collected samples currentIter = 0; times.clear(); metrics.clear(); perfValidationStage += 2; } if (!has_next) { printf("Assume that current result is valid\n"); } } else { printf("Re-measured performance result: %.2f ms\n", median_ms); } } } if (!has_next && !name.empty()) { savePerfValidationResult(name, (float)median_ms); } } #ifdef ANDROID if (log_power_checkpoints) { timeval tim; gettimeofday(&tim, NULL); unsigned long long t1 = tim.tv_sec * 1000LLU + (unsigned long long)(tim.tv_usec / 1000.f); if (currentIter == 1) RecordProperty("test_start", cv::format("%llu",t1).c_str()); if (!has_next) RecordProperty("test_complete", cv::format("%llu",t1).c_str()); } #endif if (has_next) startTimer(); // really we should measure activity from this moment, so reset start time return has_next; } void TestBase::warmup_impl(cv::Mat m, WarmUpType wtype) { switch(wtype) { case WARMUP_READ: cv::sum(m.reshape(1)); return; case WARMUP_WRITE: m.reshape(1).setTo(cv::Scalar::all(0)); return; case WARMUP_RNG: randu(m); return; default: return; } } unsigned int TestBase::getTotalInputSize() const { unsigned int res = 0; for (SizeVector::const_iterator i = inputData.begin(); i != inputData.end(); ++i) res += i->first; return res; } unsigned int TestBase::getTotalOutputSize() const { unsigned int res = 0; for (SizeVector::const_iterator i = outputData.begin(); i != outputData.end(); ++i) res += i->first; return res; } void TestBase::startTimer() { lastTime = cv::getTickCount(); } void TestBase::stopTimer() { int64 time = cv::getTickCount(); if (lastTime == 0) ADD_FAILURE() << " stopTimer() is called before startTimer()/next()"; lastTime = time - lastTime; totalTime += lastTime; lastTime -= _timeadjustment; if (lastTime < 0) lastTime = 0; times.push_back(lastTime); lastTime = 0; } performance_metrics& TestBase::calcMetrics() { CV_Assert(metrics.samples <= (unsigned int)currentIter); if ((metrics.samples == (unsigned int)currentIter) || times.size() == 0) return metrics; metrics.bytesIn = getTotalInputSize(); metrics.bytesOut = getTotalOutputSize(); metrics.frequency = cv::getTickFrequency(); metrics.samples = (unsigned int)times.size(); metrics.outliers = 0; if (metrics.terminationReason != performance_metrics::TERM_INTERRUPT && metrics.terminationReason != performance_metrics::TERM_EXCEPTION) { if (currentIter == nIters) metrics.terminationReason = performance_metrics::TERM_ITERATIONS; else if (totalTime >= timeLimit) metrics.terminationReason = performance_metrics::TERM_TIME; else metrics.terminationReason = performance_metrics::TERM_UNKNOWN; } std::sort(times.begin(), times.end()); TimeVector::const_iterator start = times.begin(); TimeVector::const_iterator end = times.end(); if (getCurrentPerformanceStrategy() == PERF_STRATEGY_BASE) { //estimate mean and stddev for log(time) double gmean = 0; double gstddev = 0; int n = 0; for(TimeVector::const_iterator i = times.begin(); i != times.end(); ++i) { double x = static_cast<double>(*i)/runsPerIteration; if (x < DBL_EPSILON) continue; double lx = log(x); ++n; double delta = lx - gmean; gmean += delta / n; gstddev += delta * (lx - gmean); } gstddev = n > 1 ? sqrt(gstddev / (n - 1)) : 0; //filter outliers assuming log-normal distribution //http://stackoverflow.com/questions/1867426/modeling-distribution-of-performance-measurements if (gstddev > DBL_EPSILON) { double minout = exp(gmean - 3 * gstddev) * runsPerIteration; double maxout = exp(gmean + 3 * gstddev) * runsPerIteration; while(*start < minout) ++start, ++metrics.outliers; do --end, ++metrics.outliers; while(*end > maxout); ++end, --metrics.outliers; } } else if (getCurrentPerformanceStrategy() == PERF_STRATEGY_SIMPLE) { metrics.outliers = static_cast<int>(times.size() * param_max_outliers / 100); for (unsigned int i = 0; i < metrics.outliers; i++) --end; } else { assert(false); } int offset = static_cast<int>(start - times.begin()); metrics.min = static_cast<double>(*start)/runsPerIteration; //calc final metrics unsigned int n = 0; double gmean = 0; double gstddev = 0; double mean = 0; double stddev = 0; unsigned int m = 0; for(; start != end; ++start) { double x = static_cast<double>(*start)/runsPerIteration; if (x > DBL_EPSILON) { double lx = log(x); ++m; double gdelta = lx - gmean; gmean += gdelta / m; gstddev += gdelta * (lx - gmean); } ++n; double delta = x - mean; mean += delta / n; stddev += delta * (x - mean); } metrics.mean = mean; metrics.gmean = exp(gmean); metrics.gstddev = m > 1 ? sqrt(gstddev / (m - 1)) : 0; metrics.stddev = n > 1 ? sqrt(stddev / (n - 1)) : 0; metrics.median = (n % 2 ? (double)times[offset + n / 2] : 0.5 * (times[offset + n / 2] + times[offset + n / 2 - 1]) ) / runsPerIteration; return metrics; } void TestBase::validateMetrics() { performance_metrics& m = calcMetrics(); if (HasFailure()) return; ASSERT_GE(m.samples, 1u) << " No time measurements was performed.\nstartTimer() and stopTimer() commands are required for performance tests."; if (getCurrentPerformanceStrategy() == PERF_STRATEGY_BASE) { EXPECT_GE(m.samples, param_min_samples) << " Only a few samples are collected.\nPlease increase number of iterations or/and time limit to get reliable performance measurements."; if (m.gstddev > DBL_EPSILON) { EXPECT_GT(/*m.gmean * */1., /*m.gmean * */ 2 * sinh(m.gstddev * param_max_deviation)) << " Test results are not reliable ((mean-sigma,mean+sigma) deviation interval is greater than measured time interval)."; } EXPECT_LE(m.outliers, std::max((unsigned int)cvCeil(m.samples * param_max_outliers / 100.), 1u)) << " Test results are not reliable (too many outliers)."; } else if (getCurrentPerformanceStrategy() == PERF_STRATEGY_SIMPLE) { double mean = metrics.mean * 1000.0f / metrics.frequency; double median = metrics.median * 1000.0f / metrics.frequency; double stddev = metrics.stddev * 1000.0f / metrics.frequency; double percents = stddev / mean * 100.f; printf("[ PERFSTAT ] (samples = %d, mean = %.2f, median = %.2f, stddev = %.2f (%.1f%%))\n", (int)metrics.samples, mean, median, stddev, percents); } else { assert(false); } } void TestBase::reportMetrics(bool toJUnitXML) { performance_metrics& m = calcMetrics(); if (m.terminationReason == performance_metrics::TERM_SKIP_TEST) { if (toJUnitXML) { RecordProperty("custom_status", "skipped"); } } else if (toJUnitXML) { RecordProperty("bytesIn", (int)m.bytesIn); RecordProperty("bytesOut", (int)m.bytesOut); RecordProperty("term", m.terminationReason); RecordProperty("samples", (int)m.samples); RecordProperty("outliers", (int)m.outliers); RecordProperty("frequency", cv::format("%.0f", m.frequency).c_str()); RecordProperty("min", cv::format("%.0f", m.min).c_str()); RecordProperty("median", cv::format("%.0f", m.median).c_str()); RecordProperty("gmean", cv::format("%.0f", m.gmean).c_str()); RecordProperty("gstddev", cv::format("%.6f", m.gstddev).c_str()); RecordProperty("mean", cv::format("%.0f", m.mean).c_str()); RecordProperty("stddev", cv::format("%.0f", m.stddev).c_str()); #ifdef CV_COLLECT_IMPL_DATA if(param_collect_impl) { RecordProperty("impl_ipp", (int)(implConf.ipp || implConf.icv)); RecordProperty("impl_ocl", (int)implConf.ocl); RecordProperty("impl_plain", (int)implConf.plain); std::string rec_line; std::vector<cv::String> rec; rec_line.clear(); rec = implConf.GetCallsForImpl(CV_IMPL_IPP|CV_IMPL_MT); for(int i=0; i<rec.size();i++ ){rec_line += rec[i].c_str(); rec_line += " ";} rec = implConf.GetCallsForImpl(CV_IMPL_IPP); for(int i=0; i<rec.size();i++ ){rec_line += rec[i].c_str(); rec_line += " ";} RecordProperty("impl_rec_ipp", rec_line.c_str()); rec_line.clear(); rec = implConf.GetCallsForImpl(CV_IMPL_OCL); for(int i=0; i<rec.size();i++ ){rec_line += rec[i].c_str(); rec_line += " ";} RecordProperty("impl_rec_ocl", rec_line.c_str()); } #endif } else { const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info(); const char* type_param = test_info->type_param(); const char* value_param = test_info->value_param(); #if defined(ANDROID) && defined(USE_ANDROID_LOGGING) LOGD("[ FAILED ] %s.%s", test_info->test_case_name(), test_info->name()); #endif if (type_param) LOGD("type = %11s", type_param); if (value_param) LOGD("params = %11s", value_param); switch (m.terminationReason) { case performance_metrics::TERM_ITERATIONS: LOGD("termination reason: reached maximum number of iterations"); break; case performance_metrics::TERM_TIME: LOGD("termination reason: reached time limit"); break; case performance_metrics::TERM_INTERRUPT: LOGD("termination reason: aborted by the performance testing framework"); break; case performance_metrics::TERM_EXCEPTION: LOGD("termination reason: unhandled exception"); break; case performance_metrics::TERM_UNKNOWN: default: LOGD("termination reason: unknown"); break; }; #ifdef CV_COLLECT_IMPL_DATA if(param_collect_impl) { LOGD("impl_ipp =%11d", (int)(implConf.ipp || implConf.icv)); LOGD("impl_ocl =%11d", (int)implConf.ocl); LOGD("impl_plain =%11d", (int)implConf.plain); std::string rec_line; std::vector<cv::String> rec; rec_line.clear(); rec = implConf.GetCallsForImpl(CV_IMPL_IPP|CV_IMPL_MT); for(int i=0; i<rec.size();i++ ){rec_line += rec[i].c_str(); rec_line += " ";} rec = implConf.GetCallsForImpl(CV_IMPL_IPP); for(int i=0; i<rec.size();i++ ){rec_line += rec[i].c_str(); rec_line += " ";} LOGD("impl_rec_ipp =%s", rec_line.c_str()); rec_line.clear(); rec = implConf.GetCallsForImpl(CV_IMPL_OCL); for(int i=0; i<rec.size();i++ ){rec_line += rec[i].c_str(); rec_line += " ";} LOGD("impl_rec_ocl =%s", rec_line.c_str()); } #endif LOGD("bytesIn =%11lu", (unsigned long)m.bytesIn); LOGD("bytesOut =%11lu", (unsigned long)m.bytesOut); if (nIters == (unsigned int)-1 || m.terminationReason == performance_metrics::TERM_ITERATIONS) LOGD("samples =%11u", m.samples); else LOGD("samples =%11u of %u", m.samples, nIters); LOGD("outliers =%11u", m.outliers); LOGD("frequency =%11.0f", m.frequency); if (m.samples > 0) { LOGD("min =%11.0f = %.2fms", m.min, m.min * 1e3 / m.frequency); LOGD("median =%11.0f = %.2fms", m.median, m.median * 1e3 / m.frequency); LOGD("gmean =%11.0f = %.2fms", m.gmean, m.gmean * 1e3 / m.frequency); LOGD("gstddev =%11.8f = %.2fms for 97%% dispersion interval", m.gstddev, m.gmean * 2 * sinh(m.gstddev * 3) * 1e3 / m.frequency); LOGD("mean =%11.0f = %.2fms", m.mean, m.mean * 1e3 / m.frequency); LOGD("stddev =%11.0f = %.2fms", m.stddev, m.stddev * 1e3 / m.frequency); } } } void TestBase::SetUp() { cv::theRNG().state = param_seed; // this rng should generate same numbers for each run if (param_threads >= 0) cv::setNumThreads(param_threads); #ifdef ANDROID if (param_affinity_mask) setCurrentThreadAffinityMask(param_affinity_mask); #endif verified = false; lastTime = 0; totalTime = 0; runsPerIteration = 1; nIters = iterationsLimitDefault; currentIter = (unsigned int)-1; timeLimit = timeLimitDefault; times.clear(); } void TestBase::TearDown() { if (metrics.terminationReason == performance_metrics::TERM_SKIP_TEST) { LOGI("\tTest was skipped"); GTEST_SUCCEED() << "Test was skipped"; } else { if (!HasFailure() && !verified) ADD_FAILURE() << "The test has no sanity checks. There should be at least one check at the end of performance test."; validateMetrics(); if (HasFailure()) { reportMetrics(false); return; } } const ::testing::TestInfo* const test_info = ::testing::UnitTest::GetInstance()->current_test_info(); const char* type_param = test_info->type_param(); const char* value_param = test_info->value_param(); if (value_param) printf("[ VALUE ] \t%s\n", value_param), fflush(stdout); if (type_param) printf("[ TYPE ] \t%s\n", type_param), fflush(stdout); #ifdef CV_COLLECT_IMPL_DATA if(param_collect_impl) { implConf.ShapeUp(); printf("[ I. FLAGS ] \t"); if(implConf.ipp_mt) { if(implConf.icv) {printf("ICV_MT "); std::vector<cv::String> fun = implConf.GetCallsForImpl(CV_IMPL_IPP|CV_IMPL_MT); printf("("); for(int i=0; i<fun.size();i++ ){printf("%s ", fun[i].c_str());} printf(") "); } if(implConf.ipp) {printf("IPP_MT "); std::vector<cv::String> fun = implConf.GetCallsForImpl(CV_IMPL_IPP|CV_IMPL_MT); printf("("); for(int i=0; i<fun.size();i++ ){printf("%s ", fun[i].c_str());} printf(") "); } } else { if(implConf.icv) {printf("ICV "); std::vector<cv::String> fun = implConf.GetCallsForImpl(CV_IMPL_IPP); printf("("); for(int i=0; i<fun.size();i++ ){printf("%s ", fun[i].c_str());} printf(") "); } if(implConf.ipp) {printf("IPP "); std::vector<cv::String> fun = implConf.GetCallsForImpl(CV_IMPL_IPP); printf("("); for(int i=0; i<fun.size();i++ ){printf("%s ", fun[i].c_str());} printf(") "); } } if(implConf.ocl) {printf("OCL "); std::vector<cv::String> fun = implConf.GetCallsForImpl(CV_IMPL_OCL); printf("("); for(int i=0; i<fun.size();i++ ){printf("%s ", fun[i].c_str());} printf(") "); } if(implConf.plain) printf("PLAIN "); if(!(implConf.ipp_mt || implConf.icv || implConf.ipp || implConf.ocl || implConf.plain)) printf("ERROR "); printf("\n"); fflush(stdout); } #endif reportMetrics(true); } std::string TestBase::getDataPath(const std::string& relativePath) { if (relativePath.empty()) { ADD_FAILURE() << " Bad path to test resource"; throw PerfEarlyExitException(); } #ifndef WINRT const char *data_path_dir = getenv("OPENCV_TEST_DATA_PATH"); #else const char *data_path_dir = OPENCV_TEST_DATA_PATH; #endif const char *path_separator = "/"; std::string path; if (data_path_dir) { int len = (int)strlen(data_path_dir) - 1; if (len < 0) len = 0; path = (data_path_dir[0] == 0 ? std::string(".") : std::string(data_path_dir)) + (data_path_dir[len] == '/' || data_path_dir[len] == '\\' ? "" : path_separator); } else { path = "."; path += path_separator; } if (relativePath[0] == '/' || relativePath[0] == '\\') path += relativePath.substr(1); else path += relativePath; FILE* fp = fopen(path.c_str(), "r"); if (fp) fclose(fp); else { ADD_FAILURE() << " Requested file \"" << path << "\" does not exist."; throw PerfEarlyExitException(); } return path; } void TestBase::RunPerfTestBody() { try { #ifdef CV_COLLECT_IMPL_DATA if(param_collect_impl) implConf.Reset(); #endif this->PerfTestBody(); #ifdef CV_COLLECT_IMPL_DATA if(param_collect_impl) implConf.GetImpl(); #endif } catch(PerfSkipTestException&) { metrics.terminationReason = performance_metrics::TERM_SKIP_TEST; return; } catch(PerfEarlyExitException&) { metrics.terminationReason = performance_metrics::TERM_INTERRUPT; return;//no additional failure logging } catch(cv::Exception& e) { metrics.terminationReason = performance_metrics::TERM_EXCEPTION; #ifdef HAVE_CUDA if (e.code == cv::Error::GpuApiCallError) cv::cuda::resetDevice(); #endif FAIL() << "Expected: PerfTestBody() doesn't throw an exception.\n Actual: it throws cv::Exception:\n " << e.what(); } catch(std::exception& e) { metrics.terminationReason = performance_metrics::TERM_EXCEPTION; FAIL() << "Expected: PerfTestBody() doesn't throw an exception.\n Actual: it throws std::exception:\n " << e.what(); } catch(...) { metrics.terminationReason = performance_metrics::TERM_EXCEPTION; FAIL() << "Expected: PerfTestBody() doesn't throw an exception.\n Actual: it throws..."; } } /*****************************************************************************************\ * ::perf::TestBase::_declareHelper \*****************************************************************************************/ TestBase::_declareHelper& TestBase::_declareHelper::iterations(unsigned int n) { test->times.clear(); test->times.reserve(n); test->nIters = std::min(n, TestBase::iterationsLimitDefault); test->currentIter = (unsigned int)-1; test->metrics.clear(); return *this; } TestBase::_declareHelper& TestBase::_declareHelper::time(double timeLimitSecs) { test->times.clear(); test->currentIter = (unsigned int)-1; test->timeLimit = (int64)(timeLimitSecs * cv::getTickFrequency()); test->metrics.clear(); return *this; } TestBase::_declareHelper& TestBase::_declareHelper::tbb_threads(int n) { cv::setNumThreads(n); return *this; } TestBase::_declareHelper& TestBase::_declareHelper::runs(unsigned int runsNumber) { test->runsPerIteration = runsNumber; return *this; } TestBase::_declareHelper& TestBase::_declareHelper::in(cv::InputOutputArray a1, WarmUpType wtype) { if (!test->times.empty()) return *this; TestBase::declareArray(test->inputData, a1, wtype); return *this; } TestBase::_declareHelper& TestBase::_declareHelper::in(cv::InputOutputArray a1, cv::InputOutputArray a2, WarmUpType wtype) { if (!test->times.empty()) return *this; TestBase::declareArray(test->inputData, a1, wtype); TestBase::declareArray(test->inputData, a2, wtype); return *this; } TestBase::_declareHelper& TestBase::_declareHelper::in(cv::InputOutputArray a1, cv::InputOutputArray a2, cv::InputOutputArray a3, WarmUpType wtype) { if (!test->times.empty()) return *this; TestBase::declareArray(test->inputData, a1, wtype); TestBase::declareArray(test->inputData, a2, wtype); TestBase::declareArray(test->inputData, a3, wtype); return *this; } TestBase::_declareHelper& TestBase::_declareHelper::in(cv::InputOutputArray a1, cv::InputOutputArray a2, cv::InputOutputArray a3, cv::InputOutputArray a4, WarmUpType wtype) { if (!test->times.empty()) return *this; TestBase::declareArray(test->inputData, a1, wtype); TestBase::declareArray(test->inputData, a2, wtype); TestBase::declareArray(test->inputData, a3, wtype); TestBase::declareArray(test->inputData, a4, wtype); return *this; } TestBase::_declareHelper& TestBase::_declareHelper::out(cv::InputOutputArray a1, WarmUpType wtype) { if (!test->times.empty()) return *this; TestBase::declareArray(test->outputData, a1, wtype); return *this; } TestBase::_declareHelper& TestBase::_declareHelper::out(cv::InputOutputArray a1, cv::InputOutputArray a2, WarmUpType wtype) { if (!test->times.empty()) return *this; TestBase::declareArray(test->outputData, a1, wtype); TestBase::declareArray(test->outputData, a2, wtype); return *this; } TestBase::_declareHelper& TestBase::_declareHelper::out(cv::InputOutputArray a1, cv::InputOutputArray a2, cv::InputOutputArray a3, WarmUpType wtype) { if (!test->times.empty()) return *this; TestBase::declareArray(test->outputData, a1, wtype); TestBase::declareArray(test->outputData, a2, wtype); TestBase::declareArray(test->outputData, a3, wtype); return *this; } TestBase::_declareHelper& TestBase::_declareHelper::out(cv::InputOutputArray a1, cv::InputOutputArray a2, cv::InputOutputArray a3, cv::InputOutputArray a4, WarmUpType wtype) { if (!test->times.empty()) return *this; TestBase::declareArray(test->outputData, a1, wtype); TestBase::declareArray(test->outputData, a2, wtype); TestBase::declareArray(test->outputData, a3, wtype); TestBase::declareArray(test->outputData, a4, wtype); return *this; } TestBase::_declareHelper& TestBase::_declareHelper::strategy(enum PERF_STRATEGY s) { test->testStrategy = s; return *this; } TestBase::_declareHelper::_declareHelper(TestBase* t) : test(t) { } /*****************************************************************************************\ * miscellaneous \*****************************************************************************************/ namespace { struct KeypointComparator { std::vector<cv::KeyPoint>& pts_; comparators::KeypointGreater cmp; KeypointComparator(std::vector<cv::KeyPoint>& pts) : pts_(pts), cmp() {} bool operator()(int idx1, int idx2) const { return cmp(pts_[idx1], pts_[idx2]); } private: const KeypointComparator& operator=(const KeypointComparator&); // quiet MSVC }; }//namespace void perf::sort(std::vector<cv::KeyPoint>& pts, cv::InputOutputArray descriptors) { cv::Mat desc = descriptors.getMat(); CV_Assert(pts.size() == (size_t)desc.rows); cv::AutoBuffer<int> idxs(desc.rows); for (int i = 0; i < desc.rows; ++i) idxs[i] = i; std::sort((int*)idxs, (int*)idxs + desc.rows, KeypointComparator(pts)); std::vector<cv::KeyPoint> spts(pts.size()); cv::Mat sdesc(desc.size(), desc.type()); for(int j = 0; j < desc.rows; ++j) { spts[j] = pts[idxs[j]]; cv::Mat row = sdesc.row(j); desc.row(idxs[j]).copyTo(row); } spts.swap(pts); sdesc.copyTo(desc); } /*****************************************************************************************\ * ::perf::GpuPerf \*****************************************************************************************/ bool perf::GpuPerf::targetDevice() { return param_impl == "cuda"; } /*****************************************************************************************\ * ::perf::PrintTo \*****************************************************************************************/ namespace perf { void PrintTo(const MatType& t, ::std::ostream* os) { switch( CV_MAT_DEPTH((int)t) ) { case CV_8U: *os << "8U"; break; case CV_8S: *os << "8S"; break; case CV_16U: *os << "16U"; break; case CV_16S: *os << "16S"; break; case CV_32S: *os << "32S"; break; case CV_32F: *os << "32F"; break; case CV_64F: *os << "64F"; break; case CV_USRTYPE1: *os << "USRTYPE1"; break; default: *os << "INVALID_TYPE"; break; } *os << 'C' << CV_MAT_CN((int)t); } } //namespace perf /*****************************************************************************************\ * ::cv::PrintTo \*****************************************************************************************/ namespace cv { void PrintTo(const String& str, ::std::ostream* os) { *os << "\"" << str << "\""; } void PrintTo(const Size& sz, ::std::ostream* os) { *os << /*"Size:" << */sz.width << "x" << sz.height; } } // namespace cv