stabilizer.cpp 11 KB
Newer Older
wester committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42
/*M///////////////////////////////////////////////////////////////////////////////////////
//
//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
//  By downloading, copying, installing or using the software you agree to this license.
//  If you do not agree to this license, do not download, install,
//  copy or use the software.
//
//
//                           License Agreement
//                For Open Source Computer Vision Library
//
// Copyright (C) 2000-2008, Intel Corporation, all rights reserved.
// Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//   * Redistribution's of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//
//   * Redistribution's in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other materials provided with the distribution.
//
//   * The name of the copyright holders may not be used to endorse or promote products
//     derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/

wester committed
43 44 45 46
#if defined __clang__
#  pragma GCC diagnostic ignored "-Wdelete-non-virtual-dtor"
#endif

wester committed
47 48 49
#include "precomp.hpp"
#include "opencv2/videostab/stabilizer.hpp"

wester committed
50
using namespace std;
wester committed
51 52 53 54 55 56 57 58

namespace cv
{
namespace videostab
{

StabilizerBase::StabilizerBase()
{
wester committed
59 60 61 62 63
    setLog(new NullLog());
    setFrameSource(new NullFrameSource());
    setMotionEstimator(new PyrLkRobustMotionEstimator());
    setDeblurer(new NullDeblurer());
    setInpainter(new NullInpainter());
wester committed
64 65 66 67 68 69 70
    setRadius(15);
    setTrimRatio(0);
    setCorrectionForInclusion(false);
    setBorderMode(BORDER_REPLICATE);
}


wester committed
71
void StabilizerBase::setUp(int cacheSize, const Mat &frame)
wester committed
72
{
wester committed
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100
    InpainterBase *_inpainter = static_cast<InpainterBase*>(inpainter_);
    doInpainting_ = dynamic_cast<NullInpainter*>(_inpainter) == 0;
    if (doInpainting_)
    {
        inpainter_->setRadius(radius_);
        inpainter_->setFrames(frames_);
        inpainter_->setMotions(motions_);
        inpainter_->setStabilizedFrames(stabilizedFrames_);
        inpainter_->setStabilizationMotions(stabilizationMotions_);
        inpainter_->update();
    }

    DeblurerBase *deblurer = static_cast<DeblurerBase*>(deblurer_);
    doDeblurring_ = dynamic_cast<NullDeblurer*>(deblurer) == 0;
    if (doDeblurring_)
    {
        blurrinessRates_.resize(cacheSize);
        float blurriness = calcBlurriness(frame);
        for (int i  = -radius_; i <= 0; ++i)
            at(i, blurrinessRates_) = blurriness;
        deblurer_->setRadius(radius_);
        deblurer_->setFrames(frames_);
        deblurer_->setMotions(motions_);
        deblurer_->setBlurrinessRates(blurrinessRates_);
        deblurer_->update();
    }

    log_->print("processing frames");
wester committed
101 102 103 104 105 106
}


Mat StabilizerBase::nextStabilizedFrame()
{
    if (curStabilizedPos_ == curPos_ && curStabilizedPos_ != -1)
wester committed
107
        return Mat(); // we've processed all frames already
wester committed
108 109 110 111 112 113

    bool processed;
    do processed = doOneIteration();
    while (processed && curStabilizedPos_ == -1);

    if (curStabilizedPos_ == -1)
wester committed
114
        return Mat(); // frame source is empty
wester committed
115

wester committed
116 117 118 119
    const Mat &stabilizedFrame = at(curStabilizedPos_, stabilizedFrames_);
    int dx = static_cast<int>(floor(trimRatio_ * stabilizedFrame.cols));
    int dy = static_cast<int>(floor(trimRatio_ * stabilizedFrame.rows));
    return stabilizedFrame(Rect(dx, dy, stabilizedFrame.cols - 2*dx, stabilizedFrame.rows - 2*dy));
wester committed
120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136
}


bool StabilizerBase::doOneIteration()
{
    Mat frame = frameSource_->nextFrame();
    if (!frame.empty())
    {
        curPos_++;

        if (curPos_ > 0)
        {
            at(curPos_, frames_) = frame;

            if (doDeblurring_)
                at(curPos_, blurrinessRates_) = calcBlurriness(frame);

wester committed
137
            estimateMotion();
wester committed
138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155

            if (curPos_ >= radius_)
            {
                curStabilizedPos_ = curPos_ - radius_;
                stabilizeFrame();
            }
        }
        else
            setUp(frame);

        log_->print(".");
        return true;
    }

    if (curStabilizedPos_ < curPos_)
    {
        curStabilizedPos_++;
        at(curStabilizedPos_ + radius_, frames_) = at(curPos_, frames_);
wester committed
156
        at(curStabilizedPos_ + radius_ - 1, motions_) = at(curPos_ - 1, motions_);
wester committed
157 158 159 160 161 162 163 164 165 166
        stabilizeFrame();

        log_->print(".");
        return true;
    }

    return false;
}


wester committed
167
void StabilizerBase::stabilizeFrame(const Mat &stabilizationMotion)
wester committed
168
{
wester committed
169
    Mat stabilizationMotion_;
wester committed
170
    if (doCorrectionForInclusion_)
wester committed
171 172 173
        stabilizationMotion_ = ensureInclusionConstraint(stabilizationMotion, frameSize_, trimRatio_);
    else
        stabilizationMotion_ = stabilizationMotion.clone();
wester committed
174

wester committed
175
    at(curStabilizedPos_, stabilizationMotions_) = stabilizationMotion_;
wester committed
176 177 178 179 180 181 182 183 184 185

    if (doDeblurring_)
    {
        at(curStabilizedPos_, frames_).copyTo(preProcessedFrame_);
        deblurer_->deblur(curStabilizedPos_, preProcessedFrame_);
    }
    else
        preProcessedFrame_ = at(curStabilizedPos_, frames_);

    // apply stabilization transformation
wester committed
186 187 188
    warpAffine(
            preProcessedFrame_, at(curStabilizedPos_, stabilizedFrames_),
            stabilizationMotion_(Rect(0,0,3,2)), frameSize_, INTER_LINEAR, borderMode_);
wester committed
189 190 191

    if (doInpainting_)
    {
wester committed
192 193 194
        warpAffine(
                frameMask_, at(curStabilizedPos_, stabilizedMasks_),
                stabilizationMotion_(Rect(0,0,3,2)), frameSize_, INTER_NEAREST);
wester committed
195 196 197 198 199 200 201 202 203 204 205 206 207 208

        erode(at(curStabilizedPos_, stabilizedMasks_), at(curStabilizedPos_, stabilizedMasks_),
              Mat());

        at(curStabilizedPos_, stabilizedMasks_).copyTo(inpaintingMask_);

        inpainter_->inpaint(
            curStabilizedPos_, at(curStabilizedPos_, stabilizedFrames_), inpaintingMask_);
    }
}


OnePassStabilizer::OnePassStabilizer()
{
wester committed
209 210
    setMotionFilter(new GaussianMotionFilter());
    resetImpl();
wester committed
211 212 213
}


wester committed
214
void OnePassStabilizer::resetImpl()
wester committed
215
{
wester committed
216 217 218 219 220 221 222 223
    curPos_ = -1;
    curStabilizedPos_ = -1;
    frames_.clear();
    motions_.clear();
    stabilizedFrames_.clear();
    stabilizationMotions_.clear();
    doDeblurring_ = false;
    doInpainting_ = false;
wester committed
224 225 226
}


wester committed
227
void OnePassStabilizer::setUp(Mat &firstFrame)
wester committed
228 229 230 231 232 233
{
    frameSize_ = firstFrame.size();
    frameMask_.create(frameSize_, CV_8U);
    frameMask_.setTo(255);

    int cacheSize = 2*radius_ + 1;
wester committed
234

wester committed
235 236 237 238 239 240 241 242 243 244 245 246 247 248
    frames_.resize(cacheSize);
    stabilizedFrames_.resize(cacheSize);
    stabilizedMasks_.resize(cacheSize);
    motions_.resize(cacheSize);
    stabilizationMotions_.resize(cacheSize);

    for (int i = -radius_; i < 0; ++i)
    {
        at(i, motions_) = Mat::eye(3, 3, CV_32F);
        at(i, frames_) = firstFrame;
    }

    at(0, frames_) = firstFrame;

wester committed
249 250
    motionFilter_->setRadius(radius_);
    motionFilter_->update();
wester committed
251

wester committed
252
    StabilizerBase::setUp(cacheSize, firstFrame);
wester committed
253 254 255
}


wester committed
256
void OnePassStabilizer::estimateMotion()
wester committed
257
{
wester committed
258 259
    at(curPos_ - 1, motions_) = motionEstimator_->estimate(
            at(curPos_ - 1, frames_), at(curPos_, frames_));
wester committed
260 261 262
}


wester committed
263
void OnePassStabilizer::stabilizeFrame()
wester committed
264
{
wester committed
265 266
    Mat stabilizationMotion = motionFilter_->stabilize(curStabilizedPos_, &motions_[0], (int)motions_.size());
    StabilizerBase::stabilizeFrame(stabilizationMotion);
wester committed
267 268 269 270 271
}


TwoPassStabilizer::TwoPassStabilizer()
{
wester committed
272
    setMotionStabilizer(new GaussianMotionFilter());
wester committed
273
    setEstimateTrimRatio(false);
wester committed
274
    resetImpl();
wester committed
275 276 277
}


wester committed
278
Mat TwoPassStabilizer::nextFrame()
wester committed
279
{
wester committed
280 281
    runPrePassIfNecessary();
    return StabilizerBase::nextStabilizedFrame();
wester committed
282 283 284
}


wester committed
285
vector<Mat> TwoPassStabilizer::motions() const
wester committed
286
{
wester committed
287 288 289 290 291
    if (frameCount_ == 0)
        return vector<Mat>();
    vector<Mat> res(frameCount_ - 1);
    copy(motions_.begin(), motions_.begin() + frameCount_ - 1, res.begin());
    return res;
wester committed
292 293 294
}


wester committed
295
void TwoPassStabilizer::resetImpl()
wester committed
296
{
wester committed
297 298 299 300 301 302 303 304 305 306
    isPrePassDone_ = false;
    frameCount_ = 0;
    curPos_ = -1;
    curStabilizedPos_ = -1;
    frames_.clear();
    motions_.clear();
    stabilizedFrames_.clear();
    stabilizationMotions_.clear();
    doDeblurring_ = false;
    doInpainting_ = false;
wester committed
307 308 309 310 311 312 313 314 315 316 317 318 319 320
}


void TwoPassStabilizer::runPrePassIfNecessary()
{
    if (!isPrePassDone_)
    {
        log_->print("first pass: estimating motions");

        Mat prevFrame, frame;

        while (!(frame = frameSource_->nextFrame()).empty())
        {
            if (frameCount_ > 0)
wester committed
321
                motions_.push_back(motionEstimator_->estimate(prevFrame, frame));
wester committed
322 323 324 325 326 327 328 329 330 331
            else
            {
                frameSize_ = frame.size();
                frameMask_.create(frameSize_, CV_8U);
                frameMask_.setTo(255);
            }

            prevFrame = frame;
            frameCount_++;

wester committed
332 333
            log_->print(".");
        }
wester committed
334 335 336

        for (int i = 0; i < radius_; ++i)
            motions_.push_back(Mat::eye(3, 3, CV_32F));
wester committed
337
        log_->print("\n");
wester committed
338

wester committed
339 340 341 342 343 344 345
        IMotionStabilizer *_motionStabilizer = static_cast<IMotionStabilizer*>(motionStabilizer_);
        MotionFilterBase *motionFilterBase = dynamic_cast<MotionFilterBase*>(_motionStabilizer);
        if (motionFilterBase)
        {
            motionFilterBase->setRadius(radius_);
            motionFilterBase->update();
        }
wester committed
346 347

        stabilizationMotions_.resize(frameCount_);
wester committed
348
        motionStabilizer_->stabilize(&motions_[0], frameCount_, &stabilizationMotions_[0]);
wester committed
349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366

        if (mustEstTrimRatio_)
        {
            trimRatio_ = 0;
            for (int i = 0; i < frameCount_; ++i)
            {
                Mat S = stabilizationMotions_[i];
                trimRatio_ = std::max(trimRatio_, estimateOptimalTrimRatio(S, frameSize_));
            }
            log_->print("estimated trim ratio: %f\n", static_cast<double>(trimRatio_));
        }

        isPrePassDone_ = true;
        frameSource_->reset();
    }
}


wester committed
367
void TwoPassStabilizer::setUp(Mat &firstFrame)
wester committed
368 369
{
    int cacheSize = 2*radius_ + 1;
wester committed
370

wester committed
371 372 373 374 375 376 377
    frames_.resize(cacheSize);
    stabilizedFrames_.resize(cacheSize);
    stabilizedMasks_.resize(cacheSize);

    for (int i = -radius_; i <= 0; ++i)
        at(i, frames_) = firstFrame;

wester committed
378
    StabilizerBase::setUp(cacheSize, firstFrame);
wester committed
379 380 381
}


wester committed
382
void TwoPassStabilizer::stabilizeFrame()
wester committed
383
{
wester committed
384
    StabilizerBase::stabilizeFrame(stabilizationMotions_[curStabilizedPos_]);
wester committed
385 386 387 388
}

} // namespace videostab
} // namespace cv