OpenBoard/src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp

#include "UBFFmpegVideoEncoder.h"

// Future proofing
#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(55,28,1)
#define av_frame_alloc  avcodec_alloc_frame
#define av_frame_free avcodec_free_frame
#endif

QString avErrorToQString(int errnum)
{
    char error[AV_ERROR_MAX_STRING_SIZE];
    av_make_error_string(error, AV_ERROR_MAX_STRING_SIZE, errnum);

    return QString(error);
}

/**
 * @brief Constructor for the ffmpeg video encoder
 *
 *
 * This class provides an interface between the screencast controller and the ffmpeg
 * back-end. It initializes the audio and video encoders and frees them when done;
 * worker threads handle the actual encoding of frames.
 *
 */
UBFFmpegVideoEncoder::UBFFmpegVideoEncoder(QObject* parent)
    : UBAbstractVideoEncoder(parent)
    , mOutputFormatContext(NULL)
    , mSwsContext(NULL)
    , mFile(NULL)
{

    mTimebase = 100 * framesPerSecond();
    qDebug() <<  "timebase: " << mTimebase;

    mVideoEncoderThread = new QThread;
    mVideoWorker = new UBFFmpegVideoEncoderWorker(this);
    mVideoWorker->moveToThread(mVideoEncoderThread);

    connect(mVideoWorker, SIGNAL(error(QString)),
            this, SLOT(setLastErrorMessage(QString)));

    connect(mVideoEncoderThread, SIGNAL(started()),
            mVideoWorker, SLOT(runEncoding()));

    connect(mVideoWorker, SIGNAL(encodingFinished()),
            mVideoEncoderThread, SLOT(quit()));

    connect(mVideoEncoderThread, SIGNAL(finished()),
            this, SLOT(finishEncoding()));
}

UBFFmpegVideoEncoder::~UBFFmpegVideoEncoder()
{
    if (mVideoWorker)
        delete mVideoWorker;

    if (mVideoEncoderThread)
        delete mVideoEncoderThread;

}

void UBFFmpegVideoEncoder::setLastErrorMessage(const QString& pMessage)
{
    qDebug() << "FFmpeg video encoder:" << pMessage;
    mLastErrorMessage = pMessage;
}

bool UBFFmpegVideoEncoder::start()
{
    bool initialized = init();

    if (initialized)
        mVideoEncoderThread->start();

    return initialized;
}

bool UBFFmpegVideoEncoder::stop()
{
    qDebug() << "Video encoder: stop requested";

    mVideoWorker->stopEncoding();

    return true;
}

bool UBFFmpegVideoEncoder::init()
{
    // Initialize ffmpeg lib
    av_register_all();
    avcodec_register_all();

    AVDictionary * options = NULL;
    int ret;

    // Output format and context
    // --------------------------------------

    if (avformat_alloc_output_context2(&mOutputFormatContext, NULL,
                                       "mp4", NULL) < 0)
    {
        setLastErrorMessage("Couldn't allocate video format context");
        return false;
    }

    // The default codecs for mp4 are h264 and aac, we use those


    // Video codec and context
    // -------------------------------------

    AVCodec * videoCodec = avcodec_find_encoder(mOutputFormatContext->oformat->video_codec);
    if (!videoCodec) {
        setLastErrorMessage("Video codec not found");
        return false;
    }

    mVideoStream = avformat_new_stream(mOutputFormatContext, 0);
    mVideoStream->time_base = {1, mTimebase};

    avcodec_get_context_defaults3(mVideoStream->codec, videoCodec);
    AVCodecContext* c = avcodec_alloc_context3(videoCodec);

    c->bit_rate = videoBitsPerSecond();
    c->width = videoSize().width();
    c->height = videoSize().height();
    c->time_base = {1, mTimebase};
    c->gop_size = 10;
    c->max_b_frames = 0;
    c->pix_fmt = AV_PIX_FMT_YUV420P;

    if (mOutputFormatContext->oformat->flags & AVFMT_GLOBALHEADER)
        c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;

    /*
     * Supported pixel formats for h264 are:
     *   AV_PIX_FMT_YUV420P
     *   AV_PIX_FMT_YUV422P
     *   AV_PIX_FMT_YUV444P
     *   AV_PIX_FMT_YUVJ420P
    */

    av_dict_set(&options, "preset", "slow", 0);
    av_dict_set(&options, "crf", "20", 0);

    ret = avcodec_open2(c, videoCodec, &options);

    if (ret < 0) {
        setLastErrorMessage(QString("Couldn't open video codec: ") + avErrorToQString(ret));
        return false;
    }

    mVideoStream->codec = c;

    // Source images are RGB32, and should be converted to YUV for h264 video
    mSwsContext = sws_getCachedContext(mSwsContext,
                                       c->width, c->height, AV_PIX_FMT_RGB32,
                                       c->width, c->height, c->pix_fmt,
                                       SWS_BICUBIC, 0, 0, 0);

    // Audio codec and context
    // -------------------------------------
    /*
    AVCodec * audioCodec = avcodec_find_encoder(mOutputFormatContext->oformat->audio_codec);
    mAudioStream = avformat_new_stream(mOutputFormatContext, audioCodec);
    */


    // Open the output file
    ret = avio_open(&(mOutputFormatContext->pb), videoFileName().toStdString().c_str(), AVIO_FLAG_WRITE);

    if (ret < 0) {
        setLastErrorMessage(QString("Couldn't open video file for writing: ") + avErrorToQString(ret));
        return false;
    }

    // Write stream header
    ret = avformat_write_header(mOutputFormatContext, NULL);

    if (ret < 0) {
        setLastErrorMessage(QString("Couldn't write header to file: ") + avErrorToQString(ret));
        return false;
    }

    return true;
}
void UBFFmpegVideoEncoder::newPixmap(const QImage &pImage, long timestamp)
{
    static bool isFirstFrame = true;
    if (isFirstFrame) {
        timestamp = 0;
        isFirstFrame = false;
    }

    if (!mVideoWorker->isRunning()) {
        qDebug() << "Encoder worker thread not running. Queuing frame.";
        mPendingFrames.enqueue({pImage, timestamp});
    }

    else {
        // First send any queued frames, then the latest one
        while (!mPendingFrames.isEmpty()) {
            AVFrame* avFrame = convertFrame(mPendingFrames.dequeue());
            if (avFrame)
                mVideoWorker->queueFrame(avFrame);
        }

        // note: if converting the frame turns out to be too slow to do here, it
        // can always be done from the worker thread (in thta case,
        // the worker's queue would contain ImageFrames rather than AVFrames)

        AVFrame* avFrame = convertFrame({pImage, timestamp});
        if (avFrame)
            mVideoWorker->queueFrame(avFrame);

        // signal the worker that frames are available
        mVideoWorker->mWaitCondition.wakeAll();
    }
}

/** Convert a frame consisting of a QImage and timestamp to an AVFrame
 * with the right pixel format and PTS
 */
AVFrame* UBFFmpegVideoEncoder::convertFrame(ImageFrame frame)
{
    AVFrame* avFrame = av_frame_alloc();

    avFrame->format = mVideoStream->codec->pix_fmt;
    avFrame->width = mVideoStream->codec->width;
    avFrame->height = mVideoStream->codec->height;
    avFrame->pts = mTimebase * frame.timestamp / 1000;

    const uchar * rgbImage = frame.image.bits();

    const int in_linesize[1] = { frame.image.bytesPerLine() };

    // Allocate the output image
    if (av_image_alloc(avFrame->data, avFrame->linesize, mVideoStream->codec->width,
                       mVideoStream->codec->height, mVideoStream->codec->pix_fmt, 32) < 0)
    {
        setLastErrorMessage("Couldn't allocate image");
        return NULL;
    }

    sws_scale(mSwsContext,
              (const uint8_t* const*)&rgbImage,
              in_linesize,
              0,
              mVideoStream->codec->height,
              avFrame->data,
              avFrame->linesize);

    return avFrame;
}

void UBFFmpegVideoEncoder::finishEncoding()
{
    qDebug() << "VideoEncoder::finishEncoding called";

    // Some frames may not be encoded, so we call avcodec_encode_video2 until they're all done

    int gotOutput;
    do {
        // TODO: get rid of duplicated code (videoWorker does almost exactly this during encoding)

        AVPacket* packet = mVideoWorker->mPacket;

        if (avcodec_encode_video2(mVideoStream->codec, packet, NULL, &gotOutput) < 0) {
            setLastErrorMessage("Couldn't encode frame to video");
            continue;
        }
        if (gotOutput) {
            AVRational codecTimebase = mVideoStream->codec->time_base;
            AVRational streamTimebase = mVideoStream->time_base;

            av_packet_rescale_ts(packet, codecTimebase, streamTimebase);
            packet->stream_index = mVideoStream->index;

            av_interleaved_write_frame(mOutputFormatContext, packet);
            av_packet_unref(packet);
        }
    } while (gotOutput);

    av_write_trailer(mOutputFormatContext);

    avio_close(mOutputFormatContext->pb);
    avcodec_close(mVideoStream->codec);
    sws_freeContext(mSwsContext);
    avformat_free_context(mOutputFormatContext);

    emit encodingFinished(true);
}

//-------------------------------------------------------------------------
// Worker
//-------------------------------------------------------------------------

UBFFmpegVideoEncoderWorker::UBFFmpegVideoEncoderWorker(UBFFmpegVideoEncoder* controller)
    : mController(controller)
{
    mStopRequested = false;
    mIsRunning = false;
    mPacket = new AVPacket();
}

UBFFmpegVideoEncoderWorker::~UBFFmpegVideoEncoderWorker()
{}

void UBFFmpegVideoEncoderWorker::stopEncoding()
{
    qDebug() << "Video worker: stop requested";
    mStopRequested = true;
    mWaitCondition.wakeAll();
}

void UBFFmpegVideoEncoderWorker::queueFrame(AVFrame* frame)
{
    mFrameQueueMutex.lock();
    mFrameQueue.enqueue(frame);
    mFrameQueueMutex.unlock();
}

/**
 * The main encoding function. Takes the queued image frames and
 * assembles them into the video
 */
void UBFFmpegVideoEncoderWorker::runEncoding()
{
    mIsRunning = true;

    while (!mStopRequested) {
        mFrameQueueMutex.lock();
        mWaitCondition.wait(&mFrameQueueMutex);

        while (!mFrameQueue.isEmpty()) {
            writeLatestVideoFrame();
        }

        /*
        while (!mAudioQueue.isEmpty()) {
            writeLatestAudioFrame();
        }
        */

        mFrameQueueMutex.unlock();
    }

    emit encodingFinished();
}

void UBFFmpegVideoEncoderWorker::writeLatestVideoFrame()
{
    AVFrame* frame = mFrameQueue.dequeue();

    int gotOutput;
    av_init_packet(mPacket);
    mPacket->data = NULL;
    mPacket->size = 0;

    // qDebug() << "Encoding frame to video. Pts: " << frame->pts << "/" << mController->mTimebase;

    if (avcodec_encode_video2(mController->mVideoStream->codec, mPacket, frame, &gotOutput) < 0)
        emit error("Error encoding frame to video");

    if (gotOutput) {
        AVRational codecTimebase = mController->mVideoStream->codec->time_base;
        AVRational streamTimebase = mController->mVideoStream->time_base;


        // recalculate the timestamp to match the stream's timebase
        av_packet_rescale_ts(mPacket, codecTimebase, streamTimebase);
        mPacket->stream_index = mController->mVideoStream->index;

        // qDebug() << "Writing encoded packet to file; pts: " << mPacket->pts << "/" << streamTimebase.den;

        av_interleaved_write_frame(mController->mOutputFormatContext, mPacket);
        av_packet_unref(mPacket);
    }

    // Duct-tape solution. I assume there's a better way of doing this, but:
    // some players like VLC show a black screen until the second frame (which
    // can be several seconds after the first one). Simply duplicating the first frame
    // seems to solve this problem, and also allows the thumbnail to be generated.

    static bool firstRun = true;
    if (firstRun) {
        firstRun = false;
        frame->pts += 1;
        mFrameQueue.enqueue(frame); // only works when the queue is empty at this point. todo: clean this up!
    }
    else
        // free the frame
        av_frame_free(&frame);
}