OpenBoard/src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp

/*
 * Copyright (C) 2015-2016 Département de l'Instruction Publique (DIP-SEM)
 *
 * This file is part of OpenBoard.
 *
 * OpenBoard is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, version 3 of the License,
 * with a specific linking exception for the OpenSSL project's
 * "OpenSSL" library (or with modified versions of it that use the
 * same license as the "OpenSSL" library).
 *
 * OpenBoard is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with OpenBoard. If not, see <http://www.gnu.org/licenses/>.
 */

#include "UBFFmpegVideoEncoder.h"

//-------------------------------------------------------------------------
// Utility functions
//-------------------------------------------------------------------------

QString avErrorToQString(int errnum)
{
    char error[AV_ERROR_MAX_STRING_SIZE];
    av_make_error_string(error, AV_ERROR_MAX_STRING_SIZE, errnum);

    return QString(error);
}

/**
 * @brief Write a given frame to the audio stream or, if a null frame is passed, flush the stream.
 *
 * @param frame An AVFrame to be written to the stream, or NULL to flush the stream
 * @param packet A (reusable) packet, used to temporarily store frame data
 * @param stream The stream to write to
 * @param outputFormatContext The output format context
 */
void writeFrame(AVFrame *frame, AVPacket *packet, AVStream *stream, AVFormatContext *outputFormatContext)
{
    int gotOutput, ret;

    av_init_packet(packet);

    do {
        if (stream->codec->codec_type == AVMEDIA_TYPE_AUDIO)
            ret = avcodec_encode_audio2(stream->codec, packet, frame, &gotOutput);
        else
            ret = avcodec_encode_video2(stream->codec, packet, frame, &gotOutput);

        if (ret < 0)
            qWarning() << "Couldn't encode audio frame: " << avErrorToQString(ret);

        else if (gotOutput) {
            AVRational codecTimebase = stream->codec->time_base;
            AVRational streamVideoTimebase = stream->time_base;

            av_packet_rescale_ts(packet, codecTimebase, streamVideoTimebase);
            packet->stream_index = stream->index;

            av_interleaved_write_frame(outputFormatContext, packet);
            av_packet_unref(packet);
        }

    } while (gotOutput && !frame);
}

void flushStream(AVPacket *packet, AVStream *stream, AVFormatContext *outputFormatContext)
{
    writeFrame(NULL, packet, stream, outputFormatContext);
}

//-------------------------------------------------------------------------
// UBFFmpegVideoEncoder
//-------------------------------------------------------------------------

UBFFmpegVideoEncoder::UBFFmpegVideoEncoder(QObject* parent)
    : UBAbstractVideoEncoder(parent)
    , mOutputFormatContext(NULL)
    , mSwsContext(NULL)
    , mShouldRecordAudio(true)
    , mAudioInput(NULL)
    , mSwrContext(NULL)
    , mAudioOutBuffer(NULL)
    , mAudioSampleRate(44100)
    , mAudioFrameCount(0)
{

    mVideoTimebase = 100 * framesPerSecond();

    mVideoEncoderThread = new QThread;
    mVideoWorker = new UBFFmpegVideoEncoderWorker(this);
    mVideoWorker->moveToThread(mVideoEncoderThread);

    connect(mVideoWorker, SIGNAL(error(QString)),
            this, SLOT(setLastErrorMessage(QString)));

    connect(mVideoEncoderThread, SIGNAL(started()),
            mVideoWorker, SLOT(runEncoding()));

    connect(mVideoWorker, SIGNAL(encodingFinished()),
            mVideoEncoderThread, SLOT(quit()));

    connect(mVideoEncoderThread, SIGNAL(finished()),
            this, SLOT(finishEncoding()));
}

UBFFmpegVideoEncoder::~UBFFmpegVideoEncoder()
{
    if (mVideoWorker)
        delete mVideoWorker;

    if (mVideoEncoderThread)
        delete mVideoEncoderThread;

    if (mAudioInput)
        delete mAudioInput;
}

void UBFFmpegVideoEncoder::setLastErrorMessage(const QString& pMessage)
{
    qWarning() << "FFmpeg video encoder:" << pMessage;
    mLastErrorMessage = pMessage;
}


bool UBFFmpegVideoEncoder::start()
{
    bool initialized = init();

    if (initialized) {
        mVideoEncoderThread->start();
        if (mShouldRecordAudio)
            mAudioInput->start();
    }

    return initialized;
}

bool UBFFmpegVideoEncoder::stop()
{
    qDebug() << "Video encoder: stop requested";

    mVideoWorker->stopEncoding();

    if (mShouldRecordAudio)
        mAudioInput->stop();

    return true;
}

bool UBFFmpegVideoEncoder::init()
{
    av_register_all();
    avcodec_register_all();

    AVDictionary * options = NULL;
    int ret;

    // Output format and context
    // --------------------------------------
    if (avformat_alloc_output_context2(&mOutputFormatContext, NULL,
                                       "mp4", NULL) < 0)
    {
        setLastErrorMessage("Couldn't allocate video format context");
        return false;
    }

    // The default codecs for mp4 are h264 and aac, we use those


    // Video codec and context
    // -------------------------------------
    mVideoStream = avformat_new_stream(mOutputFormatContext, 0);

    AVCodec * videoCodec = avcodec_find_encoder(mOutputFormatContext->oformat->video_codec);
    if (!videoCodec) {
        setLastErrorMessage("Video codec not found");
        return false;
    }

    AVCodecContext* c = avcodec_alloc_context3(videoCodec);

    c->bit_rate = videoBitsPerSecond();
    c->width = videoSize().width();
    c->height = videoSize().height();
    c->time_base = {1, mVideoTimebase};
    c->gop_size = 10;
    c->max_b_frames = 0;
    c->pix_fmt = AV_PIX_FMT_YUV420P;

    if (mOutputFormatContext->oformat->flags & AVFMT_GLOBALHEADER)
        c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;

    /*
     * Supported pixel formats for h264 are:
     *   AV_PIX_FMT_YUV420P
     *   AV_PIX_FMT_YUV422P
     *   AV_PIX_FMT_YUV444P
     *   AV_PIX_FMT_YUVJ420P
    */

    av_dict_set(&options, "preset", "slow", 0);
    av_dict_set(&options, "crf", "20", 0);

    ret = avcodec_open2(c, videoCodec, &options);

    if (ret < 0) {
        setLastErrorMessage(QString("Couldn't open video codec: ") + avErrorToQString(ret));
        return false;
    }

    mVideoStream->codec = c;

    // Source images are RGB32, and should be converted to YUV for h264 video
    mSwsContext = sws_getCachedContext(mSwsContext,
                                       c->width, c->height, AV_PIX_FMT_RGB32,
                                       c->width, c->height, c->pix_fmt,
                                       SWS_BICUBIC, 0, 0, 0);

    // Audio codec and context
    // -------------------------------------
    if (mShouldRecordAudio) {

        // Microphone input

        mAudioInput = new UBMicrophoneInput();

        connect(mAudioInput, SIGNAL(audioLevelChanged(quint8)),
                this, SIGNAL(audioLevelChanged(quint8)));

        connect(mAudioInput, SIGNAL(dataAvailable(QByteArray)),
                this, SLOT(onAudioAvailable(QByteArray)));

        if (!mAudioInput->init()) {
            setLastErrorMessage("Couldn't initialize audio input");
            return false;
        }

        int inChannelCount = mAudioInput->channelCount();
        int inSampleRate = mAudioInput->sampleRate();

        // Codec

        AVCodec * audioCodec = avcodec_find_encoder(mOutputFormatContext->oformat->audio_codec);

        if (!audioCodec) {
            setLastErrorMessage("Audio codec not found");
            return false;
        }

        mAudioStream = avformat_new_stream(mOutputFormatContext, audioCodec);
        mAudioStream->id = mOutputFormatContext->nb_streams-1;

        c = mAudioStream->codec;

        c->bit_rate = 96000;
        c->sample_fmt = audioCodec->sample_fmts[0]; // FLTP by default for AAC
        c->sample_rate = mAudioSampleRate;
        c->channels = 2;
        c->channel_layout = av_get_default_channel_layout(c->channels);
        c->profile = FF_PROFILE_AAC_MAIN;
        c->time_base = {1, mAudioSampleRate};
        c->strict_std_compliance = -2; // Enable use of experimental codec

        if (mOutputFormatContext->oformat->flags & AVFMT_GLOBALHEADER)
            c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;

        ret = avcodec_open2(c, audioCodec, NULL);

        if (ret < 0) {
            setLastErrorMessage(QString("Couldn't open audio codec: ") + avErrorToQString(ret));
            return false;
        }

        // The input (raw sound from the microphone) may not match the codec's sampling rate,
        // sample format or number of channels; we use libswresample to convert and resample it
        mSwrContext = swr_alloc();
        if (!mSwrContext) {
            setLastErrorMessage("Could not allocate resampler context");
            return false;
        }

        av_opt_set_int(mSwrContext, "in_channel_count", inChannelCount, 0);
        av_opt_set_int(mSwrContext, "in_sample_rate", inSampleRate, 0);
        av_opt_set_sample_fmt(mSwrContext, "in_sample_fmt", (AVSampleFormat)mAudioInput->sampleFormat(), 0);
        av_opt_set_int(mSwrContext, "out_channel_count", c->channels, 0);
        av_opt_set_int(mSwrContext, "out_sample_rate", c->sample_rate, 0);
        av_opt_set_sample_fmt(mSwrContext, "out_sample_fmt", c->sample_fmt, 0);

        ret = swr_init(mSwrContext);
        if (ret < 0) {
            setLastErrorMessage(QString("Couldn't initialize the resampling context: ") + avErrorToQString(ret));
            return false;
        }

        // Buffer for resampled/converted audio
        mAudioOutBuffer = av_audio_fifo_alloc(c->sample_fmt, c->channels, c->frame_size);
    }


    // Open the output file
    ret = avio_open(&(mOutputFormatContext->pb), videoFileName().toStdString().c_str(), AVIO_FLAG_WRITE);

    if (ret < 0) {
        setLastErrorMessage(QString("Couldn't open video file for writing: ") + avErrorToQString(ret));
        return false;
    }

    // Write stream header
    ret = avformat_write_header(mOutputFormatContext, NULL);

    if (ret < 0) {
        setLastErrorMessage(QString("Couldn't write header to file: ") + avErrorToQString(ret));
        return false;
    }

    return true;
}

/**
 * This function should be called every time a new "screenshot" is ready.
 * The image is converted to the right format and sent to the encoder.
 */
void UBFFmpegVideoEncoder::newPixmap(const QImage &pImage, long timestamp)
{
    if (!mVideoWorker->isRunning()) {
        qDebug() << "Encoder worker thread not running. Queuing frame.";
        mPendingFrames.enqueue({pImage, timestamp});
    }

    else {
        // First send any queued frames, then the latest one
        while (!mPendingFrames.isEmpty()) {
            AVFrame* avFrame = convertImageFrame(mPendingFrames.dequeue());
            if (avFrame)
                mVideoWorker->queueVideoFrame(avFrame);
        }

        // note: if converting the frame turns out to be too slow to do here, it
        // can always be done from the worker thread (in that case,
        // the worker's queue would contain ImageFrames rather than AVFrames)

        AVFrame* avFrame = convertImageFrame({pImage, timestamp});
        if (avFrame)
            mVideoWorker->queueVideoFrame(avFrame);

        // signal the worker that frames are available
        mVideoWorker->mWaitCondition.wakeAll();
    }
}

/**
 * Convert a frame consisting of a QImage and timestamp to an AVFrame
 * with the right pixel format and PTS
 */
AVFrame* UBFFmpegVideoEncoder::convertImageFrame(ImageFrame frame)
{
    AVFrame* avFrame = av_frame_alloc();

    avFrame->format = mVideoStream->codec->pix_fmt;
    avFrame->width = mVideoStream->codec->width;
    avFrame->height = mVideoStream->codec->height;
    avFrame->pts = mVideoTimebase * frame.timestamp / 1000;

    const uchar * rgbImage = frame.image.bits();

    const int in_linesize[1] = { frame.image.bytesPerLine() };

    // Allocate the output image
    if (av_image_alloc(avFrame->data, avFrame->linesize, mVideoStream->codec->width,
                       mVideoStream->codec->height, mVideoStream->codec->pix_fmt, 32) < 0)
    {
        qWarning() << "Couldn't allocate image";
        return NULL;
    }

    sws_scale(mSwsContext,
              (const uint8_t* const*)&rgbImage,
              in_linesize,
              0,
              mVideoStream->codec->height,
              avFrame->data,
              avFrame->linesize);

    return avFrame;
}

void UBFFmpegVideoEncoder::onAudioAvailable(QByteArray data)
{
    if (!data.isEmpty())
        processAudio(data);
}

/**
* Resample and convert audio to match the encoder's settings and queue the
* output. If enough output data is available, it is packaged into AVFrames and
* sent to the encoder thread.
*/
void UBFFmpegVideoEncoder::processAudio(QByteArray &data)
{
    int ret;
    AVCodecContext* codecContext = mAudioStream->codec;

    const char * inSamples = data.constData();

    // The number of samples (per channel) in the input
    int inSamplesCount = data.size() / ((mAudioInput->sampleSize() / 8) * mAudioInput->channelCount());

    // The number of samples we will get after conversion
    int outSamplesCount = swr_get_out_samples(mSwrContext, inSamplesCount);

    // Allocate output samples
    uint8_t ** outSamples = NULL;
    int outSamplesLineSize;

    ret = av_samples_alloc_array_and_samples(&outSamples, &outSamplesLineSize,
                                             codecContext->channels, outSamplesCount,
                                             codecContext->sample_fmt, 0);
    if (ret < 0) {
        qWarning() << "Could not allocate audio samples" << avErrorToQString(ret);
        return;
    }

    // Convert to destination format
    ret = swr_convert(mSwrContext,
                      outSamples, outSamplesCount,
                      (const uint8_t **)&inSamples, inSamplesCount);
    if (ret < 0) {
        qWarning() << "Error converting audio samples: " << avErrorToQString(ret);
        return;
    }

    // Append the converted samples to the out buffer.
    ret = av_audio_fifo_write(mAudioOutBuffer, (void**)outSamples, outSamplesCount);
    if (ret < 0) {
        qWarning() << "Could not write to FIFO queue: " << avErrorToQString(ret);
        return;
    }

    // Keep the data queued until next call if the encoder thread isn't running
    if (!mVideoWorker->isRunning())
        return;

    bool framesAdded = false;
    while (av_audio_fifo_size(mAudioOutBuffer) > codecContext->frame_size) {
        AVFrame * avFrame = av_frame_alloc();
        avFrame->nb_samples = codecContext->frame_size;
        avFrame->channel_layout = codecContext->channel_layout;
        avFrame->format = codecContext->sample_fmt;
        avFrame->sample_rate = codecContext->sample_rate;
        avFrame->pts = mAudioFrameCount;

        ret = av_frame_get_buffer(avFrame, 0);
        if (ret < 0) {
            qWarning() << "Couldn't allocate frame: " << avErrorToQString(ret);
            break;
        }

        ret = av_audio_fifo_read(mAudioOutBuffer, (void**)avFrame->data, codecContext->frame_size);
        if (ret < 0)
            qWarning() << "Could not read from FIFO queue: " << avErrorToQString(ret);

        else {
            mAudioFrameCount += codecContext->frame_size;

            mVideoWorker->queueAudioFrame(avFrame);
            framesAdded = true;
        }
    }

    if (framesAdded)
        mVideoWorker->mWaitCondition.wakeAll();
}

void UBFFmpegVideoEncoder::finishEncoding()
{
    qDebug() << "VideoEncoder::finishEncoding called";

    flushStream(mVideoWorker->mVideoPacket, mVideoStream, mOutputFormatContext);

    if (mShouldRecordAudio)
        flushStream(mVideoWorker->mAudioPacket, mAudioStream, mOutputFormatContext);

    av_write_trailer(mOutputFormatContext);
    avio_close(mOutputFormatContext->pb);

    avcodec_close(mVideoStream->codec);
    sws_freeContext(mSwsContext);

    if (mShouldRecordAudio) {
        avcodec_close(mAudioStream->codec);
        swr_free(&mSwrContext);
    }

    avformat_free_context(mOutputFormatContext);

    emit encodingFinished(true);
}


//-------------------------------------------------------------------------
// Worker
//-------------------------------------------------------------------------

UBFFmpegVideoEncoderWorker::UBFFmpegVideoEncoderWorker(UBFFmpegVideoEncoder* controller)
    : mController(controller)
{
    mStopRequested = false;
    mIsRunning = false;
    mVideoPacket = new AVPacket();
    mAudioPacket = new AVPacket();
}

UBFFmpegVideoEncoderWorker::~UBFFmpegVideoEncoderWorker()
{
    if (mVideoPacket)
        delete mVideoPacket;

    if (mAudioPacket)
        delete mAudioPacket;
}

void UBFFmpegVideoEncoderWorker::stopEncoding()
{
    qDebug() << "Video worker: stop requested";
    mStopRequested = true;
    mWaitCondition.wakeAll();
}

void UBFFmpegVideoEncoderWorker::queueVideoFrame(AVFrame* frame)
{
    if (frame) {
        mFrameQueueMutex.lock();
        mImageQueue.enqueue(frame);
        mFrameQueueMutex.unlock();
    }
}

void UBFFmpegVideoEncoderWorker::queueAudioFrame(AVFrame* frame)
{
    if (frame) {
        mFrameQueueMutex.lock();
        mAudioQueue.enqueue(frame);
        mFrameQueueMutex.unlock();
    }
}

/**
 * The main encoding function. Takes the queued frames and
 * writes them to the video and audio streams
 */
void UBFFmpegVideoEncoderWorker::runEncoding()
{
    mIsRunning = true;

    while (!mStopRequested) {
        mFrameQueueMutex.lock();
        mWaitCondition.wait(&mFrameQueueMutex);

        while (!mImageQueue.isEmpty()) {
            writeLatestVideoFrame();
        }

        while (!mAudioQueue.isEmpty()) {
            writeLatestAudioFrame();
        }

        mFrameQueueMutex.unlock();
    }

    emit encodingFinished();
}

void UBFFmpegVideoEncoderWorker::writeLatestVideoFrame()
{
    AVFrame* frame = mImageQueue.dequeue();
    writeFrame(frame, mVideoPacket, mController->mVideoStream, mController->mOutputFormatContext);
    av_frame_free(&frame);
}

void UBFFmpegVideoEncoderWorker::writeLatestAudioFrame()
{
    AVFrame *frame = mAudioQueue.dequeue();
    writeFrame(frame, mAudioPacket, mController->mAudioStream, mController->mOutputFormatContext);
    av_frame_free(&frame);
}