/*
 * Copyright (C) 2020, KylinSoft Co., Ltd.
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
#include "Synchronizer.h"

#include "src/common/Logger.h"
#include "src/common/CommandLineOptions.h"
#include "OutputManager.h"
#include "OutputSettings.h"
#include "VideoEncoder.h"
#include "AudioEncoder.h"
#include "AV/SampleCast.h"
#include "SyncDiagram.h"

// The amount of filtering applied to audio timestamps to reduce noise. Higher values reduce timestamp noise (and associated drift correction),
// but if the value is too high, it will take more time to detect gaps.
const int64_t Synchronizer::AUDIO_TIMESTAMP_FILTER = 20;

// These values change how fast the synchronizer does drift correction.
// If this value is too low, the error will not be corrected fast enough. But if the value is too high, the audio
// may get weird speed fluctuations caused by the limited accuracy of the recording timestamps.
// The difference between sample length and time length has a lot of noise and can't be used directly,
// so it is averaged out using exponential smoothing. However, since the difference tends to increase gradually over time,
// exponential smoothing would constantly lag behind, so instead of simple proportional feedback, I use a PI controller.
// For critical damping, choose I = P*P/4.
const double Synchronizer::DRIFT_CORRECTION_P = 0.3;
const double Synchronizer::DRIFT_CORRECTION_I = 0.3 * 0.3 / 4.0;

// The maximum audio/video desynchronization allowed, in seconds. If the error is greater than this value, the synchronizer will insert zeros
// rather than relying on normal drift correction. This is something that should be avoided since it will result in noticeable interruptions,
// so it should only be triggered when something is really wrong. If the error is smaller, the synchronizer will do nothing and the
// drift correction system will take care of it (eventually).
const double Synchronizer::DRIFT_ERROR_THRESHOLD = 0.05;

// The maximum block size for drift correction, in seconds. This is needed to avoid numerical problems in the feedback system.
const double Synchronizer::DRIFT_MAX_BLOCK = 0.5;

// The maximum number of video frames and audio samples that will be buffered. This should be enough to cope with the fact that video and
// audio don't arrive at the same time, but not too high because that would cause memory problems if one of the inputs fails.
// The limit for audio can be set very high, because audio uses almost no memory.
const size_t Synchronizer::MAX_VIDEO_FRAMES_BUFFERED = 30;
const size_t Synchronizer::MAX_AUDIO_SAMPLES_BUFFERED = 1000000;

// The maximum delay between video frames, in microseconds. If the delay is longer, duplicates will be inserted.
// This is needed because some video codecs/players can't handle long delays.
const int64_t Synchronizer::MAX_FRAME_DELAY = 200000;

static std::unique_ptr<AVFrameWrapper> CreateVideoFrame(unsigned int width, unsigned int height, AVPixelFormat pixel_format, const std::shared_ptr<AVFrameData>& reuse_data) {

    // get required planes
    unsigned int planes = 0;
    size_t linesize[3] = {0}, planesize[3] = {0};
    switch(pixel_format) {
        case AV_PIX_FMT_YUV444P: {
            // Y/U/V = 1 byte per pixel
            planes = 3;
            linesize[0]  = grow_align16(width); planesize[0] = linesize[0] * height;
            linesize[1]  = grow_align16(width); planesize[1] = linesize[1] * height;
            linesize[2]  = grow_align16(width); planesize[2] = linesize[2] * height;
            break;
        }
        case AV_PIX_FMT_YUV422P: {
            // Y = 1 byte per pixel, U/V = 1 byte per 2x1 pixels
            assert(width % 2 == 0);
            planes = 3;
            linesize[0]  = grow_align16(width    ); planesize[0] = linesize[0] * height;
            linesize[1]  = grow_align16(width / 2); planesize[1] = linesize[1] * height;
            linesize[2]  = grow_align16(width / 2); planesize[2] = linesize[2] * height;
            break;
        }
        case AV_PIX_FMT_YUV420P: {
            // Y = 1 byte per pixel, U/V = 1 byte per 2x2 pixels
            assert(width % 2 == 0);
            assert(height % 2 == 0);
            planes = 3;
            linesize[0]  = grow_align16(width    ); planesize[0] = linesize[0] * height    ;
            linesize[1]  = grow_align16(width / 2); planesize[1] = linesize[1] * height / 2;
            linesize[2]  = grow_align16(width / 2); planesize[2] = linesize[2] * height / 2;
            break;
        }
        case AV_PIX_FMT_NV12: {
            assert(width % 2 == 0);
            assert(height % 2 == 0);
            // planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved
            // Y = 1 byte per pixel, U/V = 1 byte per 2x2 pixels
            planes = 2;
            linesize[0]  = grow_align16(width); planesize[0] = linesize[0] * height    ;
            linesize[1]  = grow_align16(width); planesize[1] = linesize[1] * height / 2;
            break;
        }
        case AV_PIX_FMT_BGRA: {
            // BGRA = 4 bytes per pixel
            planes = 1;
            linesize[0] = grow_align16(width * 4); planesize[0] = linesize[0] * height;
            break;
        }
        case AV_PIX_FMT_BGR24:
        case AV_PIX_FMT_RGB24: {
            // BGR/RGB = 3 bytes per pixel
            planes = 1;
            linesize[0] = grow_align16(width * 3); planesize[0] = linesize[0] * height;
            break;
        }
        default: assert(false); break;
    }

    // create the frame
    size_t totalsize = 0;
    for(unsigned int p = 0; p < planes; ++p) {
        totalsize += planesize[p];
    }
    std::shared_ptr<AVFrameData> frame_data = (reuse_data == NULL)? std::make_shared<AVFrameData>(totalsize) : reuse_data;
    std::unique_ptr<AVFrameWrapper> frame(new AVFrameWrapper(frame_data));
    uint8_t *data = frame->GetRawData();
    for(unsigned int p = 0; p < planes; ++p) {
        frame->GetFrame()->data[p] = data;
        frame->GetFrame()->linesize[p] = linesize[p];
        data += planesize[p];
    }
#if SSR_USE_AVFRAME_WIDTH_HEIGHT
    frame->GetFrame()->width = width;
    frame->GetFrame()->height = height;
#endif
#if SSR_USE_AVFRAME_FORMAT
    frame->GetFrame()->format = pixel_format;
#endif
#if SSR_USE_AVFRAME_SAR
    frame->GetFrame()->sample_aspect_ratio.num = 1;
    frame->GetFrame()->sample_aspect_ratio.den = 1;
#endif

    return frame;

}

static std::unique_ptr<AVFrameWrapper> CreateAudioFrame(unsigned int channels, unsigned int sample_rate, unsigned int samples, unsigned int planes, AVSampleFormat sample_format) {

    // get required sample size
    // note: sample_size = sizeof(sampletype) * channels
    unsigned int sample_size = 0; // to keep GCC happy
    switch(sample_format) {
        case AV_SAMPLE_FMT_S16:
#if SSR_USE_AVUTIL_PLANAR_SAMPLE_FMT
        case AV_SAMPLE_FMT_S16P:
#endif
            sample_size = channels * sizeof(int16_t); break;
        case AV_SAMPLE_FMT_FLT:
#if SSR_USE_AVUTIL_PLANAR_SAMPLE_FMT
        case AV_SAMPLE_FMT_FLTP:
#endif
            sample_size = channels * sizeof(float); break;
        default: assert(false); break;
    }

    // create the frame
    size_t plane_size = grow_align16(samples * sample_size / planes);
    std::shared_ptr<AVFrameData> frame_data = std::make_shared<AVFrameData>(plane_size * planes);
    std::unique_ptr<AVFrameWrapper> frame(new AVFrameWrapper(frame_data));
    for(unsigned int p = 0; p < planes; ++p) {
        frame->GetFrame()->data[p] = frame->GetRawData() + plane_size * p;
        frame->GetFrame()->linesize[p] = samples * sample_size / planes;
    }
#if SSR_USE_AVFRAME_NB_SAMPLES
    frame->GetFrame()->nb_samples = samples;
#endif
#if SSR_USE_AVFRAME_CHANNELS
    frame->GetFrame()->channels = channels;
#endif
#if SSR_USE_AVFRAME_SAMPLE_RATE
    frame->GetFrame()->sample_rate = sample_rate;
#endif
#if SSR_USE_AVFRAME_FORMAT
    frame->GetFrame()->format = sample_format;
#endif

    return frame;

}

Synchronizer::Synchronizer(OutputManager *output_manager) {

    m_output_manager = output_manager;
    m_output_settings = m_output_manager->GetOutputSettings();
    m_output_format = m_output_manager->GetOutputFormat();
    assert(m_output_format->m_video_enabled || m_output_format->m_audio_enabled);

    try {
        Init();
    } catch(...) {
        Free();
        throw;
    }

}

Synchronizer::~Synchronizer() {

    // disconnect
    ConnectVideoSource(NULL);
    ConnectAudioSource(NULL);

    // tell the thread to stop
    if(m_thread.joinable()) {
        Logger::LogInfo("[Synchronizer::~Synchronizer] " + Logger::tr("Stopping synchronizer thread ..."));
        m_should_stop = true;
        m_thread.join();
    }

    // flush one more time
    {
        SharedLock lock(&m_shared_data);
        FlushBuffers(lock.get());
    }

    // free everything
    Free();

}

void Synchronizer::Init() {

    // initialize video
    if(m_output_format->m_video_enabled) {
        m_max_frames_skipped = (m_output_settings->video_allow_frame_skipping)? (MAX_FRAME_DELAY * m_output_format->m_video_frame_rate + 500000) / 1000000 : 0;
        VideoLock videolock(&m_video_data);
        videolock->m_last_timestamp = std::numeric_limits<int64_t>::min();
        videolock->m_next_timestamp = SINK_TIMESTAMP_ASAP;
    }

    // initialize audio
    if(m_output_format->m_audio_enabled) {
        AudioLock audiolock(&m_audio_data);
        audiolock->m_fast_resampler.reset(new FastResampler(m_output_format->m_audio_channels, 0.9f));
        InitAudioSegment(audiolock.get());
        audiolock->m_warn_desync = true;
    }

    // create sync diagram
    if(CommandLineOptions::GetSyncDiagram()) {
        m_sync_diagram.reset(new SyncDiagram(4));
        m_sync_diagram->SetChannelName(0, SyncDiagram::tr("Video in"));
        m_sync_diagram->SetChannelName(1, SyncDiagram::tr("Audio in"));
        m_sync_diagram->SetChannelName(2, SyncDiagram::tr("Video out"));
        m_sync_diagram->SetChannelName(3, SyncDiagram::tr("Audio out"));
        m_sync_diagram->show();
    }

    // initialize shared data
    {
        SharedLock lock(&m_shared_data);

        if(m_output_format->m_audio_enabled) {
            lock->m_partial_audio_frame.Alloc(m_output_format->m_audio_frame_size * m_output_format->m_audio_channels);
            lock->m_partial_audio_frame_samples = 0;
        }
        lock->m_video_pts = 0;
        lock->m_audio_samples = 0;
        lock->m_time_offset = 0;

        InitSegment(lock.get());

        lock->m_warn_drop_video = true;

    }

    // start synchronizer thread
    m_should_stop = false;
    m_error_occurred = false;
    m_thread = std::thread(&Synchronizer::SynchronizerThread, this);

}

void Synchronizer::Free() {

}

void Synchronizer::NewSegment() {

    if(m_output_format->m_audio_enabled) {
        AudioLock audiolock(&m_audio_data);
        InitAudioSegment(audiolock.get());
    }

    SharedLock lock(&m_shared_data);
    NewSegment(lock.get());

}

int64_t Synchronizer::GetTotalTime() {
    SharedLock lock(&m_shared_data);
    return GetTotalTime(lock.get());
}

int64_t Synchronizer::GetNextVideoTimestamp() {
    assert(m_output_format->m_video_enabled);
    VideoLock videolock(&m_video_data);
    return videolock->m_next_timestamp;
}

void Synchronizer::ReadVideoFrame(unsigned int width, unsigned int height, const uint8_t* data, int stride, AVPixelFormat format, int64_t timestamp) {
    assert(m_output_format->m_video_enabled);

    // add new block to sync diagram
#if DEFINE_DIAGRAM
    if(m_sync_diagram != NULL)	//bybobbi
        m_sync_diagram->AddBlock(0, (double) timestamp * 1.0e-6, (double) timestamp * 1.0e-6 + 1.0 / (double) m_output_format->m_video_frame_rate, QColor(255, 0, 0));
#endif

    VideoLock videolock(&m_video_data);

    // check the timestamp
    if(timestamp < videolock->m_last_timestamp) {
        if(timestamp < videolock->m_last_timestamp - 10000)
            Logger::LogWarning("[Synchronizer::ReadVideoFrame] " + Logger::tr("Warning: Received video frame with non-monotonic timestamp."));
        timestamp = videolock->m_last_timestamp;
    }

    // drop the frame if it is too early (before converting it)
    if(videolock->m_next_timestamp != SINK_TIMESTAMP_ASAP && timestamp < videolock->m_next_timestamp - (int64_t) (1000000 / m_output_format->m_video_frame_rate))
        return;

    // update the timestamps
    videolock->m_last_timestamp = timestamp;
    videolock->m_next_timestamp = std::max(videolock->m_next_timestamp + (int64_t) (1000000 / m_output_format->m_video_frame_rate), timestamp);

    // create the converted frame
    std::unique_ptr<AVFrameWrapper> converted_frame = CreateVideoFrame(m_output_format->m_video_width, m_output_format->m_video_height, m_output_format->m_video_pixel_format, NULL);

    // scale and convert the frame to the right format
    videolock->m_fast_scaler.Scale(width, height, format, &data, &stride,
            m_output_format->m_video_width, m_output_format->m_video_height, m_output_format->m_video_pixel_format,
            converted_frame->GetFrame()->data, converted_frame->GetFrame()->linesize);

    SharedLock lock(&m_shared_data);

    // avoid memory problems by limiting the video buffer size
    if(lock->m_video_buffer.size() >= MAX_VIDEO_FRAMES_BUFFERED) {
        if(lock->m_segment_audio_started) {
            if(lock->m_warn_drop_video) {
                lock->m_warn_drop_video = false;
                Logger::LogWarning("[Synchronizer::ReadVideoFrame] " + Logger::tr("Warning: Video buffer overflow, some frames will be lost. The audio input seems to be too slow."));
            }
            return;
        } else {
            // if the audio hasn't started yet, it makes more sense to drop the oldest frames
            lock->m_video_buffer.pop_front();
            assert(lock->m_video_buffer.size() > 0);
            lock->m_segment_video_start_time = lock->m_video_buffer.front()->GetFrame()->pts;
        }
    }

    // start video
    if(!lock->m_segment_video_started) {
        lock->m_segment_video_started = true;
        lock->m_segment_video_start_time = timestamp;
        lock->m_segment_video_stop_time = timestamp;
    }

    // store the frame
    converted_frame->GetFrame()->pts = timestamp;
    lock->m_video_buffer.push_back(std::move(converted_frame));

    // increase the segment stop time
    lock->m_segment_video_stop_time = timestamp + (int64_t) (1000000 / m_output_format->m_video_frame_rate);

}

void Synchronizer::ReadVideoPing(int64_t timestamp) {
    assert(m_output_format->m_video_enabled);

    SharedLock lock(&m_shared_data);

    // if the video has not been started, ignore it
    if(!lock->m_segment_video_started)
        return;

    // increase the segment stop time
    lock->m_segment_video_stop_time = std::max(lock->m_segment_video_stop_time, timestamp + (int64_t) (1000000 / m_output_format->m_video_frame_rate));

}

void Synchronizer::ReadAudioSamples(unsigned int channels, unsigned int sample_rate, AVSampleFormat format, unsigned int sample_count, const uint8_t* data, int64_t timestamp) {
    assert(m_output_format->m_audio_enabled);

    // sanity check
    if(sample_count == 0)
        return;

    // add new block to sync diagram
#if DEFINE_DIAGRAM
    if(m_sync_diagram != NULL) //bybobbi
        m_sync_diagram->AddBlock(1, (double) timestamp * 1.0e-6, (double) timestamp * 1.0e-6 + (double) sample_count / (double) sample_rate, QColor(0, 255, 0));
#endif

    AudioLock audiolock(&m_audio_data);

    // check the timestamp
    if(timestamp < audiolock->m_last_timestamp) {
        if(timestamp < audiolock->m_last_timestamp - 10000)
            Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Received audio samples with non-monotonic timestamp."));
        timestamp = audiolock->m_last_timestamp;
    }

    // update the timestamps
    int64_t previous_timestamp;
    if(audiolock->m_first_timestamp == (int64_t) AV_NOPTS_VALUE) {
        audiolock->m_filtered_timestamp = timestamp;
        audiolock->m_first_timestamp = timestamp;
        previous_timestamp = timestamp;
    } else {
        previous_timestamp = audiolock->m_last_timestamp;
    }
    audiolock->m_last_timestamp = timestamp;

    // filter the timestamp
    int64_t timestamp_delta = (int64_t) sample_count * (int64_t) 1000000 / (int64_t) sample_rate;
    audiolock->m_filtered_timestamp += (timestamp - audiolock->m_filtered_timestamp) / AUDIO_TIMESTAMP_FILTER;

    // calculate drift
    double current_drift = GetAudioDrift(audiolock.get());

    // if there are too many audio samples, drop some of them (unlikely unless you use PulseAudio)
    if(current_drift > DRIFT_ERROR_THRESHOLD && !audiolock->m_drop_samples) {
        audiolock->m_drop_samples = true;
        Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Too many audio samples, dropping samples to keep the audio in sync with the video."));
    }

    // if there are not enough audio samples, insert zeros
    if(current_drift < -DRIFT_ERROR_THRESHOLD && !audiolock->m_insert_samples) {
        audiolock->m_insert_samples = true;
        Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Not enough audio samples, inserting silence to keep the audio in sync with the video."));
    }

    // reset filter and recalculate drift if necessary
    if(audiolock->m_drop_samples || audiolock->m_insert_samples) {
        audiolock->m_filtered_timestamp = timestamp;
        current_drift = GetAudioDrift(audiolock.get());
    }

    // drop samples
    if(audiolock->m_drop_samples) {
        audiolock->m_drop_samples = false;

        // drop samples
        int n = (int) round(current_drift * (double) sample_rate);
        if(n > 0) {
            if(n >= (int) sample_count) {
                audiolock->m_drop_samples = true;
                return; // drop all samples
            }
            if(format == AV_SAMPLE_FMT_FLT) {
                data += n * channels * sizeof(float);
            } else if(format == AV_SAMPLE_FMT_S16) {
                data += n * channels * sizeof(int16_t);
            } else if(format == AV_SAMPLE_FMT_S32) {
                data += n * channels * sizeof(int32_t);
            } else {
                assert(false);
            }
            sample_count -= n;
        }

    }

    // insert zeros
    unsigned int sample_count_out = 0;
    if(audiolock->m_insert_samples) {
        audiolock->m_insert_samples = false;

        // how many samples should be inserted?
        int n = (int) round(-current_drift * (double) sample_rate);
        if(n > 0) {

            // insert zeros
            audiolock->m_temp_input_buffer.Alloc(n * m_output_format->m_audio_channels);
            std::fill_n(audiolock->m_temp_input_buffer.GetData(), n * m_output_format->m_audio_channels, 0.0f);
            sample_count_out = audiolock->m_fast_resampler->Resample((double) sample_rate / (double) m_output_format->m_audio_sample_rate, 1.0,
                                                                     audiolock->m_temp_input_buffer.GetData(), n, &audiolock->m_temp_output_buffer, sample_count_out);

            // recalculate drift
            current_drift = GetAudioDrift(audiolock.get(), sample_count_out);

        }

    }

    // increase filtered timestamp
    audiolock->m_filtered_timestamp += timestamp_delta;

    // do drift correction
    // The point of drift correction is to keep video and audio in sync even when the clocks are not running at exactly the same speed.
    // This can happen because the sample rate of the sound card is not always 100% accurate. Even a 0.1% error will result in audio that is
    // seconds too early or too late at the end of a one hour video. This problem doesn't occur on all computers though (I'm not sure why).
    // Another cause of desynchronization is problems/glitches with PulseAudio (e.g. jumps in time when switching between sources).
    double drift_correction_dt = fmin((double) (timestamp - previous_timestamp) * 1.0e-6, DRIFT_MAX_BLOCK);
    audiolock->m_average_drift = clamp(audiolock->m_average_drift + DRIFT_CORRECTION_I * current_drift * drift_correction_dt, -0.5, 0.5);
    if(audiolock->m_average_drift < -0.02 && audiolock->m_warn_desync) {
        audiolock->m_warn_desync = false;
        Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Audio input is more than 2% too slow!"));
    }
    if(audiolock->m_average_drift > 0.02 && audiolock->m_warn_desync) {
        audiolock->m_warn_desync = false;
        Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Audio input is more than 2% too fast!"));
    }
    double length = (double) sample_count / (double) sample_rate;
    double drift_correction = clamp(DRIFT_CORRECTION_P * current_drift + audiolock->m_average_drift, -0.5, 0.5) * fmin(1.0, DRIFT_MAX_BLOCK / length);

    //qDebug() << "current_drift" << current_drift << "average_drift" << audiolock->m_average_drift << "drift_correction" << drift_correction;

    // convert the samples
    const float *data_float = NULL; // to keep GCC happy
    if(format == AV_SAMPLE_FMT_FLT) {
        if(channels == m_output_format->m_audio_channels) {
            data_float = (const float*) data;
        } else {
            audiolock->m_temp_input_buffer.Alloc(sample_count * m_output_format->m_audio_channels);
            data_float = audiolock->m_temp_input_buffer.GetData();
            SampleChannelRemap(sample_count, (const float*) data, channels, audiolock->m_temp_input_buffer.GetData(), m_output_format->m_audio_channels);
        }
    } else if(format == AV_SAMPLE_FMT_S16) {
        audiolock->m_temp_input_buffer.Alloc(sample_count * m_output_format->m_audio_channels);
        data_float = audiolock->m_temp_input_buffer.GetData();
        SampleChannelRemap(sample_count, (const int16_t*) data, channels, audiolock->m_temp_input_buffer.GetData(), m_output_format->m_audio_channels);
    } else if(format == AV_SAMPLE_FMT_S32) {
        audiolock->m_temp_input_buffer.Alloc(sample_count * m_output_format->m_audio_channels);
        data_float = audiolock->m_temp_input_buffer.GetData();
        SampleChannelRemap(sample_count, (const int32_t*) data, channels, audiolock->m_temp_input_buffer.GetData(), m_output_format->m_audio_channels);
    } else {
        assert(false);
    }

    // resample
    sample_count_out = audiolock->m_fast_resampler->Resample((double) sample_rate / (double) m_output_format->m_audio_sample_rate, 1.0 / (1.0 - drift_correction),
                                                             data_float, sample_count, &audiolock->m_temp_output_buffer, sample_count_out);
    audiolock->m_samples_written += sample_count_out;

    SharedLock lock(&m_shared_data);

    // avoid memory problems by limiting the audio buffer size
    if(lock->m_audio_buffer.GetSize() / m_output_format->m_audio_channels >= MAX_AUDIO_SAMPLES_BUFFERED) {
        if(lock->m_segment_video_started) {
            Logger::LogWarning("[Synchronizer::ReadAudioSamples] " + Logger::tr("Warning: Audio buffer overflow, starting new segment to keep the audio in sync with the video "
                                                                                "(some video and/or audio may be lost). The video input seems to be too slow."));
            NewSegment(lock.get());
        } else {
            // If the video hasn't started yet, it makes more sense to drop the oldest samples.
            // Shifting the start time like this isn't completely accurate, but this shouldn't happen often anyway.
            // The number of samples dropped is calculated so that the buffer will be 90% full after this.
            size_t n = lock->m_audio_buffer.GetSize() / m_output_format->m_audio_channels - MAX_AUDIO_SAMPLES_BUFFERED * 9 / 10;
            lock->m_audio_buffer.Pop(n * m_output_format->m_audio_channels);
            lock->m_segment_audio_start_time += (int64_t) round((double) n / (double) m_output_format->m_audio_sample_rate * 1.0e6);
        }
    }

    // start audio
    if(!lock->m_segment_audio_started) {
        lock->m_segment_audio_started = true;
        lock->m_segment_audio_start_time = timestamp;
        lock->m_segment_audio_stop_time = timestamp;
    }

    // store the samples
    lock->m_audio_buffer.Push(audiolock->m_temp_output_buffer.GetData(), sample_count_out * m_output_format->m_audio_channels);

    // increase segment stop time
    double new_sample_length = (double) (lock->m_segment_audio_samples_read + lock->m_audio_buffer.GetSize() / m_output_format->m_audio_channels) / (double) m_output_format->m_audio_sample_rate;
    lock->m_segment_audio_stop_time = lock->m_segment_audio_start_time + (int64_t) round(new_sample_length * 1.0e6);

}

void Synchronizer::ReadAudioHole() {
    assert(m_output_format->m_audio_enabled);

    AudioLock audiolock(&m_audio_data);
    if(audiolock->m_first_timestamp != (int64_t) AV_NOPTS_VALUE) {
        audiolock->m_average_drift = 0.0;
        if(!audiolock->m_drop_samples || !audiolock->m_insert_samples) {
            Logger::LogWarning("[Synchronizer::ReadAudioHole] " + Logger::tr("Warning: Received hole in audio stream, inserting silence to keep the audio in sync with the video."));
            audiolock->m_drop_samples = true; // because PulseAudio is weird
            audiolock->m_insert_samples = true;
        }
    }

}

void Synchronizer::InitAudioSegment(AudioData* audiolock) {
    audiolock->m_last_timestamp = std::numeric_limits<int64_t>::min();
    audiolock->m_first_timestamp = AV_NOPTS_VALUE;
    audiolock->m_samples_written = 0;
    audiolock->m_average_drift = 0.0;
    audiolock->m_drop_samples = false;
    audiolock->m_insert_samples = false;
}

double Synchronizer::GetAudioDrift(AudioData* audiolock, unsigned int extra_samples) {
    double sample_length = ((double) (audiolock->m_samples_written + extra_samples) + audiolock->m_fast_resampler->GetOutputLatency()) / (double) m_output_format->m_audio_sample_rate;
    double time_length = (double) (audiolock->m_filtered_timestamp - audiolock->m_first_timestamp) * 1.0e-6;
    return sample_length - time_length;
}

void Synchronizer::NewSegment(SharedData* lock) {
    FlushBuffers(lock);
    if(lock->m_segment_video_started && lock->m_segment_audio_started) {
        int64_t segment_start_time, segment_stop_time;
        GetSegmentStartStop(lock, &segment_start_time, &segment_stop_time);
        lock->m_time_offset += std::max((int64_t) 0, segment_stop_time - segment_start_time);
    }
    lock->m_video_buffer.clear();
    lock->m_audio_buffer.Clear();
    InitSegment(lock);
}

void Synchronizer::InitSegment(SharedData* lock) {
    lock->m_segment_video_started = !m_output_format->m_video_enabled;
    lock->m_segment_audio_started = !m_output_format->m_audio_enabled;
    lock->m_segment_video_start_time = AV_NOPTS_VALUE;
    lock->m_segment_audio_start_time = AV_NOPTS_VALUE;
    lock->m_segment_video_stop_time = AV_NOPTS_VALUE;
    lock->m_segment_audio_stop_time = AV_NOPTS_VALUE;
    lock->m_segment_audio_can_drop = true;
    lock->m_segment_audio_samples_read = 0;
    lock->m_segment_video_accumulated_delay = 0;
}

int64_t Synchronizer::GetTotalTime(Synchronizer::SharedData* lock) {
    if(lock->m_segment_video_started && lock->m_segment_audio_started) {
        int64_t segment_start_time, segment_stop_time;
        GetSegmentStartStop(lock, &segment_start_time, &segment_stop_time);
        return lock->m_time_offset + std::max((int64_t) 0, segment_stop_time - segment_start_time);
    } else {
        return lock->m_time_offset;
    }
}

void Synchronizer::GetSegmentStartStop(SharedData* lock, int64_t* segment_start_time, int64_t* segment_stop_time) {
    if(!m_output_format->m_audio_enabled) {
        *segment_start_time = lock->m_segment_video_start_time;
        *segment_stop_time = lock->m_segment_video_stop_time;
    } else if(!m_output_format->m_video_enabled) {
        *segment_start_time = lock->m_segment_audio_start_time;
        *segment_stop_time = lock->m_segment_audio_stop_time;
    } else {
        *segment_start_time = std::max(lock->m_segment_video_start_time, lock->m_segment_audio_start_time);
        *segment_stop_time = std::min(lock->m_segment_video_stop_time, lock->m_segment_audio_stop_time);
    }
}

void Synchronizer::FlushBuffers(SharedData* lock) {
    if(!lock->m_segment_video_started || !lock->m_segment_audio_started)
        return;

    int64_t segment_start_time, segment_stop_time;
    GetSegmentStartStop(lock, &segment_start_time, &segment_stop_time);

    // flush video
    if(m_output_format->m_video_enabled)
        FlushVideoBuffer(lock, segment_start_time, segment_stop_time);

    // flush audio
    if(m_output_format->m_audio_enabled)
        FlushAudioBuffer(lock, segment_start_time, segment_stop_time);

}

void Synchronizer::FlushVideoBuffer(Synchronizer::SharedData* lock, int64_t segment_start_time, int64_t segment_stop_time) {

    // Sometimes long delays between video frames can occur, e.g. when a game is showing a loading screen.
    // Not all codecs/players can handle that. It's also a problem for streaming. To fix this, long delays should be avoided by
    // duplicating the previous frame a few times when needed. Whenever a video frame is sent to the encoder, it is also copied,
    // with reference counting for the actual image to minimize overhead. When there is a gap, duplicate frames are inserted.
    // Duplicate frames are always inserted with a timestamp in the past, because we don't want to drop a real frame if it is captured
    // right after the duplicate was inserted. MAX_INPUT_LATENCY simulates the latency from the capturing of a frame to the synchronizer,
    // i.e. any new frame is assumed to have a timestamp higher than the current time minus MAX_INPUT_LATENCY. The duplicate
    // frame will have a timestamp that's one frame earlier than that time, so it will never interfere with the real frame.
    // There are two situations where duplicate frames can be inserted:
    // (1) The queue is not empty, but there is a gap between frames that is too large.
    // (2) The queue is empty and the last timestamp is too long ago (relative to the end of the video segment).
    // It is perfectly possible that *both* happen, each possibly multiple times, in just one function call.

    int64_t segment_stop_video_pts = (lock->m_time_offset + (segment_stop_time - segment_start_time)) * (int64_t) m_output_format->m_video_frame_rate / (int64_t) 1000000;
    int64_t delay_time_per_frame = 1000000 / m_output_format->m_video_frame_rate;
    for( ; ; ) {

        // get/predict the timestamp of the next frame
        int64_t next_timestamp = (lock->m_video_buffer.empty())? lock->m_segment_video_stop_time - (int64_t) (1000000 / m_output_format->m_video_frame_rate) : lock->m_video_buffer.front()->GetFrame()->pts;
        int64_t next_pts = (lock->m_time_offset + (next_timestamp - segment_start_time)) * (int64_t) m_output_format->m_video_frame_rate / (int64_t) 1000000;

        // if the frame is too late, decrease the pts by one to avoid gaps
        if(next_pts > lock->m_video_pts)
            --next_pts;

        // insert delays if needed, up to the segment end
        while(lock->m_segment_video_accumulated_delay >= delay_time_per_frame && lock->m_video_pts < segment_stop_video_pts) {
            lock->m_segment_video_accumulated_delay -= delay_time_per_frame;
            lock->m_video_pts += 1;
            //Logger::LogInfo("[Synchronizer::FlushVideoBuffer] Delay [" + QString::number(lock->m_video_pts - 1) + "] acc " + QString::number(lock->m_segment_video_accumulated_delay) + ".");
        }

        // insert duplicate frames if needed, up to either the next frame or the segment end
        if(lock->m_last_video_frame_data != NULL) {
            while(lock->m_video_pts + m_max_frames_skipped < std::min(next_pts, segment_stop_video_pts)) {

                // create duplicate frame
                std::unique_ptr<AVFrameWrapper> duplicate_frame = CreateVideoFrame(m_output_format->m_video_width, m_output_format->m_video_height, m_output_format->m_video_pixel_format, lock->m_last_video_frame_data);
                duplicate_frame->GetFrame()->pts = lock->m_video_pts + m_max_frames_skipped;

                // add new block to sync diagram
#if DEFINE_DIAGRAM
                if(m_sync_diagram != NULL) {		//bybobbi
                    double t = (double) duplicate_frame->GetFrame()->pts / (double) m_output_format->m_video_frame_rate;
                    m_sync_diagram->AddBlock(2, t, t + 1.0 / (double) m_output_format->m_video_frame_rate, QColor(255, 196, 0));
                }
#endif

                // send the frame to the encoder
                lock->m_segment_video_accumulated_delay = std::max((int64_t) 0, lock->m_segment_video_accumulated_delay - m_max_frames_skipped * delay_time_per_frame);
                lock->m_video_pts = duplicate_frame->GetFrame()->pts + 1;
                //Logger::LogInfo("[Synchronizer::FlushVideoBuffer] Encoded video frame [" + QString::number(duplicate_frame->GetFrame()->pts) + "] (duplicate) acc " + QString::number(lock->m_segment_video_accumulated_delay) + ".");
                m_output_manager->AddVideoFrame(std::move(duplicate_frame));
                lock->m_segment_video_accumulated_delay += m_output_manager->GetVideoFrameDelay();

            }
        }

        // if there are no frames, or they are beyond the segment end, stop
        if(lock->m_video_buffer.empty() || next_pts >= segment_stop_video_pts)
            break;

        // get the frame
        std::unique_ptr<AVFrameWrapper> frame = std::move(lock->m_video_buffer.front());
        lock->m_video_buffer.pop_front();
        frame->GetFrame()->pts = next_pts;
        lock->m_last_video_frame_data = frame->GetFrameData();

        // if the frame is too early, drop it
        if(frame->GetFrame()->pts < lock->m_video_pts) {
            //Logger::LogInfo("[Synchronizer::FlushVideoBuffer] Dropped video frame [" + QString::number(frame->GetFrame()->pts) + "] acc " + QString::number(lock->m_segment_video_accumulated_delay) + ".");
            continue;
        }

        // if this is the first video frame, always set the pts to zero
        if(lock->m_video_pts == 0)
            frame->GetFrame()->pts = 0;

        // add new block to sync diagram
#if DEFINE_DIAGRAM
        if(m_sync_diagram != NULL) {
            double t = (double) frame->GetFrame()->pts / (double) m_output_format->m_video_frame_rate;
            m_sync_diagram->AddBlock(2, t, t + 1.0 / (double) m_output_format->m_video_frame_rate, QColor(255, 0, 0));
        }
#endif

        // send the frame to the encoder
        lock->m_segment_video_accumulated_delay = std::max((int64_t) 0, lock->m_segment_video_accumulated_delay - (frame->GetFrame()->pts - lock->m_video_pts) * delay_time_per_frame);
        lock->m_video_pts = frame->GetFrame()->pts + 1;
        //Logger::LogInfo("[Synchronizer::FlushBuffers] Encoded video frame [" + QString::number(frame->GetFrame()->pts) + "].");
        m_output_manager->AddVideoFrame(std::move(frame));
        lock->m_segment_video_accumulated_delay += m_output_manager->GetVideoFrameDelay();

    }

}

void Synchronizer::FlushAudioBuffer(Synchronizer::SharedData* lock, int64_t segment_start_time, int64_t segment_stop_time) {

    double sample_length = (double) (segment_stop_time - lock->m_segment_audio_start_time) * 1.0e-6;
    int64_t samples_max = (int64_t) ceil(sample_length * (double) m_output_format->m_audio_sample_rate) - lock->m_segment_audio_samples_read;
    if(lock->m_audio_buffer.GetSize() > 0) {

        // Normally, the correct way to calculate the position of the first sample would be:
        //     int64_t timestamp = lock->m_segment_audio_start_time + (int64_t) round((double) lock->m_segment_audio_samples_read / (double) m_audio_sample_rate * 1.0e6);
        //     int64_t pos = (int64_t) round((double) (lock->m_time_offset + (timestamp - segment_start_time)) * 1.0e-6 * (double) m_audio_sample_rate);
        // Simplified:
        //     int64_t pos = (int64_t) round((double) (lock->m_time_offset + (lock->m_segment_audio_start_time - segment_start_time)) * 1.0e-6 * (double) m_audio_sample_rate)
        //                   + lock->m_segment_audio_samples_read;
        // The first part of the expression is constant, so it only has to be calculated at the start of the segment. After that the increase in position is always
        // equal to the number of samples written. Samples are only dropped at the start of the segment, so actually
        // the position doesn't have to be calculated anymore after that, since it is assumed to be equal to lock->m_audio_samples.

        if(lock->m_segment_audio_can_drop) {

            // calculate the offset of the first sample
            int64_t pos = (int64_t) round((double) (lock->m_time_offset + (lock->m_segment_audio_start_time - segment_start_time)) * 1.0e-6 * (double) m_output_format->m_audio_sample_rate)
                          + lock->m_segment_audio_samples_read;

            // drop samples that are too early
            if(pos < lock->m_audio_samples) {
                int64_t n = std::min(lock->m_audio_samples - pos, (int64_t) lock->m_audio_buffer.GetSize() / m_output_format->m_audio_channels);
                lock->m_audio_buffer.Pop(n * m_output_format->m_audio_channels);
                lock->m_segment_audio_samples_read += n;
            }

        }

        int64_t samples_left = std::min(samples_max, (int64_t) lock->m_audio_buffer.GetSize() / m_output_format->m_audio_channels);

        // add new block to sync diagram
#if DEFINE_DIAGRAM
        if(m_sync_diagram != NULL && samples_left > 0) {	//bybobbi
            double t = (double) lock->m_audio_samples / (double) m_output_format->m_audio_sample_rate;
            m_sync_diagram->AddBlock(3, t, t + (double) samples_left / (double) m_output_format->m_audio_sample_rate, QColor(0, 255, 0));
        }
#endif

        // send the samples to the encoder
        while(samples_left > 0) {

            lock->m_segment_audio_can_drop = false;

            // copy samples until either the partial frame is full or there are no samples left
            //TODO// do direct copy/conversion to new audio frame?
            int64_t n = std::min((int64_t) (m_output_format->m_audio_frame_size - lock->m_partial_audio_frame_samples), samples_left);
            lock->m_audio_buffer.Pop(lock->m_partial_audio_frame.GetData() + lock->m_partial_audio_frame_samples * m_output_format->m_audio_channels, n * m_output_format->m_audio_channels);
            lock->m_segment_audio_samples_read += n;
            lock->m_partial_audio_frame_samples += n;
            lock->m_audio_samples += n;
            samples_left -= n;

            // is the partial frame full?
            if(lock->m_partial_audio_frame_samples == m_output_format->m_audio_frame_size) {

                // allocate a frame
#if SSR_USE_AVUTIL_PLANAR_SAMPLE_FMT
                unsigned int planes = (m_output_format->m_audio_sample_format == AV_SAMPLE_FMT_S16P ||
                                       m_output_format->m_audio_sample_format == AV_SAMPLE_FMT_FLTP)? m_output_format->m_audio_channels : 1;
#else
                unsigned int planes = 1;
#endif
                std::unique_ptr<AVFrameWrapper> audio_frame = CreateAudioFrame(m_output_format->m_audio_channels, m_output_format->m_audio_sample_rate,
                                                                               m_output_format->m_audio_frame_size, planes, m_output_format->m_audio_sample_format);
                audio_frame->GetFrame()->pts = lock->m_audio_samples;

                // copy/convert the samples
                switch(m_output_format->m_audio_sample_format) {
                    case AV_SAMPLE_FMT_S16: {
                        float *data_in = (float*) lock->m_partial_audio_frame.GetData();
                        int16_t *data_out = (int16_t*) audio_frame->GetFrame()->data[0];
                        SampleCopy(m_output_format->m_audio_frame_size * m_output_format->m_audio_channels, data_in, 1, data_out, 1);
                        break;
                    }
                    case AV_SAMPLE_FMT_FLT: {
                        float *data_in = (float*) lock->m_partial_audio_frame.GetData();
                        float *data_out = (float*) audio_frame->GetFrame()->data[0];
                        memcpy(data_out, data_in, m_output_format->m_audio_frame_size * m_output_format->m_audio_channels * sizeof(float));
                        break;
                    }
#if SSR_USE_AVUTIL_PLANAR_SAMPLE_FMT
                    case AV_SAMPLE_FMT_S16P: {
                        for(unsigned int p = 0; p < planes; ++p) {
                            float *data_in = (float*) lock->m_partial_audio_frame.GetData() + p;
                            int16_t *data_out = (int16_t*) audio_frame->GetFrame()->data[p];
                            SampleCopy(m_output_format->m_audio_frame_size, data_in, planes, data_out, 1);
                        }
                        break;
                    }
                    case AV_SAMPLE_FMT_FLTP: {
                        for(unsigned int p = 0; p < planes; ++p) {
                            float *data_in = (float*) lock->m_partial_audio_frame.GetData() + p;
                            float *data_out = (float*) audio_frame->GetFrame()->data[p];
                            SampleCopy(m_output_format->m_audio_frame_size, data_in, planes, data_out, 1);
                        }
                        break;
                    }
#endif
                    default: {
                        assert(false);
                        break;
                    }
                }
                lock->m_partial_audio_frame_samples = 0;

                //Logger::LogInfo("[Synchronizer::FlushAudioBuffer] Encoded audio frame [" + QString::number(lock->m_partial_audio_frame->pts) + "].");
                m_output_manager->AddAudioFrame(std::move(audio_frame));
            }

        }

    }

}

void Synchronizer::SynchronizerThread() {
    try {

        Logger::LogInfo("[Synchronizer::SynchronizerThread] " + Logger::tr("Synchronizer thread started."));

        while(!m_should_stop) {

            {
                SharedLock lock(&m_shared_data);
                FlushBuffers(lock.get());
                if(m_sync_diagram != NULL) {
                    double time_in = (double) hrt_time_micro() * 1.0e-6;
                    double time_out = (double) GetTotalTime(lock.get()) * 1.0e-6;
//#if DEFINE_DIAGRAM		//bybobbi
                    m_sync_diagram->SetCurrentTime(0, time_in);
                    m_sync_diagram->SetCurrentTime(1, time_in);
                    m_sync_diagram->SetCurrentTime(2, time_out);
                    m_sync_diagram->SetCurrentTime(3, time_out);
                    m_sync_diagram->Update();
//#endif
                }
            }

            usleep(20000);

        }

        Logger::LogInfo("[Synchronizer::SynchronizerThread] " + Logger::tr("Synchronizer thread stopped."));

    } catch(const std::exception& e) {
        m_error_occurred = true;
        Logger::LogError("[Synchronizer::SynchronizerThread] " + Logger::tr("Exception '%1' in synchronizer thread.").arg(e.what()));
    } catch(...) {
        m_error_occurred = true;
        Logger::LogError("[Synchronizer::SynchronizerThread] " + Logger::tr("Unknown exception in synchronizer thread."));
    }
}
