From 399a660faadf7987bda318d8dca3b18cf07310e7 Mon Sep 17 00:00:00 2001 From: zhupengfei Date: Sat, 26 Jan 2019 22:41:28 +0800 Subject: [PATCH] core/dumping: Add FFmpeg implementation Sorry for the large diff, the implementation is quite long, and I can't really find a good way to split it into commits. --- src/core/CMakeLists.txt | 11 + src/core/core.cpp | 9 + src/core/dumping/ffmpeg_backend.cpp | 530 ++++++++++++++++++++++++++++ src/core/dumping/ffmpeg_backend.h | 196 ++++++++++ 4 files changed, 746 insertions(+) create mode 100644 src/core/dumping/ffmpeg_backend.cpp create mode 100644 src/core/dumping/ffmpeg_backend.h diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 20172ea64..83056ec75 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -446,6 +446,13 @@ add_library(core STATIC tracer/recorder.h ) +if (ENABLE_FFMPEG) + target_sources(core PRIVATE + dumping/ffmpeg_backend.cpp + dumping/ffmpeg_backend.h + ) +endif() + create_target_directory_groups(core) target_link_libraries(core PUBLIC common PRIVATE audio_core network video_core) @@ -464,3 +471,7 @@ if (ARCHITECTURE_x86_64) ) target_link_libraries(core PRIVATE dynarmic) endif() + +if (ENABLE_FFMPEG) + target_link_libraries(core PRIVATE FFmpeg::avcodec FFmpeg::avformat FFmpeg::swscale FFmpeg::swresample FFmpeg::avutil) +endif() diff --git a/src/core/core.cpp b/src/core/core.cpp index b7f33d504..c7445efec 100644 --- a/src/core/core.cpp +++ b/src/core/core.cpp @@ -17,6 +17,9 @@ #include "core/core.h" #include "core/core_timing.h" #include "core/dumping/backend.h" +#ifdef ENABLE_FFMPEG +#include "core/dumping/ffmpeg_backend.h" +#endif #include "core/gdbstub/gdbstub.h" #include "core/hle/kernel/client_port.h" #include "core/hle/kernel/kernel.h" @@ -218,6 +221,12 @@ System::ResultStatus System::Init(Frontend::EmuWindow& emu_window, u32 system_mo return result; } +#ifdef ENABLE_FFMPEG + video_dumper = std::make_unique(); +#else + video_dumper = std::make_unique(); +#endif + LOG_DEBUG(Core, "Initialized OK"); // Reset counters and set time origin to current frame diff --git a/src/core/dumping/ffmpeg_backend.cpp b/src/core/dumping/ffmpeg_backend.cpp new file mode 100644 index 000000000..811a5c99b --- /dev/null +++ b/src/core/dumping/ffmpeg_backend.cpp @@ -0,0 +1,530 @@ +// Copyright 2018 Citra Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include "common/assert.h" +#include "common/file_util.h" +#include "common/logging/log.h" +#include "core/dumping/ffmpeg_backend.h" +#include "video_core/renderer_base.h" +#include "video_core/video_core.h" + +extern "C" { +#include +} + +namespace VideoDumper { + +void InitializeFFmpegLibraries() { + static bool initialized = false; + + if (initialized) + return; +#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(58, 9, 100) + av_register_all(); +#endif + avformat_network_init(); + initialized = true; +} + +FFmpegStream::~FFmpegStream() { + Free(); +} + +bool FFmpegStream::Init(AVFormatContext* format_context_) { + InitializeFFmpegLibraries(); + + format_context = format_context_; + return true; +} + +void FFmpegStream::Free() { + codec_context.reset(); +} + +void FFmpegStream::Flush() { + SendFrame(nullptr); +} + +void FFmpegStream::WritePacket(AVPacket& packet) { + if (packet.pts != static_cast(AV_NOPTS_VALUE)) { + packet.pts = av_rescale_q(packet.pts, codec_context->time_base, stream->time_base); + } + if (packet.dts != static_cast(AV_NOPTS_VALUE)) { + packet.dts = av_rescale_q(packet.dts, codec_context->time_base, stream->time_base); + } + packet.stream_index = stream->index; + av_interleaved_write_frame(format_context, &packet); +} + +void FFmpegStream::SendFrame(AVFrame* frame) { + // Initialize packet + AVPacket packet; + av_init_packet(&packet); + packet.data = nullptr; + packet.size = 0; + + // Encode frame + if (avcodec_send_frame(codec_context.get(), frame) < 0) { + LOG_ERROR(Render, "Frame dropped: could not send frame"); + return; + } + int error = 1; + while (error >= 0) { + error = avcodec_receive_packet(codec_context.get(), &packet); + if (error == AVERROR(EAGAIN) || error == AVERROR_EOF) + return; + if (error < 0) { + LOG_ERROR(Render, "Frame dropped: could not encode audio"); + return; + } else { + // Write frame to video file + WritePacket(packet); + } + } +} + +FFmpegVideoStream::~FFmpegVideoStream() { + Free(); +} + +bool FFmpegVideoStream::Init(AVFormatContext* format_context, AVOutputFormat* output_format, + const Layout::FramebufferLayout& layout_) { + + InitializeFFmpegLibraries(); + + if (!FFmpegStream::Init(format_context)) + return false; + + layout = layout_; + frame_count = 0; + + // Initialize video codec + // Ensure VP9 codec here, also to avoid patent issues + constexpr AVCodecID codec_id = AV_CODEC_ID_VP9; + const AVCodec* codec = avcodec_find_encoder(codec_id); + codec_context.reset(avcodec_alloc_context3(codec)); + if (!codec || !codec_context) { + LOG_ERROR(Render, "Could not find video encoder or allocate video codec context"); + return false; + } + + // Configure video codec context + codec_context->codec_type = AVMEDIA_TYPE_VIDEO; + codec_context->bit_rate = 2500000; + codec_context->width = layout.width; + codec_context->height = layout.height; + codec_context->time_base.num = 1; + codec_context->time_base.den = 60; + codec_context->gop_size = 12; + codec_context->pix_fmt = AV_PIX_FMT_YUV420P; + codec_context->thread_count = 8; + if (output_format->flags & AVFMT_GLOBALHEADER) + codec_context->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; + av_opt_set_int(codec_context.get(), "cpu-used", 5, 0); + + if (avcodec_open2(codec_context.get(), codec, nullptr) < 0) { + LOG_ERROR(Render, "Could not open video codec"); + return false; + } + + // Create video stream + stream = avformat_new_stream(format_context, codec); + if (!stream || avcodec_parameters_from_context(stream->codecpar, codec_context.get()) < 0) { + LOG_ERROR(Render, "Could not create video stream"); + return false; + } + + // Allocate frames + current_frame.reset(av_frame_alloc()); + scaled_frame.reset(av_frame_alloc()); + scaled_frame->format = codec_context->pix_fmt; + scaled_frame->width = layout.width; + scaled_frame->height = layout.height; + if (av_frame_get_buffer(scaled_frame.get(), 1) < 0) { + LOG_ERROR(Render, "Could not allocate frame buffer"); + return false; + } + + // Create SWS Context + auto* context = sws_getCachedContext( + sws_context.get(), layout.width, layout.height, pixel_format, layout.width, layout.height, + codec_context->pix_fmt, SWS_BICUBIC, nullptr, nullptr, nullptr); + if (context != sws_context.get()) + sws_context.reset(context); + + return true; +} + +void FFmpegVideoStream::Free() { + FFmpegStream::Free(); + + current_frame.reset(); + scaled_frame.reset(); + sws_context.reset(); +} + +void FFmpegVideoStream::ProcessFrame(VideoFrame& frame) { + if (frame.width != layout.width || frame.height != layout.height) { + LOG_ERROR(Render, "Frame dropped: resolution does not match"); + return; + } + // Prepare frame + current_frame->data[0] = frame.data.data(); + current_frame->linesize[0] = frame.stride; + current_frame->format = pixel_format; + current_frame->width = layout.width; + current_frame->height = layout.height; + + // Scale the frame + if (sws_context) { + sws_scale(sws_context.get(), current_frame->data, current_frame->linesize, 0, layout.height, + scaled_frame->data, scaled_frame->linesize); + } + scaled_frame->pts = frame_count++; + + // Encode frame + SendFrame(scaled_frame.get()); +} + +FFmpegAudioStream::~FFmpegAudioStream() { + Free(); +} + +bool FFmpegAudioStream::Init(AVFormatContext* format_context) { + InitializeFFmpegLibraries(); + + if (!FFmpegStream::Init(format_context)) + return false; + + sample_count = 0; + + // Initialize audio codec + constexpr AVCodecID codec_id = AV_CODEC_ID_VORBIS; + const AVCodec* codec = avcodec_find_encoder(codec_id); + codec_context.reset(avcodec_alloc_context3(codec)); + if (!codec || !codec_context) { + LOG_ERROR(Render, "Could not find audio encoder or allocate audio codec context"); + return false; + } + + // Configure audio codec context + codec_context->codec_type = AVMEDIA_TYPE_AUDIO; + codec_context->bit_rate = 64000; + codec_context->sample_fmt = codec->sample_fmts[0]; + codec_context->sample_rate = AudioCore::native_sample_rate; + codec_context->channel_layout = AV_CH_LAYOUT_STEREO; + codec_context->channels = 2; + + if (avcodec_open2(codec_context.get(), codec, nullptr) < 0) { + LOG_ERROR(Render, "Could not open audio codec"); + return false; + } + + // Create audio stream + stream = avformat_new_stream(format_context, codec); + if (!stream || avcodec_parameters_from_context(stream->codecpar, codec_context.get()) < 0) { + + LOG_ERROR(Render, "Could not create audio stream"); + return false; + } + + // Allocate frame + audio_frame.reset(av_frame_alloc()); + audio_frame->format = codec_context->sample_fmt; + audio_frame->channel_layout = codec_context->channel_layout; + audio_frame->channels = codec_context->channels; + + // Allocate SWR context + auto* context = + swr_alloc_set_opts(nullptr, codec_context->channel_layout, codec_context->sample_fmt, + codec_context->sample_rate, codec_context->channel_layout, + AV_SAMPLE_FMT_S16P, AudioCore::native_sample_rate, 0, nullptr); + if (!context) { + LOG_ERROR(Render, "Could not create SWR context"); + return false; + } + swr_context.reset(context); + if (swr_init(swr_context.get()) < 0) { + LOG_ERROR(Render, "Could not init SWR context"); + return false; + } + + // Allocate resampled data + int error = + av_samples_alloc_array_and_samples(&resampled_data, nullptr, codec_context->channels, + codec_context->frame_size, codec_context->sample_fmt, 0); + if (error < 0) { + LOG_ERROR(Render, "Could not allocate samples storage"); + return false; + } + + return true; +} + +void FFmpegAudioStream::Free() { + FFmpegStream::Free(); + + audio_frame.reset(); + swr_context.reset(); + // Free resampled data + if (resampled_data) { + av_freep(&resampled_data[0]); + } + av_freep(&resampled_data); +} + +void FFmpegAudioStream::ProcessFrame(VariableAudioFrame& channel0, VariableAudioFrame& channel1) { + ASSERT_MSG(channel0.size() == channel1.size(), + "Frames of the two channels must have the same number of samples"); + std::array src_data = {reinterpret_cast(channel0.data()), + reinterpret_cast(channel1.data())}; + if (swr_convert(swr_context.get(), resampled_data, channel0.size(), src_data.data(), + channel0.size()) < 0) { + + LOG_ERROR(Render, "Audio frame dropped: Could not resample data"); + return; + } + + // Prepare frame + audio_frame->nb_samples = channel0.size(); + audio_frame->data[0] = resampled_data[0]; + audio_frame->data[1] = resampled_data[1]; + audio_frame->pts = sample_count; + sample_count += channel0.size(); + + SendFrame(audio_frame.get()); +} + +std::size_t FFmpegAudioStream::GetAudioFrameSize() const { + ASSERT_MSG(codec_context, "Codec context is not initialized yet!"); + return codec_context->frame_size; +} + +FFmpegMuxer::~FFmpegMuxer() { + Free(); +} + +bool FFmpegMuxer::Init(const std::string& path, const std::string& format, + const Layout::FramebufferLayout& layout) { + + InitializeFFmpegLibraries(); + + if (!FileUtil::CreateFullPath(path)) { + return false; + } + + // Get output format + // Ensure webm here to avoid patent issues + ASSERT_MSG(format == "webm", "Only webm is allowed for frame dumping"); + auto* output_format = av_guess_format(format.c_str(), path.c_str(), "video/webm"); + if (!output_format) { + LOG_ERROR(Render, "Could not get format {}", format); + return false; + } + + // Initialize format context + auto* format_context_raw = format_context.get(); + if (avformat_alloc_output_context2(&format_context_raw, output_format, nullptr, path.c_str()) < + 0) { + + LOG_ERROR(Render, "Could not allocate output context"); + return false; + } + format_context.reset(format_context_raw); + + if (!video_stream.Init(format_context.get(), output_format, layout)) + return false; + if (!audio_stream.Init(format_context.get())) + return false; + + // Open video file + if (avio_open(&format_context->pb, path.c_str(), AVIO_FLAG_WRITE) < 0 || + avformat_write_header(format_context.get(), nullptr)) { + + LOG_ERROR(Render, "Could not open {}", path); + return false; + } + + LOG_INFO(Render, "Dumping frames to {} ({}x{})", path, layout.width, layout.height); + return true; +} + +void FFmpegMuxer::Free() { + video_stream.Free(); + audio_stream.Free(); + format_context.reset(); +} + +void FFmpegMuxer::ProcessVideoFrame(VideoFrame& frame) { + video_stream.ProcessFrame(frame); +} + +void FFmpegMuxer::ProcessAudioFrame(VariableAudioFrame& channel0, VariableAudioFrame& channel1) { + audio_stream.ProcessFrame(channel0, channel1); +} + +void FFmpegMuxer::FlushVideo() { + video_stream.Flush(); +} + +void FFmpegMuxer::FlushAudio() { + audio_stream.Flush(); +} + +std::size_t FFmpegMuxer::GetAudioFrameSize() const { + return audio_stream.GetAudioFrameSize(); +} + +void FFmpegMuxer::WriteTrailer() { + av_write_trailer(format_context.get()); +} + +FFmpegBackend::FFmpegBackend() = default; + +FFmpegBackend::~FFmpegBackend() { + ASSERT_MSG(!IsDumping(), "Dumping must be stopped first"); + + if (video_processing_thread.joinable()) + video_processing_thread.join(); + if (audio_processing_thread.joinable()) + audio_processing_thread.join(); + ffmpeg.Free(); +} + +bool FFmpegBackend::StartDumping(const std::string& path, const std::string& format, + const Layout::FramebufferLayout& layout) { + + InitializeFFmpegLibraries(); + + if (!ffmpeg.Init(path, format, layout)) { + ffmpeg.Free(); + return false; + } + + video_layout = layout; + + if (video_processing_thread.joinable()) + video_processing_thread.join(); + video_processing_thread = std::thread([&] { + event1.Set(); + while (true) { + event2.Wait(); + current_buffer = (current_buffer + 1) % 2; + next_buffer = (current_buffer + 1) % 2; + event1.Set(); + // Process this frame + auto& frame = video_frame_buffers[current_buffer]; + if (frame.width == 0 && frame.height == 0) { + // An empty frame marks the end of frame data + ffmpeg.FlushVideo(); + break; + } + ffmpeg.ProcessVideoFrame(frame); + } + // Finish audio execution first if not done yet + if (audio_processing_thread.joinable()) + audio_processing_thread.join(); + EndDumping(); + }); + + if (audio_processing_thread.joinable()) + audio_processing_thread.join(); + audio_processing_thread = std::thread([&] { + VariableAudioFrame channel0, channel1; + while (true) { + channel0 = audio_frame_queues[0].PopWait(); + channel1 = audio_frame_queues[1].PopWait(); + if (channel0.empty()) { + // An empty frame marks the end of frame data + ffmpeg.FlushAudio(); + break; + } + ffmpeg.ProcessAudioFrame(channel0, channel1); + } + }); + + VideoCore::g_renderer->PrepareVideoDumping(); + is_dumping = true; + + return true; +} + +void FFmpegBackend::AddVideoFrame(const VideoFrame& frame) { + event1.Wait(); + video_frame_buffers[next_buffer] = std::move(frame); + event2.Set(); +} + +void FFmpegBackend::AddAudioFrame(const AudioCore::StereoFrame16& frame) { + std::array, 2> refactored_frame; + for (std::size_t i = 0; i < frame.size(); i++) { + refactored_frame[0][i] = frame[i][0]; + refactored_frame[1][i] = frame[i][1]; + } + + for (auto i : {0, 1}) { + audio_buffers[i].insert(audio_buffers[i].end(), refactored_frame[i].begin(), + refactored_frame[i].end()); + } + CheckAudioBuffer(); +} + +void FFmpegBackend::AddAudioSample(const std::array& sample) { + for (auto i : {0, 1}) { + audio_buffers[i].push_back(sample[i]); + } + CheckAudioBuffer(); +} + +void FFmpegBackend::StopDumping() { + is_dumping = false; + VideoCore::g_renderer->CleanupVideoDumping(); + + // Flush the video processing queue + AddVideoFrame(VideoFrame()); + for (auto i : {0, 1}) { + // Add remaining data to audio queue + if (audio_buffers[i].size() >= 0) { + VariableAudioFrame buffer(audio_buffers[i].begin(), audio_buffers[i].end()); + audio_frame_queues[i].Push(std::move(buffer)); + audio_buffers[i].clear(); + } + // Flush the audio processing queue + audio_frame_queues[i].Push(VariableAudioFrame()); + } + // Wait until processing ends + processing_ended.Wait(); +} + +bool FFmpegBackend::IsDumping() const { + return is_dumping.load(std::memory_order_relaxed); +} + +Layout::FramebufferLayout FFmpegBackend::GetLayout() const { + return video_layout; +} + +void FFmpegBackend::EndDumping() { + LOG_INFO(Render, "Ending frame dumping"); + + ffmpeg.WriteTrailer(); + ffmpeg.Free(); + processing_ended.Set(); +} + +void FFmpegBackend::CheckAudioBuffer() { + for (auto i : {0, 1}) { + const std::size_t frame_size = ffmpeg.GetAudioFrameSize(); + // Add audio data to the queue when there is enough to form a frame + while (audio_buffers[i].size() >= frame_size) { + VariableAudioFrame buffer(audio_buffers[i].begin(), + audio_buffers[i].begin() + frame_size); + audio_frame_queues[i].Push(std::move(buffer)); + + audio_buffers[i].erase(audio_buffers[i].begin(), audio_buffers[i].begin() + frame_size); + } + } +} + +} // namespace VideoDumper diff --git a/src/core/dumping/ffmpeg_backend.h b/src/core/dumping/ffmpeg_backend.h new file mode 100644 index 000000000..0208195d5 --- /dev/null +++ b/src/core/dumping/ffmpeg_backend.h @@ -0,0 +1,196 @@ +// Copyright 2018 Citra Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include "common/common_types.h" +#include "common/thread.h" +#include "common/threadsafe_queue.h" +#include "core/dumping/backend.h" + +extern "C" { +#include +#include +#include +#include +} + +namespace VideoDumper { + +using VariableAudioFrame = std::vector; + +void InitFFmpegLibraries(); + +/** + * Wrapper around FFmpeg AVCodecContext + AVStream. + * Rescales/Resamples, encodes and writes a frame. + */ +class FFmpegStream { +public: + bool Init(AVFormatContext* format_context); + void Free(); + void Flush(); + +protected: + ~FFmpegStream(); + + void WritePacket(AVPacket& packet); + void SendFrame(AVFrame* frame); + + struct AVCodecContextDeleter { + void operator()(AVCodecContext* codec_context) const { + avcodec_free_context(&codec_context); + } + }; + + struct AVFrameDeleter { + void operator()(AVFrame* frame) const { + av_frame_free(&frame); + } + }; + + AVFormatContext* format_context{}; + std::unique_ptr codec_context{}; + AVStream* stream{}; +}; + +/** + * A FFmpegStream used for video data. + * Rescales, encodes and writes a frame. + */ +class FFmpegVideoStream : public FFmpegStream { +public: + ~FFmpegVideoStream(); + + bool Init(AVFormatContext* format_context, AVOutputFormat* output_format, + const Layout::FramebufferLayout& layout); + void Free(); + void ProcessFrame(VideoFrame& frame); + +private: + struct SwsContextDeleter { + void operator()(SwsContext* sws_context) const { + sws_freeContext(sws_context); + } + }; + + u64 frame_count{}; + + std::unique_ptr current_frame{}; + std::unique_ptr scaled_frame{}; + std::unique_ptr sws_context{}; + Layout::FramebufferLayout layout; + + /// The pixel format the frames are stored in + static constexpr AVPixelFormat pixel_format = AVPixelFormat::AV_PIX_FMT_BGRA; +}; + +/** + * A FFmpegStream used for audio data. + * Resamples (converts), encodes and writes a frame. + */ +class FFmpegAudioStream : public FFmpegStream { +public: + ~FFmpegAudioStream(); + + bool Init(AVFormatContext* format_context); + void Free(); + void ProcessFrame(VariableAudioFrame& channel0, VariableAudioFrame& channel1); + std::size_t GetAudioFrameSize() const; + +private: + struct SwrContextDeleter { + void operator()(SwrContext* swr_context) const { + swr_free(&swr_context); + } + }; + + u64 sample_count{}; + + std::unique_ptr audio_frame{}; + std::unique_ptr swr_context{}; + + u8** resampled_data{}; +}; + +/** + * Wrapper around FFmpeg AVFormatContext. + * Manages the video and audio streams, and accepts video and audio data. + */ +class FFmpegMuxer { +public: + ~FFmpegMuxer(); + + bool Init(const std::string& path, const std::string& format, + const Layout::FramebufferLayout& layout); + void Free(); + void ProcessVideoFrame(VideoFrame& frame); + void ProcessAudioFrame(VariableAudioFrame& channel0, VariableAudioFrame& channel1); + void FlushVideo(); + void FlushAudio(); + std::size_t GetAudioFrameSize() const; + void WriteTrailer(); + +private: + struct AVFormatContextDeleter { + void operator()(AVFormatContext* format_context) const { + avio_closep(&format_context->pb); + avformat_free_context(format_context); + } + }; + + FFmpegAudioStream audio_stream{}; + FFmpegVideoStream video_stream{}; + std::unique_ptr format_context{}; +}; + +/** + * FFmpeg video dumping backend. + * This class implements a double buffer, and an audio queue to keep audio data + * before enough data is received to form a frame. + */ +class FFmpegBackend : public Backend { +public: + FFmpegBackend(); + ~FFmpegBackend() override; + bool StartDumping(const std::string& path, const std::string& format, + const Layout::FramebufferLayout& layout) override; + void AddVideoFrame(const VideoFrame& frame) override; + void AddAudioFrame(const AudioCore::StereoFrame16& frame) override; + void AddAudioSample(const std::array& sample) override; + void StopDumping() override; + bool IsDumping() const override; + Layout::FramebufferLayout GetLayout() const override; + +private: + void CheckAudioBuffer(); + void EndDumping(); + + std::atomic_bool is_dumping = false; ///< Whether the backend is currently dumping + + FFmpegMuxer ffmpeg{}; + + Layout::FramebufferLayout video_layout; + std::array video_frame_buffers; + u32 current_buffer = 0, next_buffer = 1; + Common::Event event1, event2; + std::thread video_processing_thread; + + /// An audio buffer used to temporarily hold audio data, before the size is big enough + /// to be sent to the encoder as a frame + std::array audio_buffers; + std::array, 2> audio_frame_queues; + std::thread audio_processing_thread; + + Common::Event processing_ended; +}; + +} // namespace VideoDumper