#include "libs/videoDec2Decoder.h" #include "common/logging/log.h" #include #include #include #include #include #include #include #include #include extern "C" { #include #include #include #include #include #include } namespace Libs::VideoDec2::Decoder { namespace { constexpr uint32_t CODEC_TYPE_AVC = 1; constexpr uint32_t CODEC_TYPE_HEVC = 974921; constexpr uint32_t CODEC_TYPE_VP9 = 2382845; struct PacketMetadata { uint64_t pts = TIMESTAMP_INVALID; uint64_t dts = TIMESTAMP_INVALID; uint64_t attached_data = 0; }; struct StoredPicture { const Instance* owner = nullptr; PictureInfo info; }; std::mutex g_picture_mutex; std::unordered_map g_picture_infos; AVCodecID GetAvCodecId(uint32_t codec_type) { switch (codec_type) { case CODEC_TYPE_AVC: return AV_CODEC_ID_H264; case CODEC_TYPE_HEVC: return AV_CODEC_ID_HEVC; case CODEC_TYPE_VP9: return AV_CODEC_ID_VP9; default: return AV_CODEC_ID_NONE; } } const char* AvErrorString(int error) { thread_local char text[AV_ERROR_MAX_STRING_SIZE] {}; if (av_strerror(error, text, sizeof(text)) != 0) { std::strcpy(text, "unknown FFmpeg error"); } return text; } uint32_t AlignUp(uint32_t value, uint32_t alignment) { return (value + alignment - 1u) & ~(alignment - 1u); } int64_t ToAvTimestamp(uint64_t timestamp) { return timestamp == TIMESTAMP_INVALID || timestamp > static_cast(std::numeric_limits::max()) ? AV_NOPTS_VALUE : static_cast(timestamp); } } // namespace class Instance { public: explicit Instance(const Config& config): m_config(config) {} ~Instance() { ClearFrames(); ClearPictureMetadata(); if (m_sws != nullptr) { sws_freeContext(m_sws); } if (m_codec != nullptr) { avcodec_free_context(&m_codec); } } Instance(const Instance&) = delete; Instance& operator=(const Instance&) = delete; [[nodiscard]] bool Initialize() { const AVCodec* decoder = avcodec_find_decoder(GetAvCodecId(m_config.codec_type)); if (decoder == nullptr) { if (m_config.codec_type == CODEC_TYPE_VP9) { Log::WriteToConsoleAndLog( "WARNING: Videodec2: The game requested VP9 video, but FFmpeg has no VP9 " "decoder. Use FFmpeg with VP9 decoding enabled.\n"); } LOGF("Videodec2: FFmpeg decoder is unavailable for codec type %u\n", m_config.codec_type); return false; } m_codec = avcodec_alloc_context3(decoder); if (m_codec == nullptr) { LOGF("Videodec2: avcodec_alloc_context3 failed\n"); return false; } // This carries PTS/DTS/attachedData through codecs that reorder B frames. m_codec->flags |= AV_CODEC_FLAG_COPY_OPAQUE; const int result = avcodec_open2(m_codec, decoder, nullptr); if (result < 0) { LOGF("Videodec2: avcodec_open2 failed: %s (%d)\n", AvErrorString(result), result); return false; } return true; } [[nodiscard]] uint32_t CodecType() const { return m_config.codec_type; } [[nodiscard]] Result Submit(const Input& input, uint32_t* produced) { std::scoped_lock lock(m_mutex); return SubmitInput(input, produced); } [[nodiscard]] Result Receive(const FrameBuffer& frame_buffer, Output* output) { std::scoped_lock lock(m_mutex); return ReceiveOutput(frame_buffer, output); } [[nodiscard]] Result Finalize() { std::scoped_lock lock(m_mutex); if (m_finalized) return Result::Ok; const auto drained = DrainFrames(); if (drained != Result::Ok) return drained; const int result = avcodec_send_packet(m_codec, nullptr); if (result == AVERROR(EAGAIN)) return Result::InputFull; if (result < 0 && result != AVERROR_EOF) return Result::ApiFail; m_finalized = true; return DrainFrames(); } [[nodiscard]] Result DecodeInput(const Input& input, const FrameBuffer& frame_buffer, Output* output) { std::scoped_lock lock(m_mutex); *output = {}; uint32_t produced = 0; auto result = SubmitInput(input, &produced); if (result == Result::InputFull) { // The combined API supplies an output buffer together with each AU. result = ReceiveOutput(frame_buffer, output); if (result != Result::Ok) return result; return SubmitInput(input, &produced); } if (result != Result::Ok) return result; result = ReceiveOutput(frame_buffer, output); return result == Result::NoFrame ? Result::Ok : result; } [[nodiscard]] Result FlushOutput(const FrameBuffer& frame_buffer, Output* output) { std::scoped_lock lock(m_mutex); const auto result = ReceiveOutput(frame_buffer, output); return result == Result::NoFrame ? Result::Ok : result; } void ResetDecoder() { std::scoped_lock lock(m_mutex); avcodec_flush_buffers(m_codec); ClearFrames(); ClearPictureMetadata(); m_finalized = m_eof = false; } private: struct PendingFrame { AVFrame* frame; PictureInfo picture; }; void ClearFrames() { for (auto& pending: m_frames) av_frame_free(&pending.frame); m_frames.clear(); } [[nodiscard]] Result DrainFrames(uint32_t* produced = nullptr) { while (!m_eof && m_frames.size() < m_config.max_pending_frames) { AVFrame* frame = av_frame_alloc(); if (frame == nullptr) return Result::ApiFail; const int result = avcodec_receive_frame(m_codec, frame); if (result < 0) { av_frame_free(&frame); if (result == AVERROR_EOF) m_eof = true; if (result == AVERROR(EAGAIN) || m_eof) return Result::Ok; LOGF("Video decoder: avcodec_receive_frame failed: %s (%d)\n", AvErrorString(result), result); return Result::AccessUnit; } m_frames.push_back({frame, MakePictureInfo(frame)}); if (produced != nullptr) ++*produced; } return Result::Ok; } [[nodiscard]] Result SubmitInput(const Input& input, uint32_t* produced) { *produced = 0; if (m_finalized) return Result::ApiFail; const auto drained = DrainFrames(); if (drained != Result::Ok) return drained; if (m_frames.size() >= m_config.max_pending_frames) return Result::InputFull; if (input.size > static_cast(std::numeric_limits::max())) return Result::ApiFail; AVPacket* packet = av_packet_alloc(); if (packet == nullptr) return Result::ApiFail; int result = av_new_packet(packet, static_cast(input.size)); if (result < 0) { av_packet_free(&packet); return Result::ApiFail; } std::memcpy(packet->data, input.data, input.size); packet->pts = ToAvTimestamp(input.pts); packet->dts = ToAvTimestamp(input.dts); packet->opaque_ref = av_buffer_alloc(sizeof(PacketMetadata)); if (packet->opaque_ref == nullptr) { av_packet_free(&packet); return Result::ApiFail; } const PacketMetadata metadata {input.pts, input.dts, input.attached_data}; std::memcpy(packet->opaque_ref->data, &metadata, sizeof(metadata)); result = avcodec_send_packet(m_codec, packet); av_packet_free(&packet); if (result == AVERROR(EAGAIN)) return Result::InputFull; // The AU was not accepted. if (result < 0) { LOGF("Video decoder: submitting an AU failed: %s (%d)\n", AvErrorString(result), result); return Result::AccessUnit; } // Once accepted, a full output queue cannot turn this AU into a retry. return DrainFrames(produced); } [[nodiscard]] Result ReceiveOutput(const FrameBuffer& frame_buffer, Output* output) { *output = {}; const auto drained = DrainFrames(); if (drained != Result::Ok) return drained; if (m_frames.empty()) return Result::NoFrame; auto& pending = m_frames.front(); const auto result = CopyFrame(pending.frame, pending.picture, frame_buffer, output); if (result != Result::Ok) return result; output->last_frame = m_finalized && m_eof && m_frames.size() == 1u; av_frame_free(&pending.frame); m_frames.pop_front(); return Result::Ok; } [[nodiscard]] PictureInfo MakePictureInfo(const AVFrame* frame) const { PictureInfo result {}; if (frame->opaque_ref != nullptr && frame->opaque_ref->size >= sizeof(PacketMetadata)) { PacketMetadata metadata {}; std::memcpy(&metadata, frame->opaque_ref->data, sizeof(metadata)); result.pts = metadata.pts; result.dts = metadata.dts; result.attached_data = metadata.attached_data; } else { result.pts = frame->pts == AV_NOPTS_VALUE ? TIMESTAMP_INVALID : static_cast(frame->pts); result.dts = frame->pkt_dts == AV_NOPTS_VALUE ? TIMESTAMP_INVALID : static_cast(frame->pkt_dts); } result.codec_type = m_config.codec_type; result.width = static_cast(frame->width); result.height = static_cast(frame->height); result.crop_left = static_cast(frame->crop_left); result.crop_right = static_cast(frame->crop_right); result.crop_top = static_cast(frame->crop_top); result.crop_bottom = static_cast(frame->crop_bottom); result.profile = m_codec->profile > 0 ? static_cast(m_codec->profile) : 0; result.level = m_codec->level > 0 ? static_cast(m_codec->level) : 0; result.sar_width = frame->sample_aspect_ratio.num > 0 ? static_cast(std::min(frame->sample_aspect_ratio.num, 65535)) : 0; result.sar_height = frame->sample_aspect_ratio.den > 0 ? static_cast(std::min(frame->sample_aspect_ratio.den, 65535)) : 0; result.color_range = static_cast(frame->color_range); result.color_primaries = static_cast(frame->color_primaries); result.color_trc = static_cast(frame->color_trc); result.color_space = static_cast(frame->colorspace); result.key_frame = (frame->flags & AV_FRAME_FLAG_KEY) != 0; return result; } [[nodiscard]] Result CopyFrame(const AVFrame* frame, const PictureInfo& picture, const FrameBuffer& frame_buffer, Output* output) { if (frame->width <= 0 || frame->height <= 0) { return Result::ApiFail; } if ((m_config.max_width > 0 && frame->width > m_config.max_width) || (m_config.max_height > 0 && frame->height > m_config.max_height)) { return Result::OversizeDecode; } const auto width = static_cast(frame->width); const auto height = static_cast(frame->height); const auto pitch = AlignUp(width, 256); const auto chroma_rows = (static_cast(height) + 1u) / 2u; const auto required = static_cast(pitch) * height + static_cast(pitch) * chroma_rows; if (required > frame_buffer.size) { return Result::FrameBufferSize; } auto* dst = static_cast(frame_buffer.data); std::memset(dst, 0, static_cast(required)); if (frame->format == AV_PIX_FMT_NV12) { for (uint32_t y = 0; y < height; y++) { std::memcpy(dst + static_cast(y) * pitch, frame->data[0] + static_cast(y) * frame->linesize[0], width); } auto* chroma = dst + static_cast(pitch) * height; for (uint32_t y = 0; y < chroma_rows; y++) { std::memcpy(chroma + static_cast(y) * pitch, frame->data[1] + static_cast(y) * frame->linesize[1], width); } } else { m_sws = sws_getCachedContext(m_sws, frame->width, frame->height, static_cast(frame->format), frame->width, frame->height, AV_PIX_FMT_NV12, SWS_FAST_BILINEAR, nullptr, nullptr, nullptr); if (m_sws == nullptr) { return Result::ApiFail; } uint8_t* output_planes[4] = {dst, dst + static_cast(pitch) * height, nullptr, nullptr}; int output_strides[4] = {static_cast(pitch), static_cast(pitch), 0, 0}; if (sws_scale(m_sws, frame->data, frame->linesize, 0, frame->height, output_planes, output_strides) != frame->height) { return Result::ApiFail; } } output->valid = true; output->error_frame = (frame->flags & AV_FRAME_FLAG_CORRUPT) != 0; output->buffer_accepted = true; output->codec_type = m_config.codec_type; output->width = width; output->pitch = pitch; output->height = height; output->buffer = frame_buffer.data; output->buffer_size = frame_buffer.size; { std::scoped_lock lock(g_picture_mutex); g_picture_infos[frame_buffer.data] = {this, picture}; m_picture_buffers.insert(frame_buffer.data); } return Result::Ok; } void ClearPictureMetadata() { std::scoped_lock lock(g_picture_mutex); for (auto* buffer: m_picture_buffers) { const auto it = g_picture_infos.find(buffer); if (it != g_picture_infos.end() && it->second.owner == this) { g_picture_infos.erase(it); } } m_picture_buffers.clear(); } std::deque m_frames; bool m_finalized = false; bool m_eof = false; Config m_config; AVCodecContext* m_codec = nullptr; SwsContext* m_sws = nullptr; std::mutex m_mutex; std::unordered_set m_picture_buffers; }; bool IsCodecSupported(uint32_t codec_type) { return GetAvCodecId(codec_type) != AV_CODEC_ID_NONE; } Instance* Create(const Config& config) { if (!IsCodecSupported(config.codec_type) || config.max_pending_frames < 2u) { return nullptr; } auto* instance = new Instance(config); if (!instance->Initialize()) { delete instance; return nullptr; } return instance; } void Destroy(Instance* instance) { delete instance; } uint32_t GetCodecType(const Instance* instance) { return instance->CodecType(); } Result Submit(Instance* instance, const Input& input, uint32_t* produced) { return instance->Submit(input, produced); } Result Receive(Instance* instance, const FrameBuffer& frame_buffer, Output* output) { return instance->Receive(frame_buffer, output); } Result Finalize(Instance* instance) { return instance->Finalize(); } Result Decode(Instance* instance, const Input& input, const FrameBuffer& frame_buffer, Output* output) { return instance->DecodeInput(input, frame_buffer, output); } Result Flush(Instance* instance, const FrameBuffer& frame_buffer, Output* output) { return instance->FlushOutput(frame_buffer, output); } void Reset(Instance* instance) { instance->ResetDecoder(); } AvcPictureInfo MakeAvcPictureInfo(const PictureInfo& decoded, size_t size) { AvcPictureInfo picture {}; picture.this_size = size; picture.is_valid = true; picture.pts_data = decoded.pts; picture.dts_data = decoded.dts; picture.attached_data = decoded.attached_data; picture.idr_picture_flag = decoded.key_frame ? 1 : 0; picture.profile_idc = static_cast(decoded.profile); picture.level_idc = static_cast(decoded.level); picture.pic_width_in_mbs_minus1 = (decoded.width + 15u) / 16u - 1u; picture.pic_height_in_map_units_minus1 = (decoded.height + 15u) / 16u - 1u; picture.frame_mbs_only_flag = 1; picture.frame_cropping_flag = decoded.crop_left != 0 || decoded.crop_right != 0 || decoded.crop_top != 0 || decoded.crop_bottom != 0 ? 1 : 0; picture.frame_crop_left_offset = decoded.crop_left; picture.frame_crop_right_offset = decoded.crop_right; picture.frame_crop_top_offset = decoded.crop_top; picture.frame_crop_bottom_offset = decoded.crop_bottom; picture.aspect_ratio_info_present_flag = decoded.sar_width != 0 && decoded.sar_height != 0 ? 1 : 0; picture.aspect_ratio_idc = picture.aspect_ratio_info_present_flag ? 255 : 0; picture.sar_width = decoded.sar_width; picture.sar_height = decoded.sar_height; picture.video_signal_type_present_flag = 1; picture.video_format = 5; picture.video_full_range_flag = decoded.color_range == 2 ? 1 : 0; picture.colour_description_present_flag = decoded.color_primaries != 0 || decoded.color_trc != 0 || decoded.color_space != 0 ? 1 : 0; picture.colour_primaries = decoded.color_primaries; picture.transfer_characteristics = decoded.color_trc; picture.matrix_coefficients = decoded.color_space; return picture; } bool GetPictureInfo(void* frame_buffer, PictureInfo* picture_info) { std::scoped_lock lock(g_picture_mutex); const auto it = g_picture_infos.find(frame_buffer); if (it == g_picture_infos.end()) { return false; } *picture_info = it->second.info; return true; } } // namespace Libs::VideoDec2::Decoder