diff --git a/src/graphics/guest_gpu/gpu_defs.h b/src/graphics/guest_gpu/gpu_defs.h index 098174229..7515c90d2 100644 --- a/src/graphics/guest_gpu/gpu_defs.h +++ b/src/graphics/guest_gpu/gpu_defs.h @@ -374,6 +374,8 @@ enum class BufferFormat : uint32_t { k32_32_32_32UInt = 75, k32_32_32_32SInt = 76, k32_32_32_32Float = 77, + k8Srgb = 128, + k8_8Srgb = 129, k8_8_8_8Srgb = 130, k9_9_9_5Float = 132, k5_6_5UNorm = 133, diff --git a/src/graphics/guest_gpu/gpu_format.cpp b/src/graphics/guest_gpu/gpu_format.cpp index 566af8d0b..00dca527a 100644 --- a/src/graphics/guest_gpu/gpu_format.cpp +++ b/src/graphics/guest_gpu/gpu_format.cpp @@ -57,6 +57,8 @@ constexpr FormatInfo kFormatInfo[] = { {GpuEnumValue(BufferFormat::k32_32_32_32UInt), 16, 0, 16, true, true}, {GpuEnumValue(BufferFormat::k32_32_32_32SInt), 16, 0, 16, false, false}, {GpuEnumValue(BufferFormat::k32_32_32_32Float), 16, 0, 16, true, false}, + {GpuEnumValue(BufferFormat::k8Srgb), 1, 0, 0, true, false}, + {GpuEnumValue(BufferFormat::k8_8Srgb), 2, 0, 0, true, false}, {GpuEnumValue(BufferFormat::k8_8_8_8Srgb), 4, 0, 4, true, false}, {GpuEnumValue(BufferFormat::k9_9_9_5Float), 4, 0, 0, true, false}, {GpuEnumValue(BufferFormat::k5_6_5UNorm), 2, 0, 2, true, false}, diff --git a/src/graphics/host_gpu/vulkanCommon.cpp b/src/graphics/host_gpu/vulkanCommon.cpp index 74c173491..d36e51b29 100644 --- a/src/graphics/host_gpu/vulkanCommon.cpp +++ b/src/graphics/host_gpu/vulkanCommon.cpp @@ -55,6 +55,10 @@ constexpr FormatMapping kFormatMappings[] = { {Prospero::BufferFormat::k32_32_32_32UInt, vk::Format::eR32G32B32A32Uint}, {Prospero::BufferFormat::k32_32_32_32SInt, vk::Format::eR32G32B32A32Sint}, {Prospero::BufferFormat::k32_32_32_32Float, vk::Format::eR32G32B32A32Sfloat}, + // Narrow-channel sRGB formats are optional in Vulkan. Keep a same-width fallback until + // sampler-aware sRGB emulation is available. + {Prospero::BufferFormat::k8Srgb, vk::Format::eR8Unorm}, + {Prospero::BufferFormat::k8_8Srgb, vk::Format::eR8G8Unorm}, {Prospero::BufferFormat::k8_8_8_8Srgb, vk::Format::eR8G8B8A8Srgb}, {Prospero::BufferFormat::k9_9_9_5Float, vk::Format::eE5B9G9R9UfloatPack32}, {Prospero::BufferFormat::k5_6_5UNorm, vk::Format::eB5G6R5UnormPack16}, diff --git a/src/libs/libVideoDec2.cpp b/src/libs/libVideoDec2.cpp index 80cfeb33d..c97044c4b 100644 --- a/src/libs/libVideoDec2.cpp +++ b/src/libs/libVideoDec2.cpp @@ -1,10 +1,12 @@ #include "common/abi.h" #include "libs/errno.h" #include "libs/libs.h" +#include "libs/videoDec2Decoder.h" #include "loader/symbolDatabase.h" #include #include +#include #include #include @@ -14,6 +16,7 @@ LIB_VERSION("Videodec2", 1, "Videodec2", 1, 1); namespace VideoDec2 { +constexpr int32_t VIDEODEC2_ERROR_API_FAIL = -2128805632; // 0x811d0100 constexpr int32_t VIDEODEC2_ERROR_STRUCT_SIZE = -2128805631; // 0x811d0101 constexpr int32_t VIDEODEC2_ERROR_ARGUMENT_POINTER = -2128805630; // 0x811d0102 constexpr int32_t VIDEODEC2_ERROR_DECODER_INSTANCE = -2128805629; // 0x811d0103 @@ -21,13 +24,20 @@ constexpr int32_t VIDEODEC2_ERROR_MEMORY_SIZE = -2128805628; // 0x811d0 constexpr int32_t VIDEODEC2_ERROR_MEMORY_POINTER = -2128805627; // 0x811d0105 constexpr int32_t VIDEODEC2_ERROR_FRAME_BUFFER_SIZE = -2128805626; // 0x811d0106 constexpr int32_t VIDEODEC2_ERROR_FRAME_BUFFER_POINTER = -2128805625; // 0x811d0107 +constexpr int32_t VIDEODEC2_ERROR_ACCESS_UNIT_SIZE = -2128805619; // 0x811d010d +constexpr int32_t VIDEODEC2_ERROR_ACCESS_UNIT_POINTER = -2128805618; // 0x811d010e +constexpr int32_t VIDEODEC2_ERROR_OUTPUT_INFO = -2128805617; // 0x811d010f +constexpr int32_t VIDEODEC2_ERROR_COMPUTE_QUEUE = -2128805616; // 0x811d0110 constexpr int32_t VIDEODEC2_ERROR_CONFIG_INFO = -2128805376; // 0x811d0200 constexpr int32_t VIDEODEC2_ERROR_COMPUTE_PIPE_ID = -2128805375; // 0x811d0201 constexpr int32_t VIDEODEC2_ERROR_COMPUTE_QUEUE_ID = -2128805374; // 0x811d0202 constexpr int32_t VIDEODEC2_ERROR_RESOURCE_TYPE = -2128805373; // 0x811d0203 +constexpr int32_t VIDEODEC2_ERROR_CODEC_TYPE = -2128805372; // 0x811d0204 constexpr int32_t VIDEODEC2_ERROR_INPUT_QUEUE_DEPTH = -2128805370; // 0x811d0206 constexpr int32_t VIDEODEC2_ERROR_DPB_FRAME_COUNT = -2128805367; // 0x811d0209 constexpr int32_t VIDEODEC2_ERROR_FRAME_WIDTH_HEIGHT = -2128805366; // 0x811d020a +constexpr int32_t VIDEODEC2_ERROR_ACCESS_UNIT = -2128805119; // 0x811d0301 +constexpr int32_t VIDEODEC2_ERROR_OVERSIZE_DECODE = -2128805118; // 0x811d0302 constexpr uint32_t VIDEODEC2_RESOURCE_TYPE_COMPUTE = 1; constexpr size_t VIDEODEC2_MIN_MEMORY_SIZE = 16ull * 1024ull * 1024ull; @@ -101,6 +111,70 @@ struct Videodec2FrameBuffer { bool is_accepted; }; +struct Videodec2AvcPictureInfo { + size_t this_size; + bool is_valid; + + uint64_t pts_data; + uint64_t dts_data; + uint64_t attached_data; + + uint8_t idr_picture_flag; + uint8_t profile_idc; + uint8_t level_idc; + uint32_t pic_width_in_mbs_minus1; + uint32_t pic_height_in_map_units_minus1; + uint8_t frame_mbs_only_flag; + + uint8_t frame_cropping_flag; + uint32_t frame_crop_left_offset; + uint32_t frame_crop_right_offset; + uint32_t frame_crop_top_offset; + uint32_t frame_crop_bottom_offset; + + uint8_t aspect_ratio_info_present_flag; + uint8_t aspect_ratio_idc; + uint16_t sar_width; + uint16_t sar_height; + + uint8_t video_signal_type_present_flag; + uint8_t video_format; + uint8_t video_full_range_flag; + uint8_t colour_description_present_flag; + uint8_t colour_primaries; + uint8_t transfer_characteristics; + uint8_t matrix_coefficients; + + uint8_t timing_info_present_flag; + uint32_t num_units_in_tick; + uint32_t time_scale; + uint8_t fixed_frame_rate_flag; + + uint8_t bitstream_restriction_flag; + uint8_t max_dec_frame_buffering; + + uint8_t pic_struct_present_flag; + uint8_t pic_struct; + uint8_t field_pic_flag; + uint8_t bottom_field_flag; + + uint8_t sequence_parameter_set_present_flag; + uint8_t picture_parameter_set_present_flag; + uint8_t au_delimiter_present_flag; + uint8_t end_of_sequence_present_flag; + uint8_t end_of_stream_present_flag; + uint8_t filler_data_present_flag; + uint8_t picture_timing_sei_present_flag; + uint8_t buffering_period_sei_present_flag; + + uint8_t constraint_set0_flag; + uint8_t constraint_set1_flag; + uint8_t constraint_set2_flag; + uint8_t constraint_set3_flag; + uint8_t constraint_set4_flag; + uint8_t constraint_set5_flag; +}; + struct Videodec2ComputeMemoryInfo { size_t this_size; size_t cpu_gpu_memory_size; @@ -116,10 +190,7 @@ struct Videodec2ComputeConfigInfo { uint16_t reserved1; }; -struct DecoderState { - uint64_t magic; - uint32_t codec_type; -}; +using DecoderState = Decoder::Instance; static_assert(sizeof(Videodec2ComputeMemoryInfo) == 24); static_assert(sizeof(Videodec2ComputeConfigInfo) == 16); @@ -128,8 +199,7 @@ static_assert(sizeof(Videodec2DecoderMemoryInfo) == 72); static_assert(sizeof(Videodec2InputData) == 48); static_assert(sizeof(Videodec2OutputInfo) == 56); static_assert(sizeof(Videodec2FrameBuffer) == 32); - -constexpr uint64_t DECODER_MAGIC = 0x4b59545956444543ull; // KYTYVDEC +static_assert(sizeof(Videodec2AvcPictureInfo) == 120); static std::mutex g_decoder_mutex; static std::unordered_set g_decoders; @@ -156,15 +226,54 @@ static void FillNoPictureOutput(const Videodec2FrameBuffer* frame_buffer, output_info->frame_height = 0; output_info->frame_buffer = frame_buffer != nullptr ? frame_buffer->frame_buffer : nullptr; output_info->frame_buffer_size = frame_buffer != nullptr ? frame_buffer->frame_buffer_size : 0; - output_info->frame_format = VIDEODEC2_FRAME_FORMAT_DEFAULT; - output_info->frame_pitch_in_bytes = 0; + if (output_info->this_size == sizeof(Videodec2OutputInfo)) { + output_info->frame_format = VIDEODEC2_FRAME_FORMAT_DEFAULT; + output_info->frame_pitch_in_bytes = 0; + } } -static int32_t ValidateDecoderConfig(const Videodec2DecoderConfigInfo* config) { +static int32_t MapDecoderResult(Decoder::Result result) { + switch (result) { + case Decoder::Result::Ok: return OK; + case Decoder::Result::ApiFail: return VIDEODEC2_ERROR_API_FAIL; + case Decoder::Result::AccessUnit: return VIDEODEC2_ERROR_ACCESS_UNIT; + case Decoder::Result::FrameBufferSize: return VIDEODEC2_ERROR_FRAME_BUFFER_SIZE; + case Decoder::Result::OversizeDecode: return VIDEODEC2_ERROR_OVERSIZE_DECODE; + } + return VIDEODEC2_ERROR_API_FAIL; +} + +static void ApplyDecodedOutput(const Decoder::Output& decoded, Videodec2FrameBuffer* frame_buffer, + Videodec2OutputInfo* output_info) { + frame_buffer->is_accepted = decoded.buffer_accepted; + if (!decoded.valid) { + return; + } + output_info->is_valid = true; + output_info->is_error_frame = decoded.error_frame; + output_info->picture_count = 1; + output_info->codec_type = decoded.codec_type; + output_info->frame_width = decoded.width; + output_info->frame_pitch = decoded.pitch; + output_info->frame_height = decoded.height; + output_info->frame_buffer = decoded.buffer; + output_info->frame_buffer_size = decoded.buffer_size; + if (output_info->this_size == sizeof(Videodec2OutputInfo)) { + output_info->frame_format = VIDEODEC2_FRAME_FORMAT_DEFAULT; + output_info->frame_pitch_in_bytes = decoded.pitch; + } +} + +static int32_t ValidateDecoderConfig(const Videodec2DecoderConfigInfo* config, + bool require_compute_queue) { if (config->resource_type != VIDEODEC2_RESOURCE_TYPE_COMPUTE) { return VIDEODEC2_ERROR_RESOURCE_TYPE; } + if (!Decoder::IsCodecSupported(config->codec_type)) { + return VIDEODEC2_ERROR_CODEC_TYPE; + } + if (config->reserved0 != 0 || config->reserved1 != 0) { return VIDEODEC2_ERROR_CONFIG_INFO; } @@ -182,8 +291,8 @@ static int32_t ValidateDecoderConfig(const Videodec2DecoderConfigInfo* config) { return VIDEODEC2_ERROR_FRAME_WIDTH_HEIGHT; } - if (config->compute_queue == nullptr) { - return VIDEODEC2_ERROR_CONFIG_INFO; + if (require_compute_queue && config->compute_queue == nullptr) { + return VIDEODEC2_ERROR_COMPUTE_QUEUE; } return OK; @@ -243,7 +352,6 @@ static int32_t KYTY_SYSV_ABI AllocateComputeQueue( } *compute_queue = compute_memory_info->cpu_gpu_memory; - return OK; } @@ -266,7 +374,7 @@ static int32_t KYTY_SYSV_ABI QueryDecoderMemoryInfo(const Videodec2DecoderConfig return VIDEODEC2_ERROR_STRUCT_SIZE; } - const auto validation_result = ValidateDecoderConfig(config); + const auto validation_result = ValidateDecoderConfig(config, false); if (validation_result != OK) { return validation_result; } @@ -298,7 +406,7 @@ static int32_t KYTY_SYSV_ABI CreateDecoder(const Videodec2DecoderConfigInfo* con return VIDEODEC2_ERROR_STRUCT_SIZE; } - const auto validation_result = ValidateDecoderConfig(config); + const auto validation_result = ValidateDecoderConfig(config, true); if (validation_result != OK) { return validation_result; } @@ -315,9 +423,11 @@ static int32_t KYTY_SYSV_ABI CreateDecoder(const Videodec2DecoderConfigInfo* con return VIDEODEC2_ERROR_MEMORY_POINTER; } - auto* state = new DecoderState {}; - state->magic = DECODER_MAGIC; - state->codec_type = config->codec_type; + auto* state = + Decoder::Create({config->codec_type, config->max_frame_width, config->max_frame_height}); + if (state == nullptr) { + return VIDEODEC2_ERROR_API_FAIL; + } { std::scoped_lock lock(g_decoder_mutex); @@ -325,7 +435,6 @@ static int32_t KYTY_SYSV_ABI CreateDecoder(const Videodec2DecoderConfigInfo* con } *decoder = state; - return OK; } @@ -343,7 +452,7 @@ static int32_t KYTY_SYSV_ABI DeleteDecoder(Videodec2Decoder decoder) { g_decoders.erase(it); } - delete state; + Decoder::Destroy(state); return OK; } @@ -353,8 +462,8 @@ static int32_t KYTY_SYSV_ABI Decode(Videodec2Decoder decoder, const Videodec2Inp Videodec2OutputInfo* output_info) { PRINT_NAME(); - const auto* state = GetDecoder(decoder); - if (state == nullptr || state->magic != DECODER_MAGIC) { + auto* state = GetDecoder(decoder); + if (state == nullptr) { return VIDEODEC2_ERROR_DECODER_INSTANCE; } @@ -368,8 +477,12 @@ static int32_t KYTY_SYSV_ABI Decode(Videodec2Decoder decoder, const Videodec2Inp return VIDEODEC2_ERROR_STRUCT_SIZE; } - if (input_data->au_size != 0 && input_data->au_data == nullptr) { - return VIDEODEC2_ERROR_ARGUMENT_POINTER; + if (input_data->au_size == 0) { + return VIDEODEC2_ERROR_ACCESS_UNIT_SIZE; + } + + if (input_data->au_data == nullptr) { + return VIDEODEC2_ERROR_ACCESS_UNIT_POINTER; } if (frame_buffer->frame_buffer_size == 0) { @@ -381,17 +494,24 @@ static int32_t KYTY_SYSV_ABI Decode(Videodec2Decoder decoder, const Videodec2Inp } frame_buffer->is_accepted = false; - FillNoPictureOutput(frame_buffer, output_info, state->codec_type); + FillNoPictureOutput(frame_buffer, output_info, Decoder::GetCodecType(state)); - return OK; + Decoder::Output decoded {}; + const auto result = + Decoder::Decode(state, + {input_data->au_data, input_data->au_size, input_data->pts_data, + input_data->dts_data, input_data->attached_data}, + {frame_buffer->frame_buffer, frame_buffer->frame_buffer_size}, &decoded); + ApplyDecodedOutput(decoded, frame_buffer, output_info); + return MapDecoderResult(result); } static int32_t KYTY_SYSV_ABI Flush(Videodec2Decoder decoder, Videodec2FrameBuffer* frame_buffer, Videodec2OutputInfo* output_info) { PRINT_NAME(); - const auto* state = GetDecoder(decoder); - if (state == nullptr || state->magic != DECODER_MAGIC) { + auto* state = GetDecoder(decoder); + if (state == nullptr) { return VIDEODEC2_ERROR_DECODER_INSTANCE; } @@ -404,26 +524,40 @@ static int32_t KYTY_SYSV_ABI Flush(Videodec2Decoder decoder, Videodec2FrameBuffe return VIDEODEC2_ERROR_STRUCT_SIZE; } - frame_buffer->is_accepted = false; - FillNoPictureOutput(frame_buffer, output_info, state->codec_type); + if (frame_buffer->frame_buffer_size == 0) { + return VIDEODEC2_ERROR_FRAME_BUFFER_SIZE; + } - return OK; + if (frame_buffer->frame_buffer == nullptr) { + return VIDEODEC2_ERROR_FRAME_BUFFER_POINTER; + } + + frame_buffer->is_accepted = false; + FillNoPictureOutput(frame_buffer, output_info, Decoder::GetCodecType(state)); + + Decoder::Output decoded {}; + const auto result = Decoder::Flush( + state, {frame_buffer->frame_buffer, frame_buffer->frame_buffer_size}, &decoded); + ApplyDecodedOutput(decoded, frame_buffer, output_info); + return MapDecoderResult(result); } static int32_t KYTY_SYSV_ABI Reset(Videodec2Decoder decoder) { PRINT_NAME(); - const auto* state = GetDecoder(decoder); - return state != nullptr && state->magic == DECODER_MAGIC ? OK - : VIDEODEC2_ERROR_DECODER_INSTANCE; + auto* state = GetDecoder(decoder); + if (state == nullptr) { + return VIDEODEC2_ERROR_DECODER_INSTANCE; + } + Decoder::Reset(state); + return OK; } static int32_t KYTY_SYSV_ABI GetPictureInfo(const Videodec2OutputInfo* output_info, - void* /*first_picture_info*/, - void* /*second_picture_info*/) { + void* first_picture_info, void* second_picture_info) { PRINT_NAME(); - if (output_info == nullptr) { + if (output_info == nullptr || first_picture_info == nullptr) { return VIDEODEC2_ERROR_ARGUMENT_POINTER; } @@ -431,6 +565,88 @@ static int32_t KYTY_SYSV_ABI GetPictureInfo(const Videodec2OutputInfo* output_in return VIDEODEC2_ERROR_STRUCT_SIZE; } + if (!output_info->is_valid || output_info->picture_count == 0 || + output_info->frame_buffer == nullptr) { + return VIDEODEC2_ERROR_OUTPUT_INFO; + } + + Decoder::PictureInfo decoded {}; + if (!Decoder::GetPictureInfo(output_info->frame_buffer, &decoded) || + decoded.codec_type != output_info->codec_type) { + return VIDEODEC2_ERROR_OUTPUT_INFO; + } + + auto fill_common = [&decoded](void* destination, bool valid) -> int32_t { + auto* bytes = static_cast(destination); + const auto size = *static_cast(destination); + if (size < 40 || size > 256) { + return VIDEODEC2_ERROR_STRUCT_SIZE; + } + std::memset(bytes + sizeof(size_t), 0, size - sizeof(size_t)); + bytes[8] = valid ? 1 : 0; + if (valid) { + std::memcpy(bytes + 16, &decoded.pts, sizeof(decoded.pts)); + std::memcpy(bytes + 24, &decoded.dts, sizeof(decoded.dts)); + std::memcpy(bytes + 32, &decoded.attached_data, sizeof(decoded.attached_data)); + } + return OK; + }; + + if (output_info->codec_type == 1) { + const auto requested_size = *static_cast(first_picture_info); + if (requested_size != sizeof(Videodec2AvcPictureInfo) && + (requested_size | 16u) != sizeof(Videodec2AvcPictureInfo)) { + return VIDEODEC2_ERROR_STRUCT_SIZE; + } + + Videodec2AvcPictureInfo picture {}; + picture.this_size = requested_size; + picture.is_valid = true; + picture.pts_data = decoded.pts; + picture.dts_data = decoded.dts; + picture.attached_data = decoded.attached_data; + picture.idr_picture_flag = decoded.key_frame ? 1 : 0; + picture.profile_idc = static_cast(decoded.profile); + picture.level_idc = static_cast(decoded.level); + picture.pic_width_in_mbs_minus1 = (decoded.width + 15u) / 16u - 1u; + picture.pic_height_in_map_units_minus1 = (decoded.height + 15u) / 16u - 1u; + picture.frame_mbs_only_flag = 1; + picture.frame_cropping_flag = decoded.crop_left != 0 || decoded.crop_right != 0 || + decoded.crop_top != 0 || decoded.crop_bottom != 0 + ? 1 + : 0; + picture.frame_crop_left_offset = decoded.crop_left; + picture.frame_crop_right_offset = decoded.crop_right; + picture.frame_crop_top_offset = decoded.crop_top; + picture.frame_crop_bottom_offset = decoded.crop_bottom; + picture.aspect_ratio_info_present_flag = + decoded.sar_width != 0 && decoded.sar_height != 0 ? 1 : 0; + picture.aspect_ratio_idc = picture.aspect_ratio_info_present_flag ? 255 : 0; + picture.sar_width = decoded.sar_width; + picture.sar_height = decoded.sar_height; + picture.video_signal_type_present_flag = 1; + picture.video_format = 5; + picture.video_full_range_flag = decoded.color_range == 2 ? 1 : 0; + picture.colour_description_present_flag = + decoded.color_primaries != 0 || decoded.color_trc != 0 || decoded.color_space != 0 ? 1 + : 0; + picture.colour_primaries = decoded.color_primaries; + picture.transfer_characteristics = decoded.color_trc; + picture.matrix_coefficients = decoded.color_space; + std::memcpy(first_picture_info, &picture, requested_size); + } else { + const auto result = fill_common(first_picture_info, true); + if (result != OK) { + return result; + } + } + + if (second_picture_info != nullptr) { + const auto result = fill_common(second_picture_info, false); + if (result != OK) { + return result; + } + } return OK; } diff --git a/src/libs/videoDec2Decoder.cpp b/src/libs/videoDec2Decoder.cpp new file mode 100644 index 000000000..213a2ccfa --- /dev/null +++ b/src/libs/videoDec2Decoder.cpp @@ -0,0 +1,412 @@ +#include "libs/videoDec2Decoder.h" + +#include "common/logging/log.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include +#include +#include +#include +#include +#include +} + +namespace Libs::VideoDec2::Decoder { + +namespace { + +constexpr uint32_t CODEC_TYPE_AVC = 1; +constexpr uint32_t CODEC_TYPE_HEVC = 974921; +constexpr uint32_t CODEC_TYPE_VP9 = 2382845; + +struct PacketMetadata { + uint64_t pts = TIMESTAMP_INVALID; + uint64_t dts = TIMESTAMP_INVALID; + uint64_t attached_data = 0; +}; + +struct StoredPicture { + const Instance* owner = nullptr; + PictureInfo info; +}; + +std::mutex g_picture_mutex; +std::unordered_map g_picture_infos; + +AVCodecID GetAvCodecId(uint32_t codec_type) { + switch (codec_type) { + case CODEC_TYPE_AVC: return AV_CODEC_ID_H264; + case CODEC_TYPE_HEVC: return AV_CODEC_ID_HEVC; + case CODEC_TYPE_VP9: return AV_CODEC_ID_VP9; + default: return AV_CODEC_ID_NONE; + } +} + +const char* AvErrorString(int error) { + thread_local char text[AV_ERROR_MAX_STRING_SIZE] {}; + if (av_strerror(error, text, sizeof(text)) != 0) { + std::strcpy(text, "unknown FFmpeg error"); + } + return text; +} + +uint32_t AlignUp(uint32_t value, uint32_t alignment) { + return (value + alignment - 1u) & ~(alignment - 1u); +} + +int64_t ToAvTimestamp(uint64_t timestamp) { + return timestamp == TIMESTAMP_INVALID || + timestamp > static_cast(std::numeric_limits::max()) + ? AV_NOPTS_VALUE + : static_cast(timestamp); +} + +} // namespace + +class Instance { +public: + explicit Instance(const Config& config): m_config(config) {} + + ~Instance() { + ClearPictureMetadata(); + if (m_sws != nullptr) { + sws_freeContext(m_sws); + } + if (m_codec != nullptr) { + avcodec_free_context(&m_codec); + } + } + + Instance(const Instance&) = delete; + Instance& operator=(const Instance&) = delete; + + [[nodiscard]] bool Initialize() { + const AVCodec* decoder = avcodec_find_decoder(GetAvCodecId(m_config.codec_type)); + if (decoder == nullptr) { + LOGF("Videodec2: FFmpeg decoder is unavailable for codec type %u\n", + m_config.codec_type); + return false; + } + m_codec = avcodec_alloc_context3(decoder); + if (m_codec == nullptr) { + LOGF("Videodec2: avcodec_alloc_context3 failed\n"); + return false; + } + + // This carries PTS/DTS/attachedData through codecs that reorder B frames. + m_codec->flags |= AV_CODEC_FLAG_COPY_OPAQUE; + const int result = avcodec_open2(m_codec, decoder, nullptr); + if (result < 0) { + LOGF("Videodec2: avcodec_open2 failed: %s (%d)\n", AvErrorString(result), result); + return false; + } + return true; + } + + [[nodiscard]] uint32_t CodecType() const { return m_config.codec_type; } + + [[nodiscard]] Result DecodeInput(const Input& input, const FrameBuffer& frame_buffer, + Output* output) { + std::scoped_lock lock(m_mutex); + *output = {}; + m_draining = false; + + AVPacket* packet = av_packet_alloc(); + AVFrame* frame = av_frame_alloc(); + if (packet == nullptr || frame == nullptr || + input.size > static_cast(std::numeric_limits::max())) { + av_packet_free(&packet); + av_frame_free(&frame); + return Result::ApiFail; + } + + int result = av_new_packet(packet, static_cast(input.size)); + if (result < 0) { + LOGF("Videodec2: av_new_packet failed: %s (%d)\n", AvErrorString(result), result); + av_packet_free(&packet); + av_frame_free(&frame); + return Result::ApiFail; + } + std::memcpy(packet->data, input.data, input.size); + packet->pts = ToAvTimestamp(input.pts); + packet->dts = ToAvTimestamp(input.dts); + + packet->opaque_ref = av_buffer_alloc(sizeof(PacketMetadata)); + if (packet->opaque_ref == nullptr) { + av_packet_free(&packet); + av_frame_free(&frame); + return Result::ApiFail; + } + const PacketMetadata metadata {input.pts, input.dts, input.attached_data}; + std::memcpy(packet->opaque_ref->data, &metadata, sizeof(metadata)); + + bool have_pending_frame = false; + result = avcodec_send_packet(m_codec, packet); + if (result == AVERROR(EAGAIN)) { + result = avcodec_receive_frame(m_codec, frame); + if (result < 0) { + LOGF("Videodec2: decoder rejected an AU while no output was available: %s (%d)\n", + AvErrorString(result), result); + av_packet_free(&packet); + av_frame_free(&frame); + return Result::AccessUnit; + } + have_pending_frame = true; + result = avcodec_send_packet(m_codec, packet); + } + if (result < 0) { + LOGF("Videodec2: avcodec_send_packet failed: %s (%d)\n", AvErrorString(result), result); + av_packet_free(&packet); + av_frame_free(&frame); + return Result::AccessUnit; + } + + Result decode_result = Result::Ok; + if (!have_pending_frame) { + result = avcodec_receive_frame(m_codec, frame); + if (result != AVERROR(EAGAIN) && result != AVERROR_EOF) { + if (result < 0) { + LOGF("Videodec2: avcodec_receive_frame failed: %s (%d)\n", + AvErrorString(result), result); + decode_result = Result::AccessUnit; + } else { + decode_result = CopyFrame(frame, frame_buffer, output); + } + } + } else { + decode_result = CopyFrame(frame, frame_buffer, output); + } + + av_packet_free(&packet); + av_frame_free(&frame); + return decode_result; + } + + [[nodiscard]] Result FlushOutput(const FrameBuffer& frame_buffer, Output* output) { + std::scoped_lock lock(m_mutex); + *output = {}; + + AVFrame* frame = av_frame_alloc(); + if (frame == nullptr) { + return Result::ApiFail; + } + + if (!m_draining) { + const int send_result = avcodec_send_packet(m_codec, nullptr); + if (send_result == 0 || send_result == AVERROR_EOF) { + m_draining = true; + } else if (send_result != AVERROR(EAGAIN)) { + LOGF("Videodec2: flushing decoder failed: %s (%d)\n", AvErrorString(send_result), + send_result); + av_frame_free(&frame); + return Result::ApiFail; + } + } + + const int receive_result = avcodec_receive_frame(m_codec, frame); + if (receive_result == AVERROR(EAGAIN) || receive_result == AVERROR_EOF) { + av_frame_free(&frame); + return Result::Ok; + } + if (receive_result < 0) { + LOGF("Videodec2: receiving a flushed frame failed: %s (%d)\n", + AvErrorString(receive_result), receive_result); + av_frame_free(&frame); + return Result::ApiFail; + } + + const auto result = CopyFrame(frame, frame_buffer, output); + av_frame_free(&frame); + return result; + } + + void ResetDecoder() { + std::scoped_lock lock(m_mutex); + avcodec_flush_buffers(m_codec); + m_draining = false; + ClearPictureMetadata(); + } + +private: + [[nodiscard]] PictureInfo MakePictureInfo(const AVFrame* frame) const { + PictureInfo result {}; + if (frame->opaque_ref != nullptr && frame->opaque_ref->size >= sizeof(PacketMetadata)) { + PacketMetadata metadata {}; + std::memcpy(&metadata, frame->opaque_ref->data, sizeof(metadata)); + result.pts = metadata.pts; + result.dts = metadata.dts; + result.attached_data = metadata.attached_data; + } else { + result.pts = frame->pts == AV_NOPTS_VALUE ? TIMESTAMP_INVALID + : static_cast(frame->pts); + result.dts = frame->pkt_dts == AV_NOPTS_VALUE ? TIMESTAMP_INVALID + : static_cast(frame->pkt_dts); + } + result.codec_type = m_config.codec_type; + result.width = static_cast(frame->width); + result.height = static_cast(frame->height); + result.crop_left = static_cast(frame->crop_left); + result.crop_right = static_cast(frame->crop_right); + result.crop_top = static_cast(frame->crop_top); + result.crop_bottom = static_cast(frame->crop_bottom); + result.profile = m_codec->profile > 0 ? static_cast(m_codec->profile) : 0; + result.level = m_codec->level > 0 ? static_cast(m_codec->level) : 0; + result.sar_width = + frame->sample_aspect_ratio.num > 0 + ? static_cast(std::min(frame->sample_aspect_ratio.num, 65535)) + : 0; + result.sar_height = + frame->sample_aspect_ratio.den > 0 + ? static_cast(std::min(frame->sample_aspect_ratio.den, 65535)) + : 0; + result.color_range = static_cast(frame->color_range); + result.color_primaries = static_cast(frame->color_primaries); + result.color_trc = static_cast(frame->color_trc); + result.color_space = static_cast(frame->colorspace); + result.key_frame = (frame->flags & AV_FRAME_FLAG_KEY) != 0; + return result; + } + + [[nodiscard]] Result CopyFrame(const AVFrame* frame, const FrameBuffer& frame_buffer, + Output* output) { + if (frame->width <= 0 || frame->height <= 0) { + return Result::ApiFail; + } + if ((m_config.max_width > 0 && frame->width > m_config.max_width) || + (m_config.max_height > 0 && frame->height > m_config.max_height)) { + return Result::OversizeDecode; + } + + const auto width = static_cast(frame->width); + const auto height = static_cast(frame->height); + const auto pitch = AlignUp(width, 256); + const auto chroma_rows = (static_cast(height) + 1u) / 2u; + const auto required = + static_cast(pitch) * height + static_cast(pitch) * chroma_rows; + if (required > frame_buffer.size) { + return Result::FrameBufferSize; + } + + auto* dst = static_cast(frame_buffer.data); + std::memset(dst, 0, static_cast(required)); + if (frame->format == AV_PIX_FMT_NV12) { + for (uint32_t y = 0; y < height; y++) { + std::memcpy(dst + static_cast(y) * pitch, + frame->data[0] + static_cast(y) * frame->linesize[0], width); + } + auto* chroma = dst + static_cast(pitch) * height; + for (uint32_t y = 0; y < chroma_rows; y++) { + std::memcpy(chroma + static_cast(y) * pitch, + frame->data[1] + static_cast(y) * frame->linesize[1], width); + } + } else { + m_sws = sws_getCachedContext(m_sws, frame->width, frame->height, + static_cast(frame->format), frame->width, + frame->height, AV_PIX_FMT_NV12, SWS_FAST_BILINEAR, nullptr, + nullptr, nullptr); + if (m_sws == nullptr) { + return Result::ApiFail; + } + uint8_t* output_planes[4] = {dst, dst + static_cast(pitch) * height, nullptr, + nullptr}; + int output_strides[4] = {static_cast(pitch), static_cast(pitch), 0, 0}; + if (sws_scale(m_sws, frame->data, frame->linesize, 0, frame->height, output_planes, + output_strides) != frame->height) { + return Result::ApiFail; + } + } + + output->valid = true; + output->error_frame = (frame->flags & AV_FRAME_FLAG_CORRUPT) != 0; + output->buffer_accepted = true; + output->codec_type = m_config.codec_type; + output->width = width; + output->pitch = pitch; + output->height = height; + output->buffer = frame_buffer.data; + output->buffer_size = frame_buffer.size; + + { + std::scoped_lock lock(g_picture_mutex); + g_picture_infos[frame_buffer.data] = {this, MakePictureInfo(frame)}; + m_picture_buffers.insert(frame_buffer.data); + } + return Result::Ok; + } + + void ClearPictureMetadata() { + std::scoped_lock lock(g_picture_mutex); + for (auto* buffer: m_picture_buffers) { + const auto it = g_picture_infos.find(buffer); + if (it != g_picture_infos.end() && it->second.owner == this) { + g_picture_infos.erase(it); + } + } + m_picture_buffers.clear(); + } + + Config m_config; + AVCodecContext* m_codec = nullptr; + SwsContext* m_sws = nullptr; + bool m_draining = false; + std::mutex m_mutex; + std::unordered_set m_picture_buffers; +}; + +bool IsCodecSupported(uint32_t codec_type) { + return GetAvCodecId(codec_type) != AV_CODEC_ID_NONE; +} + +Instance* Create(const Config& config) { + if (!IsCodecSupported(config.codec_type)) { + return nullptr; + } + auto* instance = new Instance(config); + if (!instance->Initialize()) { + delete instance; + return nullptr; + } + return instance; +} + +void Destroy(Instance* instance) { + delete instance; +} + +uint32_t GetCodecType(const Instance* instance) { + return instance->CodecType(); +} + +Result Decode(Instance* instance, const Input& input, const FrameBuffer& frame_buffer, + Output* output) { + return instance->DecodeInput(input, frame_buffer, output); +} + +Result Flush(Instance* instance, const FrameBuffer& frame_buffer, Output* output) { + return instance->FlushOutput(frame_buffer, output); +} + +void Reset(Instance* instance) { + instance->ResetDecoder(); +} + +bool GetPictureInfo(void* frame_buffer, PictureInfo* picture_info) { + std::scoped_lock lock(g_picture_mutex); + const auto it = g_picture_infos.find(frame_buffer); + if (it == g_picture_infos.end()) { + return false; + } + *picture_info = it->second.info; + return true; +} + +} // namespace Libs::VideoDec2::Decoder diff --git a/src/libs/videoDec2Decoder.h b/src/libs/videoDec2Decoder.h new file mode 100644 index 000000000..b4c109148 --- /dev/null +++ b/src/libs/videoDec2Decoder.h @@ -0,0 +1,86 @@ +#ifndef EMULATOR_INCLUDE_EMULATOR_LIBS_VIDEODEC2DECODER_H_ +#define EMULATOR_INCLUDE_EMULATOR_LIBS_VIDEODEC2DECODER_H_ + +#include +#include + +namespace Libs::VideoDec2::Decoder { + +constexpr uint64_t TIMESTAMP_INVALID = UINT64_MAX; + +enum class Result { + Ok, + ApiFail, + AccessUnit, + FrameBufferSize, + OversizeDecode, +}; + +struct Config { + uint32_t codec_type = 0; + int32_t max_width = -1; + int32_t max_height = -1; +}; + +struct Input { + const void* data = nullptr; + size_t size = 0; + uint64_t pts = TIMESTAMP_INVALID; + uint64_t dts = TIMESTAMP_INVALID; + uint64_t attached_data = 0; +}; + +struct FrameBuffer { + void* data = nullptr; + size_t size = 0; +}; + +struct Output { + bool valid = false; + bool error_frame = false; + bool buffer_accepted = false; + uint32_t codec_type = 0; + uint32_t width = 0; + uint32_t pitch = 0; + uint32_t height = 0; + void* buffer = nullptr; + size_t buffer_size = 0; +}; + +struct PictureInfo { + uint64_t pts = TIMESTAMP_INVALID; + uint64_t dts = TIMESTAMP_INVALID; + uint64_t attached_data = 0; + uint32_t codec_type = 0; + uint32_t width = 0; + uint32_t height = 0; + uint32_t crop_left = 0; + uint32_t crop_right = 0; + uint32_t crop_top = 0; + uint32_t crop_bottom = 0; + uint32_t profile = 0; + uint32_t level = 0; + uint16_t sar_width = 0; + uint16_t sar_height = 0; + uint8_t color_range = 0; + uint8_t color_primaries = 0; + uint8_t color_trc = 0; + uint8_t color_space = 0; + bool key_frame = false; +}; + +class Instance; + +[[nodiscard]] bool IsCodecSupported(uint32_t codec_type); +[[nodiscard]] Instance* Create(const Config& config); +void Destroy(Instance* instance); +[[nodiscard]] uint32_t GetCodecType(const Instance* instance); +[[nodiscard]] Result Decode(Instance* instance, const Input& input, const FrameBuffer& frame_buffer, + Output* output); +[[nodiscard]] Result Flush(Instance* instance, const FrameBuffer& frame_buffer, Output* output); +void Reset(Instance* instance); +[[nodiscard]] bool GetPictureInfo(void* frame_buffer, PictureInfo* picture_info); + +} // namespace Libs::VideoDec2::Decoder + +#endif // EMULATOR_INCLUDE_EMULATOR_LIBS_VIDEODEC2DECODER_H_