From 17a3494d79e2ddb47d7aeb8f203e8be3281e2b8b Mon Sep 17 00:00:00 2001 From: James Hurley Date: Fri, 7 Aug 2020 09:32:45 +0200 Subject: [PATCH 01/15] Checkpoint --- .../RtmpServer/Sources/RtmpServer/main.swift | 2 +- Sources/CSwiftVideo/include/CSwiftVideo.h | 2 + Sources/CSwiftVideo/shim.cpp | 88 +++++++++++++++++++ Sources/SwiftVideo/dec.video.apple.swift | 6 +- Sources/SwiftVideo/enc.video.apple.swift | 17 +++- Sources/SwiftVideo/sample.coded.swift | 25 ++++++ 6 files changed, 135 insertions(+), 5 deletions(-) diff --git a/Examples/RtmpServer/Sources/RtmpServer/main.swift b/Examples/RtmpServer/Sources/RtmpServer/main.swift index 7c8b0fa..28d4ee0 100644 --- a/Examples/RtmpServer/Sources/RtmpServer/main.swift +++ b/Examples/RtmpServer/Sources/RtmpServer/main.swift @@ -33,7 +33,7 @@ let onConnection: LiveOnConnection = { pub, sub in // to it. Releasing the reference will close the connection an end the stream. // You can create more sophisticated graphs with decoders, buses, and other components. subs[sub.assetId()] = src >>> Tx { sample in - print("got sample \(sample.pts().toString()) type = \(sample.mediaType())") + print("got sample \(sample.pts().toString()) type = \(sample.mediaType()) size = \(sample.data().count)") return .nothing(nil) } } diff --git a/Sources/CSwiftVideo/include/CSwiftVideo.h b/Sources/CSwiftVideo/include/CSwiftVideo.h index d163182..83b9892 100644 --- a/Sources/CSwiftVideo/include/CSwiftVideo.h +++ b/Sources/CSwiftVideo/include/CSwiftVideo.h @@ -29,6 +29,8 @@ extern "C" { int aac_parse_asc(const void* data, int64_t size, int* channels, int* sample_rate, int* samples_per_frame); int h264_sps_frame_size(const void* data, int64_t size, int* width, int* height); +int vp9_is_keyframe(const void* data, int64_t size, int* is_keyframe); +int vp9_frame_size(const void* data, int64_t size, int* width, int* height); #if defined(linux) void generateRandomBytes(void* buf, size_t size); diff --git a/Sources/CSwiftVideo/shim.cpp b/Sources/CSwiftVideo/shim.cpp index bdf8c3a..9d75bef 100644 --- a/Sources/CSwiftVideo/shim.cpp +++ b/Sources/CSwiftVideo/shim.cpp @@ -270,6 +270,91 @@ extern "C" { return 1; } + /* colorspace + Unknown = 0 + BT.601 = 1 + BT.709 = 2 + SMPTE-170 = 3 + SMPTE-240 = 4 + BT.2020 = 5 + Reserved = 6 + sRGB = 7 + */ + static int vp9_bitdepth_colorspace_sampling(ExpGolomb& decoder, int64_t profile, int* bit_depth, + int* colorspace, int* yuv_range, int* subsampling_x, int* subsampling_y) { + *bit_depth = 8; + if(profile >= 2) { + *bit_depth = *decoder.get_bits(1) ? 12 : 10; + } + *colorspace = *decoder.get_bits(3); + if(*colorspace != 7) { // sRGB + *yuv_range = *decoder.get_bits(1); // movie = 0, full = 1 + if(profile == 1 || profile == 3) { + *subsampling_x = *decoder.get_bits(1); + *subsampling_y = *decoder.get_bits(1); + decoder.get_bits(1); // reserved 0 + } else { + *subsampling_x = *subsampling_y = 1; + } + } else { + *subsampling_x = *subsampling_y = 0; + return 0; + } + return 1; + } + static int vp9_frame_size(ExpGolomb& decoder, int* width, int* height) { + *width = *decoder.get_bits(16) + 1; + *height = *decoder.get_bits(16) + 1; + return 1; + } + int vp9_is_keyframe(const void* data, int64_t size, int* is_keyframe) { + auto decoder = ExpGolomb((uint8_t*)data, size); + if(*decoder.get_bits(2) != 0b10) { + return 0; + } + auto version = *decoder.get_bits(1); + auto high = *decoder.get_bits(1); + auto profile = (high << 1) + version; + if(profile == 3) { + decoder.get_bits(1); // reserved + } + if(*decoder.get_bits(1)) { // show_existing_frame - not a new frame + return 0; + } + *is_keyframe = !decoder.get_bits(1); + return 1; + } + + int vp9_frame_size(const void* data, int64_t size, int* width, int* height) { + auto decoder = ExpGolomb((uint8_t*)data, size); + if(*decoder.get_bits(2) != 0b10) { + return 0; + } + auto version = *decoder.get_bits(1); + auto high = *decoder.get_bits(1); + auto profile = (high << 1) + version; + if(profile == 3) { + decoder.get_bits(1); // reserved + } + if(*decoder.get_bits(1)) { // show_existing_frame - not a new frame + return 0; + } + auto frame_type = !*decoder.get_bits(1); + if(frame_type != 0) { + return 0; + } + int bit_depth = 8; + int colorspace = 0; + int yuv_range = 0; + int subsampling_x = 1; + int subsampling_y = 1; + if(vp9_bitdepth_colorspace_sampling(decoder, profile, &bit_depth, &colorspace, &yuv_range, &subsampling_x, &subsampling_y)) { + vp9_frame_size(decoder, width, height); + return 1; + } + return 0; + } + uint64_t test_golomb_dec() { uint8_t bytes[4] = {0}; bytes[0] = 0x1; @@ -281,3 +366,6 @@ extern "C" { return val; } } + + + diff --git a/Sources/SwiftVideo/dec.video.apple.swift b/Sources/SwiftVideo/dec.video.apple.swift index 8856a04..3bf3140 100644 --- a/Sources/SwiftVideo/dec.video.apple.swift +++ b/Sources/SwiftVideo/dec.video.apple.swift @@ -38,7 +38,7 @@ public class AppleVideoDecoder: Tx { guard let strongSelf = self else { return .gone } - guard [.avc, .hevc].contains(sample.mediaFormat()) else { + guard [.avc, .hevc, .vp9].contains(sample.mediaFormat()) else { return .error(EventError("dec.video.apple", -1, "\(sample.mediaFormat()) not supported. AppleVideoDecoder only supports AVC and HEVC")) @@ -185,6 +185,10 @@ public class AppleVideoDecoder: Tx { private let decodeQueue: DispatchQueue } +private func videoFormatFromVP9Header(_ data: Data) -> CMVideoFormatDescription? { + + return nil +} private func videoFormatFromAVCParameterSets( _ paramSets: [[UInt8]]) throws -> CMVideoFormatDescription? { let pmSetPtrs: [UnsafePointer] = try paramSets.map { try $0.withUnsafeBufferPointer { diff --git a/Sources/SwiftVideo/enc.video.apple.swift b/Sources/SwiftVideo/enc.video.apple.swift index 72d55a5..ac97e89 100644 --- a/Sources/SwiftVideo/enc.video.apple.swift +++ b/Sources/SwiftVideo/enc.video.apple.swift @@ -26,13 +26,24 @@ public class AppleVideoEncoder: Tx { } public init(format: MediaFormat, frame: CGSize, bitrate: Int = 500000) { - assert(format == .avc || format == .hevc, "AppleVideoEncoder only supports AVC and HEVC") + //assert(format == .avc || format == .hevc, "AppleVideoEncoder only supports AVC and HEVC") self.queue = DispatchQueue.init(label: "cezium.encode.video") - + let codecType = { + switch format { + case .avc: + return kCMVideoCodecType_H264 + case .hevc: + return kCMVideoCodecType_HEVC + case .vp9: + return kCMVideoCodecType_VP9 + default: + return kCMVideoCodecType_H264 + } + }() VTCompressionSessionCreate(allocator: kCFAllocatorDefault, width: Int32(frame.width), height: Int32(frame.height), - codecType: format == .avc ? kCMVideoCodecType_H264 : kCMVideoCodecType_HEVC, + codecType: codecType encoderSpecification: [kVTCompressionPropertyKey_ExpectedFrameRate: 30] as CFDictionary, imageBufferAttributes: pixelBufferOptions(frame), compressedDataAllocator: nil, diff --git a/Sources/SwiftVideo/sample.coded.swift b/Sources/SwiftVideo/sample.coded.swift index 5da6fd1..5b45722 100644 --- a/Sources/SwiftVideo/sample.coded.swift +++ b/Sources/SwiftVideo/sample.coded.swift @@ -197,6 +197,7 @@ extension CodedMediaSample: Event { enum MediaDescriptionError: Error { case unsupported case invalidMetadata + case needsKeyframe } func basicMediaDescription(_ sample: CodedMediaSample) throws -> BasicMediaDescription { @@ -224,6 +225,17 @@ func basicMediaDescription(_ sample: CodedMediaSample) throws -> BasicMediaDescr return .audio(BasicAudioDescription(sampleRate: Float(sampleRate), channelCount: Int(channels), samplesPerPacket: Int(samplesPerPacket))) + case .vp9: + guard isKeyframe(sample) else { + throw MediaDescriptionError.needsKeyframe + } + let (width, height): (Int32, Int32) = sample.data().withUnsafeBytes { + var width: Int32 = 0 + var height: Int32 = 0 + vp9_frame_size($0.baseAddress, Int64($0.count), &width, &height) + return (width, height) + } + return .video(BasicVideoDescription(size: Vector2(Float(width), Float(height)))) default: throw MediaDescriptionError.unsupported } @@ -238,6 +250,8 @@ public func isKeyframe(_ sample: CodedMediaSample) -> Bool { return isKeyframeAVC(sample) case .hevc: return false + case .vp9: + return isKeyframeVP9(sample) default: return false } @@ -251,6 +265,17 @@ private func isKeyframeAVC(_ sample: CodedMediaSample) -> Bool { return sample.data()[4] & 0x1f == 5 } +private func isKeyframeVP9(_ sample: CodedMediaSample) -> Bool { + guard sample.data().count >= 1 else { + return false + } + return sample.data().withUnsafeBytes { + var isKeyframe: Int32 = 0 + vp9_is_keyframe($0.baseAddress, Int64($0.count), &isKeyframe) + return isKeyframe == 1 + } +} + private func spsFromAVCDCR(_ sample: CodedMediaSample) throws -> Data { guard let record = sample.sideData()["config"], record.count > 8 else { From 34bd1bad88b76bd8e3492027a4a7c0174c1be295 Mon Sep 17 00:00:00 2001 From: James Hurley Date: Fri, 7 Aug 2020 09:40:35 +0200 Subject: [PATCH 02/15] Add comma --- Sources/SwiftVideo/enc.video.apple.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/SwiftVideo/enc.video.apple.swift b/Sources/SwiftVideo/enc.video.apple.swift index ac97e89..a5ed572 100644 --- a/Sources/SwiftVideo/enc.video.apple.swift +++ b/Sources/SwiftVideo/enc.video.apple.swift @@ -28,7 +28,7 @@ public class AppleVideoEncoder: Tx { public init(format: MediaFormat, frame: CGSize, bitrate: Int = 500000) { //assert(format == .avc || format == .hevc, "AppleVideoEncoder only supports AVC and HEVC") self.queue = DispatchQueue.init(label: "cezium.encode.video") - let codecType = { + let codecType = { () -> CMVideoCodecType in switch format { case .avc: return kCMVideoCodecType_H264 @@ -43,7 +43,7 @@ public class AppleVideoEncoder: Tx { VTCompressionSessionCreate(allocator: kCFAllocatorDefault, width: Int32(frame.width), height: Int32(frame.height), - codecType: codecType + codecType: codecType, encoderSpecification: [kVTCompressionPropertyKey_ExpectedFrameRate: 30] as CFDictionary, imageBufferAttributes: pixelBufferOptions(frame), compressedDataAllocator: nil, From 031555d8c8da33b5dbbd41bc179b6afe7db8b824 Mon Sep 17 00:00:00 2001 From: James Hurley Date: Fri, 7 Aug 2020 11:35:21 +0200 Subject: [PATCH 03/15] Get most of vp9 props --- Sources/CSwiftVideo/include/CSwiftVideo.h | 26 ++++++++++- Sources/CSwiftVideo/shim.cpp | 54 ++++++++++++----------- Sources/SwiftVideo/dec.video.apple.swift | 16 +++++-- Sources/SwiftVideo/sample.coded.swift | 28 +++++++----- Sources/SwiftVideo/sample.pict.swift | 9 ++++ 5 files changed, 93 insertions(+), 40 deletions(-) diff --git a/Sources/CSwiftVideo/include/CSwiftVideo.h b/Sources/CSwiftVideo/include/CSwiftVideo.h index 83b9892..1bcb653 100644 --- a/Sources/CSwiftVideo/include/CSwiftVideo.h +++ b/Sources/CSwiftVideo/include/CSwiftVideo.h @@ -27,10 +27,34 @@ extern "C" { #endif +typedef enum { + VP9ColorSpace_Unknown = 0, + VP9ColorSpace_BT_601 = 1, + VP9ColorSpace_BT_709 = 2, + VP9ColorSpace_SMPTE_170 = 3, + VP9ColorSpace_SMPTE_240 = 4, + VP9ColorSpace_BT_2020 = 5, + VP9ColorSpace_Reserved = 6, + VP9ColorSpace_SRGB = 7 +} VP9ColorSpace; + +typedef struct { + int width; + int height; + int displayWidth; + int displayHeight; + int bitDepth; + VP9ColorSpace colorSpace; + int subSamplingX; + int subSamplingY; + int fullSwingColor; + int profile; +} VP9FrameProperties; + int aac_parse_asc(const void* data, int64_t size, int* channels, int* sample_rate, int* samples_per_frame); int h264_sps_frame_size(const void* data, int64_t size, int* width, int* height); int vp9_is_keyframe(const void* data, int64_t size, int* is_keyframe); -int vp9_frame_size(const void* data, int64_t size, int* width, int* height); +int vp9_frame_properties(const void* data, int64_t size, VP9FrameProperties* props); #if defined(linux) void generateRandomBytes(void* buf, size_t size); diff --git a/Sources/CSwiftVideo/shim.cpp b/Sources/CSwiftVideo/shim.cpp index 9d75bef..8b34202 100644 --- a/Sources/CSwiftVideo/shim.cpp +++ b/Sources/CSwiftVideo/shim.cpp @@ -280,31 +280,38 @@ extern "C" { Reserved = 6 sRGB = 7 */ - static int vp9_bitdepth_colorspace_sampling(ExpGolomb& decoder, int64_t profile, int* bit_depth, - int* colorspace, int* yuv_range, int* subsampling_x, int* subsampling_y) { - *bit_depth = 8; - if(profile >= 2) { - *bit_depth = *decoder.get_bits(1) ? 12 : 10; + static int vp9_bitdepth_colorspace_sampling(ExpGolomb& decoder, VP9FrameProperties* props) { + props->bitDepth = 8; + if(props->profile >= 2) { + props->bitDepth = *decoder.get_bits(1) ? 12 : 10; } - *colorspace = *decoder.get_bits(3); - if(*colorspace != 7) { // sRGB - *yuv_range = *decoder.get_bits(1); // movie = 0, full = 1 - if(profile == 1 || profile == 3) { - *subsampling_x = *decoder.get_bits(1); - *subsampling_y = *decoder.get_bits(1); + props->colorSpace = static_cast(*decoder.get_bits(3)); + if(props->colorSpace != VP9ColorSpace_SRGB) { + props->fullSwingColor = *decoder.get_bits(1); // movie = 0, full = 1 + if(props->profile == 1 || props->profile == 3) { + props->subSamplingX = *decoder.get_bits(1); + props->subSamplingY = *decoder.get_bits(1); decoder.get_bits(1); // reserved 0 } else { - *subsampling_x = *subsampling_y = 1; + props->subSamplingX = props->subSamplingY = 1; } } else { - *subsampling_x = *subsampling_y = 0; + props->subSamplingX = props->subSamplingY = 0; return 0; } return 1; } - static int vp9_frame_size(ExpGolomb& decoder, int* width, int* height) { - *width = *decoder.get_bits(16) + 1; - *height = *decoder.get_bits(16) + 1; + static int vp9_frame_size(ExpGolomb& decoder, VP9FrameProperties* props) { + props->width = *decoder.get_bits(16) + 1; + props->height = *decoder.get_bits(16) + 1; + auto has_scaling = decoder.get_bits(1); + if(has_scaling && *has_scaling) { + props->displayWidth = *decoder.get_bits(16) + 1; + props->displayHeight = *decoder.get_bits(16) + 1; + } else { + props->displayWidth = props->width; + props->displayHeight = props->height; + } return 1; } int vp9_is_keyframe(const void* data, int64_t size, int* is_keyframe) { @@ -325,15 +332,15 @@ extern "C" { return 1; } - int vp9_frame_size(const void* data, int64_t size, int* width, int* height) { + int vp9_frame_properties(const void* data, int64_t size, VP9FrameProperties* props) { auto decoder = ExpGolomb((uint8_t*)data, size); if(*decoder.get_bits(2) != 0b10) { return 0; } auto version = *decoder.get_bits(1); auto high = *decoder.get_bits(1); - auto profile = (high << 1) + version; - if(profile == 3) { + props->profile = (high << 1) + version; + if(props->profile == 3) { decoder.get_bits(1); // reserved } if(*decoder.get_bits(1)) { // show_existing_frame - not a new frame @@ -343,13 +350,8 @@ extern "C" { if(frame_type != 0) { return 0; } - int bit_depth = 8; - int colorspace = 0; - int yuv_range = 0; - int subsampling_x = 1; - int subsampling_y = 1; - if(vp9_bitdepth_colorspace_sampling(decoder, profile, &bit_depth, &colorspace, &yuv_range, &subsampling_x, &subsampling_y)) { - vp9_frame_size(decoder, width, height); + if(vp9_bitdepth_colorspace_sampling(decoder, props)) { + vp9_frame_size(decoder, props); return 1; } return 0; diff --git a/Sources/SwiftVideo/dec.video.apple.swift b/Sources/SwiftVideo/dec.video.apple.swift index 3bf3140..e8c61a6 100644 --- a/Sources/SwiftVideo/dec.video.apple.swift +++ b/Sources/SwiftVideo/dec.video.apple.swift @@ -185,9 +185,19 @@ public class AppleVideoDecoder: Tx { private let decodeQueue: DispatchQueue } -private func videoFormatFromVP9Header(_ data: Data) -> CMVideoFormatDescription? { - - return nil +private func videoFormatFromVP9Header(_ sample: CodedMediaSample) throws -> CMVideoFormatDescription? { + let desc = try basicMediaDescription(sample) + guard case .video(let videoDesc) = desc else { + return nil + } + var formatDesc: CMVideoFormatDescription? + _ = CMVideoFormatDescriptionCreate(kCFAllocatorDefault, + kCMVideoCodecType_VP9, + videoDesc.width, + videoDesc.height, + nil, + &formatDesc) + return formatDesc } private func videoFormatFromAVCParameterSets( _ paramSets: [[UInt8]]) throws -> CMVideoFormatDescription? { let pmSetPtrs: [UnsafePointer] = try paramSets.map { diff --git a/Sources/SwiftVideo/sample.coded.swift b/Sources/SwiftVideo/sample.coded.swift index 5b45722..8787bf7 100644 --- a/Sources/SwiftVideo/sample.coded.swift +++ b/Sources/SwiftVideo/sample.coded.swift @@ -51,6 +51,10 @@ public enum EncoderSpecificSettings { public struct BasicVideoDescription { public let size: Vector2 + public let colorSpace: ColorSpace = .bt601 + public let subSampling: PixelFormat = .y420p // valid values: y420p, y422p, y444p, RGBA + public let bitDepth: Int = 8 + public let fullRangeColor: Bool = false } public struct BasicAudioDescription { @@ -226,16 +230,7 @@ func basicMediaDescription(_ sample: CodedMediaSample) throws -> BasicMediaDescr channelCount: Int(channels), samplesPerPacket: Int(samplesPerPacket))) case .vp9: - guard isKeyframe(sample) else { - throw MediaDescriptionError.needsKeyframe - } - let (width, height): (Int32, Int32) = sample.data().withUnsafeBytes { - var width: Int32 = 0 - var height: Int32 = 0 - vp9_frame_size($0.baseAddress, Int64($0.count), &width, &height) - return (width, height) - } - return .video(BasicVideoDescription(size: Vector2(Float(width), Float(height)))) + return try mediaDescriptionVP9(sample) default: throw MediaDescriptionError.unsupported } @@ -276,6 +271,19 @@ private func isKeyframeVP9(_ sample: CodedMediaSample) -> Bool { } } +private func mediaDescriptionVP9(_ sample: CodedMediaSample) throws -> BasicMediaDescription { + guard isKeyframe(sample) else { + throw MediaDescriptionError.needsKeyframe + } + let props: VP9FrameProperties = sample.data().withUnsafeBytes { + var props = VP9FrameProperties() + vp9_frame_properties($0.baseAddress, Int64($0.count), &props) + return props + } + // TODO: Fill out rest of video description + return .video(BasicVideoDescription(size: Vector2(Float(props.width), Float(props.height)))) +} + private func spsFromAVCDCR(_ sample: CodedMediaSample) throws -> Data { guard let record = sample.sideData()["config"], record.count > 8 else { diff --git a/Sources/SwiftVideo/sample.pict.swift b/Sources/SwiftVideo/sample.pict.swift index e924f9c..23ec42b 100644 --- a/Sources/SwiftVideo/sample.pict.swift +++ b/Sources/SwiftVideo/sample.pict.swift @@ -32,6 +32,15 @@ public enum PixelFormat { case invalid } +public enum ColorSpace { + case bt601 + case bt709 + case bt2020 + case smpte170 + case smpte240 + case sRGB +} + // swiftlint:disable identifier_name public enum Component { case r From ef0b2d1368850af4c42bb3a6d237fd05cab568f2 Mon Sep 17 00:00:00 2001 From: James Hurley Date: Fri, 7 Aug 2020 12:25:53 +0200 Subject: [PATCH 04/15] fix videodesc usage --- Sources/CSwiftVideo/shim.cpp | 3 --- Sources/SwiftVideo/dec.video.apple.swift | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Sources/CSwiftVideo/shim.cpp b/Sources/CSwiftVideo/shim.cpp index 8b34202..cd322c2 100644 --- a/Sources/CSwiftVideo/shim.cpp +++ b/Sources/CSwiftVideo/shim.cpp @@ -368,6 +368,3 @@ extern "C" { return val; } } - - - diff --git a/Sources/SwiftVideo/dec.video.apple.swift b/Sources/SwiftVideo/dec.video.apple.swift index e8c61a6..8148e45 100644 --- a/Sources/SwiftVideo/dec.video.apple.swift +++ b/Sources/SwiftVideo/dec.video.apple.swift @@ -193,8 +193,8 @@ private func videoFormatFromVP9Header(_ sample: CodedMediaSample) throws -> CMVi var formatDesc: CMVideoFormatDescription? _ = CMVideoFormatDescriptionCreate(kCFAllocatorDefault, kCMVideoCodecType_VP9, - videoDesc.width, - videoDesc.height, + Int32(videoDesc.size.x), + Int32(videoDesc.size.y), nil, &formatDesc) return formatDesc From 6f64dc71197a062559bdde1283bd3ead5ba1b7b5 Mon Sep 17 00:00:00 2001 From: James Hurley Date: Fri, 7 Aug 2020 12:27:22 +0200 Subject: [PATCH 05/15] Add argument labels --- Sources/SwiftVideo/dec.video.apple.swift | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Sources/SwiftVideo/dec.video.apple.swift b/Sources/SwiftVideo/dec.video.apple.swift index 8148e45..eca6ea2 100644 --- a/Sources/SwiftVideo/dec.video.apple.swift +++ b/Sources/SwiftVideo/dec.video.apple.swift @@ -191,12 +191,12 @@ private func videoFormatFromVP9Header(_ sample: CodedMediaSample) throws -> CMVi return nil } var formatDesc: CMVideoFormatDescription? - _ = CMVideoFormatDescriptionCreate(kCFAllocatorDefault, - kCMVideoCodecType_VP9, - Int32(videoDesc.size.x), - Int32(videoDesc.size.y), - nil, - &formatDesc) + _ = CMVideoFormatDescriptionCreate(allocator: kCFAllocatorDefault, + codecType: kCMVideoCodecType_VP9, + width: Int32(videoDesc.size.x), + height: Int32(videoDesc.size.y), + extensions: nil, + formatDescription: &formatDesc) return formatDesc } private func videoFormatFromAVCParameterSets( _ paramSets: [[UInt8]]) throws -> CMVideoFormatDescription? { From 276b56e8886d5ec251d1f368352bf855d6f7096f Mon Sep 17 00:00:00 2001 From: James Hurley Date: Fri, 7 Aug 2020 12:29:08 +0200 Subject: [PATCH 06/15] FormatDescriptionOut --- Sources/SwiftVideo/dec.video.apple.swift | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Sources/SwiftVideo/dec.video.apple.swift b/Sources/SwiftVideo/dec.video.apple.swift index eca6ea2..845d51f 100644 --- a/Sources/SwiftVideo/dec.video.apple.swift +++ b/Sources/SwiftVideo/dec.video.apple.swift @@ -192,11 +192,11 @@ private func videoFormatFromVP9Header(_ sample: CodedMediaSample) throws -> CMVi } var formatDesc: CMVideoFormatDescription? _ = CMVideoFormatDescriptionCreate(allocator: kCFAllocatorDefault, - codecType: kCMVideoCodecType_VP9, - width: Int32(videoDesc.size.x), - height: Int32(videoDesc.size.y), - extensions: nil, - formatDescription: &formatDesc) + codecType: kCMVideoCodecType_VP9, + width: Int32(videoDesc.size.x), + height: Int32(videoDesc.size.y), + extensions: nil, + formatDescriptionOut: &formatDesc) return formatDesc } private func videoFormatFromAVCParameterSets( _ paramSets: [[UInt8]]) throws -> CMVideoFormatDescription? { From ddb0572bce3b6b3354f5e5ab4b838a2b86ab7654 Mon Sep 17 00:00:00 2001 From: James Hurley Date: Fri, 7 Aug 2020 12:58:52 +0200 Subject: [PATCH 07/15] keep format description from initial creation --- Sources/SwiftVideo/dec.video.apple.swift | 132 ++++++++++++----------- 1 file changed, 71 insertions(+), 61 deletions(-) diff --git a/Sources/SwiftVideo/dec.video.apple.swift b/Sources/SwiftVideo/dec.video.apple.swift index 845d51f..112dd58 100644 --- a/Sources/SwiftVideo/dec.video.apple.swift +++ b/Sources/SwiftVideo/dec.video.apple.swift @@ -22,6 +22,7 @@ import Dispatch enum DecodeError: Error { case invalidBase case osstatus(Int32) + case unsupportedCodec } public class AppleVideoDecoder: Tx { @@ -79,68 +80,58 @@ public class AppleVideoDecoder: Tx { blockBufferOut: &buffer) return buffer } - guard let dataBuffer = dataBufferWk, let dcr = sample.sideData()["config"] else { + guard let dataBuffer = dataBufferWk else { print("Failed to create buffer") return .error(EventError("dec.video.apple", -2, "Failed to create buffer")) } - - do { - // TODO: HEVC Support - let paramSets = parameterSetsFromAVCC(dcr) - let formatDesc = try videoFormatFromAVCParameterSets(paramSets) - let pts = rescale(sample.pts(), 90000).value - let dts = rescale(sample.dts(), 90000).value - var videoSampleTimingInfo = CMSampleTimingInfo(duration: CMTimeMake(value: 1, timescale: 1), - presentationTimeStamp: CMTimeMake(value: pts, timescale: 90000), - decodeTimeStamp: CMTimeMake(value: dts, timescale: 90000)) - var sampleBuffer: CMSampleBuffer? - var sampleSize = [sample.data().count] - CMSampleBufferCreate(allocator: kCFAllocatorDefault, - dataBuffer: dataBuffer, - dataReady: true, - makeDataReadyCallback: nil, - refcon: nil, - formatDescription: formatDesc, - sampleCount: 1, - sampleTimingEntryCount: 1, - sampleTimingArray: &videoSampleTimingInfo, - sampleSizeEntryCount: 1, - sampleSizeArray: &sampleSize, - sampleBufferOut: &sampleBuffer) - if let sampleBuffer = sampleBuffer { - let flags: VTDecodeFrameFlags = [._1xRealTimePlayback, - ._EnableTemporalProcessing, - ._EnableAsynchronousDecompression] - let result = VTDecompressionSessionDecodeFrame(session, - sampleBuffer: sampleBuffer, - flags: flags, - infoFlagsOut: nil) { [weak self] (_, _, imgBuffer, pts, _) in - guard let strongSelf = self, let imgBuffer = imgBuffer else { - return - } - let ptsTime = TimePoint(pts.value, Int64(pts.timescale)) - let img = ImageBuffer(imgBuffer) - let pic = PictureSample(img, - assetId: sample.assetId(), - workspaceId: sample.workspaceId(), - time: strongSelf.clock.current(), - pts: ptsTime) - strongSelf.sampleQueue.sync { - strongSelf.output.append(pic) - strongSelf.output = strongSelf - .output.sorted { $0.pts() < $1.pts() } - } - } - if result != 0 { - print("result=\(result)") - return .error(EventError("dec.video.apple", Int(result), "OSStatus, decode failed")) + let pts = rescale(sample.pts(), 90000).value + let dts = rescale(sample.dts(), 90000).value + var videoSampleTimingInfo = CMSampleTimingInfo(duration: CMTimeMake(value: 1, timescale: 1), + presentationTimeStamp: CMTimeMake(value: pts, timescale: 90000), + decodeTimeStamp: CMTimeMake(value: dts, timescale: 90000)) + var sampleBuffer: CMSampleBuffer? + var sampleSize = [sample.data().count] + CMSampleBufferCreate(allocator: kCFAllocatorDefault, + dataBuffer: dataBuffer, + dataReady: true, + makeDataReadyCallback: nil, + refcon: nil, + formatDescription: formatDesc, + sampleCount: 1, + sampleTimingEntryCount: 1, + sampleTimingArray: &videoSampleTimingInfo, + sampleSizeEntryCount: 1, + sampleSizeArray: &sampleSize, + sampleBufferOut: &sampleBuffer) + if let sampleBuffer = sampleBuffer { + let flags: VTDecodeFrameFlags = [._1xRealTimePlayback, + ._EnableTemporalProcessing, + ._EnableAsynchronousDecompression] + let result = VTDecompressionSessionDecodeFrame(session, + sampleBuffer: sampleBuffer, + flags: flags, + infoFlagsOut: nil) { [weak self] (_, _, imgBuffer, pts, _) in + guard let strongSelf = self, let imgBuffer = imgBuffer else { + return + } + let ptsTime = TimePoint(pts.value, Int64(pts.timescale)) + let img = ImageBuffer(imgBuffer) + let pic = PictureSample(img, + assetId: sample.assetId(), + workspaceId: sample.workspaceId(), + time: strongSelf.clock.current(), + pts: ptsTime) + strongSelf.sampleQueue.sync { + strongSelf.output.append(pic) + strongSelf.output = strongSelf + .output.sorted { $0.pts() < $1.pts() } + } } + if result != 0 { + print("result=\(result)") + return .error(EventError("dec.video.apple", Int(result), "OSStatus, decode failed")) } - } catch { - // error - return .error(EventError("dec.video.apple", -4, "Caught decode error \(error)")) } - return self.sampleQueue.sync { if let outSample = self.output[safe:0], self.output.count >= 5 { self.output.removeFirst() @@ -152,15 +143,14 @@ public class AppleVideoDecoder: Tx { } private func setupSession(_ sample: CodedMediaSample) throws { let desc = try basicMediaDescription(sample) - guard case .video(let videoDesc) = desc, let dcr = sample.sideData()["config"] else { + guard case .video(let videoDesc) = desc else { return } let pixelBufferOptions = [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, kCVPixelBufferWidthKey as String: Int(videoDesc.size.x), kCVPixelBufferHeightKey as String: Int(videoDesc.size.y)] as CFDictionary - let paramSets = parameterSetsFromAVCC(dcr) - let formatDesc = try videoFormatFromAVCParameterSets(paramSets) + formatDesc = try videoFormatDescription(sample) if let format = formatDesc { var session: VTDecompressionSession? @@ -180,25 +170,45 @@ public class AppleVideoDecoder: Tx { } private var output: [PictureSample] private var session: VTDecompressionSession? + private var formatDesc: CMVideoFormatDescription? private let clock: Clock private let sampleQueue: DispatchQueue private let decodeQueue: DispatchQueue } -private func videoFormatFromVP9Header(_ sample: CodedMediaSample) throws -> CMVideoFormatDescription? { +private func videoFormatDescription(_ sample: CodedMediaSample) throws -> CMVideoFormatDescription? { + switch(sample.mediaFormat()) { + case .avc: + guard let dcr = sample.sideData()["config"] else { + throw MediaDescriptionError.invalidMetadata + } + let paramSets = parameterSetsFromAVCC(dcr) + return try videoFormatFromAVCParameterSets(paramSets) + case .vp9: + return try videoFormatFromVP9(sample) + default: + throw DecodeError.unsupportedCodec + } +} + +private func videoFormatFromVP9(_ sample: CodedMediaSample) throws -> CMVideoFormatDescription? { let desc = try basicMediaDescription(sample) guard case .video(let videoDesc) = desc else { return nil } var formatDesc: CMVideoFormatDescription? + let extensions = [ + kCMFormatDescriptionExtension_FullRangeVideo as String: videoDesc.fullRangeColor + ] as CFDictionary _ = CMVideoFormatDescriptionCreate(allocator: kCFAllocatorDefault, codecType: kCMVideoCodecType_VP9, width: Int32(videoDesc.size.x), height: Int32(videoDesc.size.y), - extensions: nil, + extensions: extensions, formatDescriptionOut: &formatDesc) return formatDesc } + private func videoFormatFromAVCParameterSets( _ paramSets: [[UInt8]]) throws -> CMVideoFormatDescription? { let pmSetPtrs: [UnsafePointer] = try paramSets.map { try $0.withUnsafeBufferPointer { From f352a7592df02d99d74db697c29427b213237c1d Mon Sep 17 00:00:00 2001 From: James Hurley Date: Fri, 7 Aug 2020 17:06:31 +0200 Subject: [PATCH 08/15] some improvements --- Sources/CSwiftVideo/shim.cpp | 21 ++++++++------------- Sources/SwiftVideo/enc.video.apple.swift | 2 +- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/Sources/CSwiftVideo/shim.cpp b/Sources/CSwiftVideo/shim.cpp index cd322c2..1e49533 100644 --- a/Sources/CSwiftVideo/shim.cpp +++ b/Sources/CSwiftVideo/shim.cpp @@ -270,16 +270,6 @@ extern "C" { return 1; } - /* colorspace - Unknown = 0 - BT.601 = 1 - BT.709 = 2 - SMPTE-170 = 3 - SMPTE-240 = 4 - BT.2020 = 5 - Reserved = 6 - sRGB = 7 - */ static int vp9_bitdepth_colorspace_sampling(ExpGolomb& decoder, VP9FrameProperties* props) { props->bitDepth = 8; if(props->profile >= 2) { @@ -297,6 +287,7 @@ extern "C" { } } else { props->subSamplingX = props->subSamplingY = 0; + decoder.get_bits(1); // reserved 0 return 0; } return 1; @@ -328,7 +319,7 @@ extern "C" { if(*decoder.get_bits(1)) { // show_existing_frame - not a new frame return 0; } - *is_keyframe = !decoder.get_bits(1); + *is_keyframe = !(*decoder.get_bits(1)); return 1; } @@ -346,11 +337,15 @@ extern "C" { if(*decoder.get_bits(1)) { // show_existing_frame - not a new frame return 0; } - auto frame_type = !*decoder.get_bits(1); + auto frame_type = *decoder.get_bits(1); if(frame_type != 0) { return 0; } - if(vp9_bitdepth_colorspace_sampling(decoder, props)) { + decoder.get_bits(1); // show_frame + decoder.get_bits(1); // error_resilient_mode + auto sync_code = *decoder.get_bits(24); + if(sync_code = 0x498342 && vp9_bitdepth_colorspace_sampling(decoder, props)) { + auto refresh_frames_flag = *decoder.get_bits(8); vp9_frame_size(decoder, props); return 1; } diff --git a/Sources/SwiftVideo/enc.video.apple.swift b/Sources/SwiftVideo/enc.video.apple.swift index a5ed572..397855e 100644 --- a/Sources/SwiftVideo/enc.video.apple.swift +++ b/Sources/SwiftVideo/enc.video.apple.swift @@ -40,7 +40,7 @@ public class AppleVideoEncoder: Tx { return kCMVideoCodecType_H264 } }() - VTCompressionSessionCreate(allocator: kCFAllocatorDefault, + let result = VTCompressionSessionCreate(allocator: kCFAllocatorDefault, width: Int32(frame.width), height: Int32(frame.height), codecType: codecType, From 61a4f623e9df529555ebed6e07fb5ca5a3840a1b Mon Sep 17 00:00:00 2001 From: James Hurley Date: Sat, 8 Aug 2020 16:51:29 +0200 Subject: [PATCH 09/15] Fix parsing --- Sources/CSwiftVideo/shim.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Sources/CSwiftVideo/shim.cpp b/Sources/CSwiftVideo/shim.cpp index 1e49533..d2528af 100644 --- a/Sources/CSwiftVideo/shim.cpp +++ b/Sources/CSwiftVideo/shim.cpp @@ -113,6 +113,11 @@ class ExpGolomb { ptr = (uint8_t*)(base + (pos / 8)); return accumulator; } + + int64_t alignment() const { + return pos % 8; + } + private: int64_t zeroes() { uint8_t* p = ptr; @@ -288,7 +293,9 @@ extern "C" { } else { props->subSamplingX = props->subSamplingY = 0; decoder.get_bits(1); // reserved 0 - return 0; + if(profile != 1 && profile != 3) { + return 0; + } } return 1; } @@ -345,7 +352,9 @@ extern "C" { decoder.get_bits(1); // error_resilient_mode auto sync_code = *decoder.get_bits(24); if(sync_code = 0x498342 && vp9_bitdepth_colorspace_sampling(decoder, props)) { - auto refresh_frames_flag = *decoder.get_bits(8); + if(decoder.alignment() > 0) { + decoder.get_bits(8 - decoder.alignment()); + } vp9_frame_size(decoder, props); return 1; } From 5999124861dde35e2594f397a4241b9883fc4336 Mon Sep 17 00:00:00 2001 From: James Hurley Date: Sat, 8 Aug 2020 16:54:56 +0200 Subject: [PATCH 10/15] fix build --- Sources/CSwiftVideo/shim.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/CSwiftVideo/shim.cpp b/Sources/CSwiftVideo/shim.cpp index d2528af..ed4e104 100644 --- a/Sources/CSwiftVideo/shim.cpp +++ b/Sources/CSwiftVideo/shim.cpp @@ -293,7 +293,7 @@ extern "C" { } else { props->subSamplingX = props->subSamplingY = 0; decoder.get_bits(1); // reserved 0 - if(profile != 1 && profile != 3) { + if(props->profile != 1 && props->profile != 3) { return 0; } } @@ -351,7 +351,7 @@ extern "C" { decoder.get_bits(1); // show_frame decoder.get_bits(1); // error_resilient_mode auto sync_code = *decoder.get_bits(24); - if(sync_code = 0x498342 && vp9_bitdepth_colorspace_sampling(decoder, props)) { + if(sync_code == 0x498342 && vp9_bitdepth_colorspace_sampling(decoder, props)) { if(decoder.alignment() > 0) { decoder.get_bits(8 - decoder.alignment()); } From b453bbe0348dfaf5939671ddb05d1fd9d0740a54 Mon Sep 17 00:00:00 2001 From: James Hurley Date: Thu, 13 Aug 2020 12:42:34 +0200 Subject: [PATCH 11/15] Register vp9 on macos --- Sources/SwiftVideo/dec.video.apple.swift | 3 +++ Sources/SwiftVideo/enc.video.apple.swift | 19 +++++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Sources/SwiftVideo/dec.video.apple.swift b/Sources/SwiftVideo/dec.video.apple.swift index 112dd58..e4cc1fa 100644 --- a/Sources/SwiftVideo/dec.video.apple.swift +++ b/Sources/SwiftVideo/dec.video.apple.swift @@ -155,6 +155,9 @@ public class AppleVideoDecoder: Tx { if let format = formatDesc { var session: VTDecompressionSession? let decoderSpecification = [:] as CFDictionary + #if os(macOS) + VTRegisterSupplementalVideoDecoderIfAvailable(kCMVideoCodecType_VP9) + #endif let result = VTDecompressionSessionCreate(allocator: kCFAllocatorDefault, formatDescription: format, decoderSpecification: decoderSpecification, diff --git a/Sources/SwiftVideo/enc.video.apple.swift b/Sources/SwiftVideo/enc.video.apple.swift index 397855e..c7d075f 100644 --- a/Sources/SwiftVideo/enc.video.apple.swift +++ b/Sources/SwiftVideo/enc.video.apple.swift @@ -30,16 +30,19 @@ public class AppleVideoEncoder: Tx { self.queue = DispatchQueue.init(label: "cezium.encode.video") let codecType = { () -> CMVideoCodecType in switch format { - case .avc: - return kCMVideoCodecType_H264 - case .hevc: - return kCMVideoCodecType_HEVC - case .vp9: - return kCMVideoCodecType_VP9 - default: - return kCMVideoCodecType_H264 + case .avc: + return kCMVideoCodecType_H264 + case .hevc: + return kCMVideoCodecType_HEVC + case .vp9: + return kCMVideoCodecType_VP9 + default: + return kCMVideoCodecType_H264 } }() + #if os(macOS) + VTRegisterSupplementalVideoDecoderIfAvailable(kCMVideoCodecType_VP9) + #endif let result = VTCompressionSessionCreate(allocator: kCFAllocatorDefault, width: Int32(frame.width), height: Int32(frame.height), From 26cbffe697a0993c1e213c4a9ebadd2448ce4568 Mon Sep 17 00:00:00 2001 From: James Hurley Date: Thu, 13 Aug 2020 12:52:02 +0200 Subject: [PATCH 12/15] mark if available --- Sources/SwiftVideo/dec.video.apple.swift | 6 +++--- Sources/SwiftVideo/enc.video.apple.swift | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Sources/SwiftVideo/dec.video.apple.swift b/Sources/SwiftVideo/dec.video.apple.swift index e4cc1fa..97d9a0f 100644 --- a/Sources/SwiftVideo/dec.video.apple.swift +++ b/Sources/SwiftVideo/dec.video.apple.swift @@ -155,9 +155,9 @@ public class AppleVideoDecoder: Tx { if let format = formatDesc { var session: VTDecompressionSession? let decoderSpecification = [:] as CFDictionary - #if os(macOS) - VTRegisterSupplementalVideoDecoderIfAvailable(kCMVideoCodecType_VP9) - #endif + if #available(macOS 10.11, *) { + VTRegisterSupplementalVideoDecoderIfAvailable(kCMVideoCodecType_VP9) + } let result = VTDecompressionSessionCreate(allocator: kCFAllocatorDefault, formatDescription: format, decoderSpecification: decoderSpecification, diff --git a/Sources/SwiftVideo/enc.video.apple.swift b/Sources/SwiftVideo/enc.video.apple.swift index c7d075f..4d6a1d9 100644 --- a/Sources/SwiftVideo/enc.video.apple.swift +++ b/Sources/SwiftVideo/enc.video.apple.swift @@ -40,9 +40,9 @@ public class AppleVideoEncoder: Tx { return kCMVideoCodecType_H264 } }() - #if os(macOS) - VTRegisterSupplementalVideoDecoderIfAvailable(kCMVideoCodecType_VP9) - #endif + if #available(macOS 10.11, *) { + VTRegisterSupplementalVideoDecoderIfAvailable(kCMVideoCodecType_VP9) + } let result = VTCompressionSessionCreate(allocator: kCFAllocatorDefault, width: Int32(frame.width), height: Int32(frame.height), From 1039327364c66cf3d920e58b0e0931aa07ff8ffc Mon Sep 17 00:00:00 2001 From: James Hurley Date: Thu, 13 Aug 2020 12:55:46 +0200 Subject: [PATCH 13/15] macOS 11.0 not 10.11 --- Sources/SwiftVideo/dec.video.apple.swift | 2 +- Sources/SwiftVideo/enc.video.apple.swift | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Sources/SwiftVideo/dec.video.apple.swift b/Sources/SwiftVideo/dec.video.apple.swift index 97d9a0f..3e14ba4 100644 --- a/Sources/SwiftVideo/dec.video.apple.swift +++ b/Sources/SwiftVideo/dec.video.apple.swift @@ -155,7 +155,7 @@ public class AppleVideoDecoder: Tx { if let format = formatDesc { var session: VTDecompressionSession? let decoderSpecification = [:] as CFDictionary - if #available(macOS 10.11, *) { + if #available(macOS 11.0, *) { VTRegisterSupplementalVideoDecoderIfAvailable(kCMVideoCodecType_VP9) } let result = VTDecompressionSessionCreate(allocator: kCFAllocatorDefault, diff --git a/Sources/SwiftVideo/enc.video.apple.swift b/Sources/SwiftVideo/enc.video.apple.swift index 4d6a1d9..0e0ab42 100644 --- a/Sources/SwiftVideo/enc.video.apple.swift +++ b/Sources/SwiftVideo/enc.video.apple.swift @@ -40,8 +40,8 @@ public class AppleVideoEncoder: Tx { return kCMVideoCodecType_H264 } }() - if #available(macOS 10.11, *) { - VTRegisterSupplementalVideoDecoderIfAvailable(kCMVideoCodecType_VP9) + if #available(macOS 11.0, *) { + VTRegisterSupplementalVideoEncoderIfAvailable(kCMVideoCodecType_VP9) } let result = VTCompressionSessionCreate(allocator: kCFAllocatorDefault, width: Int32(frame.width), From f456259820235e6e4d138f48dc9ba9b1c913c6ff Mon Sep 17 00:00:00 2001 From: James Hurley Date: Thu, 13 Aug 2020 12:57:11 +0200 Subject: [PATCH 14/15] rm from encoder --- Sources/SwiftVideo/enc.video.apple.swift | 3 --- 1 file changed, 3 deletions(-) diff --git a/Sources/SwiftVideo/enc.video.apple.swift b/Sources/SwiftVideo/enc.video.apple.swift index 0e0ab42..c5b5860 100644 --- a/Sources/SwiftVideo/enc.video.apple.swift +++ b/Sources/SwiftVideo/enc.video.apple.swift @@ -40,9 +40,6 @@ public class AppleVideoEncoder: Tx { return kCMVideoCodecType_H264 } }() - if #available(macOS 11.0, *) { - VTRegisterSupplementalVideoEncoderIfAvailable(kCMVideoCodecType_VP9) - } let result = VTCompressionSessionCreate(allocator: kCFAllocatorDefault, width: Int32(frame.width), height: Int32(frame.height), From bd6b6e99337bf97248d94a1293db4a61df2933cf Mon Sep 17 00:00:00 2001 From: James Hurley Date: Wed, 16 Sep 2020 09:09:51 +0200 Subject: [PATCH 15/15] do not build macos code --- Sources/SwiftVideo/dec.video.apple.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Sources/SwiftVideo/dec.video.apple.swift b/Sources/SwiftVideo/dec.video.apple.swift index 3e14ba4..473d222 100644 --- a/Sources/SwiftVideo/dec.video.apple.swift +++ b/Sources/SwiftVideo/dec.video.apple.swift @@ -155,9 +155,11 @@ public class AppleVideoDecoder: Tx { if let format = formatDesc { var session: VTDecompressionSession? let decoderSpecification = [:] as CFDictionary + #if os(macOS) if #available(macOS 11.0, *) { VTRegisterSupplementalVideoDecoderIfAvailable(kCMVideoCodecType_VP9) } + #endif let result = VTDecompressionSessionCreate(allocator: kCFAllocatorDefault, formatDescription: format, decoderSpecification: decoderSpecification,