WIP Apple VideoToolbox VP9 support - To be merged after Xcode 12 is released

Examples/RtmpServer/Sources/RtmpServer/main.swift

@@ -33,7 +33,7 @@ let onConnection: LiveOnConnection = { pub, sub in
         // to it. Releasing the reference will close the connection an end the stream.
         // You can create more sophisticated graphs with decoders, buses, and other components.
         subs[sub.assetId()] = src >>> Tx { sample in
-            print("got sample \(sample.pts().toString()) type = \(sample.mediaType())")
+            print("got sample \(sample.pts().toString()) type = \(sample.mediaType()) size = \(sample.data().count)")
             return .nothing(nil)
         }
     }

Sources/CSwiftVideo/include/CSwiftVideo.h

@@ -27,8 +27,34 @@
 extern "C" {
 #endif
 
+typedef enum {
+   VP9ColorSpace_Unknown = 0,
+   VP9ColorSpace_BT_601 = 1,
+   VP9ColorSpace_BT_709 = 2,
+   VP9ColorSpace_SMPTE_170 = 3,
+   VP9ColorSpace_SMPTE_240 = 4,
+   VP9ColorSpace_BT_2020 = 5,
+   VP9ColorSpace_Reserved = 6,
+   VP9ColorSpace_SRGB = 7
+} VP9ColorSpace;
+
+typedef struct {
+   int width;
+   int height;
+   int displayWidth;
+   int displayHeight;
+   int bitDepth;
+   VP9ColorSpace colorSpace;
+   int subSamplingX;
+   int subSamplingY;
+   int fullSwingColor;
+   int profile;
+} VP9FrameProperties;
+
 int aac_parse_asc(const void* data, int64_t size, int* channels, int* sample_rate, int* samples_per_frame);
 int h264_sps_frame_size(const void* data, int64_t size, int* width, int* height);
+int vp9_is_keyframe(const void* data, int64_t size, int* is_keyframe);
+int vp9_frame_properties(const void* data, int64_t size, VP9FrameProperties* props);
 
 #if defined(linux)
 void generateRandomBytes(void* buf, size_t size);

Sources/CSwiftVideo/shim.cpp

@@ -113,6 +113,11 @@ class ExpGolomb {
         ptr = (uint8_t*)(base + (pos / 8));
         return accumulator;
     }
+
+    int64_t alignment() const {
+        return pos % 8;
+    }
+
 private:
     int64_t zeroes() {
         uint8_t* p = ptr;
@@ -270,6 +275,92 @@ extern "C" {
         return 1;
     }
 
+    static int vp9_bitdepth_colorspace_sampling(ExpGolomb& decoder, VP9FrameProperties* props) {
+        props->bitDepth = 8;
+        if(props->profile >= 2) {
+            props->bitDepth = *decoder.get_bits(1) ? 12 : 10;
+        }
+        props->colorSpace = static_cast<VP9ColorSpace>(*decoder.get_bits(3));
+        if(props->colorSpace != VP9ColorSpace_SRGB) {
+            props->fullSwingColor = *decoder.get_bits(1); // movie = 0, full = 1
+            if(props->profile == 1 || props->profile == 3) {
+                props->subSamplingX = *decoder.get_bits(1);
+                props->subSamplingY = *decoder.get_bits(1);
+                decoder.get_bits(1); // reserved 0
+            } else {
+                props->subSamplingX = props->subSamplingY = 1;
+            }
+        } else {
+            props->subSamplingX = props->subSamplingY = 0;
+            decoder.get_bits(1); // reserved 0
+            if(props->profile != 1 && props->profile != 3) {
+                return 0;
+            }
+        }
+        return 1;
+    }
+    static int vp9_frame_size(ExpGolomb& decoder, VP9FrameProperties* props) {
+        props->width = *decoder.get_bits(16) + 1;
+        props->height = *decoder.get_bits(16) + 1;
+        auto has_scaling = decoder.get_bits(1);
+        if(has_scaling && *has_scaling) {
+            props->displayWidth = *decoder.get_bits(16) + 1;
+            props->displayHeight = *decoder.get_bits(16) + 1;
+        } else {
+            props->displayWidth = props->width;
+            props->displayHeight = props->height;
+        }
+        return 1;
+    }
+    int vp9_is_keyframe(const void* data, int64_t size, int* is_keyframe) {
+        auto decoder = ExpGolomb((uint8_t*)data, size);
+        if(*decoder.get_bits(2) != 0b10) {
+            return 0;
+        }
+        auto version = *decoder.get_bits(1);
+        auto high = *decoder.get_bits(1);
+        auto profile = (high << 1) + version;
+        if(profile == 3) {
+            decoder.get_bits(1); // reserved
+        }
+        if(*decoder.get_bits(1)) { // show_existing_frame - not a new frame
+            return 0;
+        }
+        *is_keyframe = !(*decoder.get_bits(1));
+        return 1;
+    }
+
+    int vp9_frame_properties(const void* data, int64_t size, VP9FrameProperties* props) {
+        auto decoder = ExpGolomb((uint8_t*)data, size);
+        if(*decoder.get_bits(2) != 0b10) {
+            return 0;
+        }
+        auto version = *decoder.get_bits(1);
+        auto high = *decoder.get_bits(1);
+        props->profile = (high << 1) + version;
+        if(props->profile == 3) {
+            decoder.get_bits(1); // reserved
+        }
+        if(*decoder.get_bits(1)) { // show_existing_frame - not a new frame
+            return 0;
+        }
+        auto frame_type = *decoder.get_bits(1);
+        if(frame_type != 0) {
+            return 0;
+        }
+        decoder.get_bits(1); // show_frame
+        decoder.get_bits(1); // error_resilient_mode
+        auto sync_code = *decoder.get_bits(24);
+        if(sync_code == 0x498342 && vp9_bitdepth_colorspace_sampling(decoder, props)) {
+            if(decoder.alignment() > 0) {
+                decoder.get_bits(8 - decoder.alignment());
+            }
+            vp9_frame_size(decoder, props);
+            return 1;
+        }
+        return 0;
+    }
+
     uint64_t test_golomb_dec() {
         uint8_t bytes[4] = {0};
         bytes[0] = 0x1;

Sources/SwiftVideo/dec.video.apple.swift

@@ -22,6 +22,7 @@ import Dispatch
 enum DecodeError: Error {
     case invalidBase
     case osstatus(Int32)
+    case unsupportedCodec
 }
 
 public class AppleVideoDecoder: Tx<CodedMediaSample, PictureSample> {
@@ -38,7 +39,7 @@ public class AppleVideoDecoder: Tx<CodedMediaSample, PictureSample> {
             guard let strongSelf = self else {
                 return .gone
             }
-            guard [.avc, .hevc].contains(sample.mediaFormat()) else {
+            guard [.avc, .hevc, .vp9].contains(sample.mediaFormat()) else {
                 return .error(EventError("dec.video.apple",
                    -1,
                    "\(sample.mediaFormat()) not supported. AppleVideoDecoder only supports AVC and HEVC"))
@@ -79,68 +80,58 @@ public class AppleVideoDecoder: Tx<CodedMediaSample, PictureSample> {
                                                blockBufferOut: &buffer)
             return buffer
         }
-        guard let dataBuffer = dataBufferWk, let dcr = sample.sideData()["config"] else {
+        guard let dataBuffer = dataBufferWk else {
             print("Failed to create buffer")
             return .error(EventError("dec.video.apple", -2, "Failed to create buffer"))
         }
-
-        do {
-            // TODO: HEVC Support
-            let paramSets = parameterSetsFromAVCC(dcr)
-            let formatDesc = try videoFormatFromAVCParameterSets(paramSets)
-            let pts = rescale(sample.pts(), 90000).value
-            let dts = rescale(sample.dts(), 90000).value
-            var videoSampleTimingInfo = CMSampleTimingInfo(duration: CMTimeMake(value: 1, timescale: 1),
-                                           presentationTimeStamp: CMTimeMake(value: pts, timescale: 90000),
-                                           decodeTimeStamp: CMTimeMake(value: dts, timescale: 90000))
-            var sampleBuffer: CMSampleBuffer?
-            var sampleSize = [sample.data().count]
-            CMSampleBufferCreate(allocator: kCFAllocatorDefault,
-                                 dataBuffer: dataBuffer,
-                                 dataReady: true,
-                                 makeDataReadyCallback: nil,
-                                 refcon: nil,
-                                 formatDescription: formatDesc,
-                                 sampleCount: 1,
-                                 sampleTimingEntryCount: 1,
-                                 sampleTimingArray: &videoSampleTimingInfo,
-                                 sampleSizeEntryCount: 1,
-                                 sampleSizeArray: &sampleSize,
-                                 sampleBufferOut: &sampleBuffer)
-            if let sampleBuffer = sampleBuffer {
-                let flags: VTDecodeFrameFlags = [._1xRealTimePlayback,
-                                                 ._EnableTemporalProcessing,
-                                                 ._EnableAsynchronousDecompression]
-                let result = VTDecompressionSessionDecodeFrame(session,
-                                                  sampleBuffer: sampleBuffer,
-                                                  flags: flags,
-                                                  infoFlagsOut: nil) { [weak self] (_, _, imgBuffer, pts, _) in
-                                                    guard let strongSelf = self, let imgBuffer = imgBuffer else {
-                                                        return
-                                                    }
-                                                    let ptsTime = TimePoint(pts.value, Int64(pts.timescale))
-                                                    let img = ImageBuffer(imgBuffer)
-                                                    let pic = PictureSample(img,
-                                                                            assetId: sample.assetId(),
-                                                                            workspaceId: sample.workspaceId(),
-                                                                            time: strongSelf.clock.current(),
-                                                                            pts: ptsTime)
-                                                    strongSelf.sampleQueue.sync {
-                                                        strongSelf.output.append(pic)
-                                                        strongSelf.output = strongSelf
-                                                            .output.sorted { $0.pts() < $1.pts() }
-                                                    }
-                    }
-                if result != 0 {
-                    print("result=\(result)")
-                    return .error(EventError("dec.video.apple", Int(result), "OSStatus, decode failed"))
+        let pts = rescale(sample.pts(), 90000).value
+        let dts = rescale(sample.dts(), 90000).value
+        var videoSampleTimingInfo = CMSampleTimingInfo(duration: CMTimeMake(value: 1, timescale: 1),
+                                       presentationTimeStamp: CMTimeMake(value: pts, timescale: 90000),
+                                       decodeTimeStamp: CMTimeMake(value: dts, timescale: 90000))
+        var sampleBuffer: CMSampleBuffer?
+        var sampleSize = [sample.data().count]
+        CMSampleBufferCreate(allocator: kCFAllocatorDefault,
+                             dataBuffer: dataBuffer,
+                             dataReady: true,
+                             makeDataReadyCallback: nil,
+                             refcon: nil,
+                             formatDescription: formatDesc,
+                             sampleCount: 1,
+                             sampleTimingEntryCount: 1,
+                             sampleTimingArray: &videoSampleTimingInfo,
+                             sampleSizeEntryCount: 1,
+                             sampleSizeArray: &sampleSize,
+                             sampleBufferOut: &sampleBuffer)
+        if let sampleBuffer = sampleBuffer {
+            let flags: VTDecodeFrameFlags = [._1xRealTimePlayback,
+                                             ._EnableTemporalProcessing,
+                                             ._EnableAsynchronousDecompression]
+            let result = VTDecompressionSessionDecodeFrame(session,
+                                              sampleBuffer: sampleBuffer,
+                                              flags: flags,
+                                              infoFlagsOut: nil) { [weak self] (_, _, imgBuffer, pts, _) in
+                                                guard let strongSelf = self, let imgBuffer = imgBuffer else {
+                                                    return
+                                                }
+                                                let ptsTime = TimePoint(pts.value, Int64(pts.timescale))
+                                                let img = ImageBuffer(imgBuffer)
+                                                let pic = PictureSample(img,
+                                                                        assetId: sample.assetId(),
+                                                                        workspaceId: sample.workspaceId(),
+                                                                        time: strongSelf.clock.current(),
+                                                                        pts: ptsTime)
+                                                strongSelf.sampleQueue.sync {
+                                                    strongSelf.output.append(pic)
+                                                    strongSelf.output = strongSelf
+                                                        .output.sorted { $0.pts() < $1.pts() }
+                                                }
                 }
+            if result != 0 {
+                print("result=\(result)")
+                return .error(EventError("dec.video.apple", Int(result), "OSStatus, decode failed"))
             }
-        } catch {
-            // error
-            return .error(EventError("dec.video.apple", -4, "Caught decode error \(error)"))
         }
-
         return self.sampleQueue.sync {
             if let outSample = self.output[safe:0], self.output.count >= 5 {
                 self.output.removeFirst()
@@ -152,19 +143,23 @@ public class AppleVideoDecoder: Tx<CodedMediaSample, PictureSample> {
     }
     private func setupSession(_ sample: CodedMediaSample) throws {
         let desc = try basicMediaDescription(sample)
-        guard case .video(let videoDesc) = desc, let dcr = sample.sideData()["config"] else {
+        guard case .video(let videoDesc) = desc else {
             return
         }
         let pixelBufferOptions =
             [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange,
              kCVPixelBufferWidthKey as String: Int(videoDesc.size.x),
              kCVPixelBufferHeightKey as String: Int(videoDesc.size.y)] as CFDictionary
-        let paramSets = parameterSetsFromAVCC(dcr)
-        let formatDesc = try videoFormatFromAVCParameterSets(paramSets)
+        formatDesc = try videoFormatDescription(sample)
 
         if let format = formatDesc {
             var session: VTDecompressionSession?
             let decoderSpecification = [:] as CFDictionary
+            #if os(macOS)
+            if #available(macOS 11.0, *) {
+                VTRegisterSupplementalVideoDecoderIfAvailable(kCMVideoCodecType_VP9)
+            }
+            #endif
             let result = VTDecompressionSessionCreate(allocator: kCFAllocatorDefault,
                                           formatDescription: format,
                                           decoderSpecification: decoderSpecification,
@@ -180,11 +175,45 @@ public class AppleVideoDecoder: Tx<CodedMediaSample, PictureSample> {
     }
     private var output: [PictureSample]
     private var session: VTDecompressionSession?
+    private var formatDesc: CMVideoFormatDescription?
     private let clock: Clock
     private let sampleQueue: DispatchQueue
     private let decodeQueue: DispatchQueue
 }
 
+private func videoFormatDescription(_ sample: CodedMediaSample) throws -> CMVideoFormatDescription? {
+    switch(sample.mediaFormat()) {
+    case .avc:
+        guard let dcr = sample.sideData()["config"] else {
+            throw MediaDescriptionError.invalidMetadata
+        }
+        let paramSets = parameterSetsFromAVCC(dcr)
+        return try videoFormatFromAVCParameterSets(paramSets)
+    case .vp9:
+        return try videoFormatFromVP9(sample)
+    default:
+        throw DecodeError.unsupportedCodec
+    }
+}
+
+private func videoFormatFromVP9(_ sample: CodedMediaSample) throws -> CMVideoFormatDescription? {
+    let desc = try basicMediaDescription(sample)
+    guard case .video(let videoDesc) = desc else {
+        return nil
+    }
+    var formatDesc: CMVideoFormatDescription?
+    let extensions = [
+        kCMFormatDescriptionExtension_FullRangeVideo as String: videoDesc.fullRangeColor
+    ] as CFDictionary
+    _ = CMVideoFormatDescriptionCreate(allocator: kCFAllocatorDefault,
+          codecType: kCMVideoCodecType_VP9,
+          width: Int32(videoDesc.size.x),
+          height: Int32(videoDesc.size.y),
+          extensions: extensions,
+          formatDescriptionOut: &formatDesc)
+    return formatDesc
+}
+
 private func videoFormatFromAVCParameterSets( _ paramSets: [[UInt8]]) throws -> CMVideoFormatDescription? {
     let pmSetPtrs: [UnsafePointer<UInt8>] = try paramSets.map {
         try $0.withUnsafeBufferPointer {

Sources/SwiftVideo/enc.video.apple.swift

@@ -26,13 +26,24 @@ public class AppleVideoEncoder: Tx<PictureSample, [CodedMediaSample]> {
     }
 
     public init(format: MediaFormat, frame: CGSize, bitrate: Int = 500000) {
-        assert(format == .avc || format == .hevc, "AppleVideoEncoder only supports AVC and HEVC")
+        //assert(format == .avc || format == .hevc, "AppleVideoEncoder only supports AVC and HEVC")
         self.queue = DispatchQueue.init(label: "cezium.encode.video")
-
-        VTCompressionSessionCreate(allocator: kCFAllocatorDefault,
+        let codecType = { () -> CMVideoCodecType in
+            switch format {
+            case .avc:
+                return kCMVideoCodecType_H264
+            case .hevc:
+                return kCMVideoCodecType_HEVC
+            case .vp9:
+                return kCMVideoCodecType_VP9
+            default:
+                return kCMVideoCodecType_H264
+            }
+        }()
+        let result = VTCompressionSessionCreate(allocator: kCFAllocatorDefault,
            width: Int32(frame.width),
            height: Int32(frame.height),
-           codecType: format == .avc ? kCMVideoCodecType_H264 : kCMVideoCodecType_HEVC,
+           codecType: codecType,
            encoderSpecification: [kVTCompressionPropertyKey_ExpectedFrameRate: 30] as CFDictionary,
            imageBufferAttributes: pixelBufferOptions(frame),
            compressedDataAllocator: nil,