diff --git a/Examples/RtmpServer/Sources/RtmpServer/main.swift b/Examples/RtmpServer/Sources/RtmpServer/main.swift index 7c8b0fa..28d4ee0 100644 --- a/Examples/RtmpServer/Sources/RtmpServer/main.swift +++ b/Examples/RtmpServer/Sources/RtmpServer/main.swift @@ -33,7 +33,7 @@ let onConnection: LiveOnConnection = { pub, sub in // to it. Releasing the reference will close the connection an end the stream. // You can create more sophisticated graphs with decoders, buses, and other components. subs[sub.assetId()] = src >>> Tx { sample in - print("got sample \(sample.pts().toString()) type = \(sample.mediaType())") + print("got sample \(sample.pts().toString()) type = \(sample.mediaType()) size = \(sample.data().count)") return .nothing(nil) } } diff --git a/Sources/CSwiftVideo/include/CSwiftVideo.h b/Sources/CSwiftVideo/include/CSwiftVideo.h index d163182..1bcb653 100644 --- a/Sources/CSwiftVideo/include/CSwiftVideo.h +++ b/Sources/CSwiftVideo/include/CSwiftVideo.h @@ -27,8 +27,34 @@ extern "C" { #endif +typedef enum { + VP9ColorSpace_Unknown = 0, + VP9ColorSpace_BT_601 = 1, + VP9ColorSpace_BT_709 = 2, + VP9ColorSpace_SMPTE_170 = 3, + VP9ColorSpace_SMPTE_240 = 4, + VP9ColorSpace_BT_2020 = 5, + VP9ColorSpace_Reserved = 6, + VP9ColorSpace_SRGB = 7 +} VP9ColorSpace; + +typedef struct { + int width; + int height; + int displayWidth; + int displayHeight; + int bitDepth; + VP9ColorSpace colorSpace; + int subSamplingX; + int subSamplingY; + int fullSwingColor; + int profile; +} VP9FrameProperties; + int aac_parse_asc(const void* data, int64_t size, int* channels, int* sample_rate, int* samples_per_frame); int h264_sps_frame_size(const void* data, int64_t size, int* width, int* height); +int vp9_is_keyframe(const void* data, int64_t size, int* is_keyframe); +int vp9_frame_properties(const void* data, int64_t size, VP9FrameProperties* props); #if defined(linux) void generateRandomBytes(void* buf, size_t size); diff --git a/Sources/CSwiftVideo/shim.cpp b/Sources/CSwiftVideo/shim.cpp index bdf8c3a..ed4e104 100644 --- a/Sources/CSwiftVideo/shim.cpp +++ b/Sources/CSwiftVideo/shim.cpp @@ -113,6 +113,11 @@ class ExpGolomb { ptr = (uint8_t*)(base + (pos / 8)); return accumulator; } + + int64_t alignment() const { + return pos % 8; + } + private: int64_t zeroes() { uint8_t* p = ptr; @@ -270,6 +275,92 @@ extern "C" { return 1; } + static int vp9_bitdepth_colorspace_sampling(ExpGolomb& decoder, VP9FrameProperties* props) { + props->bitDepth = 8; + if(props->profile >= 2) { + props->bitDepth = *decoder.get_bits(1) ? 12 : 10; + } + props->colorSpace = static_cast(*decoder.get_bits(3)); + if(props->colorSpace != VP9ColorSpace_SRGB) { + props->fullSwingColor = *decoder.get_bits(1); // movie = 0, full = 1 + if(props->profile == 1 || props->profile == 3) { + props->subSamplingX = *decoder.get_bits(1); + props->subSamplingY = *decoder.get_bits(1); + decoder.get_bits(1); // reserved 0 + } else { + props->subSamplingX = props->subSamplingY = 1; + } + } else { + props->subSamplingX = props->subSamplingY = 0; + decoder.get_bits(1); // reserved 0 + if(props->profile != 1 && props->profile != 3) { + return 0; + } + } + return 1; + } + static int vp9_frame_size(ExpGolomb& decoder, VP9FrameProperties* props) { + props->width = *decoder.get_bits(16) + 1; + props->height = *decoder.get_bits(16) + 1; + auto has_scaling = decoder.get_bits(1); + if(has_scaling && *has_scaling) { + props->displayWidth = *decoder.get_bits(16) + 1; + props->displayHeight = *decoder.get_bits(16) + 1; + } else { + props->displayWidth = props->width; + props->displayHeight = props->height; + } + return 1; + } + int vp9_is_keyframe(const void* data, int64_t size, int* is_keyframe) { + auto decoder = ExpGolomb((uint8_t*)data, size); + if(*decoder.get_bits(2) != 0b10) { + return 0; + } + auto version = *decoder.get_bits(1); + auto high = *decoder.get_bits(1); + auto profile = (high << 1) + version; + if(profile == 3) { + decoder.get_bits(1); // reserved + } + if(*decoder.get_bits(1)) { // show_existing_frame - not a new frame + return 0; + } + *is_keyframe = !(*decoder.get_bits(1)); + return 1; + } + + int vp9_frame_properties(const void* data, int64_t size, VP9FrameProperties* props) { + auto decoder = ExpGolomb((uint8_t*)data, size); + if(*decoder.get_bits(2) != 0b10) { + return 0; + } + auto version = *decoder.get_bits(1); + auto high = *decoder.get_bits(1); + props->profile = (high << 1) + version; + if(props->profile == 3) { + decoder.get_bits(1); // reserved + } + if(*decoder.get_bits(1)) { // show_existing_frame - not a new frame + return 0; + } + auto frame_type = *decoder.get_bits(1); + if(frame_type != 0) { + return 0; + } + decoder.get_bits(1); // show_frame + decoder.get_bits(1); // error_resilient_mode + auto sync_code = *decoder.get_bits(24); + if(sync_code == 0x498342 && vp9_bitdepth_colorspace_sampling(decoder, props)) { + if(decoder.alignment() > 0) { + decoder.get_bits(8 - decoder.alignment()); + } + vp9_frame_size(decoder, props); + return 1; + } + return 0; + } + uint64_t test_golomb_dec() { uint8_t bytes[4] = {0}; bytes[0] = 0x1; diff --git a/Sources/SwiftVideo/dec.video.apple.swift b/Sources/SwiftVideo/dec.video.apple.swift index 8856a04..473d222 100644 --- a/Sources/SwiftVideo/dec.video.apple.swift +++ b/Sources/SwiftVideo/dec.video.apple.swift @@ -22,6 +22,7 @@ import Dispatch enum DecodeError: Error { case invalidBase case osstatus(Int32) + case unsupportedCodec } public class AppleVideoDecoder: Tx { @@ -38,7 +39,7 @@ public class AppleVideoDecoder: Tx { guard let strongSelf = self else { return .gone } - guard [.avc, .hevc].contains(sample.mediaFormat()) else { + guard [.avc, .hevc, .vp9].contains(sample.mediaFormat()) else { return .error(EventError("dec.video.apple", -1, "\(sample.mediaFormat()) not supported. AppleVideoDecoder only supports AVC and HEVC")) @@ -79,68 +80,58 @@ public class AppleVideoDecoder: Tx { blockBufferOut: &buffer) return buffer } - guard let dataBuffer = dataBufferWk, let dcr = sample.sideData()["config"] else { + guard let dataBuffer = dataBufferWk else { print("Failed to create buffer") return .error(EventError("dec.video.apple", -2, "Failed to create buffer")) } - - do { - // TODO: HEVC Support - let paramSets = parameterSetsFromAVCC(dcr) - let formatDesc = try videoFormatFromAVCParameterSets(paramSets) - let pts = rescale(sample.pts(), 90000).value - let dts = rescale(sample.dts(), 90000).value - var videoSampleTimingInfo = CMSampleTimingInfo(duration: CMTimeMake(value: 1, timescale: 1), - presentationTimeStamp: CMTimeMake(value: pts, timescale: 90000), - decodeTimeStamp: CMTimeMake(value: dts, timescale: 90000)) - var sampleBuffer: CMSampleBuffer? - var sampleSize = [sample.data().count] - CMSampleBufferCreate(allocator: kCFAllocatorDefault, - dataBuffer: dataBuffer, - dataReady: true, - makeDataReadyCallback: nil, - refcon: nil, - formatDescription: formatDesc, - sampleCount: 1, - sampleTimingEntryCount: 1, - sampleTimingArray: &videoSampleTimingInfo, - sampleSizeEntryCount: 1, - sampleSizeArray: &sampleSize, - sampleBufferOut: &sampleBuffer) - if let sampleBuffer = sampleBuffer { - let flags: VTDecodeFrameFlags = [._1xRealTimePlayback, - ._EnableTemporalProcessing, - ._EnableAsynchronousDecompression] - let result = VTDecompressionSessionDecodeFrame(session, - sampleBuffer: sampleBuffer, - flags: flags, - infoFlagsOut: nil) { [weak self] (_, _, imgBuffer, pts, _) in - guard let strongSelf = self, let imgBuffer = imgBuffer else { - return - } - let ptsTime = TimePoint(pts.value, Int64(pts.timescale)) - let img = ImageBuffer(imgBuffer) - let pic = PictureSample(img, - assetId: sample.assetId(), - workspaceId: sample.workspaceId(), - time: strongSelf.clock.current(), - pts: ptsTime) - strongSelf.sampleQueue.sync { - strongSelf.output.append(pic) - strongSelf.output = strongSelf - .output.sorted { $0.pts() < $1.pts() } - } - } - if result != 0 { - print("result=\(result)") - return .error(EventError("dec.video.apple", Int(result), "OSStatus, decode failed")) + let pts = rescale(sample.pts(), 90000).value + let dts = rescale(sample.dts(), 90000).value + var videoSampleTimingInfo = CMSampleTimingInfo(duration: CMTimeMake(value: 1, timescale: 1), + presentationTimeStamp: CMTimeMake(value: pts, timescale: 90000), + decodeTimeStamp: CMTimeMake(value: dts, timescale: 90000)) + var sampleBuffer: CMSampleBuffer? + var sampleSize = [sample.data().count] + CMSampleBufferCreate(allocator: kCFAllocatorDefault, + dataBuffer: dataBuffer, + dataReady: true, + makeDataReadyCallback: nil, + refcon: nil, + formatDescription: formatDesc, + sampleCount: 1, + sampleTimingEntryCount: 1, + sampleTimingArray: &videoSampleTimingInfo, + sampleSizeEntryCount: 1, + sampleSizeArray: &sampleSize, + sampleBufferOut: &sampleBuffer) + if let sampleBuffer = sampleBuffer { + let flags: VTDecodeFrameFlags = [._1xRealTimePlayback, + ._EnableTemporalProcessing, + ._EnableAsynchronousDecompression] + let result = VTDecompressionSessionDecodeFrame(session, + sampleBuffer: sampleBuffer, + flags: flags, + infoFlagsOut: nil) { [weak self] (_, _, imgBuffer, pts, _) in + guard let strongSelf = self, let imgBuffer = imgBuffer else { + return + } + let ptsTime = TimePoint(pts.value, Int64(pts.timescale)) + let img = ImageBuffer(imgBuffer) + let pic = PictureSample(img, + assetId: sample.assetId(), + workspaceId: sample.workspaceId(), + time: strongSelf.clock.current(), + pts: ptsTime) + strongSelf.sampleQueue.sync { + strongSelf.output.append(pic) + strongSelf.output = strongSelf + .output.sorted { $0.pts() < $1.pts() } + } } + if result != 0 { + print("result=\(result)") + return .error(EventError("dec.video.apple", Int(result), "OSStatus, decode failed")) } - } catch { - // error - return .error(EventError("dec.video.apple", -4, "Caught decode error \(error)")) } - return self.sampleQueue.sync { if let outSample = self.output[safe:0], self.output.count >= 5 { self.output.removeFirst() @@ -152,19 +143,23 @@ public class AppleVideoDecoder: Tx { } private func setupSession(_ sample: CodedMediaSample) throws { let desc = try basicMediaDescription(sample) - guard case .video(let videoDesc) = desc, let dcr = sample.sideData()["config"] else { + guard case .video(let videoDesc) = desc else { return } let pixelBufferOptions = [kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, kCVPixelBufferWidthKey as String: Int(videoDesc.size.x), kCVPixelBufferHeightKey as String: Int(videoDesc.size.y)] as CFDictionary - let paramSets = parameterSetsFromAVCC(dcr) - let formatDesc = try videoFormatFromAVCParameterSets(paramSets) + formatDesc = try videoFormatDescription(sample) if let format = formatDesc { var session: VTDecompressionSession? let decoderSpecification = [:] as CFDictionary + #if os(macOS) + if #available(macOS 11.0, *) { + VTRegisterSupplementalVideoDecoderIfAvailable(kCMVideoCodecType_VP9) + } + #endif let result = VTDecompressionSessionCreate(allocator: kCFAllocatorDefault, formatDescription: format, decoderSpecification: decoderSpecification, @@ -180,11 +175,45 @@ public class AppleVideoDecoder: Tx { } private var output: [PictureSample] private var session: VTDecompressionSession? + private var formatDesc: CMVideoFormatDescription? private let clock: Clock private let sampleQueue: DispatchQueue private let decodeQueue: DispatchQueue } +private func videoFormatDescription(_ sample: CodedMediaSample) throws -> CMVideoFormatDescription? { + switch(sample.mediaFormat()) { + case .avc: + guard let dcr = sample.sideData()["config"] else { + throw MediaDescriptionError.invalidMetadata + } + let paramSets = parameterSetsFromAVCC(dcr) + return try videoFormatFromAVCParameterSets(paramSets) + case .vp9: + return try videoFormatFromVP9(sample) + default: + throw DecodeError.unsupportedCodec + } +} + +private func videoFormatFromVP9(_ sample: CodedMediaSample) throws -> CMVideoFormatDescription? { + let desc = try basicMediaDescription(sample) + guard case .video(let videoDesc) = desc else { + return nil + } + var formatDesc: CMVideoFormatDescription? + let extensions = [ + kCMFormatDescriptionExtension_FullRangeVideo as String: videoDesc.fullRangeColor + ] as CFDictionary + _ = CMVideoFormatDescriptionCreate(allocator: kCFAllocatorDefault, + codecType: kCMVideoCodecType_VP9, + width: Int32(videoDesc.size.x), + height: Int32(videoDesc.size.y), + extensions: extensions, + formatDescriptionOut: &formatDesc) + return formatDesc +} + private func videoFormatFromAVCParameterSets( _ paramSets: [[UInt8]]) throws -> CMVideoFormatDescription? { let pmSetPtrs: [UnsafePointer] = try paramSets.map { try $0.withUnsafeBufferPointer { diff --git a/Sources/SwiftVideo/enc.video.apple.swift b/Sources/SwiftVideo/enc.video.apple.swift index 72d55a5..c5b5860 100644 --- a/Sources/SwiftVideo/enc.video.apple.swift +++ b/Sources/SwiftVideo/enc.video.apple.swift @@ -26,13 +26,24 @@ public class AppleVideoEncoder: Tx { } public init(format: MediaFormat, frame: CGSize, bitrate: Int = 500000) { - assert(format == .avc || format == .hevc, "AppleVideoEncoder only supports AVC and HEVC") + //assert(format == .avc || format == .hevc, "AppleVideoEncoder only supports AVC and HEVC") self.queue = DispatchQueue.init(label: "cezium.encode.video") - - VTCompressionSessionCreate(allocator: kCFAllocatorDefault, + let codecType = { () -> CMVideoCodecType in + switch format { + case .avc: + return kCMVideoCodecType_H264 + case .hevc: + return kCMVideoCodecType_HEVC + case .vp9: + return kCMVideoCodecType_VP9 + default: + return kCMVideoCodecType_H264 + } + }() + let result = VTCompressionSessionCreate(allocator: kCFAllocatorDefault, width: Int32(frame.width), height: Int32(frame.height), - codecType: format == .avc ? kCMVideoCodecType_H264 : kCMVideoCodecType_HEVC, + codecType: codecType, encoderSpecification: [kVTCompressionPropertyKey_ExpectedFrameRate: 30] as CFDictionary, imageBufferAttributes: pixelBufferOptions(frame), compressedDataAllocator: nil, diff --git a/Sources/SwiftVideo/sample.coded.swift b/Sources/SwiftVideo/sample.coded.swift index 5da6fd1..8787bf7 100644 --- a/Sources/SwiftVideo/sample.coded.swift +++ b/Sources/SwiftVideo/sample.coded.swift @@ -51,6 +51,10 @@ public enum EncoderSpecificSettings { public struct BasicVideoDescription { public let size: Vector2 + public let colorSpace: ColorSpace = .bt601 + public let subSampling: PixelFormat = .y420p // valid values: y420p, y422p, y444p, RGBA + public let bitDepth: Int = 8 + public let fullRangeColor: Bool = false } public struct BasicAudioDescription { @@ -197,6 +201,7 @@ extension CodedMediaSample: Event { enum MediaDescriptionError: Error { case unsupported case invalidMetadata + case needsKeyframe } func basicMediaDescription(_ sample: CodedMediaSample) throws -> BasicMediaDescription { @@ -224,6 +229,8 @@ func basicMediaDescription(_ sample: CodedMediaSample) throws -> BasicMediaDescr return .audio(BasicAudioDescription(sampleRate: Float(sampleRate), channelCount: Int(channels), samplesPerPacket: Int(samplesPerPacket))) + case .vp9: + return try mediaDescriptionVP9(sample) default: throw MediaDescriptionError.unsupported } @@ -238,6 +245,8 @@ public func isKeyframe(_ sample: CodedMediaSample) -> Bool { return isKeyframeAVC(sample) case .hevc: return false + case .vp9: + return isKeyframeVP9(sample) default: return false } @@ -251,6 +260,30 @@ private func isKeyframeAVC(_ sample: CodedMediaSample) -> Bool { return sample.data()[4] & 0x1f == 5 } +private func isKeyframeVP9(_ sample: CodedMediaSample) -> Bool { + guard sample.data().count >= 1 else { + return false + } + return sample.data().withUnsafeBytes { + var isKeyframe: Int32 = 0 + vp9_is_keyframe($0.baseAddress, Int64($0.count), &isKeyframe) + return isKeyframe == 1 + } +} + +private func mediaDescriptionVP9(_ sample: CodedMediaSample) throws -> BasicMediaDescription { + guard isKeyframe(sample) else { + throw MediaDescriptionError.needsKeyframe + } + let props: VP9FrameProperties = sample.data().withUnsafeBytes { + var props = VP9FrameProperties() + vp9_frame_properties($0.baseAddress, Int64($0.count), &props) + return props + } + // TODO: Fill out rest of video description + return .video(BasicVideoDescription(size: Vector2(Float(props.width), Float(props.height)))) +} + private func spsFromAVCDCR(_ sample: CodedMediaSample) throws -> Data { guard let record = sample.sideData()["config"], record.count > 8 else { diff --git a/Sources/SwiftVideo/sample.pict.swift b/Sources/SwiftVideo/sample.pict.swift index e924f9c..23ec42b 100644 --- a/Sources/SwiftVideo/sample.pict.swift +++ b/Sources/SwiftVideo/sample.pict.swift @@ -32,6 +32,15 @@ public enum PixelFormat { case invalid } +public enum ColorSpace { + case bt601 + case bt709 + case bt2020 + case smpte170 + case smpte240 + case sRGB +} + // swiftlint:disable identifier_name public enum Component { case r