diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc index 73040a485f..cb9892a9d2 100644 --- a/media/engine/simulcast_encoder_adapter.cc +++ b/media/engine/simulcast_encoder_adapter.cc @@ -951,6 +951,12 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const { encoder_info.scaling_settings = VideoEncoder::ScalingSettings::kOff; std::vector encoder_names; + // SEA can keep multiple stream contexts alive even when runtime bitrate + // allocation has paused all but one spatial layer. Track that state + // explicitly so we can preserve simulcast-specific aggregation while still + // forwarding the active encoder's single-layer adaptation hints. + size_t active_stream_count = 0; + std::optional active_stream_info; for (size_t i = 0; i < stream_contexts_.size(); ++i) { VideoEncoder::EncoderInfo encoder_impl_info = @@ -958,6 +964,11 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const { // Encoder name indicates names of all active sub-encoders. if (!stream_contexts_[i].is_paused()) { + // If exactly one layer stays unpaused after SetRates(), this is the + // encoder whose runtime adaptation fields should be exposed to the rest + // of WebRTC. + ++active_stream_count; + active_stream_info = encoder_impl_info; encoder_names.push_back(encoder_impl_info.implementation_name); } if (i == 0) { @@ -1011,6 +1022,27 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const { encoder_info.implementation_name += implementation_name_builder.Release(); } + if (active_stream_count == 1) { + RTC_DCHECK(active_stream_info.has_value()); + // Keep the aggregated SEA view for simulcast-specific fields such as + // implementation_name, fps_allocation and alignment, but make the adapter + // behave like single-layer publishing for the runtime-sensitive fields + // consumed by adaptation and frame handling. + encoder_info.scaling_settings = active_stream_info->scaling_settings; + encoder_info.supports_native_handle = + active_stream_info->supports_native_handle; + encoder_info.has_trusted_rate_controller = + active_stream_info->has_trusted_rate_controller; + encoder_info.is_hardware_accelerated = + active_stream_info->is_hardware_accelerated; + encoder_info.is_qp_trusted = active_stream_info->is_qp_trusted; + encoder_info.resolution_bitrate_limits = + active_stream_info->resolution_bitrate_limits; + encoder_info.min_qp = active_stream_info->min_qp; + encoder_info.preferred_pixel_formats = + active_stream_info->preferred_pixel_formats; + } + OverrideFromFieldTrial(&encoder_info); return encoder_info; diff --git a/media/engine/simulcast_encoder_adapter_unittest.cc b/media/engine/simulcast_encoder_adapter_unittest.cc index a332003054..16fd83ea9c 100644 --- a/media/engine/simulcast_encoder_adapter_unittest.cc +++ b/media/engine/simulcast_encoder_adapter_unittest.cc @@ -286,6 +286,7 @@ class MockVideoEncoder : public VideoEncoder { info.supports_simulcast = supports_simulcast_; info.is_qp_trusted = is_qp_trusted_; info.resolution_bitrate_limits = resolution_bitrate_limits; + info.min_qp = min_qp_; return info; } @@ -365,6 +366,8 @@ class MockVideoEncoder : public VideoEncoder { resolution_bitrate_limits = limits; } + void set_min_qp(std::optional min_qp) { min_qp_ = min_qp; } + bool supports_simulcast() const { return supports_simulcast_; } SdpVideoFormat video_format() const { return video_format_; } @@ -384,6 +387,7 @@ class MockVideoEncoder : public VideoEncoder { FramerateFractions fps_allocation_; bool supports_simulcast_ = false; std::optional is_qp_trusted_; + std::optional min_qp_; SdpVideoFormat video_format_; std::vector resolution_bitrate_limits; @@ -1692,6 +1696,143 @@ TEST_F(TestSimulcastEncoderAdapterFake, ReportsFpsAllocation) { ::testing::ElementsAreArray(expected_fps_allocation)); } +TEST_F(TestSimulcastEncoderAdapterFake, + ForwardsRuntimeSensitiveEncoderInfoForSingleUnpausedLayer) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 3; + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->RegisterEncodeCompleteCallback(this); + ASSERT_EQ(3u, helper_->factory()->encoders().size()); + + auto* low_encoder = helper_->factory()->encoders()[0]; + auto* mid_encoder = helper_->factory()->encoders()[1]; + auto* high_encoder = helper_->factory()->encoders()[2]; + + low_encoder->set_scaling_settings(VideoEncoder::ScalingSettings(10, 20, 111)); + low_encoder->set_supports_native_handle(false); + low_encoder->set_is_qp_trusted(true); + low_encoder->set_resolution_bitrate_limits( + {VideoEncoder::ResolutionBitrateLimits(111, 1111, 2222, 3333)}); + low_encoder->set_min_qp(10); + low_encoder->set_fps_allocation( + FramerateFractions{EncoderInfo::kMaxFramerateFraction / 2}); + + mid_encoder->set_scaling_settings(VideoEncoder::ScalingSettings(30, 40, 222)); + mid_encoder->set_supports_native_handle(true); + mid_encoder->set_is_qp_trusted(false); + mid_encoder->set_resolution_bitrate_limits( + {VideoEncoder::ResolutionBitrateLimits(222, 4444, 5555, 6666)}); + mid_encoder->set_min_qp(20); + mid_encoder->set_fps_allocation( + FramerateFractions{EncoderInfo::kMaxFramerateFraction / 3, + EncoderInfo::kMaxFramerateFraction}); + + high_encoder->set_scaling_settings( + VideoEncoder::ScalingSettings(50, 60, 333)); + high_encoder->set_supports_native_handle(false); + high_encoder->set_is_qp_trusted(true); + high_encoder->set_resolution_bitrate_limits( + {VideoEncoder::ResolutionBitrateLimits(333, 7777, 8888, 9999)}); + high_encoder->set_min_qp(30); + high_encoder->set_fps_allocation( + FramerateFractions{EncoderInfo::kMaxFramerateFraction}); + + // Only keep the middle spatial layer active. SEA still has three stream + // contexts, so this exercises the runtime state that used to incorrectly + // report aggregated simulcast encoder info with scaling disabled. + VideoBitrateAllocation allocation; + ASSERT_TRUE(allocation.SetBitrate(1, 0, 500000)); + adapter_->SetRates(VideoEncoder::RateControlParameters(allocation, 30.0)); + + const auto info = adapter_->GetEncoderInfo(); + // Runtime-sensitive fields should come from the only unpaused encoder. + ASSERT_TRUE(info.scaling_settings.thresholds.has_value()); + EXPECT_EQ(30, info.scaling_settings.thresholds->low); + EXPECT_EQ(40, info.scaling_settings.thresholds->high); + EXPECT_EQ(222, info.scaling_settings.min_pixels_per_frame); + EXPECT_TRUE(info.supports_native_handle); + EXPECT_EQ(std::optional(false), info.is_qp_trusted); + EXPECT_EQ(std::optional(20), info.min_qp); + EXPECT_EQ(info.resolution_bitrate_limits, + std::vector( + {VideoEncoder::ResolutionBitrateLimits(222, 4444, 5555, 6666)})); + // Simulcast-specific fields must remain in SEA's aggregated spatial-slot + // layout even when runtime-sensitive fields are forwarded from one encoder. + EXPECT_THAT(info.fps_allocation[0], + ::testing::ElementsAre(EncoderInfo::kMaxFramerateFraction / 2)); + EXPECT_THAT(info.fps_allocation[1], + ::testing::ElementsAre(EncoderInfo::kMaxFramerateFraction / 3, + EncoderInfo::kMaxFramerateFraction)); + EXPECT_THAT(info.fps_allocation[2], + ::testing::ElementsAre(EncoderInfo::kMaxFramerateFraction)); +} + +TEST_F(TestSimulcastEncoderAdapterFake, + RestoresAggregatedEncoderInfoWhenMultipleLayersUnpause) { + SimulcastTestFixtureImpl::DefaultSettings( + &codec_, static_cast(kTestTemporalLayerProfile), + kVideoCodecVP8); + codec_.numberOfSimulcastStreams = 3; + EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); + adapter_->RegisterEncodeCompleteCallback(this); + ASSERT_EQ(3u, helper_->factory()->encoders().size()); + + auto* low_encoder = helper_->factory()->encoders()[0]; + auto* mid_encoder = helper_->factory()->encoders()[1]; + auto* high_encoder = helper_->factory()->encoders()[2]; + + low_encoder->set_scaling_settings(VideoEncoder::ScalingSettings(10, 20, 111)); + low_encoder->set_supports_native_handle(false); + low_encoder->set_fps_allocation( + FramerateFractions{EncoderInfo::kMaxFramerateFraction / 2}); + + mid_encoder->set_scaling_settings(VideoEncoder::ScalingSettings(30, 40, 222)); + mid_encoder->set_supports_native_handle(true); + mid_encoder->set_fps_allocation( + FramerateFractions{EncoderInfo::kMaxFramerateFraction / 3, + EncoderInfo::kMaxFramerateFraction}); + + high_encoder->set_scaling_settings( + VideoEncoder::ScalingSettings(50, 60, 333)); + high_encoder->set_supports_native_handle(false); + high_encoder->set_fps_allocation( + FramerateFractions{EncoderInfo::kMaxFramerateFraction}); + + // First collapse to a single active spatial layer and verify the forwarded + // encoder info. + VideoBitrateAllocation one_layer_allocation; + ASSERT_TRUE(one_layer_allocation.SetBitrate(1, 0, 500000)); + adapter_->SetRates( + VideoEncoder::RateControlParameters(one_layer_allocation, 30.0)); + + auto info = adapter_->GetEncoderInfo(); + ASSERT_TRUE(info.scaling_settings.thresholds.has_value()); + EXPECT_EQ(30, info.scaling_settings.thresholds->low); + EXPECT_EQ(40, info.scaling_settings.thresholds->high); + EXPECT_TRUE(info.supports_native_handle); + + // Then enable another layer. SEA should immediately return to its normal + // aggregated simulcast view without requiring a re-init. + VideoBitrateAllocation two_layer_allocation; + ASSERT_TRUE(two_layer_allocation.SetBitrate(1, 0, 500000)); + ASSERT_TRUE(two_layer_allocation.SetBitrate(2, 0, 700000)); + adapter_->SetRates( + VideoEncoder::RateControlParameters(two_layer_allocation, 30.0)); + + info = adapter_->GetEncoderInfo(); + EXPECT_FALSE(info.scaling_settings.thresholds.has_value()); + EXPECT_TRUE(info.supports_native_handle); + EXPECT_THAT(info.fps_allocation[0], + ::testing::ElementsAre(EncoderInfo::kMaxFramerateFraction / 2)); + EXPECT_THAT(info.fps_allocation[1], + ::testing::ElementsAre(EncoderInfo::kMaxFramerateFraction / 3, + EncoderInfo::kMaxFramerateFraction)); + EXPECT_THAT(info.fps_allocation[2], + ::testing::ElementsAre(EncoderInfo::kMaxFramerateFraction)); +} + TEST_F(TestSimulcastEncoderAdapterFake, SetRateDistributesBandwithAllocation) { SimulcastTestFixtureImpl::DefaultSettings( &codec_, static_cast(kTestTemporalLayerProfile), diff --git a/sdk/objc/native/src/objc_video_encoder_factory.mm b/sdk/objc/native/src/objc_video_encoder_factory.mm index ce6d124532..fd75d72afa 100644 --- a/sdk/objc/native/src/objc_video_encoder_factory.mm +++ b/sdk/objc/native/src/objc_video_encoder_factory.mm @@ -134,8 +134,17 @@ void SetRates(const RateControlParameters ¶meters) override { RTC_OBJC_TYPE(RTCVideoEncoderQpThresholds) *qp_thresholds = [encoder_ scalingSettings]; + // The default kDefaultMinPixelsPerFrame (320*180 = 57600) is too + // conservative for iOS hardware encoders (VideoToolbox). It prevents + // the quality scaler from reducing resolution below ~180x320 (the + // quarter-resolution simulcast layer for 720p), which limits adaptation + // range on constrained networks (e.g. 3G). VideoToolbox can encode at + // resolutions well below this threshold, so we use a lower floor to + // allow meaningful quality adaptation in bandwidth-limited scenarios. + constexpr int kObjCEncoderMinPixelsPerFrame = 90 * 160; info.scaling_settings = qp_thresholds ? - ScalingSettings(qp_thresholds.low, qp_thresholds.high) : + ScalingSettings(qp_thresholds.low, qp_thresholds.high, + kObjCEncoderMinPixelsPerFrame) : ScalingSettings::kOff; info.requested_resolution_alignment = encoder_.resolutionAlignment > 0 ?: 1; diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc index ae575de690..eaef23e77e 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.cc +++ b/video/adaptation/video_stream_encoder_resource_manager.cc @@ -559,16 +559,86 @@ void VideoStreamEncoderResourceManager::UpdateBandwidthQualityScalerSettings( } } +void VideoStreamEncoderResourceManager::ResetAdaptationsForSimulcastChange() { + RTC_DCHECK_RUN_ON(encoder_queue_); + std::vector> quality_resources; + for (const auto& resource_and_reason : resources_) { + if (resource_and_reason.second == VideoAdaptationReason::kQuality) { + quality_resources.push_back(resource_and_reason.first); + } + } + + if (quality_resources.empty()) { + return; + } + + for (const auto& resource : quality_resources) { + if (resource == quality_scaler_resource_) { + RTC_LOG(LS_INFO) << "Clearing quality scaler restrictions for simulcast " + "active-layer transition."; + quality_scaler_resource_->StopCheckForOveruse(); + RemoveResource(resource); + initial_frame_dropper_->OnQualityScalerSettingsUpdated(); + } else if (resource == bandwidth_quality_scaler_resource_) { + RTC_LOG(LS_INFO) + << "Clearing bandwidth quality scaler restrictions for simulcast " + "active-layer transition."; + bandwidth_quality_scaler_resource_->StopCheckForOveruse(); + RemoveResource(resource); + } else { + RTC_LOG(LS_INFO) << "Resetting quality adaptation resource \"" + << resource->Name() + << "\" for simulcast active-layer transition."; + RemoveResource(resource); + AddResource(resource, VideoAdaptationReason::kQuality); + } + } + + UpdateStatsAdaptationSettings(); +} + +void VideoStreamEncoderResourceManager::SetSimulcastActive( + bool simulcast_active) { + RTC_DCHECK_RUN_ON(encoder_queue_); + if (simulcast_active_ != simulcast_active) { + RTC_LOG(LS_INFO) << "SetSimulcastActive: " << simulcast_active_ + << " -> " << simulcast_active; + } + simulcast_active_ = simulcast_active; +} + void VideoStreamEncoderResourceManager::ConfigureQualityScaler( const VideoEncoder::EncoderInfo& encoder_info) { RTC_DCHECK_RUN_ON(encoder_queue_); const auto scaling_settings = encoder_info.scaling_settings; + // Quality scaling (adapting input resolution based on encoded QP) is allowed + // only when ALL of the following are true: + // 1. The degradation preference permits resolution changes. + // 2. Simulcast is NOT active — during simulcast, the SFU handles quality + // adaptation by activating/deactivating layers. Allowing the quality + // scaler to run would shrink the input resolution and cause all + // simulcast layer dimensions to be recomputed from degraded input. + // 3. The encoder reports QP thresholds or the encoder config explicitly + // allows quality scaling (is_quality_scaling_allowed). + // 4. The encoder's QP values are trusted for scaling decisions. + const bool resolution_scaling_enabled = + IsResolutionScalingEnabled(degradation_preference_); + const bool has_scaling_thresholds = + scaling_settings.thresholds.has_value() || + (encoder_settings_.has_value() && + encoder_settings_->encoder_config().is_quality_scaling_allowed); + const bool qp_trusted = encoder_info.is_qp_trusted.value_or(true); const bool quality_scaling_allowed = - IsResolutionScalingEnabled(degradation_preference_) && - (scaling_settings.thresholds.has_value() || - (encoder_settings_.has_value() && - encoder_settings_->encoder_config().is_quality_scaling_allowed)) && - encoder_info.is_qp_trusted.value_or(true); + resolution_scaling_enabled && !simulcast_active_ && + has_scaling_thresholds && qp_trusted; + + RTC_LOG(LS_INFO) << "ConfigureQualityScaler: allowed=" << quality_scaling_allowed + << " (resolution_scaling=" << resolution_scaling_enabled + << ", simulcast_active=" << simulcast_active_ + << ", has_thresholds=" << has_scaling_thresholds + << ", qp_trusted=" << qp_trusted + << ", scaler_running=" << quality_scaler_resource_->is_started() + << ")"; // TODO(https://crbug.com/webrtc/11222): Should this move to // QualityScalerResource? @@ -609,6 +679,7 @@ void VideoStreamEncoderResourceManager::ConfigureBandwidthQualityScaler( RTC_DCHECK_RUN_ON(encoder_queue_); const bool bandwidth_quality_scaling_allowed = IsResolutionScalingEnabled(degradation_preference_) && + !simulcast_active_ && (encoder_settings_.has_value() && encoder_settings_->encoder_config().is_quality_scaling_allowed) && !encoder_info.is_qp_trusted.value_or(true); diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h index 1520bd5aef..b448aa85d5 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.h +++ b/video/adaptation/video_stream_encoder_resource_manager.h @@ -109,6 +109,21 @@ class VideoStreamEncoderResourceManager // TODO(https://crbug.com/webrtc/11338): This can be made private if we // configure on SetDegredationPreference and SetEncoderSettings. void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info); + + // Controls whether the quality scaler is suppressed due to simulcast. + // When multiple simulcast layers are active, resolution-based quality + // adaptation must be disabled because the quality scaler would shrink the + // input resolution, causing all simulcast layer dimensions to be derived + // from the degraded input rather than the original capture resolution. + // In simulcast mode, quality adaptation is handled by the SFU + // activating/deactivating layers instead. + void SetSimulcastActive(bool simulcast_active); + + // Stops the quality scaler and clears its accumulated adaptation + // restrictions. Called when the number of active simulcast layers increases + // from <=1 to >1, so that the source provides full-resolution frames for + // the new multi-layer configuration. + void ResetAdaptationsForSimulcastChange(); void ConfigureBandwidthQualityScaler( const VideoEncoder::EncoderInfo& encoder_info); @@ -222,6 +237,13 @@ class VideoStreamEncoderResourceManager std::optional encoder_settings_ RTC_GUARDED_BY(encoder_queue_); + // True when multiple simulcast layers are active. While set, the quality + // scaler is suppressed to prevent input resolution degradation that would + // corrupt simulcast layer dimensions. Set by VideoStreamEncoder from the + // current runtime layer allocation, with encoder-config fallback before the + // first SetRates() arrives. + bool simulcast_active_ RTC_GUARDED_BY(encoder_queue_) = false; + // Ties a resource to a reason for statistical reporting. This AdaptReason is // also used by this module to make decisions about how to adapt up/down. std::map, VideoAdaptationReason> resources_ diff --git a/video/config/encoder_stream_factory.cc b/video/config/encoder_stream_factory.cc index 558427478a..2baf10943b 100644 --- a/video/config/encoder_stream_factory.cc +++ b/video/config/encoder_stream_factory.cc @@ -69,13 +69,15 @@ bool IsTemporalLayersSupported(VideoCodecType codec_type) { } size_t FindRequiredActiveLayers(const VideoEncoderConfig& encoder_config) { - // Need enough layers so that at least the first active one is present. + // Need enough layers so that all active ones are present. + // Return the position after the highest active layer. + size_t highest = 0; for (size_t i = 0; i < encoder_config.number_of_streams; ++i) { if (encoder_config.simulcast_layers[i].active) { - return i + 1; + highest = i + 1; } } - return 0; + return highest; } // The selected thresholds for QVGA and VGA corresponded to a QP around 10. diff --git a/video/config/encoder_stream_factory_unittest.cc b/video/config/encoder_stream_factory_unittest.cc index a36efaab01..85f28a500c 100644 --- a/video/config/encoder_stream_factory_unittest.cc +++ b/video/config/encoder_stream_factory_unittest.cc @@ -340,12 +340,68 @@ TEST(EncoderStreamFactory, ReducesStreamCountWhenResolutionIsLow) { SizeIs(1)); } -TEST(EncoderStreamFactory, ReducesStreamCountDownToFirstActiveStream) { +TEST(EncoderStreamFactory, KeepsStreamCountToIncludeHighestActiveLayer) { EXPECT_THAT( CreateStreamResolutions({.number_of_streams = 3, .resolution = {.width = 100, .height = 100}, .first_active_layer_idx = 1}), - SizeIs(2)); + SizeIs(3)); +} + +TEST(EncoderStreamFactory, KeepsAllStreamsForSparseActivePattern) { + ExplicitKeyValueConfig field_trials(""); + VideoEncoderConfig encoder_config; + encoder_config.codec_type = VideoCodecType::kVideoCodecVP8; + encoder_config.number_of_streams = 3; + encoder_config.simulcast_layers.resize(3); + encoder_config.simulcast_layers[0].active = true; + encoder_config.simulcast_layers[1].active = false; + encoder_config.simulcast_layers[2].active = true; + auto streams = CreateEncoderStreams( + field_trials, {.width = 100, .height = 100}, encoder_config); + EXPECT_THAT(streams, SizeIs(3)); +} + +TEST(EncoderStreamFactory, KeepsStreamsForHighAndLowActive) { + ExplicitKeyValueConfig field_trials(""); + VideoEncoderConfig encoder_config; + encoder_config.codec_type = VideoCodecType::kVideoCodecVP8; + encoder_config.number_of_streams = 3; + encoder_config.simulcast_layers.resize(3); + encoder_config.simulcast_layers[0].active = true; + encoder_config.simulcast_layers[1].active = true; + encoder_config.simulcast_layers[2].active = false; + auto streams = CreateEncoderStreams( + field_trials, {.width = 100, .height = 100}, encoder_config); + EXPECT_THAT(streams, SizeIs(2)); +} + +TEST(EncoderStreamFactory, KeepsStreamsForOnlyHighestActive) { + ExplicitKeyValueConfig field_trials(""); + VideoEncoderConfig encoder_config; + encoder_config.codec_type = VideoCodecType::kVideoCodecVP8; + encoder_config.number_of_streams = 3; + encoder_config.simulcast_layers.resize(3); + encoder_config.simulcast_layers[0].active = false; + encoder_config.simulcast_layers[1].active = false; + encoder_config.simulcast_layers[2].active = true; + auto streams = CreateEncoderStreams( + field_trials, {.width = 100, .height = 100}, encoder_config); + EXPECT_THAT(streams, SizeIs(3)); +} + +TEST(EncoderStreamFactory, ReducesToOneStreamWhenOnlyLowestActive) { + ExplicitKeyValueConfig field_trials(""); + VideoEncoderConfig encoder_config; + encoder_config.codec_type = VideoCodecType::kVideoCodecVP8; + encoder_config.number_of_streams = 3; + encoder_config.simulcast_layers.resize(3); + encoder_config.simulcast_layers[0].active = true; + encoder_config.simulcast_layers[1].active = false; + encoder_config.simulcast_layers[2].active = false; + auto streams = CreateEncoderStreams( + field_trials, {.width = 100, .height = 100}, encoder_config); + EXPECT_THAT(streams, SizeIs(1)); } TEST(EncoderStreamFactory, diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 5ddaf24f66..dbcea866d8 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -1059,6 +1059,8 @@ void VideoStreamEncoder::ReconfigureEncoder() { AlignmentAdjuster::GetAlignmentAndMaybeAdjustScaleFactors( encoder_->GetEncoderInfo(), &encoder_config_, std::nullopt); + UpdateSimulcastAdaptationState(GetNumActiveSimulcastLayers()); + std::vector streams; if (encoder_config_.video_stream_factory) { // Note: only tests set their own EncoderStreamFactory... @@ -1760,6 +1762,8 @@ void VideoStreamEncoder::SetEncoderRates( last_encoder_rate_settings_ = rate_settings; } + UpdateSimulcastAdaptationState(GetNumActiveSimulcastLayers()); + if (!encoder_) return; @@ -1812,6 +1816,78 @@ void VideoStreamEncoder::SetEncoderRates( } } +size_t VideoStreamEncoder::GetNumActiveSimulcastLayers() const { + RTC_DCHECK_RUN_ON(encoder_queue_.get()); + if (last_encoder_rate_settings_.has_value()) { + size_t num_active_layers = 0; + for (size_t i = 0; i < encoder_config_.number_of_streams; ++i) { + if (last_encoder_rate_settings_->rate_control.target_bitrate + .GetSpatialLayerSum(i) > 0) { + ++num_active_layers; + } + } + return num_active_layers; + } + + size_t num_active_layers = 0; + for (size_t i = 0; i < encoder_config_.number_of_streams; ++i) { + if (encoder_config_.simulcast_layers[i].active) { + ++num_active_layers; + } + } + return num_active_layers; +} + +void VideoStreamEncoder::UpdateSimulcastAdaptationState( + size_t num_active_layers) { + RTC_DCHECK_RUN_ON(encoder_queue_.get()); + // Determine whether we are effectively in simulcast mode based on the + // number of active runtime layers, not only the configured streams. The + // encoder config may keep 3 streams configured while SetRates() collapses + // publishing down to a single layer, or expands it back to simulcast, + // without a fresh ConfigureEncoder() call. + const bool is_simulcast = num_active_layers > 1; + const bool was_simulcast = prev_num_active_simulcast_layers_ > 1; + + RTC_LOG(LS_INFO) << "[VSE] Simulcast state: active_layers=" + << num_active_layers + << ", prev_active_layers=" + << prev_num_active_simulcast_layers_ + << ", streams=" << encoder_config_.number_of_streams + << ", is_simulcast=" << is_simulcast + << ", was_simulcast=" << was_simulcast; + + if (is_simulcast && + (!was_simulcast || + num_active_layers > prev_num_active_simulcast_layers_)) { + // Whenever the active simulcast layer count increases, disable the + // quality scaler and clear any accumulated resolution restrictions so the + // source can return to full-resolution input for the larger runtime layer + // set. The guard on prev > 0 avoids clearing on the very first encoder + // configuration where no restrictions exist yet. + RTC_LOG(LS_INFO) << "[VSE] Active-layer increase -> simulcast: disabling " + "quality scaler, clearing accumulated restrictions."; + stream_resource_manager_.SetSimulcastActive(true); + if (prev_num_active_simulcast_layers_ > 0) { + stream_resource_manager_.ResetAdaptationsForSimulcastChange(); + } + } else if (!is_simulcast && was_simulcast) { + RTC_LOG(LS_INFO) << "[VSE] Transition -> single stream: quality scaler " + "will be re-enabled."; + stream_resource_manager_.SetSimulcastActive(false); + } else if (is_simulcast) { + RTC_LOG(LS_INFO) << "[VSE] Staying in simulcast: quality scaler remains " + "disabled."; + stream_resource_manager_.SetSimulcastActive(true); + } else { + RTC_LOG(LS_INFO) << "[VSE] Staying in single stream: quality scaler " + "remains enabled."; + stream_resource_manager_.SetSimulcastActive(false); + } + + prev_num_active_simulcast_layers_ = num_active_layers; +} + void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame, int64_t time_when_posted_us) { RTC_DCHECK_RUN_ON(encoder_queue_.get()); diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 917d928149..c8217a9d57 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -245,6 +245,9 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, uint32_t GetInputFramerateFps() RTC_RUN_ON(encoder_queue_); void SetEncoderRates(const EncoderRateSettings& rate_settings) RTC_RUN_ON(encoder_queue_); + size_t GetNumActiveSimulcastLayers() const RTC_RUN_ON(encoder_queue_); + void UpdateSimulcastAdaptationState(size_t num_active_layers) + RTC_RUN_ON(encoder_queue_); void RunPostEncode(const EncodedImage& encoded_image, int64_t time_sent_us, @@ -313,6 +316,10 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, std::optional last_frame_info_ RTC_GUARDED_BY(encoder_queue_); int crop_width_ RTC_GUARDED_BY(encoder_queue_) = 0; int crop_height_ RTC_GUARDED_BY(encoder_queue_) = 0; + // Tracks the runtime number of active spatial layers so quality-scaler + // transitions follow the current rate allocation, not only the latest + // encoder configuration. + size_t prev_num_active_simulcast_layers_ RTC_GUARDED_BY(encoder_queue_) = 0; std::optional encoder_target_bitrate_bps_ RTC_GUARDED_BY(encoder_queue_); size_t max_data_payload_length_ RTC_GUARDED_BY(encoder_queue_) = 0; diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index 17f4ea1a2c..150654f926 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -10,6 +10,7 @@ #include "video/video_stream_encoder.h" #include +#include #include #include #include @@ -177,6 +178,40 @@ const uint8_t kCodedFrameVp8Qp25[] = { const DataRate kDefaultH265Bitrate180p = DataRate::KilobitsPerSec(150); #endif +VideoEncoderConfig Make3LayerVp8SimulcastConfig( + const std::array& active_layers) { + VideoEncoderConfig config; + test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3, &config); + config.video_stream_factory = nullptr; + for (size_t i = 0; i < active_layers.size(); ++i) { + config.simulcast_layers[i].active = active_layers[i]; + config.simulcast_layers[i].num_temporal_layers = 1; + config.simulcast_layers[i].max_framerate = kDefaultFramerate; + } + config.max_bitrate_bps = kSimulcastTargetBitrate.bps(); + config.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo; + return config; +} + +size_t CountActiveSpatialLayers(const VideoBitrateAllocation& bitrate, + size_t num_streams) { + size_t active_layers = 0; + for (size_t i = 0; i < num_streams; ++i) { + if (bitrate.GetSpatialLayerSum(i) > 0) { + ++active_layers; + } + } + return active_layers; +} + +bool HasAdaptationResourceNamed( + const std::vector>& resources, + const char* name) { + return absl::c_any_of(resources, [name](const scoped_refptr& r) { + return r->Name() == name; + }); +} + VideoFrame CreateSimpleNV12Frame() { return VideoFrame::Builder() .set_video_frame_buffer(make_ref_counted( @@ -1120,8 +1155,13 @@ class VideoStreamEncoderTest : public ::testing::Test { EncoderInfo info = FakeEncoder::GetEncoderInfo(); if (initialized_ == EncoderState::kInitialized) { if (quality_scaling_) { - info.scaling_settings = VideoEncoder::ScalingSettings( - kQpLow, kQpHigh, kMinPixelsPerFrame); + const bool allow_quality_scaling = + !quality_scaling_follows_active_spatial_layers_ || + active_spatial_layers_ <= 1; + info.scaling_settings = allow_quality_scaling + ? VideoEncoder::ScalingSettings( + kQpLow, kQpHigh, kMinPixelsPerFrame) + : VideoEncoder::ScalingSettings::kOff; } info.is_hardware_accelerated = is_hardware_accelerated_; for (int i = 0; i < kMaxSpatialLayers; ++i) { @@ -1166,6 +1206,11 @@ class VideoStreamEncoderTest : public ::testing::Test { quality_scaling_ = b; } + void SetQualityScalingFollowsActiveSpatialLayers(bool enabled) { + MutexLock lock(&local_mutex_); + quality_scaling_follows_active_spatial_layers_ = enabled; + } + void SetRequestedResolutionAlignment( uint32_t requested_resolution_alignment) { MutexLock lock(&local_mutex_); @@ -1377,6 +1422,8 @@ class VideoStreamEncoderTest : public ::testing::Test { void SetRates(const RateControlParameters& parameters) { MutexLock lock(&local_mutex_); num_set_rates_++; + active_spatial_layers_ = + CountActiveSpatialLayers(parameters.bitrate, kMaxSpatialLayers); VideoBitrateAllocation adjusted_rate_allocation; for (size_t si = 0; si < kMaxSpatialLayers; ++si) { for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) { @@ -1407,6 +1454,9 @@ class VideoStreamEncoderTest : public ::testing::Test { int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0; int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0; bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true; + bool quality_scaling_follows_active_spatial_layers_ + RTC_GUARDED_BY(local_mutex_) = false; + size_t active_spatial_layers_ RTC_GUARDED_BY(local_mutex_) = 0; uint32_t requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1; bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) = false; @@ -6815,6 +6865,174 @@ TEST_F(VideoStreamEncoderTest, video_stream_encoder_->Stop(); } +TEST_F(VideoStreamEncoderTest, + QualityScalerRestrictionsResetWhenActiveLayersIncrease) { + // Set up 3-stream simulcast with only the highest layer active (1:1 call). + ResetEncoder("VP8", 3, 1, 1, false); + fake_encoder_.SetQualityScaling(true); + const int kWidth = 1280; + const int kHeight = 720; + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate, + 0, 0, 0); + + VideoEncoderConfig config_1_active = + Make3LayerVp8SimulcastConfig({false, false, true}); + video_stream_encoder_->ConfigureEncoder(config_1_active.Copy(), + kMaxPayloadLength); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + + // Send a frame and trigger quality low to accumulate a resolution adaptation. + video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight)); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + video_stream_encoder_->TriggerQualityLow(); + + video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight)); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + + EXPECT_THAT(video_source_.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight))); + + // Now transition to multi-layer: activate all 3 layers (3rd participant). + VideoEncoderConfig video_encoder_config = + Make3LayerVp8SimulcastConfig({true, true, true}); + video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(), + kMaxPayloadLength); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + AdvanceTime(TimeDelta::Zero()); + + // Quality scaler restrictions should have been cleared on the active-layer + // increase so the source can provide full-resolution frames. + EXPECT_THAT(video_source_.sink_wants(), ResolutionMax()); + + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, + QualityScalerRestrictionsResetOnTwoToThreeLayerIncrease) { + // Set up 3-stream simulcast with 2 layers active. + ResetEncoder("VP8", 3, 1, 1, false); + fake_encoder_.SetQualityScaling(true); + const int kWidth = 1280; + const int kHeight = 720; + + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate, + 0, 0, 0); + + // Start with 2 active layers. + VideoEncoderConfig config_2_active = + Make3LayerVp8SimulcastConfig({true, true, false}); + video_stream_encoder_->ConfigureEncoder(config_2_active.Copy(), + kMaxPayloadLength); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + + // Send a frame and trigger quality low to accumulate a restriction. + video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight)); + WaitForEncodedFrame(1); + video_stream_encoder_->TriggerQualityLow(); + + video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight)); + WaitForEncodedFrame(2); + + EXPECT_THAT(video_source_.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight))); + + // Now increase to 3 active layers. + VideoEncoderConfig config_3_active = + Make3LayerVp8SimulcastConfig({true, true, true}); + video_stream_encoder_->ConfigureEncoder(config_3_active.Copy(), + kMaxPayloadLength); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + AdvanceTime(TimeDelta::Zero()); + + // Restrictions should be cleared on the 2 -> 3 active-layer increase. + EXPECT_THAT(video_source_.sink_wants(), ResolutionMax()); + + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, + BandwidthQualityScalerRemovedWhenTransitioningToSimulcast) { + fake_encoder_.SetQualityScaling(false); + fake_encoder_.SetIsQpTrusted(false); + + VideoEncoderConfig single_layer_config = + Make3LayerVp8SimulcastConfig({false, false, true}); + single_layer_config.is_quality_scaling_allowed = true; + ConfigureEncoder(single_layer_config.Copy()); + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate, + 0, 0, 0); + video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + + EXPECT_TRUE(HasAdaptationResourceNamed( + video_stream_encoder_->GetAdaptationResources(), + "BandwidthQualityScalerResource")); + + VideoEncoderConfig simulcast_config = + Make3LayerVp8SimulcastConfig({true, true, true}); + simulcast_config.is_quality_scaling_allowed = true; + video_stream_encoder_->ConfigureEncoder(simulcast_config.Copy(), + kMaxPayloadLength); + video_stream_encoder_->WaitUntilTaskQueueIsIdle(); + + EXPECT_FALSE(HasAdaptationResourceNamed( + video_stream_encoder_->GetAdaptationResources(), + "BandwidthQualityScalerResource")); + + video_stream_encoder_->Stop(); +} + +TEST_F(VideoStreamEncoderTest, + QualityScalerResourceTracksRuntimeLayerAllocation) { + fake_encoder_.SetQualityScaling(true); + fake_encoder_.SetQualityScalingFollowsActiveSpatialLayers(true); + + VideoEncoderConfig simulcast_config = + Make3LayerVp8SimulcastConfig({true, true, true}); + ConfigureEncoder(simulcast_config.Copy()); + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate, + 0, 0, 0); + video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720)); + WaitForEncodedFrame(1); + + auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings(); + ASSERT_TRUE(rate_settings.has_value()); + EXPECT_GT(CountActiveSpatialLayers(rate_settings->bitrate, + simulcast_config.number_of_streams), + 1u); + + EXPECT_FALSE(HasAdaptationResourceNamed( + video_stream_encoder_->GetAdaptationResources(), "QualityScalerResource")); + + const std::array single_layer_rates = { + DataRate::KilobitsPerSec(100), DataRate::KilobitsPerSec(150), + DataRate::KilobitsPerSec(200), DataRate::KilobitsPerSec(300), + DataRate::KilobitsPerSec(500)}; + bool found_single_layer_rate = false; + for (DataRate rate : single_layer_rates) { + video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( + rate, rate, rate, 0, 0, 0); + rate_settings = fake_encoder_.GetAndResetLastRateControlSettings(); + ASSERT_TRUE(rate_settings.has_value()); + if (CountActiveSpatialLayers(rate_settings->bitrate, + simulcast_config.number_of_streams) == 1u) { + found_single_layer_rate = true; + break; + } + } + ASSERT_TRUE(found_single_layer_rate); + + video_source_.IncomingCapturedFrame(CreateFrame(2, 1280, 720)); + WaitForEncodedFrame(2); + EXPECT_TRUE(HasAdaptationResourceNamed( + video_stream_encoder_->GetAdaptationResources(), "QualityScalerResource")); + + video_stream_encoder_->Stop(); +} + TEST_F(VideoStreamEncoderTest, ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) { const int kTooSmallWidth = 10;