[Enhancement]Extend SimulcastEncoderAdapter to support adaptive streaming

media/engine/simulcast_encoder_adapter.cc

@@ -951,13 +951,24 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const {
 
   encoder_info.scaling_settings = VideoEncoder::ScalingSettings::kOff;
   std::vector<std::string> encoder_names;
+  // SEA can keep multiple stream contexts alive even when runtime bitrate
+  // allocation has paused all but one spatial layer. Track that state
+  // explicitly so we can preserve simulcast-specific aggregation while still
+  // forwarding the active encoder's single-layer adaptation hints.
+  size_t active_stream_count = 0;
+  std::optional<VideoEncoder::EncoderInfo> active_stream_info;
 
   for (size_t i = 0; i < stream_contexts_.size(); ++i) {
     VideoEncoder::EncoderInfo encoder_impl_info =
         stream_contexts_[i].encoder().GetEncoderInfo();
 
     // Encoder name indicates names of all active sub-encoders.
     if (!stream_contexts_[i].is_paused()) {
+      // If exactly one layer stays unpaused after SetRates(), this is the
+      // encoder whose runtime adaptation fields should be exposed to the rest
+      // of WebRTC.
+      ++active_stream_count;
+      active_stream_info = encoder_impl_info;
       encoder_names.push_back(encoder_impl_info.implementation_name);
     }
     if (i == 0) {
@@ -1011,6 +1022,27 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const {
     encoder_info.implementation_name += implementation_name_builder.Release();
   }
 
+  if (active_stream_count == 1) {
+    RTC_DCHECK(active_stream_info.has_value());
+    // Keep the aggregated SEA view for simulcast-specific fields such as
+    // implementation_name, fps_allocation and alignment, but make the adapter
+    // behave like single-layer publishing for the runtime-sensitive fields
+    // consumed by adaptation and frame handling.
+    encoder_info.scaling_settings = active_stream_info->scaling_settings;
+    encoder_info.supports_native_handle =
+        active_stream_info->supports_native_handle;
+    encoder_info.has_trusted_rate_controller =
+        active_stream_info->has_trusted_rate_controller;
+    encoder_info.is_hardware_accelerated =
+        active_stream_info->is_hardware_accelerated;
+    encoder_info.is_qp_trusted = active_stream_info->is_qp_trusted;
+    encoder_info.resolution_bitrate_limits =
+        active_stream_info->resolution_bitrate_limits;
+    encoder_info.min_qp = active_stream_info->min_qp;
+    encoder_info.preferred_pixel_formats =
+        active_stream_info->preferred_pixel_formats;
+  }
+
   OverrideFromFieldTrial(&encoder_info);
 
   return encoder_info;

media/engine/simulcast_encoder_adapter_unittest.cc

@@ -286,6 +286,7 @@ class MockVideoEncoder : public VideoEncoder {
     info.supports_simulcast = supports_simulcast_;
     info.is_qp_trusted = is_qp_trusted_;
     info.resolution_bitrate_limits = resolution_bitrate_limits;
+    info.min_qp = min_qp_;
     return info;
   }
 
@@ -365,6 +366,8 @@ class MockVideoEncoder : public VideoEncoder {
     resolution_bitrate_limits = limits;
   }
 
+  void set_min_qp(std::optional<int> min_qp) { min_qp_ = min_qp; }
+
   bool supports_simulcast() const { return supports_simulcast_; }
 
   SdpVideoFormat video_format() const { return video_format_; }
@@ -384,6 +387,7 @@ class MockVideoEncoder : public VideoEncoder {
   FramerateFractions fps_allocation_;
   bool supports_simulcast_ = false;
   std::optional<bool> is_qp_trusted_;
+  std::optional<int> min_qp_;
   SdpVideoFormat video_format_;
   std::vector<VideoEncoder::ResolutionBitrateLimits> resolution_bitrate_limits;
 
@@ -1692,6 +1696,143 @@ TEST_F(TestSimulcastEncoderAdapterFake, ReportsFpsAllocation) {
               ::testing::ElementsAreArray(expected_fps_allocation));
 }
 
+TEST_F(TestSimulcastEncoderAdapterFake,
+       ForwardsRuntimeSensitiveEncoderInfoForSingleUnpausedLayer) {
+  SimulcastTestFixtureImpl::DefaultSettings(
+      &codec_, static_cast<const int*>(kTestTemporalLayerProfile),
+      kVideoCodecVP8);
+  codec_.numberOfSimulcastStreams = 3;
+  EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings));
+  adapter_->RegisterEncodeCompleteCallback(this);
+  ASSERT_EQ(3u, helper_->factory()->encoders().size());
+
+  auto* low_encoder = helper_->factory()->encoders()[0];
+  auto* mid_encoder = helper_->factory()->encoders()[1];
+  auto* high_encoder = helper_->factory()->encoders()[2];
+
+  low_encoder->set_scaling_settings(VideoEncoder::ScalingSettings(10, 20, 111));
+  low_encoder->set_supports_native_handle(false);
+  low_encoder->set_is_qp_trusted(true);
+  low_encoder->set_resolution_bitrate_limits(
+      {VideoEncoder::ResolutionBitrateLimits(111, 1111, 2222, 3333)});
+  low_encoder->set_min_qp(10);
+  low_encoder->set_fps_allocation(
+      FramerateFractions{EncoderInfo::kMaxFramerateFraction / 2});
+
+  mid_encoder->set_scaling_settings(VideoEncoder::ScalingSettings(30, 40, 222));
+  mid_encoder->set_supports_native_handle(true);
+  mid_encoder->set_is_qp_trusted(false);
+  mid_encoder->set_resolution_bitrate_limits(
+      {VideoEncoder::ResolutionBitrateLimits(222, 4444, 5555, 6666)});
+  mid_encoder->set_min_qp(20);
+  mid_encoder->set_fps_allocation(
+      FramerateFractions{EncoderInfo::kMaxFramerateFraction / 3,
+                         EncoderInfo::kMaxFramerateFraction});
+
+  high_encoder->set_scaling_settings(
+      VideoEncoder::ScalingSettings(50, 60, 333));
+  high_encoder->set_supports_native_handle(false);
+  high_encoder->set_is_qp_trusted(true);
+  high_encoder->set_resolution_bitrate_limits(
+      {VideoEncoder::ResolutionBitrateLimits(333, 7777, 8888, 9999)});
+  high_encoder->set_min_qp(30);
+  high_encoder->set_fps_allocation(
+      FramerateFractions{EncoderInfo::kMaxFramerateFraction});
+
+  // Only keep the middle spatial layer active. SEA still has three stream
+  // contexts, so this exercises the runtime state that used to incorrectly
+  // report aggregated simulcast encoder info with scaling disabled.
+  VideoBitrateAllocation allocation;
+  ASSERT_TRUE(allocation.SetBitrate(1, 0, 500000));
+  adapter_->SetRates(VideoEncoder::RateControlParameters(allocation, 30.0));
+
+  const auto info = adapter_->GetEncoderInfo();
+  // Runtime-sensitive fields should come from the only unpaused encoder.
+  ASSERT_TRUE(info.scaling_settings.thresholds.has_value());
+  EXPECT_EQ(30, info.scaling_settings.thresholds->low);
+  EXPECT_EQ(40, info.scaling_settings.thresholds->high);
+  EXPECT_EQ(222, info.scaling_settings.min_pixels_per_frame);
+  EXPECT_TRUE(info.supports_native_handle);
+  EXPECT_EQ(std::optional<bool>(false), info.is_qp_trusted);
+  EXPECT_EQ(std::optional<int>(20), info.min_qp);
+  EXPECT_EQ(info.resolution_bitrate_limits,
+            std::vector<VideoEncoder::ResolutionBitrateLimits>(
+                {VideoEncoder::ResolutionBitrateLimits(222, 4444, 5555, 6666)}));
+  // Simulcast-specific fields must remain in SEA's aggregated spatial-slot
+  // layout even when runtime-sensitive fields are forwarded from one encoder.
+  EXPECT_THAT(info.fps_allocation[0],
+              ::testing::ElementsAre(EncoderInfo::kMaxFramerateFraction / 2));
+  EXPECT_THAT(info.fps_allocation[1],
+              ::testing::ElementsAre(EncoderInfo::kMaxFramerateFraction / 3,
+                                     EncoderInfo::kMaxFramerateFraction));
+  EXPECT_THAT(info.fps_allocation[2],
+              ::testing::ElementsAre(EncoderInfo::kMaxFramerateFraction));
+}
+
+TEST_F(TestSimulcastEncoderAdapterFake,
+       RestoresAggregatedEncoderInfoWhenMultipleLayersUnpause) {
+  SimulcastTestFixtureImpl::DefaultSettings(
+      &codec_, static_cast<const int*>(kTestTemporalLayerProfile),
+      kVideoCodecVP8);
+  codec_.numberOfSimulcastStreams = 3;
+  EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings));
+  adapter_->RegisterEncodeCompleteCallback(this);
+  ASSERT_EQ(3u, helper_->factory()->encoders().size());
+
+  auto* low_encoder = helper_->factory()->encoders()[0];
+  auto* mid_encoder = helper_->factory()->encoders()[1];
+  auto* high_encoder = helper_->factory()->encoders()[2];
+
+  low_encoder->set_scaling_settings(VideoEncoder::ScalingSettings(10, 20, 111));
+  low_encoder->set_supports_native_handle(false);
+  low_encoder->set_fps_allocation(
+      FramerateFractions{EncoderInfo::kMaxFramerateFraction / 2});
+
+  mid_encoder->set_scaling_settings(VideoEncoder::ScalingSettings(30, 40, 222));
+  mid_encoder->set_supports_native_handle(true);
+  mid_encoder->set_fps_allocation(
+      FramerateFractions{EncoderInfo::kMaxFramerateFraction / 3,
+                         EncoderInfo::kMaxFramerateFraction});
+
+  high_encoder->set_scaling_settings(
+      VideoEncoder::ScalingSettings(50, 60, 333));
+  high_encoder->set_supports_native_handle(false);
+  high_encoder->set_fps_allocation(
+      FramerateFractions{EncoderInfo::kMaxFramerateFraction});
+
+  // First collapse to a single active spatial layer and verify the forwarded
+  // encoder info.
+  VideoBitrateAllocation one_layer_allocation;
+  ASSERT_TRUE(one_layer_allocation.SetBitrate(1, 0, 500000));
+  adapter_->SetRates(
+      VideoEncoder::RateControlParameters(one_layer_allocation, 30.0));
+
+  auto info = adapter_->GetEncoderInfo();
+  ASSERT_TRUE(info.scaling_settings.thresholds.has_value());
+  EXPECT_EQ(30, info.scaling_settings.thresholds->low);
+  EXPECT_EQ(40, info.scaling_settings.thresholds->high);
+  EXPECT_TRUE(info.supports_native_handle);
+
+  // Then enable another layer. SEA should immediately return to its normal
+  // aggregated simulcast view without requiring a re-init.
+  VideoBitrateAllocation two_layer_allocation;
+  ASSERT_TRUE(two_layer_allocation.SetBitrate(1, 0, 500000));
+  ASSERT_TRUE(two_layer_allocation.SetBitrate(2, 0, 700000));
+  adapter_->SetRates(
+      VideoEncoder::RateControlParameters(two_layer_allocation, 30.0));
+
+  info = adapter_->GetEncoderInfo();
+  EXPECT_FALSE(info.scaling_settings.thresholds.has_value());
+  EXPECT_TRUE(info.supports_native_handle);
+  EXPECT_THAT(info.fps_allocation[0],
+              ::testing::ElementsAre(EncoderInfo::kMaxFramerateFraction / 2));
+  EXPECT_THAT(info.fps_allocation[1],
+              ::testing::ElementsAre(EncoderInfo::kMaxFramerateFraction / 3,
+                                     EncoderInfo::kMaxFramerateFraction));
+  EXPECT_THAT(info.fps_allocation[2],
+              ::testing::ElementsAre(EncoderInfo::kMaxFramerateFraction));
+}
+
 TEST_F(TestSimulcastEncoderAdapterFake, SetRateDistributesBandwithAllocation) {
   SimulcastTestFixtureImpl::DefaultSettings(
       &codec_, static_cast<const int*>(kTestTemporalLayerProfile),

sdk/objc/native/src/objc_video_encoder_factory.mm

@@ -134,8 +134,17 @@ void SetRates(const RateControlParameters &parameters) override {
 
     RTC_OBJC_TYPE(RTCVideoEncoderQpThresholds) *qp_thresholds =
         [encoder_ scalingSettings];
+    // The default kDefaultMinPixelsPerFrame (320*180 = 57600) is too
+    // conservative for iOS hardware encoders (VideoToolbox). It prevents
+    // the quality scaler from reducing resolution below ~180x320 (the
+    // quarter-resolution simulcast layer for 720p), which limits adaptation
+    // range on constrained networks (e.g. 3G). VideoToolbox can encode at
+    // resolutions well below this threshold, so we use a lower floor to
+    // allow meaningful quality adaptation in bandwidth-limited scenarios.
+    constexpr int kObjCEncoderMinPixelsPerFrame = 90 * 160;
     info.scaling_settings = qp_thresholds ?
-        ScalingSettings(qp_thresholds.low, qp_thresholds.high) :
+        ScalingSettings(qp_thresholds.low, qp_thresholds.high,
+                        kObjCEncoderMinPixelsPerFrame) :
         ScalingSettings::kOff;
 
     info.requested_resolution_alignment = encoder_.resolutionAlignment > 0 ?: 1;

video/adaptation/video_stream_encoder_resource_manager.cc

@@ -559,16 +559,86 @@ void VideoStreamEncoderResourceManager::UpdateBandwidthQualityScalerSettings(
   }
 }
 
+void VideoStreamEncoderResourceManager::ResetAdaptationsForSimulcastChange() {
+  RTC_DCHECK_RUN_ON(encoder_queue_);
+  std::vector<scoped_refptr<Resource>> quality_resources;
+  for (const auto& resource_and_reason : resources_) {
+    if (resource_and_reason.second == VideoAdaptationReason::kQuality) {
+      quality_resources.push_back(resource_and_reason.first);
+    }
+  }
+
+  if (quality_resources.empty()) {
+    return;
+  }
+
+  for (const auto& resource : quality_resources) {
+    if (resource == quality_scaler_resource_) {
+      RTC_LOG(LS_INFO) << "Clearing quality scaler restrictions for simulcast "
+                          "active-layer transition.";
+      quality_scaler_resource_->StopCheckForOveruse();
+      RemoveResource(resource);
+      initial_frame_dropper_->OnQualityScalerSettingsUpdated();
+    } else if (resource == bandwidth_quality_scaler_resource_) {
+      RTC_LOG(LS_INFO)
+          << "Clearing bandwidth quality scaler restrictions for simulcast "
+             "active-layer transition.";
+      bandwidth_quality_scaler_resource_->StopCheckForOveruse();
+      RemoveResource(resource);
+    } else {
+      RTC_LOG(LS_INFO) << "Resetting quality adaptation resource \""
+                       << resource->Name()
+                       << "\" for simulcast active-layer transition.";
+      RemoveResource(resource);
+      AddResource(resource, VideoAdaptationReason::kQuality);
+    }
+  }
+
+  UpdateStatsAdaptationSettings();
+}
+
+void VideoStreamEncoderResourceManager::SetSimulcastActive(
+    bool simulcast_active) {
+  RTC_DCHECK_RUN_ON(encoder_queue_);
+  if (simulcast_active_ != simulcast_active) {
+    RTC_LOG(LS_INFO) << "SetSimulcastActive: " << simulcast_active_
+                     << " -> " << simulcast_active;
+  }
+  simulcast_active_ = simulcast_active;
+}
+
 void VideoStreamEncoderResourceManager::ConfigureQualityScaler(
     const VideoEncoder::EncoderInfo& encoder_info) {
   RTC_DCHECK_RUN_ON(encoder_queue_);
   const auto scaling_settings = encoder_info.scaling_settings;
+  // Quality scaling (adapting input resolution based on encoded QP) is allowed
+  // only when ALL of the following are true:
+  //  1. The degradation preference permits resolution changes.
+  //  2. Simulcast is NOT active — during simulcast, the SFU handles quality
+  //     adaptation by activating/deactivating layers. Allowing the quality
+  //     scaler to run would shrink the input resolution and cause all
+  //     simulcast layer dimensions to be recomputed from degraded input.
+  //  3. The encoder reports QP thresholds or the encoder config explicitly
+  //     allows quality scaling (is_quality_scaling_allowed).
+  //  4. The encoder's QP values are trusted for scaling decisions.
+  const bool resolution_scaling_enabled =
+      IsResolutionScalingEnabled(degradation_preference_);
+  const bool has_scaling_thresholds =
+      scaling_settings.thresholds.has_value() ||
+      (encoder_settings_.has_value() &&
+       encoder_settings_->encoder_config().is_quality_scaling_allowed);
+  const bool qp_trusted = encoder_info.is_qp_trusted.value_or(true);
   const bool quality_scaling_allowed =
-      IsResolutionScalingEnabled(degradation_preference_) &&
-      (scaling_settings.thresholds.has_value() ||
-       (encoder_settings_.has_value() &&
-        encoder_settings_->encoder_config().is_quality_scaling_allowed)) &&
-      encoder_info.is_qp_trusted.value_or(true);
+      resolution_scaling_enabled && !simulcast_active_ &&
+      has_scaling_thresholds && qp_trusted;
+
+  RTC_LOG(LS_INFO) << "ConfigureQualityScaler: allowed=" << quality_scaling_allowed
+                   << " (resolution_scaling=" << resolution_scaling_enabled
+                   << ", simulcast_active=" << simulcast_active_
+                   << ", has_thresholds=" << has_scaling_thresholds
+                   << ", qp_trusted=" << qp_trusted
+                   << ", scaler_running=" << quality_scaler_resource_->is_started()
+                   << ")";
 
   // TODO(https://crbug.com/webrtc/11222): Should this move to
   // QualityScalerResource?
@@ -609,6 +679,7 @@ void VideoStreamEncoderResourceManager::ConfigureBandwidthQualityScaler(
   RTC_DCHECK_RUN_ON(encoder_queue_);
   const bool bandwidth_quality_scaling_allowed =
       IsResolutionScalingEnabled(degradation_preference_) &&
+      !simulcast_active_ &&
       (encoder_settings_.has_value() &&
        encoder_settings_->encoder_config().is_quality_scaling_allowed) &&
       !encoder_info.is_qp_trusted.value_or(true);

video/adaptation/video_stream_encoder_resource_manager.h

@@ -109,6 +109,21 @@ class VideoStreamEncoderResourceManager
   // TODO(https://crbug.com/webrtc/11338): This can be made private if we
   // configure on SetDegredationPreference and SetEncoderSettings.
   void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info);
+
+  // Controls whether the quality scaler is suppressed due to simulcast.
+  // When multiple simulcast layers are active, resolution-based quality
+  // adaptation must be disabled because the quality scaler would shrink the
+  // input resolution, causing all simulcast layer dimensions to be derived
+  // from the degraded input rather than the original capture resolution.
+  // In simulcast mode, quality adaptation is handled by the SFU
+  // activating/deactivating layers instead.
+  void SetSimulcastActive(bool simulcast_active);
+
+  // Stops the quality scaler and clears its accumulated adaptation
+  // restrictions. Called when the number of active simulcast layers increases
+  // from <=1 to >1, so that the source provides full-resolution frames for
+  // the new multi-layer configuration.
+  void ResetAdaptationsForSimulcastChange();
   void ConfigureBandwidthQualityScaler(
       const VideoEncoder::EncoderInfo& encoder_info);
 
@@ -222,6 +237,13 @@ class VideoStreamEncoderResourceManager
   std::optional<EncoderSettings> encoder_settings_
       RTC_GUARDED_BY(encoder_queue_);
 
+  // True when multiple simulcast layers are active. While set, the quality
+  // scaler is suppressed to prevent input resolution degradation that would
+  // corrupt simulcast layer dimensions. Set by VideoStreamEncoder from the
+  // current runtime layer allocation, with encoder-config fallback before the
+  // first SetRates() arrives.
+  bool simulcast_active_ RTC_GUARDED_BY(encoder_queue_) = false;
+
   // Ties a resource to a reason for statistical reporting. This AdaptReason is
   // also used by this module to make decisions about how to adapt up/down.
   std::map<scoped_refptr<Resource>, VideoAdaptationReason> resources_

video/config/encoder_stream_factory.cc

@@ -69,13 +69,15 @@ bool IsTemporalLayersSupported(VideoCodecType codec_type) {
 }
 
 size_t FindRequiredActiveLayers(const VideoEncoderConfig& encoder_config) {
-  // Need enough layers so that at least the first active one is present.
+  // Need enough layers so that all active ones are present.
+  // Return the position after the highest active layer.
+  size_t highest = 0;
   for (size_t i = 0; i < encoder_config.number_of_streams; ++i) {
     if (encoder_config.simulcast_layers[i].active) {
-      return i + 1;
+      highest = i + 1;
     }
   }
-  return 0;
+  return highest;
 }
 
 // The selected thresholds for QVGA and VGA corresponded to a QP around 10.