File: quality_scaler.cc

package info (click to toggle)
thunderbird 1%3A143.0.1-1
  • links: PTS, VCS
  • area: main
  • in suites: experimental
  • size: 4,703,968 kB
  • sloc: cpp: 7,770,492; javascript: 5,943,842; ansic: 3,918,754; python: 1,418,263; xml: 653,354; asm: 474,045; java: 183,079; sh: 111,238; makefile: 20,410; perl: 14,359; objc: 13,059; yacc: 4,583; pascal: 3,405; lex: 1,720; ruby: 999; exp: 762; sql: 715; awk: 580; php: 436; lisp: 430; sed: 69; csh: 10
file content (326 lines) | stat: -rw-r--r-- 11,843 bytes parent folder | download | duplicates (3)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
/*
 *  Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "modules/video_coding/utility/quality_scaler.h"

#include <cstddef>
#include <cstdint>
#include <memory>
#include <optional>

#include "api/field_trials_view.h"
#include "api/sequence_checker.h"
#include "api/task_queue/task_queue_base.h"
#include "api/units/time_delta.h"
#include "api/video_codecs/video_encoder.h"
#include "rtc_base/checks.h"
#include "rtc_base/experiments/quality_scaler_settings.h"
#include "rtc_base/experiments/quality_scaling_experiment.h"
#include "rtc_base/logging.h"
#include "rtc_base/numerics/exp_filter.h"
#include "rtc_base/weak_ptr.h"

namespace webrtc {

namespace {
// Threshold constant used until first downscale (to permit fast rampup).
static const int kMeasureMs = 2000;
static const float kSamplePeriodScaleFactor = 2.5;
static const int kFramedropPercentThreshold = 60;
static const size_t kMinFramesNeededToScale = 2 * 30;

}  // namespace

class QualityScaler::QpSmoother {
 public:
  explicit QpSmoother(float alpha)
      : alpha_(alpha),
        // The initial value of last_sample_ms doesn't matter since the smoother
        // will ignore the time delta for the first update.
        last_sample_ms_(0),
        smoother_(alpha) {}

  std::optional<int> GetAvg() const {
    float value = smoother_.filtered();
    if (value == ExpFilter::kValueUndefined) {
      return std::nullopt;
    }
    return static_cast<int>(value);
  }

  void Add(float sample, int64_t time_sent_us) {
    int64_t now_ms = time_sent_us / 1000;
    smoother_.Apply(static_cast<float>(now_ms - last_sample_ms_), sample);
    last_sample_ms_ = now_ms;
  }

  void Reset() { smoother_.Reset(alpha_); }

 private:
  const float alpha_;
  int64_t last_sample_ms_;
  ExpFilter smoother_;
};

// The QualityScaler checks for QP periodically by queuing CheckQpTasks. The
// task will either run to completion and trigger a new task being queued, or it
// will be destroyed because the QualityScaler is destroyed.
//
// When high or low QP is reported, the task will be pending until a callback is
// invoked. This lets the QualityScalerQpUsageHandlerInterface react to QP usage
// asynchronously and prevents checking for QP until the stream has potentially
// been reconfigured.
class QualityScaler::CheckQpTask {
 public:
  // The result of one CheckQpTask may influence the delay of the next
  // CheckQpTask.
  struct Result {
    bool observed_enough_frames = false;
    bool qp_usage_reported = false;
  };

  CheckQpTask(QualityScaler* quality_scaler, Result previous_task_result)
      : quality_scaler_(quality_scaler),
        state_(State::kNotStarted),
        previous_task_result_(previous_task_result),
        weak_ptr_factory_(this) {}

  void StartDelayedTask() {
    RTC_DCHECK_EQ(state_, State::kNotStarted);
    state_ = State::kCheckingQp;
    TaskQueueBase::Current()->PostDelayedTask(
        [this_weak_ptr = weak_ptr_factory_.GetWeakPtr(), this] {
          if (!this_weak_ptr) {
            // The task has been cancelled through destruction.
            return;
          }
          RTC_DCHECK_EQ(state_, State::kCheckingQp);
          RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_);
          switch (quality_scaler_->CheckQp()) {
            case QualityScaler::CheckQpResult::kInsufficientSamples: {
              result_.observed_enough_frames = false;
              // After this line, `this` may be deleted.
              break;
            }
            case QualityScaler::CheckQpResult::kNormalQp: {
              result_.observed_enough_frames = true;
              break;
            }
            case QualityScaler::CheckQpResult::kHighQp: {
              result_.observed_enough_frames = true;
              result_.qp_usage_reported = true;
              quality_scaler_->fast_rampup_ = false;
              quality_scaler_->handler_->OnReportQpUsageHigh();
              quality_scaler_->ClearSamples();
              break;
            }
            case QualityScaler::CheckQpResult::kLowQp: {
              result_.observed_enough_frames = true;
              result_.qp_usage_reported = true;
              quality_scaler_->handler_->OnReportQpUsageLow();
              quality_scaler_->ClearSamples();
              break;
            }
          }
          state_ = State::kCompleted;
          // Starting the next task deletes the pending task. After this line,
          // `this` has been deleted.
          quality_scaler_->StartNextCheckQpTask();
        },
        TimeDelta::Millis(GetCheckingQpDelayMs()));
  }

  bool HasCompletedTask() const { return state_ == State::kCompleted; }

  Result result() const {
    RTC_DCHECK(HasCompletedTask());
    return result_;
  }

 private:
  enum class State {
    kNotStarted,
    kCheckingQp,
    kCompleted,
  };

  // Determines the sampling period of CheckQpTasks.
  int64_t GetCheckingQpDelayMs() const {
    RTC_DCHECK_RUN_ON(&quality_scaler_->task_checker_);
    if (quality_scaler_->fast_rampup_) {
      return quality_scaler_->sampling_period_ms_;
    }
    if (quality_scaler_->experiment_enabled_ &&
        !previous_task_result_.observed_enough_frames) {
      // Use half the interval while waiting for enough frames.
      return quality_scaler_->sampling_period_ms_ / 2;
    }
    if (quality_scaler_->scale_factor_ &&
        !previous_task_result_.qp_usage_reported) {
      // Last CheckQp did not call AdaptDown/Up, possibly reduce interval.
      return quality_scaler_->sampling_period_ms_ *
             quality_scaler_->scale_factor_.value();
    }
    return quality_scaler_->sampling_period_ms_ *
           quality_scaler_->initial_scale_factor_;
  }

  QualityScaler* const quality_scaler_;
  State state_;
  const Result previous_task_result_;
  Result result_;

  WeakPtrFactory<CheckQpTask> weak_ptr_factory_;
};

QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler,
                             VideoEncoder::QpThresholds thresholds,
                             const FieldTrialsView& field_trials)
    : QualityScaler(handler, thresholds, field_trials, kMeasureMs) {}

// Protected ctor, should not be called directly.
QualityScaler::QualityScaler(QualityScalerQpUsageHandlerInterface* handler,
                             VideoEncoder::QpThresholds thresholds,
                             const FieldTrialsView& field_trials,
                             int64_t default_sampling_period_ms)
    : handler_(handler),
      thresholds_(thresholds),
      sampling_period_ms_(QualityScalerSettings(field_trials)
                              .SamplingPeriodMs()
                              .value_or(default_sampling_period_ms)),
      fast_rampup_(true),
      // Arbitrarily choose size based on 30 fps for 5 seconds.
      average_qp_(QualityScalerSettings(field_trials)
                      .AverageQpWindow()
                      .value_or(5 * 30)),
      framedrop_percent_media_opt_(5 * 30),
      framedrop_percent_all_(5 * 30),
      experiment_enabled_(QualityScalingExperiment::Enabled(field_trials)),
      min_frames_needed_(QualityScalerSettings(field_trials)
                             .MinFrames()
                             .value_or(kMinFramesNeededToScale)),
      initial_scale_factor_(QualityScalerSettings(field_trials)
                                .InitialScaleFactor()
                                .value_or(kSamplePeriodScaleFactor)),
      scale_factor_(QualityScalerSettings(field_trials).ScaleFactor()) {
  RTC_DCHECK_RUN_ON(&task_checker_);
  if (experiment_enabled_) {
    config_ = QualityScalingExperiment::GetConfig(field_trials);
    qp_smoother_high_.reset(new QpSmoother(config_.alpha_high));
    qp_smoother_low_.reset(new QpSmoother(config_.alpha_low));
  }
  RTC_DCHECK(handler_ != nullptr);
  StartNextCheckQpTask();
  RTC_LOG(LS_INFO) << "QP thresholds: low: " << thresholds_.low
                   << ", high: " << thresholds_.high;
}

QualityScaler::~QualityScaler() {
  RTC_DCHECK_RUN_ON(&task_checker_);
}

void QualityScaler::StartNextCheckQpTask() {
  RTC_DCHECK_RUN_ON(&task_checker_);
  RTC_DCHECK(!pending_qp_task_ || pending_qp_task_->HasCompletedTask())
      << "A previous CheckQpTask has not completed yet!";
  CheckQpTask::Result previous_task_result;
  if (pending_qp_task_) {
    previous_task_result = pending_qp_task_->result();
  }
  pending_qp_task_ = std::make_unique<CheckQpTask>(this, previous_task_result);
  pending_qp_task_->StartDelayedTask();
}

void QualityScaler::SetQpThresholds(VideoEncoder::QpThresholds thresholds) {
  RTC_DCHECK_RUN_ON(&task_checker_);
  thresholds_ = thresholds;
}

void QualityScaler::ReportDroppedFrameByMediaOpt() {
  RTC_DCHECK_RUN_ON(&task_checker_);
  framedrop_percent_media_opt_.AddSample(100);
  framedrop_percent_all_.AddSample(100);
}

void QualityScaler::ReportDroppedFrameByEncoder() {
  RTC_DCHECK_RUN_ON(&task_checker_);
  framedrop_percent_all_.AddSample(100);
}

void QualityScaler::ReportQp(int qp, int64_t time_sent_us) {
  RTC_DCHECK_RUN_ON(&task_checker_);
  framedrop_percent_media_opt_.AddSample(0);
  framedrop_percent_all_.AddSample(0);
  average_qp_.AddSample(qp);
  if (qp_smoother_high_)
    qp_smoother_high_->Add(qp, time_sent_us);
  if (qp_smoother_low_)
    qp_smoother_low_->Add(qp, time_sent_us);
}

QualityScaler::CheckQpResult QualityScaler::CheckQp() const {
  RTC_DCHECK_RUN_ON(&task_checker_);
  // Should be set through InitEncode -> Should be set by now.
  RTC_DCHECK_GE(thresholds_.low, 0);

  // If we have not observed at least this many frames we can't make a good
  // scaling decision.
  const size_t frames = config_.use_all_drop_reasons
                            ? framedrop_percent_all_.Size()
                            : framedrop_percent_media_opt_.Size();
  if (frames < min_frames_needed_) {
    return CheckQpResult::kInsufficientSamples;
  }

  // Check if we should scale down due to high frame drop.
  const std::optional<int> drop_rate =
      config_.use_all_drop_reasons
          ? framedrop_percent_all_.GetAverageRoundedDown()
          : framedrop_percent_media_opt_.GetAverageRoundedDown();
  if (drop_rate && *drop_rate >= kFramedropPercentThreshold) {
    RTC_LOG(LS_INFO) << "Reporting high QP, framedrop percent " << *drop_rate;
    return CheckQpResult::kHighQp;
  }

  // Check if we should scale up or down based on QP.
  const std::optional<int> avg_qp_high =
      qp_smoother_high_ ? qp_smoother_high_->GetAvg()
                        : average_qp_.GetAverageRoundedDown();
  const std::optional<int> avg_qp_low =
      qp_smoother_low_ ? qp_smoother_low_->GetAvg()
                       : average_qp_.GetAverageRoundedDown();
  if (avg_qp_high && avg_qp_low) {
    RTC_LOG(LS_INFO) << "Checking average QP " << *avg_qp_high << " ("
                     << *avg_qp_low << ").";
    if (*avg_qp_high > thresholds_.high) {
      return CheckQpResult::kHighQp;
    }
    if (*avg_qp_low <= thresholds_.low) {
      // QP has been low. We want to try a higher resolution.
      return CheckQpResult::kLowQp;
    }
  }
  return CheckQpResult::kNormalQp;
}

void QualityScaler::ClearSamples() {
  RTC_DCHECK_RUN_ON(&task_checker_);
  framedrop_percent_media_opt_.Reset();
  framedrop_percent_all_.Reset();
  average_qp_.Reset();
  if (qp_smoother_high_)
    qp_smoother_high_->Reset();
  if (qp_smoother_low_)
    qp_smoother_low_->Reset();
}

QualityScalerQpUsageHandlerInterface::~QualityScalerQpUsageHandlerInterface() {}

}  // namespace webrtc