| 12
 3
 4
 5
 6
 7
 8
 9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
 100
 101
 102
 103
 104
 105
 106
 107
 108
 109
 110
 111
 112
 113
 114
 115
 116
 117
 118
 119
 120
 121
 122
 123
 124
 125
 126
 127
 128
 129
 130
 131
 132
 133
 134
 135
 136
 137
 138
 139
 140
 141
 142
 143
 144
 145
 146
 147
 148
 149
 150
 151
 152
 153
 154
 155
 156
 157
 158
 159
 160
 161
 162
 163
 164
 165
 166
 167
 168
 169
 170
 171
 172
 173
 174
 175
 176
 177
 178
 179
 180
 181
 182
 183
 184
 185
 186
 187
 188
 189
 190
 191
 192
 193
 194
 195
 196
 197
 198
 199
 200
 201
 202
 203
 204
 205
 206
 207
 208
 209
 210
 211
 212
 213
 214
 215
 216
 217
 218
 219
 220
 221
 222
 223
 224
 225
 226
 227
 228
 229
 230
 231
 232
 233
 234
 235
 236
 237
 238
 239
 240
 241
 242
 243
 244
 245
 246
 247
 248
 249
 250
 251
 252
 253
 254
 255
 256
 257
 258
 259
 260
 261
 262
 263
 264
 265
 266
 267
 268
 269
 270
 271
 272
 273
 274
 275
 276
 277
 278
 279
 280
 281
 282
 283
 284
 285
 286
 287
 288
 289
 290
 291
 292
 293
 294
 295
 296
 297
 298
 299
 300
 301
 302
 303
 304
 305
 306
 307
 308
 309
 310
 311
 312
 313
 314
 315
 316
 317
 318
 319
 320
 321
 322
 323
 324
 325
 326
 327
 328
 329
 330
 331
 332
 333
 334
 335
 336
 337
 338
 339
 340
 341
 342
 343
 344
 345
 346
 347
 348
 349
 350
 
 | /*
 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */
#ifndef MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_
#define MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_
#include <math.h>
#include <stdlib.h>
#include <memory>
#include "modules/video_coding/internal_defines.h"
#include "rtc_base/experiments/rate_control_settings.h"
#include "rtc_base/numerics/exp_filter.h"
namespace webrtc {
namespace media_optimization {
// Number of time periods used for (max) window filter for packet loss
// TODO(marpan): set reasonable window size for filtered packet loss,
// adjustment should be based on logged/real data of loss stats/correlation.
constexpr int kLossPrHistorySize = 10;
// 1000 ms, total filter length is (kLossPrHistorySize * 1000) ms
constexpr int kLossPrShortFilterWinMs = 1000;
// The type of filter used on the received packet loss reports.
enum FilterPacketLossMode {
  kNoFilter,   // No filtering on received loss.
  kAvgFilter,  // Recursive average filter.
  kMaxFilter   // Max-window filter, over the time interval of:
               // (kLossPrHistorySize * kLossPrShortFilterWinMs) ms.
};
// Thresholds for hybrid NACK/FEC
// common to media optimization and the jitter buffer.
constexpr int64_t kLowRttNackMs = 20;
// If the RTT is higher than this an extra RTT wont be added to to the jitter
// buffer delay.
constexpr int kMaxRttDelayThreshold = 500;
struct VCMProtectionParameters {
  VCMProtectionParameters();
  int64_t rtt;
  float lossPr;
  float bitRate;
  float packetsPerFrame;
  float packetsPerFrameKey;
  float frameRate;
  float keyFrameSize;
  uint8_t fecRateDelta;
  uint8_t fecRateKey;
  uint16_t codecWidth;
  uint16_t codecHeight;
  int numLayers;
};
/******************************/
/* VCMProtectionMethod class  */
/******************************/
enum VCMProtectionMethodEnum { kNack, kFec, kNackFec, kNone };
class VCMLossProbabilitySample {
 public:
  VCMLossProbabilitySample() : lossPr255(0), timeMs(-1) {}
  uint8_t lossPr255;
  int64_t timeMs;
};
class VCMProtectionMethod {
 public:
  VCMProtectionMethod();
  virtual ~VCMProtectionMethod();
  // Updates the efficiency of the method using the parameters provided
  //
  // Input:
  //         - parameters         : Parameters used to calculate efficiency
  //
  // Return value                 : True if this method is recommended in
  //                                the given conditions.
  virtual bool UpdateParameters(const VCMProtectionParameters* parameters) = 0;
  // Returns the protection type
  //
  // Return value                 : The protection type
  VCMProtectionMethodEnum Type() const;
  // Returns the effective packet loss for ER, required by this protection
  // method
  //
  // Return value                 : Required effective packet loss
  virtual uint8_t RequiredPacketLossER();
  // Extracts the FEC protection factor for Key frame, required by this
  // protection method
  //
  // Return value                 : Required protectionFactor for Key frame
  virtual uint8_t RequiredProtectionFactorK();
  // Extracts the FEC protection factor for Delta frame, required by this
  // protection method
  //
  // Return value                 : Required protectionFactor for delta frame
  virtual uint8_t RequiredProtectionFactorD();
  // Extracts whether the FEC Unequal protection (UEP) is used for Key frame.
  //
  // Return value                 : Required Unequal protection on/off state.
  virtual bool RequiredUepProtectionK();
  // Extracts whether the the FEC Unequal protection (UEP) is used for Delta
  // frame.
  //
  // Return value                 : Required Unequal protection on/off state.
  virtual bool RequiredUepProtectionD();
  virtual int MaxFramesFec() const;
 protected:
  uint8_t _effectivePacketLoss;
  uint8_t _protectionFactorK;
  uint8_t _protectionFactorD;
  // Estimation of residual loss after the FEC
  float _scaleProtKey;
  int32_t _maxPayloadSize;
  bool _useUepProtectionK;
  bool _useUepProtectionD;
  float _corrFecCost;
  VCMProtectionMethodEnum _type;
};
class VCMNackMethod : public VCMProtectionMethod {
 public:
  VCMNackMethod();
  ~VCMNackMethod() override;
  bool UpdateParameters(const VCMProtectionParameters* parameters) override;
  // Get the effective packet loss
  bool EffectivePacketLoss(const VCMProtectionParameters* parameter);
};
class VCMFecMethod : public VCMProtectionMethod {
 public:
  VCMFecMethod();
  ~VCMFecMethod() override;
  bool UpdateParameters(const VCMProtectionParameters* parameters) override;
  // Get the effective packet loss for ER
  bool EffectivePacketLoss(const VCMProtectionParameters* parameters);
  // Get the FEC protection factors
  bool ProtectionFactor(const VCMProtectionParameters* parameters);
  // Get the boost for key frame protection
  uint8_t BoostCodeRateKey(uint8_t packetFrameDelta,
                           uint8_t packetFrameKey) const;
  // Convert the rates: defined relative to total# packets or source# packets
  uint8_t ConvertFECRate(uint8_t codeRate) const;
  // Get the average effective recovery from FEC: for random loss model
  float AvgRecoveryFEC(const VCMProtectionParameters* parameters) const;
  // Update FEC with protectionFactorD
  void UpdateProtectionFactorD(uint8_t protectionFactorD);
  // Update FEC with protectionFactorK
  void UpdateProtectionFactorK(uint8_t protectionFactorK);
  // Compute the bits per frame. Account for temporal layers when applicable.
  int BitsPerFrame(const VCMProtectionParameters* parameters);
 protected:
  static constexpr int kUpperLimitFramesFec = 6;
  // Thresholds values for the bytes/frame and round trip time, below which we
  // may turn off FEC, depending on `_numLayers` and `_maxFramesFec`.
  // Max bytes/frame for VGA, corresponds to ~140k at 25fps.
  static constexpr int kMaxBytesPerFrameForFec = 700;
  // Max bytes/frame for CIF and lower: corresponds to ~80k at 25fps.
  static constexpr int kMaxBytesPerFrameForFecLow = 400;
  // Max bytes/frame for frame size larger than VGA, ~200k at 25fps.
  static constexpr int kMaxBytesPerFrameForFecHigh = 1000;
  const RateControlSettings rate_control_settings_;
};
class VCMNackFecMethod : public VCMFecMethod {
 public:
  VCMNackFecMethod(int64_t lowRttNackThresholdMs,
                   int64_t highRttNackThresholdMs);
  ~VCMNackFecMethod() override;
  bool UpdateParameters(const VCMProtectionParameters* parameters) override;
  // Get the effective packet loss for ER
  bool EffectivePacketLoss(const VCMProtectionParameters* parameters);
  // Get the protection factors
  bool ProtectionFactor(const VCMProtectionParameters* parameters);
  // Get the max number of frames the FEC is allowed to be based on.
  int MaxFramesFec() const override;
  // Turn off the FEC based on low bitrate and other factors.
  bool BitRateTooLowForFec(const VCMProtectionParameters* parameters);
 private:
  int ComputeMaxFramesFec(const VCMProtectionParameters* parameters);
  int64_t _lowRttNackMs;
  int64_t _highRttNackMs;
  int _maxFramesFec;
};
class VCMLossProtectionLogic {
 public:
  explicit VCMLossProtectionLogic(int64_t nowMs);
  ~VCMLossProtectionLogic();
  // Set the protection method to be used
  //
  // Input:
  //        - newMethodType    : New requested protection method type. If one
  //                           is already set, it will be deleted and replaced
  void SetMethod(VCMProtectionMethodEnum newMethodType);
  // Update the round-trip time
  //
  // Input:
  //          - rtt           : Round-trip time in seconds.
  void UpdateRtt(int64_t rtt);
  // Update the filtered packet loss.
  //
  // Input:
  //          - packetLossEnc :  The reported packet loss filtered
  //                             (max window or average)
  void UpdateFilteredLossPr(uint8_t packetLossEnc);
  // Update the current target bit rate.
  //
  // Input:
  //          - bitRate          : The current target bit rate in kbits/s
  void UpdateBitRate(float bitRate);
  // Update the number of packets per frame estimate, for delta frames
  //
  // Input:
  //          - nPackets         : Number of packets in the latest sent frame.
  void UpdatePacketsPerFrame(float nPackets, int64_t nowMs);
  // Update the number of packets per frame estimate, for key frames
  //
  // Input:
  //          - nPackets         : umber of packets in the latest sent frame.
  void UpdatePacketsPerFrameKey(float nPackets, int64_t nowMs);
  // Update the keyFrameSize estimate
  //
  // Input:
  //          - keyFrameSize     : The size of the latest sent key frame.
  void UpdateKeyFrameSize(float keyFrameSize);
  // Update the frame rate
  //
  // Input:
  //          - frameRate        : The current target frame rate.
  void UpdateFrameRate(float frameRate) { _frameRate = frameRate; }
  // Update the frame size
  //
  // Input:
  //          - width        : The codec frame width.
  //          - height       : The codec frame height.
  void UpdateFrameSize(size_t width, size_t height);
  // Update the number of active layers
  //
  // Input:
  //          - numLayers    : Number of layers used.
  void UpdateNumLayers(int numLayers);
  // The amount of packet loss to cover for with FEC.
  //
  // Input:
  //          - fecRateKey      : Packet loss to cover for with FEC when
  //                              sending key frames.
  //          - fecRateDelta    : Packet loss to cover for with FEC when
  //                              sending delta frames.
  void UpdateFECRates(uint8_t fecRateKey, uint8_t fecRateDelta) {
    _fecRateKey = fecRateKey;
    _fecRateDelta = fecRateDelta;
  }
  // Update the protection methods with the current VCMProtectionParameters
  // and set the requested protection settings.
  // Return value     : Returns true on update
  bool UpdateMethod();
  // Returns the method currently selected.
  //
  // Return value                 : The protection method currently selected.
  VCMProtectionMethod* SelectedMethod() const;
  // Return the protection type of the currently selected method
  VCMProtectionMethodEnum SelectedType() const;
  // Updates the filtered loss for the average and max window packet loss,
  // and returns the filtered loss probability in the interval [0, 255].
  // The returned filtered loss value depends on the parameter `filter_mode`.
  // The input parameter `lossPr255` is the received packet loss.
  // Return value                 : The filtered loss probability
  uint8_t FilteredLoss(int64_t nowMs,
                       FilterPacketLossMode filter_mode,
                       uint8_t lossPr255);
  void Reset(int64_t nowMs);
  void Release();
 private:
  // Sets the available loss protection methods.
  void UpdateMaxLossHistory(uint8_t lossPr255, int64_t now);
  uint8_t MaxFilteredLossPr(int64_t nowMs) const;
  std::unique_ptr<VCMProtectionMethod> _selectedMethod;
  VCMProtectionParameters _currentParameters;
  int64_t _rtt;
  float _lossPr;
  float _bitRate;
  float _frameRate;
  float _keyFrameSize;
  uint8_t _fecRateKey;
  uint8_t _fecRateDelta;
  int64_t _lastPrUpdateT;
  int64_t _lastPacketPerFrameUpdateT;
  int64_t _lastPacketPerFrameUpdateTKey;
  rtc::ExpFilter _lossPr255;
  VCMLossProbabilitySample _lossPrHistory[kLossPrHistorySize];
  uint8_t _shortMaxLossPr255;
  rtc::ExpFilter _packetsPerFrame;
  rtc::ExpFilter _packetsPerFrameKey;
  size_t _codecWidth;
  size_t _codecHeight;
  int _numLayers;
};
}  // namespace media_optimization
}  // namespace webrtc
#endif  // MODULES_VIDEO_CODING_MEDIA_OPT_UTIL_H_
 |