From: Andreas Pehrson <apehrson@mozilla.com>
Date: Tue, 23 Nov 2021 14:11:00 +0000
Subject: Bug 1742181 - libwebrtc: Implement packetsDiscarded bookkeeping for
 received video. r=ng

Depends on D131707

Differential Revision: https://phabricator.services.mozilla.com/D131708
Mercurial Revision: https://hg.mozilla.org/mozilla-central/rev/d0196a45a1f449874fc2a759e85e403c45c25575

Also includes:

Bug 1804288 - (fix-de7ae5755b) reimplement Bug 1742181 - libwebrtc: Implement packetsDiscarded bookkeeping for received video. r=pehrsons

Differential Revision: https://phabricator.services.mozilla.com/D163959
Mercurial Revision: https://hg.mozilla.org/mozilla-central/rev/ee566d1bfb654d36e5d58dce637fb0580b989ac1
---
 api/video/frame_buffer.cc               | 25 ++++++++++++++++++++++---
 api/video/frame_buffer.h                |  4 ++++
 call/video_receive_stream.h             |  2 ++
 modules/video_coding/packet_buffer.cc   | 10 +++++++---
 modules/video_coding/packet_buffer.h    |  5 ++++-
 video/receive_statistics_proxy.cc       |  5 +++++
 video/receive_statistics_proxy.h        |  1 +
 video/rtp_video_stream_receiver2.cc     |  5 ++++-
 video/rtp_video_stream_receiver2.h      |  3 +++
 video/video_receive_stream2.cc          |  1 +
 video/video_stream_buffer_controller.cc | 12 ++++++++++++
 video/video_stream_buffer_controller.h  |  5 +++++
 12 files changed, 70 insertions(+), 8 deletions(-)

diff --git a/api/video/frame_buffer.cc b/api/video/frame_buffer.cc
index 17a030b2c4..251b464f37 100644
--- a/api/video/frame_buffer.cc
+++ b/api/video/frame_buffer.cc
@@ -149,14 +149,29 @@ void FrameBuffer::DropNextDecodableTemporalUnit() {
   }
 
   auto end_it = std::next(next_decodable_temporal_unit_->last_frame);
-  num_dropped_frames_ += std::count_if(
-      frames_.begin(), end_it,
-      [](const auto& f) { return f.second.encoded_frame != nullptr; });
+
+  UpdateDroppedFramesAndDiscardedPackets(frames_.begin(), end_it);
 
   frames_.erase(frames_.begin(), end_it);
   FindNextAndLastDecodableTemporalUnit();
 }
 
+void FrameBuffer::UpdateDroppedFramesAndDiscardedPackets(FrameIterator begin_it,
+                                                         FrameIterator end_it) {
+  unsigned int num_discarded_packets = 0;
+  unsigned int num_dropped_frames =
+      std::count_if(begin_it, end_it, [&](const auto& f) {
+        if (f.second.encoded_frame) {
+          const auto& packetInfos = f.second.encoded_frame->PacketInfos();
+          num_discarded_packets += packetInfos.size();
+        }
+        return f.second.encoded_frame != nullptr;
+      });
+
+  num_dropped_frames_ += num_dropped_frames;
+  num_discarded_packets_ += num_discarded_packets;
+}
+
 std::optional<int64_t> FrameBuffer::LastContinuousFrameId() const {
   return last_continuous_frame_id_;
 }
@@ -176,6 +191,9 @@ int FrameBuffer::GetTotalNumberOfContinuousTemporalUnits() const {
 int FrameBuffer::GetTotalNumberOfDroppedFrames() const {
   return num_dropped_frames_;
 }
+int FrameBuffer::GetTotalNumberOfDiscardedPackets() const {
+  return num_discarded_packets_;
+}
 
 size_t FrameBuffer::CurrentSize() const {
   return frames_.size();
@@ -278,6 +296,7 @@ void FrameBuffer::FindNextAndLastDecodableTemporalUnit() {
 }
 
 void FrameBuffer::Clear() {
+  UpdateDroppedFramesAndDiscardedPackets(frames_.begin(), frames_.end());
   frames_.clear();
   next_decodable_temporal_unit_.reset();
   decodable_temporal_units_info_.reset();
diff --git a/api/video/frame_buffer.h b/api/video/frame_buffer.h
index e607a182c2..5f2c5d0f2a 100644
--- a/api/video/frame_buffer.h
+++ b/api/video/frame_buffer.h
@@ -67,6 +67,7 @@ class FrameBuffer {
 
   int GetTotalNumberOfContinuousTemporalUnits() const;
   int GetTotalNumberOfDroppedFrames() const;
+  int GetTotalNumberOfDiscardedPackets() const;
   size_t CurrentSize() const;
 
  private:
@@ -88,6 +89,8 @@ class FrameBuffer {
   void PropagateContinuity(const FrameIterator& frame_it);
   void FindNextAndLastDecodableTemporalUnit();
   void Clear();
+  void UpdateDroppedFramesAndDiscardedPackets(FrameIterator begin_it,
+                                              FrameIterator end_it);
 
   const bool legacy_frame_id_jump_behavior_;
   const size_t max_size_;
@@ -100,6 +103,7 @@ class FrameBuffer {
 
   int num_continuous_temporal_units_ = 0;
   int num_dropped_frames_ = 0;
+  int num_discarded_packets_ = 0;
 };
 
 }  // namespace webrtc
diff --git a/call/video_receive_stream.h b/call/video_receive_stream.h
index 17c56bb2b1..8ae86e11e2 100644
--- a/call/video_receive_stream.h
+++ b/call/video_receive_stream.h
@@ -117,6 +117,8 @@ class VideoReceiveStreamInterface : public MediaReceiveStreamInterface {
     // https://www.w3.org/TR/webrtc-stats/#dom-rtcvideoreceiverstats-framesdropped
     uint32_t frames_dropped = 0;
     uint32_t frames_decoded = 0;
+    // https://w3c.github.io/webrtc-stats/#dom-rtcreceivedrtpstreamstats-packetsdiscarded
+    uint64_t packets_discarded = 0;
     // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totaldecodetime
     TimeDelta total_decode_time = TimeDelta::Zero();
     // https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-totalprocessingdelay
diff --git a/modules/video_coding/packet_buffer.cc b/modules/video_coding/packet_buffer.cc
index cfc16d5854..17d7cc769c 100644
--- a/modules/video_coding/packet_buffer.cc
+++ b/modules/video_coding/packet_buffer.cc
@@ -127,24 +127,26 @@ PacketBuffer::InsertResult PacketBuffer::InsertPacket(
   return result;
 }
 
-void PacketBuffer::ClearTo(uint16_t seq_num) {
+uint32_t PacketBuffer::ClearTo(uint16_t seq_num) {
   // We have already cleared past this sequence number, no need to do anything.
   if (AheadOf<uint16_t>(first_seq_num_, seq_num)) {
-    return;
+    return 0;
   }
 
   // If the packet buffer was cleared between a frame was created and returned.
   if (!first_packet_received_)
-    return;
+    return 0;
 
   // Avoid iterating over the buffer more than once by capping the number of
   // iterations to the `size_` of the buffer.
   ++seq_num;
+  uint32_t num_cleared_packets = 0;
   size_t diff = ForwardDiff<uint16_t>(first_seq_num_, seq_num);
   size_t iterations = std::min(diff, buffer_.size());
   for (size_t i = 0; i < iterations; ++i) {
     auto& stored = buffer_[first_seq_num_ % buffer_.size()];
     if (stored != nullptr && AheadOf<uint16_t>(seq_num, stored->seq_num())) {
+      ++num_cleared_packets;
       stored = nullptr;
     }
     ++first_seq_num_;
@@ -160,6 +162,8 @@ void PacketBuffer::ClearTo(uint16_t seq_num) {
 
   received_padding_.erase(received_padding_.begin(),
                           received_padding_.lower_bound(seq_num));
+
+  return num_cleared_packets;
 }
 
 void PacketBuffer::Clear() {
diff --git a/modules/video_coding/packet_buffer.h b/modules/video_coding/packet_buffer.h
index f17b00efb1..e205032fa6 100644
--- a/modules/video_coding/packet_buffer.h
+++ b/modules/video_coding/packet_buffer.h
@@ -79,7 +79,10 @@ class PacketBuffer {
   ABSL_MUST_USE_RESULT InsertResult
   InsertPacket(std::unique_ptr<Packet> packet);
   ABSL_MUST_USE_RESULT InsertResult InsertPadding(uint16_t seq_num);
-  void ClearTo(uint16_t seq_num);
+
+  // Clear all packets older than |seq_num|. Returns the number of packets
+  // cleared.
+  uint32_t ClearTo(uint16_t seq_num);
   void Clear();
 
   void ForceSpsPpsIdrIsH264Keyframe();
diff --git a/video/receive_statistics_proxy.cc b/video/receive_statistics_proxy.cc
index aae6b60f2d..8afbc0e176 100644
--- a/video/receive_statistics_proxy.cc
+++ b/video/receive_statistics_proxy.cc
@@ -798,6 +798,11 @@ void ReceiveStatisticsProxy::OnDroppedFrames(uint32_t frames_dropped) {
       }));
 }
 
+void ReceiveStatisticsProxy::OnDiscardedPackets(uint32_t packets_discarded) {
+  RTC_DCHECK_RUN_ON(&main_thread_);
+  stats_.packets_discarded += packets_discarded;
+}
+
 void ReceiveStatisticsProxy::OnPreDecode(VideoCodecType codec_type, int qp) {
   RTC_DCHECK_RUN_ON(&main_thread_);
   last_codec_type_ = codec_type;
diff --git a/video/receive_statistics_proxy.h b/video/receive_statistics_proxy.h
index 023f4fa5f2..9c4df5c5ea 100644
--- a/video/receive_statistics_proxy.h
+++ b/video/receive_statistics_proxy.h
@@ -103,6 +103,7 @@ class ReceiveStatisticsProxy : public VideoStreamBufferControllerStatsObserver,
   void OnDecodableFrame(TimeDelta jitter_buffer_delay,
                         TimeDelta target_delay,
                         TimeDelta minimum_delay) override;
+  void OnDiscardedPackets(uint32_t packets_discarded) override;
   void OnFrameBufferTimingsUpdated(int estimated_max_decode_time_ms,
                                    int current_delay_ms,
                                    int target_delay_ms,
diff --git a/video/rtp_video_stream_receiver2.cc b/video/rtp_video_stream_receiver2.cc
index 262dea79b9..f0fe21a638 100644
--- a/video/rtp_video_stream_receiver2.cc
+++ b/video/rtp_video_stream_receiver2.cc
@@ -286,6 +286,7 @@ RtpVideoStreamReceiver2::RtpVideoStreamReceiver2(
     RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer,
     RtcpCnameCallback* rtcp_cname_callback,
     NackPeriodicProcessor* nack_periodic_processor,
+    VideoStreamBufferControllerStatsObserver* vcm_receive_statistics,
     OnCompleteFrameCallback* complete_frame_callback,
     scoped_refptr<FrameDecryptorInterface> frame_decryptor,
     scoped_refptr<FrameTransformerInterface> frame_transformer)
@@ -329,6 +330,7 @@ RtpVideoStreamReceiver2::RtpVideoStreamReceiver2(
                                             config_.rtp.nack,
                                             &rtcp_feedback_buffer_,
                                             &rtcp_feedback_buffer_)),
+      vcm_receive_statistics_(vcm_receive_statistics),
       packet_buffer_(kPacketBufferStartSize,
                      PacketBufferMaxSize(env_.field_trials())),
       reference_finder_(std::make_unique<RtpFrameReferenceFinder>()),
@@ -1397,7 +1399,8 @@ void RtpVideoStreamReceiver2::FrameDecoded(int64_t picture_id) {
     int64_t unwrapped_rtp_seq_num = rtp_seq_num_unwrapper_.Unwrap(seq_num);
     packet_infos_.erase(packet_infos_.begin(),
                         packet_infos_.upper_bound(unwrapped_rtp_seq_num));
-    packet_buffer_.ClearTo(seq_num);
+    uint32_t num_packets_cleared = packet_buffer_.ClearTo(seq_num);
+    vcm_receive_statistics_->OnDiscardedPackets(num_packets_cleared);
     reference_finder_->ClearTo(seq_num);
   }
 }
diff --git a/video/rtp_video_stream_receiver2.h b/video/rtp_video_stream_receiver2.h
index d75c303fc4..fc3bd1b941 100644
--- a/video/rtp_video_stream_receiver2.h
+++ b/video/rtp_video_stream_receiver2.h
@@ -68,6 +68,7 @@
 #include "rtc_base/thread_annotations.h"
 #include "video/buffered_frame_decryptor.h"
 #include "video/unique_timestamp_counter.h"
+#include "video/video_stream_buffer_controller.h"
 
 namespace webrtc {
 
@@ -110,6 +111,7 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender,
       RtcpPacketTypeCounterObserver* rtcp_packet_type_counter_observer,
       RtcpCnameCallback* rtcp_cname_callback,
       NackPeriodicProcessor* nack_periodic_processor,
+      VideoStreamBufferControllerStatsObserver* vcm_receive_statistics,
       // The KeyFrameRequestSender is optional; if not provided, key frame
       // requests are sent via the internal RtpRtcp module.
       OnCompleteFrameCallback* complete_frame_callback,
@@ -394,6 +396,7 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender,
   std::unique_ptr<LossNotificationController> loss_notification_controller_
       RTC_GUARDED_BY(packet_sequence_checker_);
 
+  VideoStreamBufferControllerStatsObserver* const vcm_receive_statistics_;
   video_coding::PacketBuffer packet_buffer_
       RTC_GUARDED_BY(packet_sequence_checker_);
   // h26x_packet_buffer_ is applicable to H.264 and H.265. For H.265 it is
diff --git a/video/video_receive_stream2.cc b/video/video_receive_stream2.cc
index e4e2cb9835..af9ec0108e 100644
--- a/video/video_receive_stream2.cc
+++ b/video/video_receive_stream2.cc
@@ -257,6 +257,7 @@ VideoReceiveStream2::VideoReceiveStream2(
                                  &stats_proxy_,
                                  &stats_proxy_,
                                  nack_periodic_processor,
+                                 &stats_proxy_,
                                  this,  // OnCompleteFrameCallback
                                  std::move(config_.frame_decryptor),
                                  std::move(config_.frame_transformer)),
diff --git a/video/video_stream_buffer_controller.cc b/video/video_stream_buffer_controller.cc
index 83de75ac3f..01063b12f7 100644
--- a/video/video_stream_buffer_controller.cc
+++ b/video/video_stream_buffer_controller.cc
@@ -259,6 +259,7 @@ void VideoStreamBufferController::OnFrameReady(
 
   // Update stats.
   UpdateDroppedFrames();
+  UpdateDiscardedPackets();
   UpdateFrameBufferTimings(min_receive_time, now);
   UpdateTimingFrameInfo();
 
@@ -324,6 +325,17 @@ void VideoStreamBufferController::UpdateDroppedFrames()
       buffer_->GetTotalNumberOfDroppedFrames();
 }
 
+void VideoStreamBufferController::UpdateDiscardedPackets()
+    RTC_RUN_ON(&worker_sequence_checker_) {
+  const int discarded_packets = buffer_->GetTotalNumberOfDiscardedPackets() -
+                                packets_discarded_before_last_new_frame_;
+  if (discarded_packets > 0) {
+    stats_proxy_->OnDiscardedPackets(discarded_packets);
+  }
+  packets_discarded_before_last_new_frame_ =
+      buffer_->GetTotalNumberOfDiscardedPackets();
+}
+
 void VideoStreamBufferController::UpdateFrameBufferTimings(
     Timestamp min_receive_time,
     Timestamp now) {
diff --git a/video/video_stream_buffer_controller.h b/video/video_stream_buffer_controller.h
index 43d4544f71..3f3d6e465c 100644
--- a/video/video_stream_buffer_controller.h
+++ b/video/video_stream_buffer_controller.h
@@ -52,6 +52,8 @@ class VideoStreamBufferControllerStatsObserver {
                                 TimeDelta target_delay,
                                 TimeDelta minimum_delay) = 0;
 
+  virtual void OnDiscardedPackets(uint32_t packets_discarded) = 0;
+
   // Various jitter buffer delays determined by VCMTiming.
   virtual void OnFrameBufferTimingsUpdated(int estimated_max_decode_time_ms,
                                            int current_delay_ms,
@@ -94,6 +96,7 @@ class VideoStreamBufferController {
   void OnTimeout(TimeDelta delay);
   void FrameReadyForDecode(uint32_t rtp_timestamp, Timestamp render_time);
   void UpdateDroppedFrames() RTC_RUN_ON(&worker_sequence_checker_);
+  void UpdateDiscardedPackets() RTC_RUN_ON(&worker_sequence_checker_);
   void UpdateFrameBufferTimings(Timestamp min_receive_time, Timestamp now);
   void UpdateTimingFrameInfo();
   bool IsTooManyFramesQueued() const RTC_RUN_ON(&worker_sequence_checker_);
@@ -120,6 +123,8 @@ class VideoStreamBufferController {
       RTC_GUARDED_BY(&worker_sequence_checker_);
   int frames_dropped_before_last_new_frame_
       RTC_GUARDED_BY(&worker_sequence_checker_) = 0;
+  int packets_discarded_before_last_new_frame_
+      RTC_GUARDED_BY(&worker_sequence_checker_) = 0;
   VCMVideoProtection protection_mode_
       RTC_GUARDED_BY(&worker_sequence_checker_) = kProtectionNack;