// Copyright 2020 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "cast/standalone_sender/streaming_opus_encoder.h" #include #include #include #include "util/chrono_helpers.h" namespace openscreen { namespace cast { using openscreen::operator<<; // To pretty-print chrono values. namespace { // The bitrate at which virtually all stereo audio can be encoded and decoded // without human-perceivable artifacts. Source: // https://wiki.hydrogenaud.io/index.php?title=Opus#Bitrate_performance constexpr opus_int32 kTransparentBitrate = 160000; // The maximum number of Cast audio frames the encoder may fall behind by before // skipping-ahead the RTP timestamps to compensate. constexpr int kMaxCastFramesBeforeSkip = 3; } // namespace StreamingOpusEncoder::StreamingOpusEncoder(int num_channels, int cast_frames_per_second, Sender* sender) : num_channels_(num_channels), sender_(sender), samples_per_cast_frame_(sample_rate() / cast_frames_per_second), approximate_cast_frame_duration_(Clock::to_duration(seconds(1)) / cast_frames_per_second), encoder_storage_(new uint8_t[opus_encoder_get_size(num_channels_)]), input_(new float[num_channels_ * samples_per_cast_frame_]), output_(new uint8_t[kOpusMaxPayloadSize]) { OSP_CHECK_GT(cast_frames_per_second, 0); OSP_DCHECK(sender_); OSP_CHECK_GT(samples_per_cast_frame_, 0); OSP_CHECK_EQ(sample_rate() % cast_frames_per_second, 0); OSP_CHECK(approximate_cast_frame_duration_ > Clock::duration::zero()); frame_.dependency = EncodedFrame::KEY_FRAME; const auto init_result = opus_encoder_init( encoder(), sample_rate(), num_channels_, OPUS_APPLICATION_AUDIO); OSP_CHECK_EQ(init_result, OPUS_OK); UseStandardQuality(); } StreamingOpusEncoder::~StreamingOpusEncoder() = default; int StreamingOpusEncoder::GetBitrate() const { opus_int32 bitrate; const auto ctl_result = opus_encoder_ctl(encoder(), OPUS_GET_BITRATE(&bitrate)); OSP_CHECK_EQ(ctl_result, OPUS_OK); return bitrate; } void StreamingOpusEncoder::UseStandardQuality() { const auto ctl_result = opus_encoder_ctl(encoder(), OPUS_SET_BITRATE(OPUS_AUTO)); OSP_CHECK_EQ(ctl_result, OPUS_OK); UpdateCodecDelay(); } void StreamingOpusEncoder::UseHighQuality() { // kTransparentBitrate assumes stereo audio. Scale it by the actual number of // channels. const opus_int32 bitrate = kTransparentBitrate * num_channels_ / 2; const auto ctl_result = opus_encoder_ctl(encoder(), OPUS_SET_BITRATE(bitrate)); OSP_CHECK_EQ(ctl_result, OPUS_OK); UpdateCodecDelay(); } void StreamingOpusEncoder::EncodeAndSend(const float* interleaved_samples, int num_samples, Clock::time_point reference_time) { OSP_DCHECK(interleaved_samples); OSP_DCHECK_GT(num_samples, 0); ResolveTimestampsAndMaybeSkip(reference_time); while (num_samples > 0) { const int samples_copied = FillInputBuffer(interleaved_samples, num_samples); num_samples -= samples_copied; interleaved_samples += num_channels_ * samples_copied; if (num_samples_queued_ < samples_per_cast_frame_) { return; // Not enough yet for a full Cast audio frame. } const opus_int32 packet_size_or_error = opus_encode_float(encoder(), input_.get(), num_samples_queued_, output_.get(), kOpusMaxPayloadSize); num_samples_queued_ = 0; if (packet_size_or_error < 0) { OSP_LOG_FATAL << "AUDIO[" << sender_->ssrc() << "] Error code from opus_encode_float(): " << packet_size_or_error; return; } frame_.frame_id = sender_->GetNextFrameId(); frame_.referenced_frame_id = frame_.frame_id; // Note: It's possible for Opus to encode a zero byte packet. Send a Cast // audio frame anyway, to represent the passage of silence and to send other // stream metadata. frame_.data = absl::Span(output_.get(), packet_size_or_error); last_sent_frame_reference_time_ = frame_.reference_time; switch (sender_->EnqueueFrame(frame_)) { case Sender::OK: break; case Sender::PAYLOAD_TOO_LARGE: OSP_NOTREACHED(); // The Opus packet cannot possibly be too large. break; case Sender::REACHED_ID_SPAN_LIMIT: OSP_LOG_WARN << "AUDIO[" << sender_->ssrc() << "] Dropping: FrameId span limit reached."; break; case Sender::MAX_DURATION_IN_FLIGHT: OSP_LOG_INFO << "AUDIO[" << sender_->ssrc() << "] Dropping: In-flight duration would be too high."; break; } frame_.rtp_timestamp += RtpTimeDelta::FromTicks(samples_per_cast_frame_); frame_.reference_time += approximate_cast_frame_duration_; } } void StreamingOpusEncoder::UpdateCodecDelay() { opus_int32 lookahead = 0; const auto ctl_result = opus_encoder_ctl(encoder(), OPUS_GET_LOOKAHEAD(&lookahead)); OSP_CHECK_EQ(ctl_result, OPUS_OK); codec_delay_ = RtpTimeDelta::FromTicks(lookahead).ToDuration( sample_rate()); } void StreamingOpusEncoder::ResolveTimestampsAndMaybeSkip( Clock::time_point reference_time) { // Back-track the reference time to account for the audio delay introduced by // the codec. reference_time -= codec_delay_; // Special case: Nothing special for the first frame's timestamps. if (start_time_ == Clock::time_point::min()) { frame_.rtp_timestamp = RtpTimeTicks(); frame_.reference_time = start_time_ = reference_time; last_sent_frame_reference_time_ = reference_time - approximate_cast_frame_duration_; return; } const RtpTimeTicks current_position = frame_.rtp_timestamp + RtpTimeDelta::FromTicks(num_samples_queued_); const RtpTimeTicks reference_position = RtpTimeTicks::FromTimeSinceOrigin( reference_time - start_time_, sample_rate()); const RtpTimeDelta rtp_advancement = reference_position - current_position; const RtpTimeDelta skip_threshold = RtpTimeDelta::FromTicks(samples_per_cast_frame_) * kMaxCastFramesBeforeSkip; if (rtp_advancement > skip_threshold) { OSP_LOG_WARN << "Detected audio gap " << rtp_advancement.ToDuration(sample_rate()) << ", skipping ahead..."; num_samples_queued_ = 0; frame_.rtp_timestamp = reference_position; } // Further back-track the reference time to account for the already-queued // samples. reference_time -= RtpTimeDelta::FromTicks(num_samples_queued_) .ToDuration(sample_rate()); // Frame reference times must be monotonically increasing. A little noise in // the negative direction is okay to cap-off. Log a warning if there's a // bigger problem (at the source). const Clock::time_point lower_bound = last_sent_frame_reference_time_ + RtpTimeDelta::FromTicks(1).ToDuration(sample_rate()); if (reference_time < lower_bound) { const Clock::duration backwards_amount = last_sent_frame_reference_time_ - reference_time; OSP_LOG_IF(WARN, backwards_amount >= approximate_cast_frame_duration_) << "Reference time went *backwards* too much (" << backwards_amount << " in wrong direction). A/V sync may suffer at the Receiver!"; reference_time = lower_bound; } frame_.reference_time = reference_time; } int StreamingOpusEncoder::FillInputBuffer(const float* interleaved_samples, int num_samples) { const int samples_needed = samples_per_cast_frame_ - num_samples_queued_; const int samples_to_copy = std::min(num_samples, samples_needed); std::copy(interleaved_samples, interleaved_samples + num_channels_ * samples_to_copy, input_.get() + num_channels_ * num_samples_queued_); num_samples_queued_ += samples_to_copy; return samples_to_copy; } // static constexpr int StreamingOpusEncoder::kDefaultCastAudioFramesPerSecond; // static constexpr int StreamingOpusEncoder::kOpusMaxPayloadSize; } // namespace cast } // namespace openscreen