/work/obj-fuzz/dist/include/TrackEncoder.h

Source (jump to first uncovered line)
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
 * You can obtain one at http://mozilla.org/MPL/2.0/. */

#ifndef TrackEncoder_h_
#define TrackEncoder_h_

#include "AudioSegment.h"
#include "EncodedFrameContainer.h"
#include "MediaStreamGraph.h"
#include "StreamTracks.h"
#include "TrackMetadataBase.h"
#include "VideoSegment.h"

namespace mozilla {

class AbstractThread;
class TrackEncoder;

class TrackEncoderListener
{
public:
  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TrackEncoderListener)

  /**
   * Called when the TrackEncoder's underlying encoder has been successfully
   * initialized and there's non-null data ready to be encoded.
   */
  virtual void Initialized(TrackEncoder* aEncoder) = 0;

  /**
   * Called when there's new data ready to be encoded.
   * Always called after Initialized().
   */
  virtual void DataAvailable(TrackEncoder* aEncoder) = 0;

  /**
   * Called after the TrackEncoder hit an unexpected error, causing it to
   * abort operation.
   */
  virtual void Error(TrackEncoder* aEncoder) = 0;
protected:
  virtual ~TrackEncoderListener() {}
};

/**
 * Base class of AudioTrackEncoder and VideoTrackEncoder. Lifetime managed by
 * MediaEncoder. All methods are to be called only on the worker thread.
 *
 * MediaStreamTrackListeners will get store raw data in mIncomingBuffer, so
 * mIncomingBuffer is protected by a lock. The control APIs are all called by
 * MediaEncoder on its dedicated thread, where GetEncodedTrack is called
 * periodically to swap out mIncomingBuffer, feed it to the encoder, and return
 * the encoded data.
 */
class TrackEncoder
{
  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TrackEncoder);

public:
  explicit TrackEncoder(TrackRate aTrackRate);

  virtual void Suspend(TimeStamp aTime) = 0;

  virtual void Resume(TimeStamp aTime) = 0;

  /**
   * Called by MediaEncoder to cancel the encoding.
   */
  virtual void Cancel() = 0;

  /**
   * Notifies us that we have reached the end of the stream and no more data
   * will be appended.
   */
  virtual void NotifyEndOfStream() = 0;

  /**
   * MediaStreamGraph notifies us about the time of the track's start.
   * This gets called on the MediaEncoder thread after a dispatch.
   */
  virtual void SetStartOffset(StreamTime aStartOffset) = 0;

  /**
   * Dispatched from MediaStreamGraph when it has run an iteration where the
   * input track of the track this TrackEncoder is associated with didn't have
   * any data.
   */
  virtual void AdvanceBlockedInput(StreamTime aDuration) = 0;

  /**
   * MediaStreamGraph notifies us about the duration of data that has just been
   * processed. This gets called on the MediaEncoder thread after a dispatch.
   */
  virtual void AdvanceCurrentTime(StreamTime aDuration) = 0;

  /**
   * Creates and sets up meta data for a specific codec, called on the worker
   * thread.
   */
  virtual already_AddRefed<TrackMetadataBase> GetMetadata() = 0;

  /**
   * Encodes raw segments. Result data is returned in aData, and called on the
   * worker thread.
   */
  virtual nsresult GetEncodedTrack(EncodedFrameContainer& aData) = 0;

  /**
   * Returns true once this TrackEncoder is initialized.
   */
  bool IsInitialized();

  /**
   * True if the track encoder has encoded all source segments coming from
   * MediaStreamGraph. Call on the worker thread.
   */
  bool IsEncodingComplete();

  /**
   * If this TrackEncoder was not already initialized, it is set to initialized
   * and listeners are notified.
   */
  void SetInitialized();

  /**
   * Notifies listeners that there is data available for encoding.
   */
  void OnDataAvailable();

  /**
   * Called after an error. Cancels the encoding and notifies listeners.
   */
  void OnError();

  /**
   * Registers a listener to events from this TrackEncoder.
   * We hold a strong reference to the listener.
   */
  void RegisterListener(TrackEncoderListener* aListener);

  /**
   * Unregisters a listener from events from this TrackEncoder.
   * The listener will stop receiving events synchronously.
   */
  bool UnregisterListener(TrackEncoderListener* aListener);

  virtual void SetBitrate(const uint32_t aBitrate) = 0;

  /**
   * It's optional to set the worker thread, but if you do we'll assert that
   * we are in the worker thread in every method that gets called.
   */
  void SetWorkerThread(AbstractThread* aWorkerThread);

  /**
   * Measure size of internal buffers.
   */
  virtual size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) = 0;

protected:
  virtual ~TrackEncoder()
  {
    MOZ_ASSERT(mListeners.IsEmpty());
  }

  /**
   * True if the track encoder has encoded all source data.
   */
  bool mEncodingComplete;

  /**
   * True if flag of EOS or any form of indicating EOS has set in the codec-
   * encoder.
   */
  bool mEosSetInEncoder;

  /**
   * True if the track encoder has been initialized successfully.
   */
  bool mInitialized;

  /**
   * True once all data until the end of the input track has been received.
   */
  bool mEndOfStream;

  /**
   * True once this encoding has been cancelled.
   */
  bool mCanceled;

  /**
   * The latest current time reported to us from the MSG.
   */
  StreamTime mCurrentTime;

  // How many times we have tried to initialize the encoder.
  uint32_t mInitCounter;
  StreamTime mNotInitDuration;

  bool mSuspended;

  /**
   * The track rate of source media.
   */
  TrackRate mTrackRate;

  /**
   * If set we assert that all methods are called on this thread.
   */
  RefPtr<AbstractThread> mWorkerThread;

  nsTArray<RefPtr<TrackEncoderListener>> mListeners;
};

class AudioTrackEncoder : public TrackEncoder
{
public:
  explicit AudioTrackEncoder(TrackRate aTrackRate)
    : TrackEncoder(aTrackRate)
    , mChannels(0)
    , mSamplingRate(0)
    , mAudioBitrate(0)
    , mDirectConnected(false)
  {}

  /**
   * Suspends encoding from mCurrentTime, i.e., all audio data until the next
   * Resume() will be dropped.
   */
  void Suspend(TimeStamp aTime) override;

  /**
   * Resumes encoding starting at mCurrentTime.
   */
  void Resume(TimeStamp aTime) override;

  /**
   * Appends and consumes track data from aSegment.
   */
  void AppendAudioSegment(AudioSegment&& aSegment);

  /**
   * Takes track data from the last time TakeTrackData ran until mCurrentTime
   * and moves it to aSegment.
   */
  void TakeTrackData(AudioSegment& aSegment);

  template<typename T>
  static
  void InterleaveTrackData(nsTArray<const T*>& aInput,
                           int32_t aDuration,
                           uint32_t aOutputChannels,
                           AudioDataValue* aOutput,
                           float aVolume)
  {
    if (aInput.Length() < aOutputChannels) {
      // Up-mix. This might make the mChannelData have more than aChannels.
      AudioChannelsUpMix(&aInput, aOutputChannels, SilentChannel::ZeroChannel<T>());
    }

    if (aInput.Length() > aOutputChannels) {
      DownmixAndInterleave(aInput, aDuration,
                           aVolume, aOutputChannels, aOutput);
    } else {
      InterleaveAndConvertBuffer(aInput.Elements(), aDuration, aVolume,
                                 aOutputChannels, aOutput);
    }
  }

  /**
   * Interleaves the track data and stores the result into aOutput. Might need
   * to up-mix or down-mix the channel data if the channels number of this chunk
   * is different from aOutputChannels. The channel data from aChunk might be
   * modified by up-mixing.
   */
  static void InterleaveTrackData(AudioChunk& aChunk, int32_t aDuration,
                                  uint32_t aOutputChannels,
                                  AudioDataValue* aOutput);

  /**
   * De-interleaves the aInput data and stores the result into aOutput.
   * No up-mix or down-mix operations inside.
   */
  static void DeInterleaveTrackData(AudioDataValue* aInput, int32_t aDuration,
                                    int32_t aChannels, AudioDataValue* aOutput);

  /**
   * Measure size of internal buffers.
   */
  size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) override;

  void SetBitrate(const uint32_t aBitrate) override
  {
    mAudioBitrate = aBitrate;
  }

  /**
   * Tries to initiate the AudioEncoder based on data in aSegment.
   * This can be re-called often, as it will exit early should we already be
   * initiated. mInitiated will only be set if there was enough data in
   * aSegment to infer metadata. If mInitiated gets set, listeners are notified.
   *
   * Not having enough data in aSegment to initiate the encoder for an accumulated aDuration of one second will make us initiate with a default number of channels.
   *
   * If we attempt to initiate the underlying encoder but fail, we Cancel() and
   * notify listeners.
   */
  void TryInit(const AudioSegment& aSegment, StreamTime aDuration);

  void Cancel() override;

  /**
   * Dispatched from MediaStreamGraph when we have finished feeding data to
   * mIncomingBuffer.
   */
  void NotifyEndOfStream() override;

  void SetStartOffset(StreamTime aStartOffset) override;

  /**
   * Dispatched from MediaStreamGraph when it has run an iteration where the
   * input track of the track this TrackEncoder is associated with didn't have
   * any data.
   *
   * Since we sometimes use a direct listener for AudioSegments we miss periods
   * of time for which the source didn't have any data. This ensures that the
   * latest frame gets displayed while we wait for more data to be pushed.
   */
  void AdvanceBlockedInput(StreamTime aDuration) override;

  /**
   * Dispatched from MediaStreamGraph when it has run an iteration so we can
   * hand more data to the encoder.
   */
  void AdvanceCurrentTime(StreamTime aDuration) override;
protected:
  /**
   * Number of samples per channel in a pcm buffer. This is also the value of
   * frame size required by audio encoder, and listeners will be notified when
   * at least this much data has been added to mOutgoingBuffer.
   */
  virtual int GetPacketDuration() { return 0; }

  /**
   * Initializes the audio encoder. The call of this method is delayed until we
   * have received the first valid track from MediaStreamGraph.
   */
  virtual nsresult Init(int aChannels, int aSamplingRate) = 0;

  /**
   * The number of channels are used for processing PCM data in the audio encoder.
   * This value comes from the first valid audio chunk. If encoder can't support
   * the channels in the chunk, downmix PCM stream can be performed.
   * This value also be used to initialize the audio encoder.
   */
  int mChannels;

  /**
   * The sampling rate of source audio data.
   */
  int mSamplingRate;

  /**
   * A segment queue of incoming audio track data, from listeners.
   * The duration of mIncomingBuffer is strictly increasing as it gets fed more
   * data. Consumed data is replaced by null data.
   */
  AudioSegment mIncomingBuffer;

  /**
   * A segment queue of outgoing audio track data to the encoder.
   * The contents of mOutgoingBuffer will always be what has been consumed from
   * mIncomingBuffer (up to mCurrentTime) but not yet consumed by the encoder
   * sub class.
   */
  AudioSegment mOutgoingBuffer;

  uint32_t mAudioBitrate;

  // This may only be accessed on the MSG thread.
  // I.e., in the regular NotifyQueuedChanges for audio to avoid adding data
  // from that callback when the direct one is active.
  bool mDirectConnected;
};

enum class FrameDroppingMode {
  ALLOW, // Allowed to drop frames to keep up under load
  DISALLOW, // Must not drop any frames, even if it means we will OOM
};

class VideoTrackEncoder : public TrackEncoder
{
public:
  explicit VideoTrackEncoder(TrackRate aTrackRate, FrameDroppingMode aFrameDroppingMode);

  /**
   * Suspends encoding from aTime, i.e., all video frame with a timestamp
   * between aTime and the timestamp of the next Resume() will be dropped.
   */
  void Suspend(TimeStamp aTime) override;

  /**
   * Resumes encoding starting at aTime.
   */
  void Resume(TimeStamp aTime) override;

  /**
   * Appends source video frames to mIncomingBuffer. We only append the source
   * chunk if the image is different from mLastChunk's image. Called on the
   * MediaStreamGraph thread.
   */
  void AppendVideoSegment(VideoSegment&& aSegment);

  /**
   * Takes track data from the last time TakeTrackData ran until mCurrentTime
   * and moves it to aSegment.
   */
  void TakeTrackData(VideoSegment& aSegment);

  /**
   * Measure size of internal buffers.
   */
  size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) override;

  void SetBitrate(const uint32_t aBitrate) override
  {
    mVideoBitrate = aBitrate;
  }

  /**
   * Tries to initiate the VideoEncoder based on data in aSegment.
   * This can be re-called often, as it will exit early should we already be
   * initiated. mInitiated will only be set if there was enough data in
   * aSegment to infer metadata. If mInitiated gets set, listeners are notified.
   *
   * Failing to initiate the encoder for an accumulated aDuration of 30 seconds
   * is seen as an error and will cancel the current encoding.
   */
  void Init(const VideoSegment& aSegment, StreamTime aDuration);

  StreamTime SecondsToMediaTime(double aS) const
  {
    NS_ASSERTION(0 <= aS && aS <= TRACK_TICKS_MAX/TRACK_RATE_MAX,
                 "Bad seconds");
    return mTrackRate * aS;
  }

  void Cancel() override;

  /**
   * Notifies us that we have reached the end of the stream and no more data
   * will be appended to mIncomingBuffer.
   */
  void NotifyEndOfStream() override;

  void SetStartOffset(StreamTime aStartOffset) override;

  /**
   * Dispatched from MediaStreamGraph when it has run an iteration where the
   * input track of the track this TrackEncoder is associated with didn't have
   * any data.
   *
   * Since we use a direct listener for VideoSegments we miss periods of time
   * for which the source didn't have any data. This ensures that the latest
   * frame gets displayed while we wait for more data to be pushed.
   */
  void AdvanceBlockedInput(StreamTime aDuration) override;

  /**
   * Dispatched from MediaStreamGraph when it has run an iteration so we can
   * hand more data to the encoder.
   */
  void AdvanceCurrentTime(StreamTime aDuration) override;

  /**
   * Set desired keyframe interval defined in milliseconds.
   */
  void SetKeyFrameInterval(int32_t aKeyFrameInterval);

protected:
  /**
   * Initialize the video encoder. In order to collect the value of width and
   * height of source frames, this initialization is delayed until we have
   * received the first valid video frame from MediaStreamGraph.
   * Listeners will be notified after it has been successfully initialized.
   */
  virtual nsresult Init(int aWidth, int aHeight, int aDisplayWidth,
                        int aDisplayHeight) = 0;

  /**
   * The width of source video frame, ceiled if the source width is odd.
   */
  int mFrameWidth;

  /**
   * The height of source video frame, ceiled if the source height is odd.
   */
  int mFrameHeight;

  /**
   * The display width of source video frame.
   */
  int mDisplayWidth;

  /**
   * The display height of source video frame.
   */
  int mDisplayHeight;

  /**
   * The last unique frame and duration so far handled by NotifyAdvanceCurrentTime.
   * When a new frame is detected, mLastChunk is added to mOutgoingBuffer.
   */
  VideoChunk mLastChunk;

  /**
   * A segment queue of incoming video track data, from listeners.
   * The duration of mIncomingBuffer is strictly increasing as it gets fed more
   * data. Consumed data is replaced by null data.
   */
  VideoSegment mIncomingBuffer;

  /**
   * A segment queue of outgoing video track data to the encoder.
   * The contents of mOutgoingBuffer will always be what has been consumed from
   * mIncomingBuffer (up to mCurrentTime) but not yet consumed by the encoder
   * sub class. There won't be any null data at the beginning of mOutgoingBuffer
   * unless explicitly pushed by the producer.
   */
  VideoSegment mOutgoingBuffer;

  /**
   * The number of mTrackRate ticks we have passed to mOutgoingBuffer.
   */
  StreamTime mEncodedTicks;

  /**
   * The time of the first real video frame passed to mOutgoingBuffer (at t=0).
   *
   * Note that this time will progress during suspension, to make sure the
   * incoming frames stay in sync with the output.
   */
  TimeStamp mStartTime;

  /**
   * The time Suspend was called on the MediaRecorder, so we can calculate the
   * duration on the next Resume().
   */
  TimeStamp mSuspendTime;

  uint32_t mVideoBitrate;

  /**
   * ALLOW to drop frames under load.
   * DISALLOW to encode all frames, mainly for testing.
   */
  FrameDroppingMode mFrameDroppingMode;

  /**
   * The desired keyframe interval defined in milliseconds.
   */
  int32_t mKeyFrameInterval;
};

} // namespace mozilla

#endif

Coverage Report

Created: 2018-09-25 14:53

Line	Count	Source (jump to first uncovered line)
1		/* -- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 --*/
2		/* This Source Code Form is subject to the terms of the Mozilla Public
3		* License, v. 2.0. If a copy of the MPL was not distributed with this file,
4		* You can obtain one at http://mozilla.org/MPL/2.0/. */
5
6		#ifndef TrackEncoder_h_
7		#define TrackEncoder_h_
8
9		#include "AudioSegment.h"
10		#include "EncodedFrameContainer.h"
11		#include "MediaStreamGraph.h"
12		#include "StreamTracks.h"
13		#include "TrackMetadataBase.h"
14		#include "VideoSegment.h"
15
16		namespace mozilla {
17
18		class AbstractThread;
19		class TrackEncoder;
20
21		class TrackEncoderListener
22		{
23		public:
24		NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TrackEncoderListener)
25
26		/**
27		* Called when the TrackEncoder's underlying encoder has been successfully
28		* initialized and there's non-null data ready to be encoded.
29		*/
30		virtual void Initialized(TrackEncoder* aEncoder) = 0;
31
32		/**
33		* Called when there's new data ready to be encoded.
34		* Always called after Initialized().
35		*/
36		virtual void DataAvailable(TrackEncoder* aEncoder) = 0;
37
38		/**
39		* Called after the TrackEncoder hit an unexpected error, causing it to
40		* abort operation.
41		*/
42		virtual void Error(TrackEncoder* aEncoder) = 0;
43		protected:
44	0	virtual ~TrackEncoderListener() {}
45		};
46
47		/**
48		* Base class of AudioTrackEncoder and VideoTrackEncoder. Lifetime managed by
49		* MediaEncoder. All methods are to be called only on the worker thread.
50		*
51		* MediaStreamTrackListeners will get store raw data in mIncomingBuffer, so
52		* mIncomingBuffer is protected by a lock. The control APIs are all called by
53		* MediaEncoder on its dedicated thread, where GetEncodedTrack is called
54		* periodically to swap out mIncomingBuffer, feed it to the encoder, and return
55		* the encoded data.
56		*/
57		class TrackEncoder
58		{
59		NS_INLINE_DECL_THREADSAFE_REFCOUNTING(TrackEncoder);
60
61		public:
62		explicit TrackEncoder(TrackRate aTrackRate);
63
64		virtual void Suspend(TimeStamp aTime) = 0;
65
66		virtual void Resume(TimeStamp aTime) = 0;
67
68		/**
69		* Called by MediaEncoder to cancel the encoding.
70		*/
71		virtual void Cancel() = 0;
72
73		/**
74		* Notifies us that we have reached the end of the stream and no more data
75		* will be appended.
76		*/
77		virtual void NotifyEndOfStream() = 0;
78
79		/**
80		* MediaStreamGraph notifies us about the time of the track's start.
81		* This gets called on the MediaEncoder thread after a dispatch.
82		*/
83		virtual void SetStartOffset(StreamTime aStartOffset) = 0;
84
85		/**
86		* Dispatched from MediaStreamGraph when it has run an iteration where the
87		* input track of the track this TrackEncoder is associated with didn't have
88		* any data.
89		*/
90		virtual void AdvanceBlockedInput(StreamTime aDuration) = 0;
91
92		/**
93		* MediaStreamGraph notifies us about the duration of data that has just been
94		* processed. This gets called on the MediaEncoder thread after a dispatch.
95		*/
96		virtual void AdvanceCurrentTime(StreamTime aDuration) = 0;
97
98		/**
99		* Creates and sets up meta data for a specific codec, called on the worker
100		* thread.
101		*/
102		virtual already_AddRefed<TrackMetadataBase> GetMetadata() = 0;
103
104		/**
105		* Encodes raw segments. Result data is returned in aData, and called on the
106		* worker thread.
107		*/
108		virtual nsresult GetEncodedTrack(EncodedFrameContainer& aData) = 0;
109
110		/**
111		* Returns true once this TrackEncoder is initialized.
112		*/
113		bool IsInitialized();
114
115		/**
116		* True if the track encoder has encoded all source segments coming from
117		* MediaStreamGraph. Call on the worker thread.
118		*/
119		bool IsEncodingComplete();
120
121		/**
122		* If this TrackEncoder was not already initialized, it is set to initialized
123		* and listeners are notified.
124		*/
125		void SetInitialized();
126
127		/**
128		* Notifies listeners that there is data available for encoding.
129		*/
130		void OnDataAvailable();
131
132		/**
133		* Called after an error. Cancels the encoding and notifies listeners.
134		*/
135		void OnError();
136
137		/**
138		* Registers a listener to events from this TrackEncoder.
139		* We hold a strong reference to the listener.
140		*/
141		void RegisterListener(TrackEncoderListener* aListener);
142
143		/**
144		* Unregisters a listener from events from this TrackEncoder.
145		* The listener will stop receiving events synchronously.
146		*/
147		bool UnregisterListener(TrackEncoderListener* aListener);
148
149		virtual void SetBitrate(const uint32_t aBitrate) = 0;
150
151		/**
152		* It's optional to set the worker thread, but if you do we'll assert that
153		* we are in the worker thread in every method that gets called.
154		*/
155		void SetWorkerThread(AbstractThread* aWorkerThread);
156
157		/**
158		* Measure size of internal buffers.
159		*/
160		virtual size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) = 0;
161
162		protected:
163		virtual ~TrackEncoder()
164	0	{
165	0	MOZ_ASSERT(mListeners.IsEmpty());
166	0	}
167
168		/**
169		* True if the track encoder has encoded all source data.
170		*/
171		bool mEncodingComplete;
172
173		/**
174		* True if flag of EOS or any form of indicating EOS has set in the codec-
175		* encoder.
176		*/
177		bool mEosSetInEncoder;
178
179		/**
180		* True if the track encoder has been initialized successfully.
181		*/
182		bool mInitialized;
183
184		/**
185		* True once all data until the end of the input track has been received.
186		*/
187		bool mEndOfStream;
188
189		/**
190		* True once this encoding has been cancelled.
191		*/
192		bool mCanceled;
193
194		/**
195		* The latest current time reported to us from the MSG.
196		*/
197		StreamTime mCurrentTime;
198
199		// How many times we have tried to initialize the encoder.
200		uint32_t mInitCounter;
201		StreamTime mNotInitDuration;
202
203		bool mSuspended;
204
205		/**
206		* The track rate of source media.
207		*/
208		TrackRate mTrackRate;
209
210		/**
211		* If set we assert that all methods are called on this thread.
212		*/
213		RefPtr<AbstractThread> mWorkerThread;
214
215		nsTArray<RefPtr<TrackEncoderListener>> mListeners;
216		};
217
218		class AudioTrackEncoder : public TrackEncoder
219		{
220		public:
221		explicit AudioTrackEncoder(TrackRate aTrackRate)
222		: TrackEncoder(aTrackRate)
223		, mChannels(0)
224		, mSamplingRate(0)
225		, mAudioBitrate(0)
226		, mDirectConnected(false)
227	0	{}
228
229		/**
230		* Suspends encoding from mCurrentTime, i.e., all audio data until the next
231		* Resume() will be dropped.
232		*/
233		void Suspend(TimeStamp aTime) override;
234
235		/**
236		* Resumes encoding starting at mCurrentTime.
237		*/
238		void Resume(TimeStamp aTime) override;
239
240		/**
241		* Appends and consumes track data from aSegment.
242		*/
243		void AppendAudioSegment(AudioSegment&& aSegment);
244
245		/**
246		* Takes track data from the last time TakeTrackData ran until mCurrentTime
247		* and moves it to aSegment.
248		*/
249		void TakeTrackData(AudioSegment& aSegment);
250
251		template<typename T>
252		static
253		void InterleaveTrackData(nsTArray<const T*>& aInput,
254		int32_t aDuration,
255		uint32_t aOutputChannels,
256		AudioDataValue* aOutput,
257		float aVolume)
258	0	{
259	0	if (aInput.Length() < aOutputChannels) {
260	0	// Up-mix. This might make the mChannelData have more than aChannels.
261	0	AudioChannelsUpMix(&aInput, aOutputChannels, SilentChannel::ZeroChannel<T>());
262	0	}
263	0
264	0	if (aInput.Length() > aOutputChannels) {
265	0	DownmixAndInterleave(aInput, aDuration,
266	0	aVolume, aOutputChannels, aOutput);
267	0	} else {
268	0	InterleaveAndConvertBuffer(aInput.Elements(), aDuration, aVolume,
269	0	aOutputChannels, aOutput);
270	0	}
271	0	} Unexecuted instantiation: void mozilla::AudioTrackEncoder::InterleaveTrackData<short>(nsTArray<short const>&, int, unsigned int, float, float) Unexecuted instantiation: void mozilla::AudioTrackEncoder::InterleaveTrackData<float>(nsTArray<float const>&, int, unsigned int, float, float)
272
273		/**
274		* Interleaves the track data and stores the result into aOutput. Might need
275		* to up-mix or down-mix the channel data if the channels number of this chunk
276		* is different from aOutputChannels. The channel data from aChunk might be
277		* modified by up-mixing.
278		*/
279		static void InterleaveTrackData(AudioChunk& aChunk, int32_t aDuration,
280		uint32_t aOutputChannels,
281		AudioDataValue* aOutput);
282
283		/**
284		* De-interleaves the aInput data and stores the result into aOutput.
285		* No up-mix or down-mix operations inside.
286		*/
287		static void DeInterleaveTrackData(AudioDataValue* aInput, int32_t aDuration,
288		int32_t aChannels, AudioDataValue* aOutput);
289
290		/**
291		* Measure size of internal buffers.
292		*/
293		size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) override;
294
295		void SetBitrate(const uint32_t aBitrate) override
296	0	{
297	0	mAudioBitrate = aBitrate;
298	0	}
299
300		/**
301		* Tries to initiate the AudioEncoder based on data in aSegment.
302		* This can be re-called often, as it will exit early should we already be
303		* initiated. mInitiated will only be set if there was enough data in
304		* aSegment to infer metadata. If mInitiated gets set, listeners are notified.
305		*
306		* Not having enough data in aSegment to initiate the encoder for an accumulated aDuration of one second will make us initiate with a default number of channels.
307		*
308		* If we attempt to initiate the underlying encoder but fail, we Cancel() and
309		* notify listeners.
310		*/
311		void TryInit(const AudioSegment& aSegment, StreamTime aDuration);
312
313		void Cancel() override;
314
315		/**
316		* Dispatched from MediaStreamGraph when we have finished feeding data to
317		* mIncomingBuffer.
318		*/
319		void NotifyEndOfStream() override;
320
321		void SetStartOffset(StreamTime aStartOffset) override;
322
323		/**
324		* Dispatched from MediaStreamGraph when it has run an iteration where the
325		* input track of the track this TrackEncoder is associated with didn't have
326		* any data.
327		*
328		* Since we sometimes use a direct listener for AudioSegments we miss periods
329		* of time for which the source didn't have any data. This ensures that the
330		* latest frame gets displayed while we wait for more data to be pushed.
331		*/
332		void AdvanceBlockedInput(StreamTime aDuration) override;
333
334		/**
335		* Dispatched from MediaStreamGraph when it has run an iteration so we can
336		* hand more data to the encoder.
337		*/
338		void AdvanceCurrentTime(StreamTime aDuration) override;
339		protected:
340		/**
341		* Number of samples per channel in a pcm buffer. This is also the value of
342		* frame size required by audio encoder, and listeners will be notified when
343		* at least this much data has been added to mOutgoingBuffer.
344		*/
345	0	virtual int GetPacketDuration() { return 0; }
346
347		/**
348		* Initializes the audio encoder. The call of this method is delayed until we
349		* have received the first valid track from MediaStreamGraph.
350		*/
351		virtual nsresult Init(int aChannels, int aSamplingRate) = 0;
352
353		/**
354		* The number of channels are used for processing PCM data in the audio encoder.
355		* This value comes from the first valid audio chunk. If encoder can't support
356		* the channels in the chunk, downmix PCM stream can be performed.
357		* This value also be used to initialize the audio encoder.
358		*/
359		int mChannels;
360
361		/**
362		* The sampling rate of source audio data.
363		*/
364		int mSamplingRate;
365
366		/**
367		* A segment queue of incoming audio track data, from listeners.
368		* The duration of mIncomingBuffer is strictly increasing as it gets fed more
369		* data. Consumed data is replaced by null data.
370		*/
371		AudioSegment mIncomingBuffer;
372
373		/**
374		* A segment queue of outgoing audio track data to the encoder.
375		* The contents of mOutgoingBuffer will always be what has been consumed from
376		* mIncomingBuffer (up to mCurrentTime) but not yet consumed by the encoder
377		* sub class.
378		*/
379		AudioSegment mOutgoingBuffer;
380
381		uint32_t mAudioBitrate;
382
383		// This may only be accessed on the MSG thread.
384		// I.e., in the regular NotifyQueuedChanges for audio to avoid adding data
385		// from that callback when the direct one is active.
386		bool mDirectConnected;
387		};
388
389		enum class FrameDroppingMode {
390		ALLOW, // Allowed to drop frames to keep up under load
391		DISALLOW, // Must not drop any frames, even if it means we will OOM
392		};
393
394		class VideoTrackEncoder : public TrackEncoder
395		{
396		public:
397		explicit VideoTrackEncoder(TrackRate aTrackRate, FrameDroppingMode aFrameDroppingMode);
398
399		/**
400		* Suspends encoding from aTime, i.e., all video frame with a timestamp
401		* between aTime and the timestamp of the next Resume() will be dropped.
402		*/
403		void Suspend(TimeStamp aTime) override;
404
405		/**
406		* Resumes encoding starting at aTime.
407		*/
408		void Resume(TimeStamp aTime) override;
409
410		/**
411		* Appends source video frames to mIncomingBuffer. We only append the source
412		* chunk if the image is different from mLastChunk's image. Called on the
413		* MediaStreamGraph thread.
414		*/
415		void AppendVideoSegment(VideoSegment&& aSegment);
416
417		/**
418		* Takes track data from the last time TakeTrackData ran until mCurrentTime
419		* and moves it to aSegment.
420		*/
421		void TakeTrackData(VideoSegment& aSegment);
422
423		/**
424		* Measure size of internal buffers.
425		*/
426		size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) override;
427
428		void SetBitrate(const uint32_t aBitrate) override
429	0	{
430	0	mVideoBitrate = aBitrate;
431	0	}
432
433		/**
434		* Tries to initiate the VideoEncoder based on data in aSegment.
435		* This can be re-called often, as it will exit early should we already be
436		* initiated. mInitiated will only be set if there was enough data in
437		* aSegment to infer metadata. If mInitiated gets set, listeners are notified.
438		*
439		* Failing to initiate the encoder for an accumulated aDuration of 30 seconds
440		* is seen as an error and will cancel the current encoding.
441		*/
442		void Init(const VideoSegment& aSegment, StreamTime aDuration);
443
444		StreamTime SecondsToMediaTime(double aS) const
445	0	{
446	0	NS_ASSERTION(0 <= aS && aS <= TRACK_TICKS_MAX/TRACK_RATE_MAX,
447	0	"Bad seconds");
448	0	return mTrackRate * aS;
449	0	}
450
451		void Cancel() override;
452
453		/**
454		* Notifies us that we have reached the end of the stream and no more data
455		* will be appended to mIncomingBuffer.
456		*/
457		void NotifyEndOfStream() override;
458
459		void SetStartOffset(StreamTime aStartOffset) override;
460
461		/**
462		* Dispatched from MediaStreamGraph when it has run an iteration where the
463		* input track of the track this TrackEncoder is associated with didn't have
464		* any data.
465		*
466		* Since we use a direct listener for VideoSegments we miss periods of time
467		* for which the source didn't have any data. This ensures that the latest
468		* frame gets displayed while we wait for more data to be pushed.
469		*/
470		void AdvanceBlockedInput(StreamTime aDuration) override;
471
472		/**
473		* Dispatched from MediaStreamGraph when it has run an iteration so we can
474		* hand more data to the encoder.
475		*/
476		void AdvanceCurrentTime(StreamTime aDuration) override;
477
478		/**
479		* Set desired keyframe interval defined in milliseconds.
480		*/
481		void SetKeyFrameInterval(int32_t aKeyFrameInterval);
482
483		protected:
484		/**
485		* Initialize the video encoder. In order to collect the value of width and
486		* height of source frames, this initialization is delayed until we have
487		* received the first valid video frame from MediaStreamGraph.
488		* Listeners will be notified after it has been successfully initialized.
489		*/
490		virtual nsresult Init(int aWidth, int aHeight, int aDisplayWidth,
491		int aDisplayHeight) = 0;
492
493		/**
494		* The width of source video frame, ceiled if the source width is odd.
495		*/
496		int mFrameWidth;
497
498		/**
499		* The height of source video frame, ceiled if the source height is odd.
500		*/
501		int mFrameHeight;
502
503		/**
504		* The display width of source video frame.
505		*/
506		int mDisplayWidth;
507
508		/**
509		* The display height of source video frame.
510		*/
511		int mDisplayHeight;
512
513		/**
514		* The last unique frame and duration so far handled by NotifyAdvanceCurrentTime.
515		* When a new frame is detected, mLastChunk is added to mOutgoingBuffer.
516		*/
517		VideoChunk mLastChunk;
518
519		/**
520		* A segment queue of incoming video track data, from listeners.
521		* The duration of mIncomingBuffer is strictly increasing as it gets fed more
522		* data. Consumed data is replaced by null data.
523		*/
524		VideoSegment mIncomingBuffer;
525
526		/**
527		* A segment queue of outgoing video track data to the encoder.
528		* The contents of mOutgoingBuffer will always be what has been consumed from
529		* mIncomingBuffer (up to mCurrentTime) but not yet consumed by the encoder
530		* sub class. There won't be any null data at the beginning of mOutgoingBuffer
531		* unless explicitly pushed by the producer.
532		*/
533		VideoSegment mOutgoingBuffer;
534
535		/**
536		* The number of mTrackRate ticks we have passed to mOutgoingBuffer.
537		*/
538		StreamTime mEncodedTicks;
539
540		/**
541		* The time of the first real video frame passed to mOutgoingBuffer (at t=0).
542		*
543		* Note that this time will progress during suspension, to make sure the
544		* incoming frames stay in sync with the output.
545		*/
546		TimeStamp mStartTime;
547
548		/**
549		* The time Suspend was called on the MediaRecorder, so we can calculate the
550		* duration on the next Resume().
551		*/
552		TimeStamp mSuspendTime;
553
554		uint32_t mVideoBitrate;
555
556		/**
557		* ALLOW to drop frames under load.
558		* DISALLOW to encode all frames, mainly for testing.
559		*/
560		FrameDroppingMode mFrameDroppingMode;
561
562		/**
563		* The desired keyframe interval defined in milliseconds.
564		*/
565		int32_t mKeyFrameInterval;
566		};
567
568		} // namespace mozilla
569
570		#endif