/src/mozilla-central/dom/media/webrtc/MediaEngineDefault.cpp

Source (jump to first uncovered line)
/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
 * You can obtain one at http://mozilla.org/MPL/2.0/. */

#include "MediaEngineDefault.h"

#include "ImageContainer.h"
#include "ImageTypes.h"
#include "Layers.h"
#include "MediaStreamGraph.h"
#include "MediaTrackConstraints.h"
#include "mozilla/dom/File.h"
#include "mozilla/UniquePtr.h"
#include "nsCOMPtr.h"
#include "nsContentUtils.h"
#include "nsIFile.h"
#include "nsIFilePicker.h"
#include "nsIPrefBranch.h"
#include "nsIPrefService.h"
#include "Tracing.h"

#ifdef MOZ_WIDGET_ANDROID
#include "nsISupportsUtils.h"
#endif

#ifdef MOZ_WEBRTC
#include "YuvStamper.h"
#endif

#define DEFAULT_AUDIO_TIMER_MS 10
namespace mozilla {

using namespace mozilla::gfx;

/**
 * Default video source.
 */

MediaEngineDefaultVideoSource::MediaEngineDefaultVideoSource()
  : mTimer(nullptr)
  , mMutex("MediaEngineDefaultVideoSource::mMutex")
{}

MediaEngineDefaultVideoSource::~MediaEngineDefaultVideoSource()
{}

nsString
MediaEngineDefaultVideoSource::GetName() const
{
  return NS_LITERAL_STRING(u"Default Video Device");
}

nsCString
MediaEngineDefaultVideoSource::GetUUID() const
{
  return NS_LITERAL_CSTRING("1041FCBD-3F12-4F7B-9E9B-1EC556DD5676");
}

uint32_t
MediaEngineDefaultVideoSource::GetBestFitnessDistance(
    const nsTArray<const NormalizedConstraintSet*>& aConstraintSets,
    const nsString& aDeviceId) const
{
  AssertIsOnOwningThread();

  uint32_t distance = 0;
#ifdef MOZ_WEBRTC
  for (const auto* cs : aConstraintSets) {
    distance = MediaConstraintsHelper::GetMinimumFitnessDistance(*cs, aDeviceId);
    break; // distance is read from first entry only
  }
#endif
  return distance;
}

nsresult
MediaEngineDefaultVideoSource::Allocate(const dom::MediaTrackConstraints &aConstraints,
                                        const MediaEnginePrefs &aPrefs,
                                        const nsString& aDeviceId,
                                        const mozilla::ipc::PrincipalInfo& aPrincipalInfo,
                                        AllocationHandle** aOutHandle,
                                        const char** aOutBadConstraint)
{
  AssertIsOnOwningThread();

  MOZ_ASSERT(mState == kReleased);

  FlattenedConstraints c(aConstraints);

  // Mock failure for automated tests.
  if (c.mDeviceId.mIdeal.find(NS_LITERAL_STRING("bad device")) !=
      c.mDeviceId.mIdeal.end()) {
    return NS_ERROR_FAILURE;
  }

  // emulator debug is very, very slow; reduce load on it with smaller/slower fake video
  mOpts = aPrefs;
  mOpts.mWidth = c.mWidth.Get(aPrefs.mWidth ? aPrefs.mWidth :
#ifdef DEBUG
                              MediaEnginePrefs::DEFAULT_43_VIDEO_WIDTH/2
#else
                              MediaEnginePrefs::DEFAULT_43_VIDEO_WIDTH
#endif
                              );
  mOpts.mHeight = c.mHeight.Get(aPrefs.mHeight ? aPrefs.mHeight :
#ifdef DEBUG
                                MediaEnginePrefs::DEFAULT_43_VIDEO_HEIGHT/2
#else
                                MediaEnginePrefs::DEFAULT_43_VIDEO_HEIGHT
#endif
                                );
  mOpts.mWidth = std::max(160, std::min(mOpts.mWidth, 4096)) & ~1;
  mOpts.mHeight = std::max(90, std::min(mOpts.mHeight, 2160)) & ~1;
  *aOutHandle = nullptr;

  MutexAutoLock lock(mMutex);
  mState = kAllocated;
  return NS_OK;
}

nsresult
MediaEngineDefaultVideoSource::Deallocate(const RefPtr<const AllocationHandle>& aHandle)
{
  AssertIsOnOwningThread();

  MOZ_ASSERT(!aHandle);
  MOZ_ASSERT(!mImage);
  MOZ_ASSERT(mState == kStopped || mState == kAllocated);

  MutexAutoLock lock(mMutex);
  if (mStream && IsTrackIDExplicit(mTrackID)) {
    mStream->EndTrack(mTrackID);
    mStream = nullptr;
    mTrackID = TRACK_NONE;
  }
  mState = kReleased;
  mImageContainer = nullptr;

  return NS_OK;
}

static void AllocateSolidColorFrame(layers::PlanarYCbCrData& aData,
                                    int aWidth, int aHeight,
                                    int aY, int aCb, int aCr)
{
  MOZ_ASSERT(!(aWidth&1));
  MOZ_ASSERT(!(aHeight&1));
  // Allocate a single frame with a solid color
  int yLen = aWidth*aHeight;
  int cbLen = yLen>>2;
  int crLen = cbLen;
  uint8_t* frame = (uint8_t*) malloc(yLen+cbLen+crLen);
  memset(frame, aY, yLen);
  memset(frame+yLen, aCb, cbLen);
  memset(frame+yLen+cbLen, aCr, crLen);

  aData.mYChannel = frame;
  aData.mYSize = IntSize(aWidth, aHeight);
  aData.mYStride = aWidth;
  aData.mCbCrStride = aWidth>>1;
  aData.mCbChannel = frame + yLen;
  aData.mCrChannel = aData.mCbChannel + cbLen;
  aData.mCbCrSize = IntSize(aWidth>>1, aHeight>>1);
  aData.mPicX = 0;
  aData.mPicY = 0;
  aData.mPicSize = IntSize(aWidth, aHeight);
  aData.mStereoMode = StereoMode::MONO;
}

static void ReleaseFrame(layers::PlanarYCbCrData& aData)
{
  free(aData.mYChannel);
}

nsresult
MediaEngineDefaultVideoSource::SetTrack(const RefPtr<const AllocationHandle>& aHandle,
                                        const RefPtr<SourceMediaStream>& aStream,
                                        TrackID aTrackID,
                                        const PrincipalHandle& aPrincipal)
{
  AssertIsOnOwningThread();

  MOZ_ASSERT(mState == kAllocated);
  MOZ_ASSERT(!mStream);
  MOZ_ASSERT(mTrackID == TRACK_NONE);

  {
    MutexAutoLock lock(mMutex);
    mStream = aStream;
    mTrackID = aTrackID;
  }
  aStream->AddTrack(aTrackID, 0, new VideoSegment(),
                    SourceMediaStream::ADDTRACK_QUEUED);
  return NS_OK;
}

nsresult
MediaEngineDefaultVideoSource::Start(const RefPtr<const AllocationHandle>& aHandle)
{
  AssertIsOnOwningThread();

  MOZ_ASSERT(mState == kAllocated || mState == kStopped);
  MOZ_ASSERT(mStream, "SetTrack() must happen before Start()");
  MOZ_ASSERT(IsTrackIDExplicit(mTrackID), "SetTrack() must happen before Start()");

  mTimer = NS_NewTimer();
  if (!mTimer) {
    return NS_ERROR_FAILURE;
  }

  if (!mImageContainer) {
    mImageContainer =
      layers::LayerManager::CreateImageContainer(layers::ImageContainer::ASYNCHRONOUS);
  }

  // Start timer for subsequent frames
  uint32_t interval;
#if defined(MOZ_WIDGET_ANDROID) && defined(DEBUG)
// emulator debug is very, very slow and has problems dealing with realtime audio inputs
  interval = 10 * (1000 / mOpts.mFPS);
#else
  interval = 1000 / mOpts.mFPS;
#endif
  mTimer->InitWithNamedFuncCallback([](nsITimer* aTimer, void* aClosure) {
      RefPtr<MediaEngineDefaultVideoSource> source =
        static_cast<MediaEngineDefaultVideoSource*>(aClosure);
      source->GenerateFrame();
    }, this, interval, nsITimer::TYPE_REPEATING_SLACK,
    "MediaEngineDefaultVideoSource::GenerateFrame");

  MutexAutoLock lock(mMutex);
  mState = kStarted;
  return NS_OK;
}

nsresult
MediaEngineDefaultVideoSource::Stop(const RefPtr<const AllocationHandle>& aHandle)
{
  AssertIsOnOwningThread();

  if (mState == kStopped || mState == kAllocated) {
    return NS_OK;
  }

  MOZ_ASSERT(mState == kStarted);
  MOZ_ASSERT(mTimer);
  MOZ_ASSERT(mStream);
  MOZ_ASSERT(IsTrackIDExplicit(mTrackID));

  mTimer->Cancel();
  mTimer = nullptr;

  MutexAutoLock lock(mMutex);

  mImage = nullptr;
  mState = kStopped;

  return NS_OK;
}

nsresult
MediaEngineDefaultVideoSource::Reconfigure(
    const RefPtr<AllocationHandle>& aHandle,
    const dom::MediaTrackConstraints& aConstraints,
    const MediaEnginePrefs &aPrefs,
    const nsString& aDeviceId,
    const char** aOutBadConstraint)
{
  return NS_OK;
}

void
MediaEngineDefaultVideoSource::GenerateFrame()
{
  AssertIsOnOwningThread();

  // Update the target color
  if (mCr <= 16) {
    if (mCb < 240) {
      mCb++;
    } else {
      mCr++;
    }
  } else if (mCb >= 240) {
    if (mCr < 240) {
      mCr++;
    } else {
      mCb--;
    }
  } else if (mCr >= 240) {
    if (mCb > 16) {
      mCb--;
    } else {
      mCr--;
    }
  } else {
    mCr--;
  }

  // Allocate a single solid color image
  RefPtr<layers::PlanarYCbCrImage> ycbcr_image = mImageContainer->CreatePlanarYCbCrImage();
  layers::PlanarYCbCrData data;
  AllocateSolidColorFrame(data, mOpts.mWidth, mOpts.mHeight, 0x80, mCb, mCr);

#ifdef MOZ_WEBRTC
  uint64_t timestamp = PR_Now();
  YuvStamper::Encode(mOpts.mWidth, mOpts.mHeight, mOpts.mWidth,
         data.mYChannel,
         reinterpret_cast<unsigned char*>(&timestamp), sizeof(timestamp),
         0, 0);
#endif

  bool setData = ycbcr_image->CopyData(data);
  MOZ_ASSERT(setData);

  // SetData copies data, so we can free the frame
  ReleaseFrame(data);

  if (!setData) {
    return;
  }

  MutexAutoLock lock(mMutex);
  mImage = std::move(ycbcr_image);
}

void
MediaEngineDefaultVideoSource::Pull(const RefPtr<const AllocationHandle>& aHandle,
                                    const RefPtr<SourceMediaStream>& aStream,
                                    TrackID aTrackID,
                                    StreamTime aDesiredTime,
                                    const PrincipalHandle& aPrincipalHandle)
{
  TRACE_AUDIO_CALLBACK_COMMENT("SourceMediaStream %p track %i",
                               aStream.get(), aTrackID);
  // AppendFrame takes ownership of `segment`
  VideoSegment segment;

  RefPtr<layers::Image> image;
  {
    MutexAutoLock lock(mMutex);
    // Started - append real image
    // Stopped - append null
    // Released - Track is ended, safe to ignore
    //            Can happen because NotifyPull comes from a stream listener
    if (mState == kReleased) {
      return;
    }
    MOZ_ASSERT(mState != kAllocated);
    if (mState == kStarted) {
      MOZ_ASSERT(mStream == aStream);
      MOZ_ASSERT(mTrackID == aTrackID);
      image = mImage;
    }
  }

  StreamTime delta = aDesiredTime - aStream->GetEndOfAppendedData(aTrackID);
  if (delta > 0) {
    // nullptr images are allowed
    IntSize size(mOpts.mWidth, mOpts.mHeight);
    segment.AppendFrame(image.forget(), delta, size, aPrincipalHandle);
    // This can fail if either a) we haven't added the track yet, or b)
    // we've removed or finished the track.
    aStream->AppendToTrack(aTrackID, &segment);
  }
}

/**
 * Default audio source.
 */

MediaEngineDefaultAudioSource::MediaEngineDefaultAudioSource()
  : mMutex("MediaEngineDefaultAudioSource::mMutex")
{}

MediaEngineDefaultAudioSource::~MediaEngineDefaultAudioSource()
{}

nsString
MediaEngineDefaultAudioSource::GetName() const
{
  return NS_LITERAL_STRING(u"Default Audio Device");
}

nsCString
MediaEngineDefaultAudioSource::GetUUID() const
{
  return NS_LITERAL_CSTRING("B7CBD7C1-53EF-42F9-8353-73F61C70C092");
}

uint32_t
MediaEngineDefaultAudioSource::GetBestFitnessDistance(
    const nsTArray<const NormalizedConstraintSet*>& aConstraintSets,
    const nsString& aDeviceId) const
{
  uint32_t distance = 0;
#ifdef MOZ_WEBRTC
  for (const auto* cs : aConstraintSets) {
    distance = MediaConstraintsHelper::GetMinimumFitnessDistance(*cs, aDeviceId);
    break; // distance is read from first entry only
  }
#endif
  return distance;
}

bool
MediaEngineDefaultAudioSource::IsAvailable() const
{
  AssertIsOnOwningThread();

  return mState == kReleased;
}

nsresult
MediaEngineDefaultAudioSource::Allocate(const dom::MediaTrackConstraints &aConstraints,
                                        const MediaEnginePrefs &aPrefs,
                                        const nsString& aDeviceId,
                                        const mozilla::ipc::PrincipalInfo& aPrincipalInfo,
                                        AllocationHandle** aOutHandle,
                                        const char** aOutBadConstraint)
{
  AssertIsOnOwningThread();

  MOZ_ASSERT(mState == kReleased);

  // Mock failure for automated tests.
  if (aConstraints.mDeviceId.IsString() &&
      aConstraints.mDeviceId.GetAsString().EqualsASCII("bad device")) {
    return NS_ERROR_FAILURE;
  }

  mFreq = aPrefs.mFreq ? aPrefs.mFreq : 1000;
  *aOutHandle = nullptr;

  MutexAutoLock lock(mMutex);
  mState = kAllocated;
  return NS_OK;
}

nsresult
MediaEngineDefaultAudioSource::Deallocate(const RefPtr<const AllocationHandle>& aHandle)
{
  AssertIsOnOwningThread();

  MOZ_ASSERT(!aHandle);
  MOZ_ASSERT(mState == kStopped || mState == kAllocated);

  MutexAutoLock lock(mMutex);
  if (mStream && IsTrackIDExplicit(mTrackID)) {
    mStream->EndTrack(mTrackID);
    mStream = nullptr;
    mTrackID = TRACK_NONE;
  }
  mState = kReleased;
  return NS_OK;
}

nsresult
MediaEngineDefaultAudioSource::SetTrack(const RefPtr<const AllocationHandle>& aHandle,
                                        const RefPtr<SourceMediaStream>& aStream,
                                        TrackID aTrackID,
                                        const PrincipalHandle& aPrincipal)
{
  AssertIsOnOwningThread();

  MOZ_ASSERT(mState == kAllocated);
  MOZ_ASSERT(!mStream);
  MOZ_ASSERT(mTrackID == TRACK_NONE);

  // AddAudioTrack will take ownership of segment
  mStream = aStream;
  mTrackID = aTrackID;
  aStream->AddAudioTrack(aTrackID,
                         aStream->GraphRate(),
                         0,
                         new AudioSegment(),
                         SourceMediaStream::ADDTRACK_QUEUED);
  return NS_OK;
}

nsresult
MediaEngineDefaultAudioSource::Start(const RefPtr<const AllocationHandle>& aHandle)
{
  AssertIsOnOwningThread();

  MOZ_ASSERT(mState == kAllocated || mState == kStopped);
  MOZ_ASSERT(mStream, "SetTrack() must happen before Start()");
  MOZ_ASSERT(IsTrackIDExplicit(mTrackID), "SetTrack() must happen before Start()");

  if (!mSineGenerator) {
    // generate sine wave (default 1KHz)
    mSineGenerator = new SineWaveGenerator(mStream->GraphRate(), mFreq);
  }

  MutexAutoLock lock(mMutex);
  mState = kStarted;
  return NS_OK;
}

nsresult
MediaEngineDefaultAudioSource::Stop(const RefPtr<const AllocationHandle>& aHandle)
{
  AssertIsOnOwningThread();

  if (mState == kStopped || mState == kAllocated) {
    return NS_OK;
  }

  MOZ_ASSERT(mState == kStarted);

  MutexAutoLock lock(mMutex);
  mState = kStopped;
  return NS_OK;
}

nsresult
MediaEngineDefaultAudioSource::Reconfigure(
    const RefPtr<AllocationHandle>& aHandle,
    const dom::MediaTrackConstraints& aConstraints,
    const MediaEnginePrefs &aPrefs,
    const nsString& aDeviceId,
    const char** aOutBadConstraint)
{
  return NS_OK;
}

void
MediaEngineDefaultAudioSource::AppendToSegment(AudioSegment& aSegment,
                                               TrackTicks aSamples,
                                               const PrincipalHandle& aPrincipalHandle)
{
  RefPtr<SharedBuffer> buffer = SharedBuffer::Create(aSamples * sizeof(int16_t));
  int16_t* dest = static_cast<int16_t*>(buffer->Data());

  mSineGenerator->generate(dest, aSamples);
  AutoTArray<const int16_t*,1> channels;
  channels.AppendElement(dest);
  aSegment.AppendFrames(buffer.forget(), channels, aSamples, aPrincipalHandle);
}

void
MediaEngineDefaultAudioSource::Pull(const RefPtr<const AllocationHandle>& aHandle,
                                    const RefPtr<SourceMediaStream>& aStream,
                                    TrackID aTrackID,
                                    StreamTime aDesiredTime,
                                    const PrincipalHandle& aPrincipalHandle)
{
  TRACE_AUDIO_CALLBACK_COMMENT("SourceMediaStream %p track %i",
                               aStream.get(), aTrackID);
  AudioSegment segment;
  // avoid accumulating rounding errors
  TrackTicks desired = aStream->TimeToTicksRoundUp(aStream->GraphRate(), aDesiredTime);
  TrackTicks delta = desired - mLastNotify;
  mLastNotify += delta;
  AppendToSegment(segment, delta, aPrincipalHandle);
  aStream->AppendToTrack(aTrackID, &segment);
}

void
MediaEngineDefault::EnumerateDevices(uint64_t aWindowId,
                                     dom::MediaSourceEnum aMediaSource,
                                     MediaSinkEnum aMediaSink,
                                     nsTArray<RefPtr<MediaDevice>>* aDevices)
{
  AssertIsOnOwningThread();

  switch (aMediaSource) {
    case dom::MediaSourceEnum::Camera: {
      // Only supports camera video sources. See Bug 1038241.

      // We once had code here to find a VideoSource with the same settings and
      // re-use that. This is no longer possible since the resolution gets set
      // in Allocate().

      nsTArray<RefPtr<MediaEngineSource>>*
        devicesForThisWindow = mVSources.LookupOrAdd(aWindowId);
      auto newSource = MakeRefPtr<MediaEngineDefaultVideoSource>();
      devicesForThisWindow->AppendElement(newSource);
      aDevices->AppendElement(MakeRefPtr<MediaDevice>(
                                newSource,
                                newSource->GetName(),
                                NS_ConvertUTF8toUTF16(newSource->GetUUID())));
      return;
    }
    case dom::MediaSourceEnum::Microphone: {
      nsTArray<RefPtr<MediaEngineDefaultAudioSource>>*
        devicesForThisWindow = mASources.LookupOrAdd(aWindowId);
      for (const RefPtr<MediaEngineDefaultAudioSource>& source : *devicesForThisWindow) {
        if (source->IsAvailable()) {
          aDevices->AppendElement(MakeRefPtr<MediaDevice>(
                                    source,
                                    source->GetName(),
                                    NS_ConvertUTF8toUTF16(source->GetUUID())));
        }
      }

      if (aDevices->IsEmpty()) {
        // All streams are currently busy, just make a new one.
        auto newSource = MakeRefPtr<MediaEngineDefaultAudioSource>();
        devicesForThisWindow->AppendElement(newSource);
        aDevices->AppendElement(MakeRefPtr<MediaDevice>(
                                  newSource,
                                  newSource->GetName(),
                                  NS_ConvertUTF8toUTF16(newSource->GetUUID())));
      }
      return;
    }
    default:
      MOZ_ASSERT_UNREACHABLE("Unsupported source type");
      return;
  }

  if (aMediaSink == MediaSinkEnum::Speaker) {
    NS_WARNING("No default implementation for MediaSinkEnum::Speaker");
  }
}

void
MediaEngineDefault::ReleaseResourcesForWindow(uint64_t aWindowId)
{
  nsTArray<RefPtr<MediaEngineDefaultAudioSource>>* audioDevicesForThisWindow =
   mASources.Get(aWindowId);

  if (audioDevicesForThisWindow) {
    for (const RefPtr<MediaEngineDefaultAudioSource>& source :
         *audioDevicesForThisWindow) {
      source->Shutdown();
    }
  }

  mASources.Remove(aWindowId);

  nsTArray<RefPtr<MediaEngineSource>>* videoDevicesForThisWindow =
    mVSources.Get(aWindowId);

  if (videoDevicesForThisWindow) {
    for (const RefPtr<MediaEngineSource>& source :
         *videoDevicesForThisWindow) {
      source->Shutdown();
    }
  }

  mVSources.Remove(aWindowId);
}

void
MediaEngineDefault::Shutdown()
{
  AssertIsOnOwningThread();

  for (auto iter = mVSources.Iter(); !iter.Done(); iter.Next()) {
    for (const RefPtr<MediaEngineSource>& source : *iter.UserData()) {
      if (source) {
        source->Shutdown();
      }
    }
  }
  for (auto iter = mASources.Iter(); !iter.Done(); iter.Next()) {
    for (const RefPtr<MediaEngineDefaultAudioSource>& source : *iter.UserData()) {
      if (source) {
        source->Shutdown();
      }
    }
  }
  mVSources.Clear();
  mASources.Clear();
};

} // namespace mozilla

Coverage Report

Created: 2018-09-25 14:53

Line	Count	Source (jump to first uncovered line)
1		/* This Source Code Form is subject to the terms of the Mozilla Public
2		* License, v. 2.0. If a copy of the MPL was not distributed with this file,
3		* You can obtain one at http://mozilla.org/MPL/2.0/. */
4
5		#include "MediaEngineDefault.h"
6
7		#include "ImageContainer.h"
8		#include "ImageTypes.h"
9		#include "Layers.h"
10		#include "MediaStreamGraph.h"
11		#include "MediaTrackConstraints.h"
12		#include "mozilla/dom/File.h"
13		#include "mozilla/UniquePtr.h"
14		#include "nsCOMPtr.h"
15		#include "nsContentUtils.h"
16		#include "nsIFile.h"
17		#include "nsIFilePicker.h"
18		#include "nsIPrefBranch.h"
19		#include "nsIPrefService.h"
20		#include "Tracing.h"
21
22		#ifdef MOZ_WIDGET_ANDROID
23		#include "nsISupportsUtils.h"
24		#endif
25
26		#ifdef MOZ_WEBRTC
27		#include "YuvStamper.h"
28		#endif
29
30		#define DEFAULT_AUDIO_TIMER_MS 10
31		namespace mozilla {
32
33		using namespace mozilla::gfx;
34
35		/**
36		* Default video source.
37		*/
38
39		MediaEngineDefaultVideoSource::MediaEngineDefaultVideoSource()
40		: mTimer(nullptr)
41		, mMutex("MediaEngineDefaultVideoSource::mMutex")
42	0	{}
43
44		MediaEngineDefaultVideoSource::~MediaEngineDefaultVideoSource()
45	0	{}
46
47		nsString
48		MediaEngineDefaultVideoSource::GetName() const
49	0	{
50	0	return NS_LITERAL_STRING(u"Default Video Device");
51	0	}
52
53		nsCString
54		MediaEngineDefaultVideoSource::GetUUID() const
55	0	{
56	0	return NS_LITERAL_CSTRING("1041FCBD-3F12-4F7B-9E9B-1EC556DD5676");
57	0	}
58
59		uint32_t
60		MediaEngineDefaultVideoSource::GetBestFitnessDistance(
61		const nsTArray<const NormalizedConstraintSet*>& aConstraintSets,
62		const nsString& aDeviceId) const
63	0	{
64	0	AssertIsOnOwningThread();
65	0
66	0	uint32_t distance = 0;
67	0	#ifdef MOZ_WEBRTC
68	0	for (const auto* cs : aConstraintSets) {
69	0	distance = MediaConstraintsHelper::GetMinimumFitnessDistance(*cs, aDeviceId);
70	0	break; // distance is read from first entry only
71	0	}
72	0	#endif
73	0	return distance;
74	0	}
75
76		nsresult
77		MediaEngineDefaultVideoSource::Allocate(const dom::MediaTrackConstraints &aConstraints,
78		const MediaEnginePrefs &aPrefs,
79		const nsString& aDeviceId,
80		const mozilla::ipc::PrincipalInfo& aPrincipalInfo,
81		AllocationHandle** aOutHandle,
82		const char** aOutBadConstraint)
83	0	{
84	0	AssertIsOnOwningThread();
85	0
86	0	MOZ_ASSERT(mState == kReleased);
87	0
88	0	FlattenedConstraints c(aConstraints);
89	0
90	0	// Mock failure for automated tests.
91	0	if (c.mDeviceId.mIdeal.find(NS_LITERAL_STRING("bad device")) !=
92	0	c.mDeviceId.mIdeal.end()) {
93	0	return NS_ERROR_FAILURE;
94	0	}
95	0
96	0	// emulator debug is very, very slow; reduce load on it with smaller/slower fake video
97	0	mOpts = aPrefs;
98	0	mOpts.mWidth = c.mWidth.Get(aPrefs.mWidth ? aPrefs.mWidth :
99		#ifdef DEBUG
100		MediaEnginePrefs::DEFAULT_43_VIDEO_WIDTH/2
101		#else
102	0	MediaEnginePrefs::DEFAULT_43_VIDEO_WIDTH
103	0	#endif
104	0	);
105	0	mOpts.mHeight = c.mHeight.Get(aPrefs.mHeight ? aPrefs.mHeight :
106		#ifdef DEBUG
107		MediaEnginePrefs::DEFAULT_43_VIDEO_HEIGHT/2
108		#else
109	0	MediaEnginePrefs::DEFAULT_43_VIDEO_HEIGHT
110	0	#endif
111	0	);
112	0	mOpts.mWidth = std::max(160, std::min(mOpts.mWidth, 4096)) & ~1;
113	0	mOpts.mHeight = std::max(90, std::min(mOpts.mHeight, 2160)) & ~1;
114	0	*aOutHandle = nullptr;
115	0
116	0	MutexAutoLock lock(mMutex);
117	0	mState = kAllocated;
118	0	return NS_OK;
119	0	}
120
121		nsresult
122		MediaEngineDefaultVideoSource::Deallocate(const RefPtr<const AllocationHandle>& aHandle)
123	0	{
124	0	AssertIsOnOwningThread();
125	0
126	0	MOZ_ASSERT(!aHandle);
127	0	MOZ_ASSERT(!mImage);
128	0	MOZ_ASSERT(mState == kStopped \|\| mState == kAllocated);
129	0
130	0	MutexAutoLock lock(mMutex);
131	0	if (mStream && IsTrackIDExplicit(mTrackID)) {
132	0	mStream->EndTrack(mTrackID);
133	0	mStream = nullptr;
134	0	mTrackID = TRACK_NONE;
135	0	}
136	0	mState = kReleased;
137	0	mImageContainer = nullptr;
138	0
139	0	return NS_OK;
140	0	}
141
142		static void AllocateSolidColorFrame(layers::PlanarYCbCrData& aData,
143		int aWidth, int aHeight,
144		int aY, int aCb, int aCr)
145	0	{
146	0	MOZ_ASSERT(!(aWidth&1));
147	0	MOZ_ASSERT(!(aHeight&1));
148	0	// Allocate a single frame with a solid color
149	0	int yLen = aWidth*aHeight;
150	0	int cbLen = yLen>>2;
151	0	int crLen = cbLen;
152	0	uint8_t* frame = (uint8_t*) malloc(yLen+cbLen+crLen);
153	0	memset(frame, aY, yLen);
154	0	memset(frame+yLen, aCb, cbLen);
155	0	memset(frame+yLen+cbLen, aCr, crLen);
156	0
157	0	aData.mYChannel = frame;
158	0	aData.mYSize = IntSize(aWidth, aHeight);
159	0	aData.mYStride = aWidth;
160	0	aData.mCbCrStride = aWidth>>1;
161	0	aData.mCbChannel = frame + yLen;
162	0	aData.mCrChannel = aData.mCbChannel + cbLen;
163	0	aData.mCbCrSize = IntSize(aWidth>>1, aHeight>>1);
164	0	aData.mPicX = 0;
165	0	aData.mPicY = 0;
166	0	aData.mPicSize = IntSize(aWidth, aHeight);
167	0	aData.mStereoMode = StereoMode::MONO;
168	0	}
169
170		static void ReleaseFrame(layers::PlanarYCbCrData& aData)
171	0	{
172	0	free(aData.mYChannel);
173	0	}
174
175		nsresult
176		MediaEngineDefaultVideoSource::SetTrack(const RefPtr<const AllocationHandle>& aHandle,
177		const RefPtr<SourceMediaStream>& aStream,
178		TrackID aTrackID,
179		const PrincipalHandle& aPrincipal)
180	0	{
181	0	AssertIsOnOwningThread();
182	0
183	0	MOZ_ASSERT(mState == kAllocated);
184	0	MOZ_ASSERT(!mStream);
185	0	MOZ_ASSERT(mTrackID == TRACK_NONE);
186	0
187	0	{
188	0	MutexAutoLock lock(mMutex);
189	0	mStream = aStream;
190	0	mTrackID = aTrackID;
191	0	}
192	0	aStream->AddTrack(aTrackID, 0, new VideoSegment(),
193	0	SourceMediaStream::ADDTRACK_QUEUED);
194	0	return NS_OK;
195	0	}
196
197		nsresult
198		MediaEngineDefaultVideoSource::Start(const RefPtr<const AllocationHandle>& aHandle)
199	0	{
200	0	AssertIsOnOwningThread();
201	0
202	0	MOZ_ASSERT(mState == kAllocated \|\| mState == kStopped);
203	0	MOZ_ASSERT(mStream, "SetTrack() must happen before Start()");
204	0	MOZ_ASSERT(IsTrackIDExplicit(mTrackID), "SetTrack() must happen before Start()");
205	0
206	0	mTimer = NS_NewTimer();
207	0	if (!mTimer) {
208	0	return NS_ERROR_FAILURE;
209	0	}
210	0
211	0	if (!mImageContainer) {
212	0	mImageContainer =
213	0	layers::LayerManager::CreateImageContainer(layers::ImageContainer::ASYNCHRONOUS);
214	0	}
215	0
216	0	// Start timer for subsequent frames
217	0	uint32_t interval;
218		#if defined(MOZ_WIDGET_ANDROID) && defined(DEBUG)
219		// emulator debug is very, very slow and has problems dealing with realtime audio inputs
220		interval = 10 * (1000 / mOpts.mFPS);
221		#else
222		interval = 1000 / mOpts.mFPS;
223	0	#endif
224	0	mTimer->InitWithNamedFuncCallback([](nsITimer* aTimer, void* aClosure) {
225	0	RefPtr<MediaEngineDefaultVideoSource> source =
226	0	static_cast<MediaEngineDefaultVideoSource*>(aClosure);
227	0	source->GenerateFrame();
228	0	}, this, interval, nsITimer::TYPE_REPEATING_SLACK,
229	0	"MediaEngineDefaultVideoSource::GenerateFrame");
230	0
231	0	MutexAutoLock lock(mMutex);
232	0	mState = kStarted;
233	0	return NS_OK;
234	0	}
235
236		nsresult
237		MediaEngineDefaultVideoSource::Stop(const RefPtr<const AllocationHandle>& aHandle)
238	0	{
239	0	AssertIsOnOwningThread();
240	0
241	0	if (mState == kStopped \|\| mState == kAllocated) {
242	0	return NS_OK;
243	0	}
244	0
245	0	MOZ_ASSERT(mState == kStarted);
246	0	MOZ_ASSERT(mTimer);
247	0	MOZ_ASSERT(mStream);
248	0	MOZ_ASSERT(IsTrackIDExplicit(mTrackID));
249	0
250	0	mTimer->Cancel();
251	0	mTimer = nullptr;
252	0
253	0	MutexAutoLock lock(mMutex);
254	0
255	0	mImage = nullptr;
256	0	mState = kStopped;
257	0
258	0	return NS_OK;
259	0	}
260
261		nsresult
262		MediaEngineDefaultVideoSource::Reconfigure(
263		const RefPtr<AllocationHandle>& aHandle,
264		const dom::MediaTrackConstraints& aConstraints,
265		const MediaEnginePrefs &aPrefs,
266		const nsString& aDeviceId,
267		const char** aOutBadConstraint)
268	0	{
269	0	return NS_OK;
270	0	}
271
272		void
273		MediaEngineDefaultVideoSource::GenerateFrame()
274	0	{
275	0	AssertIsOnOwningThread();
276	0
277	0	// Update the target color
278	0	if (mCr <= 16) {
279	0	if (mCb < 240) {
280	0	mCb++;
281	0	} else {
282	0	mCr++;
283	0	}
284	0	} else if (mCb >= 240) {
285	0	if (mCr < 240) {
286	0	mCr++;
287	0	} else {
288	0	mCb--;
289	0	}
290	0	} else if (mCr >= 240) {
291	0	if (mCb > 16) {
292	0	mCb--;
293	0	} else {
294	0	mCr--;
295	0	}
296	0	} else {
297	0	mCr--;
298	0	}
299	0
300	0	// Allocate a single solid color image
301	0	RefPtr<layers::PlanarYCbCrImage> ycbcr_image = mImageContainer->CreatePlanarYCbCrImage();
302	0	layers::PlanarYCbCrData data;
303	0	AllocateSolidColorFrame(data, mOpts.mWidth, mOpts.mHeight, 0x80, mCb, mCr);
304	0
305	0	#ifdef MOZ_WEBRTC
306	0	uint64_t timestamp = PR_Now();
307	0	YuvStamper::Encode(mOpts.mWidth, mOpts.mHeight, mOpts.mWidth,
308	0	data.mYChannel,
309	0	reinterpret_cast<unsigned char*>(&timestamp), sizeof(timestamp),
310	0	0, 0);
311	0	#endif
312	0
313	0	bool setData = ycbcr_image->CopyData(data);
314	0	MOZ_ASSERT(setData);
315	0
316	0	// SetData copies data, so we can free the frame
317	0	ReleaseFrame(data);
318	0
319	0	if (!setData) {
320	0	return;
321	0	}
322	0
323	0	MutexAutoLock lock(mMutex);
324	0	mImage = std::move(ycbcr_image);
325	0	}
326
327		void
328		MediaEngineDefaultVideoSource::Pull(const RefPtr<const AllocationHandle>& aHandle,
329		const RefPtr<SourceMediaStream>& aStream,
330		TrackID aTrackID,
331		StreamTime aDesiredTime,
332		const PrincipalHandle& aPrincipalHandle)
333	0	{
334	0	TRACE_AUDIO_CALLBACK_COMMENT("SourceMediaStream %p track %i",
335	0	aStream.get(), aTrackID);
336	0	// AppendFrame takes ownership of `segment`
337	0	VideoSegment segment;
338	0
339	0	RefPtr<layers::Image> image;
340	0	{
341	0	MutexAutoLock lock(mMutex);
342	0	// Started - append real image
343	0	// Stopped - append null
344	0	// Released - Track is ended, safe to ignore
345	0	// Can happen because NotifyPull comes from a stream listener
346	0	if (mState == kReleased) {
347	0	return;
348	0	}
349	0	MOZ_ASSERT(mState != kAllocated);
350	0	if (mState == kStarted) {
351	0	MOZ_ASSERT(mStream == aStream);
352	0	MOZ_ASSERT(mTrackID == aTrackID);
353	0	image = mImage;
354	0	}
355	0	}
356	0
357	0	StreamTime delta = aDesiredTime - aStream->GetEndOfAppendedData(aTrackID);
358	0	if (delta > 0) {
359	0	// nullptr images are allowed
360	0	IntSize size(mOpts.mWidth, mOpts.mHeight);
361	0	segment.AppendFrame(image.forget(), delta, size, aPrincipalHandle);
362	0	// This can fail if either a) we haven't added the track yet, or b)
363	0	// we've removed or finished the track.
364	0	aStream->AppendToTrack(aTrackID, &segment);
365	0	}
366	0	}
367
368		/**
369		* Default audio source.
370		*/
371
372		MediaEngineDefaultAudioSource::MediaEngineDefaultAudioSource()
373		: mMutex("MediaEngineDefaultAudioSource::mMutex")
374	0	{}
375
376		MediaEngineDefaultAudioSource::~MediaEngineDefaultAudioSource()
377	0	{}
378
379		nsString
380		MediaEngineDefaultAudioSource::GetName() const
381	0	{
382	0	return NS_LITERAL_STRING(u"Default Audio Device");
383	0	}
384
385		nsCString
386		MediaEngineDefaultAudioSource::GetUUID() const
387	0	{
388	0	return NS_LITERAL_CSTRING("B7CBD7C1-53EF-42F9-8353-73F61C70C092");
389	0	}
390
391		uint32_t
392		MediaEngineDefaultAudioSource::GetBestFitnessDistance(
393		const nsTArray<const NormalizedConstraintSet*>& aConstraintSets,
394		const nsString& aDeviceId) const
395	0	{
396	0	uint32_t distance = 0;
397	0	#ifdef MOZ_WEBRTC
398	0	for (const auto* cs : aConstraintSets) {
399	0	distance = MediaConstraintsHelper::GetMinimumFitnessDistance(*cs, aDeviceId);
400	0	break; // distance is read from first entry only
401	0	}
402	0	#endif
403	0	return distance;
404	0	}
405
406		bool
407		MediaEngineDefaultAudioSource::IsAvailable() const
408	0	{
409	0	AssertIsOnOwningThread();
410	0
411	0	return mState == kReleased;
412	0	}
413
414		nsresult
415		MediaEngineDefaultAudioSource::Allocate(const dom::MediaTrackConstraints &aConstraints,
416		const MediaEnginePrefs &aPrefs,
417		const nsString& aDeviceId,
418		const mozilla::ipc::PrincipalInfo& aPrincipalInfo,
419		AllocationHandle** aOutHandle,
420		const char** aOutBadConstraint)
421	0	{
422	0	AssertIsOnOwningThread();
423	0
424	0	MOZ_ASSERT(mState == kReleased);
425	0
426	0	// Mock failure for automated tests.
427	0	if (aConstraints.mDeviceId.IsString() &&
428	0	aConstraints.mDeviceId.GetAsString().EqualsASCII("bad device")) {
429	0	return NS_ERROR_FAILURE;
430	0	}
431	0
432	0	mFreq = aPrefs.mFreq ? aPrefs.mFreq : 1000;
433	0	*aOutHandle = nullptr;
434	0
435	0	MutexAutoLock lock(mMutex);
436	0	mState = kAllocated;
437	0	return NS_OK;
438	0	}
439
440		nsresult
441		MediaEngineDefaultAudioSource::Deallocate(const RefPtr<const AllocationHandle>& aHandle)
442	0	{
443	0	AssertIsOnOwningThread();
444	0
445	0	MOZ_ASSERT(!aHandle);
446	0	MOZ_ASSERT(mState == kStopped \|\| mState == kAllocated);
447	0
448	0	MutexAutoLock lock(mMutex);
449	0	if (mStream && IsTrackIDExplicit(mTrackID)) {
450	0	mStream->EndTrack(mTrackID);
451	0	mStream = nullptr;
452	0	mTrackID = TRACK_NONE;
453	0	}
454	0	mState = kReleased;
455	0	return NS_OK;
456	0	}
457
458		nsresult
459		MediaEngineDefaultAudioSource::SetTrack(const RefPtr<const AllocationHandle>& aHandle,
460		const RefPtr<SourceMediaStream>& aStream,
461		TrackID aTrackID,
462		const PrincipalHandle& aPrincipal)
463	0	{
464	0	AssertIsOnOwningThread();
465	0
466	0	MOZ_ASSERT(mState == kAllocated);
467	0	MOZ_ASSERT(!mStream);
468	0	MOZ_ASSERT(mTrackID == TRACK_NONE);
469	0
470	0	// AddAudioTrack will take ownership of segment
471	0	mStream = aStream;
472	0	mTrackID = aTrackID;
473	0	aStream->AddAudioTrack(aTrackID,
474	0	aStream->GraphRate(),
475	0	0,
476	0	new AudioSegment(),
477	0	SourceMediaStream::ADDTRACK_QUEUED);
478	0	return NS_OK;
479	0	}
480
481		nsresult
482		MediaEngineDefaultAudioSource::Start(const RefPtr<const AllocationHandle>& aHandle)
483	0	{
484	0	AssertIsOnOwningThread();
485	0
486	0	MOZ_ASSERT(mState == kAllocated \|\| mState == kStopped);
487	0	MOZ_ASSERT(mStream, "SetTrack() must happen before Start()");
488	0	MOZ_ASSERT(IsTrackIDExplicit(mTrackID), "SetTrack() must happen before Start()");
489	0
490	0	if (!mSineGenerator) {
491	0	// generate sine wave (default 1KHz)
492	0	mSineGenerator = new SineWaveGenerator(mStream->GraphRate(), mFreq);
493	0	}
494	0
495	0	MutexAutoLock lock(mMutex);
496	0	mState = kStarted;
497	0	return NS_OK;
498	0	}
499
500		nsresult
501		MediaEngineDefaultAudioSource::Stop(const RefPtr<const AllocationHandle>& aHandle)
502	0	{
503	0	AssertIsOnOwningThread();
504	0
505	0	if (mState == kStopped \|\| mState == kAllocated) {
506	0	return NS_OK;
507	0	}
508	0
509	0	MOZ_ASSERT(mState == kStarted);
510	0
511	0	MutexAutoLock lock(mMutex);
512	0	mState = kStopped;
513	0	return NS_OK;
514	0	}
515
516		nsresult
517		MediaEngineDefaultAudioSource::Reconfigure(
518		const RefPtr<AllocationHandle>& aHandle,
519		const dom::MediaTrackConstraints& aConstraints,
520		const MediaEnginePrefs &aPrefs,
521		const nsString& aDeviceId,
522		const char** aOutBadConstraint)
523	0	{
524	0	return NS_OK;
525	0	}
526
527		void
528		MediaEngineDefaultAudioSource::AppendToSegment(AudioSegment& aSegment,
529		TrackTicks aSamples,
530		const PrincipalHandle& aPrincipalHandle)
531	0	{
532	0	RefPtr<SharedBuffer> buffer = SharedBuffer::Create(aSamples * sizeof(int16_t));
533	0	int16_t* dest = static_cast<int16_t*>(buffer->Data());
534	0
535	0	mSineGenerator->generate(dest, aSamples);
536	0	AutoTArray<const int16_t*,1> channels;
537	0	channels.AppendElement(dest);
538	0	aSegment.AppendFrames(buffer.forget(), channels, aSamples, aPrincipalHandle);
539	0	}
540
541		void
542		MediaEngineDefaultAudioSource::Pull(const RefPtr<const AllocationHandle>& aHandle,
543		const RefPtr<SourceMediaStream>& aStream,
544		TrackID aTrackID,
545		StreamTime aDesiredTime,
546		const PrincipalHandle& aPrincipalHandle)
547	0	{
548	0	TRACE_AUDIO_CALLBACK_COMMENT("SourceMediaStream %p track %i",
549	0	aStream.get(), aTrackID);
550	0	AudioSegment segment;
551	0	// avoid accumulating rounding errors
552	0	TrackTicks desired = aStream->TimeToTicksRoundUp(aStream->GraphRate(), aDesiredTime);
553	0	TrackTicks delta = desired - mLastNotify;
554	0	mLastNotify += delta;
555	0	AppendToSegment(segment, delta, aPrincipalHandle);
556	0	aStream->AppendToTrack(aTrackID, &segment);
557	0	}
558
559		void
560		MediaEngineDefault::EnumerateDevices(uint64_t aWindowId,
561		dom::MediaSourceEnum aMediaSource,
562		MediaSinkEnum aMediaSink,
563		nsTArray<RefPtr<MediaDevice>>* aDevices)
564	0	{
565	0	AssertIsOnOwningThread();
566	0
567	0	switch (aMediaSource) {
568	0	case dom::MediaSourceEnum::Camera: {
569	0	// Only supports camera video sources. See Bug 1038241.
570	0
571	0	// We once had code here to find a VideoSource with the same settings and
572	0	// re-use that. This is no longer possible since the resolution gets set
573	0	// in Allocate().
574	0
575	0	nsTArray<RefPtr<MediaEngineSource>>*
576	0	devicesForThisWindow = mVSources.LookupOrAdd(aWindowId);
577	0	auto newSource = MakeRefPtr<MediaEngineDefaultVideoSource>();
578	0	devicesForThisWindow->AppendElement(newSource);
579	0	aDevices->AppendElement(MakeRefPtr<MediaDevice>(
580	0	newSource,
581	0	newSource->GetName(),
582	0	NS_ConvertUTF8toUTF16(newSource->GetUUID())));
583	0	return;
584	0	}
585	0	case dom::MediaSourceEnum::Microphone: {
586	0	nsTArray<RefPtr<MediaEngineDefaultAudioSource>>*
587	0	devicesForThisWindow = mASources.LookupOrAdd(aWindowId);
588	0	for (const RefPtr<MediaEngineDefaultAudioSource>& source : *devicesForThisWindow) {
589	0	if (source->IsAvailable()) {
590	0	aDevices->AppendElement(MakeRefPtr<MediaDevice>(
591	0	source,
592	0	source->GetName(),
593	0	NS_ConvertUTF8toUTF16(source->GetUUID())));
594	0	}
595	0	}
596	0
597	0	if (aDevices->IsEmpty()) {
598	0	// All streams are currently busy, just make a new one.
599	0	auto newSource = MakeRefPtr<MediaEngineDefaultAudioSource>();
600	0	devicesForThisWindow->AppendElement(newSource);
601	0	aDevices->AppendElement(MakeRefPtr<MediaDevice>(
602	0	newSource,
603	0	newSource->GetName(),
604	0	NS_ConvertUTF8toUTF16(newSource->GetUUID())));
605	0	}
606	0	return;
607	0	}
608	0	default:
609	0	MOZ_ASSERT_UNREACHABLE("Unsupported source type");
610	0	return;
611	0	}
612	0
613	0	if (aMediaSink == MediaSinkEnum::Speaker) {
614	0	NS_WARNING("No default implementation for MediaSinkEnum::Speaker");
615	0	}
616	0	}
617
618		void
619		MediaEngineDefault::ReleaseResourcesForWindow(uint64_t aWindowId)
620	0	{
621	0	nsTArray<RefPtr<MediaEngineDefaultAudioSource>>* audioDevicesForThisWindow =
622	0	mASources.Get(aWindowId);
623	0
624	0	if (audioDevicesForThisWindow) {
625	0	for (const RefPtr<MediaEngineDefaultAudioSource>& source :
626	0	*audioDevicesForThisWindow) {
627	0	source->Shutdown();
628	0	}
629	0	}
630	0
631	0	mASources.Remove(aWindowId);
632	0
633	0	nsTArray<RefPtr<MediaEngineSource>>* videoDevicesForThisWindow =
634	0	mVSources.Get(aWindowId);
635	0
636	0	if (videoDevicesForThisWindow) {
637	0	for (const RefPtr<MediaEngineSource>& source :
638	0	*videoDevicesForThisWindow) {
639	0	source->Shutdown();
640	0	}
641	0	}
642	0
643	0	mVSources.Remove(aWindowId);
644	0	}
645
646		void
647		MediaEngineDefault::Shutdown()
648	0	{
649	0	AssertIsOnOwningThread();
650	0
651	0	for (auto iter = mVSources.Iter(); !iter.Done(); iter.Next()) {
652	0	for (const RefPtr<MediaEngineSource>& source : *iter.UserData()) {
653	0	if (source) {
654	0	source->Shutdown();
655	0	}
656	0	}
657	0	}
658	0	for (auto iter = mASources.Iter(); !iter.Done(); iter.Next()) {
659	0	for (const RefPtr<MediaEngineDefaultAudioSource>& source : *iter.UserData()) {
660	0	if (source) {
661	0	source->Shutdown();
662	0	}
663	0	}
664	0	}
665	0	mVSources.Clear();
666	0	mASources.Clear();
667	0	};
668
669		} // namespace mozilla