Coverage Report

Created: 2018-09-25 14:53

/src/mozilla-central/gfx/vr/gfxVRPuppet.cpp
Line
Count
Source (jump to first uncovered line)
1
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2
/* vim: set ts=8 sts=2 et sw=2 tw=80: */
3
/* This Source Code Form is subject to the terms of the Mozilla Public
4
 * License, v. 2.0. If a copy of the MPL was not distributed with this
5
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7
#if defined(XP_WIN)
8
#include "CompositorD3D11.h"
9
#include "TextureD3D11.h"
10
#include "mozilla/gfx/DeviceManagerDx.h"
11
#elif defined(XP_MACOSX)
12
#include "mozilla/gfx/MacIOSurface.h"
13
#endif
14
15
#include "mozilla/Base64.h"
16
#include "mozilla/gfx/DataSurfaceHelpers.h"
17
#include "gfxPrefs.h"
18
#include "gfxUtils.h"
19
#include "gfxVRPuppet.h"
20
#include "VRManager.h"
21
#include "VRThread.h"
22
23
#include "mozilla/dom/GamepadEventTypes.h"
24
#include "mozilla/dom/GamepadBinding.h"
25
26
// See CompositorD3D11Shaders.h
27
namespace mozilla {
28
namespace layers {
29
struct ShaderBytes { const void* mData; size_t mLength; };
30
extern ShaderBytes sRGBShader;
31
extern ShaderBytes sLayerQuadVS;
32
} // namespace layers
33
} // namespace mozilla
34
35
using namespace mozilla;
36
using namespace mozilla::gfx;
37
using namespace mozilla::gfx::impl;
38
using namespace mozilla::layers;
39
40
// Reminder: changing the order of these buttons may break web content
41
static const uint64_t kPuppetButtonMask[] = {
42
  1,
43
  2,
44
  4,
45
  8
46
};
47
static const uint32_t kNumPuppetButtonMask = sizeof(kPuppetButtonMask) /
48
                                             sizeof(uint64_t);
49
static const uint32_t kNumPuppetAxis = 3;
50
static const uint32_t kNumPuppetHaptcs = 1;
51
52
VRDisplayPuppet::VRDisplayPuppet()
53
 : VRDisplayLocal(VRDeviceType::Puppet)
54
 , mIsPresenting(false)
55
 , mSensorState{}
56
0
{
57
0
  MOZ_COUNT_CTOR_INHERITED(VRDisplayPuppet, VRDisplayLocal);
58
0
59
0
  VRDisplayState& state = mDisplayInfo.mDisplayState;
60
0
  strncpy(state.mDisplayName, "Puppet HMD", kVRDisplayNameMaxLen);
61
0
  state.mIsConnected = true;
62
0
  state.mIsMounted = false;
63
0
  state.mCapabilityFlags = VRDisplayCapabilityFlags::Cap_None |
64
0
                           VRDisplayCapabilityFlags::Cap_Orientation |
65
0
                           VRDisplayCapabilityFlags::Cap_Position |
66
0
                           VRDisplayCapabilityFlags::Cap_External |
67
0
                           VRDisplayCapabilityFlags::Cap_Present |
68
0
                           VRDisplayCapabilityFlags::Cap_StageParameters;
69
0
  state.mEyeResolution.width = 1836; // 1080 * 1.7
70
0
  state.mEyeResolution.height = 2040; // 1200 * 1.7
71
0
72
0
  // SteamVR gives the application a single FOV to use; it's not configurable as with Oculus
73
0
  for (uint32_t eye = 0; eye < 2; ++eye) {
74
0
    state.mEyeTranslation[eye].x = 0.0f;
75
0
    state.mEyeTranslation[eye].y = 0.0f;
76
0
    state.mEyeTranslation[eye].z = 0.0f;
77
0
    state.mEyeFOV[eye] = VRFieldOfView(45.0, 45.0, 45.0, 45.0);
78
0
  }
79
0
80
0
  // default: 1m x 1m space, 0.75m high in seated position
81
0
  state.mStageSize.width = 1.0f;
82
0
  state.mStageSize.height = 1.0f;
83
0
84
0
  state.mSittingToStandingTransform[0] = 1.0f;
85
0
  state.mSittingToStandingTransform[1] = 0.0f;
86
0
  state.mSittingToStandingTransform[2] = 0.0f;
87
0
  state.mSittingToStandingTransform[3] = 0.0f;
88
0
89
0
  state.mSittingToStandingTransform[4] = 0.0f;
90
0
  state.mSittingToStandingTransform[5] = 1.0f;
91
0
  state.mSittingToStandingTransform[6] = 0.0f;
92
0
  state.mSittingToStandingTransform[7] = 0.0f;
93
0
94
0
  state.mSittingToStandingTransform[8] = 0.0f;
95
0
  state.mSittingToStandingTransform[9] = 0.0f;
96
0
  state.mSittingToStandingTransform[10] = 1.0f;
97
0
  state.mSittingToStandingTransform[11] = 0.0f;
98
0
99
0
  state.mSittingToStandingTransform[12] = 0.0f;
100
0
  state.mSittingToStandingTransform[13] = 0.75f;
101
0
  state.mSittingToStandingTransform[14] = 0.0f;
102
0
  state.mSittingToStandingTransform[15] = 1.0f;
103
0
104
0
  gfx::Quaternion rot;
105
0
106
0
  mSensorState.flags |= VRDisplayCapabilityFlags::Cap_Orientation;
107
0
  mSensorState.pose.orientation[0] = rot.x;
108
0
  mSensorState.pose.orientation[1] = rot.y;
109
0
  mSensorState.pose.orientation[2] = rot.z;
110
0
  mSensorState.pose.orientation[3] = rot.w;
111
0
  mSensorState.pose.angularVelocity[0] = 0.0f;
112
0
  mSensorState.pose.angularVelocity[1] = 0.0f;
113
0
  mSensorState.pose.angularVelocity[2] = 0.0f;
114
0
115
0
  mSensorState.flags |= VRDisplayCapabilityFlags::Cap_Position;
116
0
  mSensorState.pose.position[0] = 0.0f;
117
0
  mSensorState.pose.position[1] = 0.0f;
118
0
  mSensorState.pose.position[2] = 0.0f;
119
0
  mSensorState.pose.linearVelocity[0] = 0.0f;
120
0
  mSensorState.pose.linearVelocity[1] = 0.0f;
121
0
  mSensorState.pose.linearVelocity[2] = 0.0f;
122
0
}
123
124
VRDisplayPuppet::~VRDisplayPuppet()
125
0
{
126
0
  MOZ_COUNT_DTOR_INHERITED(VRDisplayPuppet, VRDisplayLocal);
127
0
}
128
129
void
130
VRDisplayPuppet::SetDisplayInfo(const VRDisplayInfo& aDisplayInfo)
131
0
{
132
0
  // We are only interested in the eye and mount info of the display info.
133
0
  VRDisplayState& state = mDisplayInfo.mDisplayState;
134
0
  state.mEyeResolution = aDisplayInfo.mDisplayState.mEyeResolution;
135
0
  state.mIsMounted = aDisplayInfo.mDisplayState.mIsMounted;
136
0
  memcpy(&state.mEyeFOV, &aDisplayInfo.mDisplayState.mEyeFOV,
137
0
         sizeof(state.mEyeFOV[0]) * VRDisplayState::NumEyes);
138
0
  memcpy(&state.mEyeTranslation, &aDisplayInfo.mDisplayState.mEyeTranslation,
139
0
         sizeof(state.mEyeTranslation[0]) * VRDisplayState::NumEyes);
140
0
}
141
142
void
143
VRDisplayPuppet::Destroy()
144
0
{
145
0
  StopPresentation();
146
0
}
147
148
void
149
VRDisplayPuppet::ZeroSensor()
150
0
{
151
0
}
152
153
VRHMDSensorState
154
VRDisplayPuppet::GetSensorState()
155
0
{
156
0
  mSensorState.inputFrameID = mDisplayInfo.mFrameId;
157
0
158
0
  Matrix4x4 matHeadToEye[2];
159
0
  for (uint32_t eye = 0; eye < 2; ++eye) {
160
0
    matHeadToEye[eye].PreTranslate(mDisplayInfo.GetEyeTranslation(eye));
161
0
  }
162
0
  mSensorState.CalcViewMatrices(matHeadToEye);
163
0
164
0
  return mSensorState;
165
0
}
166
167
void
168
VRDisplayPuppet::SetSensorState(const VRHMDSensorState& aSensorState)
169
0
{
170
0
  memcpy(&mSensorState, &aSensorState, sizeof(mSensorState));
171
0
}
172
173
void
174
VRDisplayPuppet::StartPresentation()
175
0
{
176
0
  if (mIsPresenting) {
177
0
    return;
178
0
  }
179
0
  mIsPresenting = true;
180
0
181
#if defined(XP_WIN)
182
  if (!CreateD3DObjects()) {
183
    return;
184
  }
185
186
  if (FAILED(mDevice->CreateVertexShader(sLayerQuadVS.mData,
187
                      sLayerQuadVS.mLength, nullptr, &mQuadVS))) {
188
    NS_WARNING("Failed to create vertex shader for Puppet");
189
    return;
190
  }
191
192
  if (FAILED(mDevice->CreatePixelShader(sRGBShader.mData,
193
                      sRGBShader.mLength, nullptr, &mQuadPS))) {
194
    NS_WARNING("Failed to create pixel shader for Puppet");
195
    return;
196
  }
197
198
  CD3D11_BUFFER_DESC cBufferDesc(sizeof(layers::VertexShaderConstants),
199
                                 D3D11_BIND_CONSTANT_BUFFER,
200
                                 D3D11_USAGE_DYNAMIC,
201
                                 D3D11_CPU_ACCESS_WRITE);
202
203
  if (FAILED(mDevice->CreateBuffer(&cBufferDesc, nullptr, getter_AddRefs(mVSConstantBuffer)))) {
204
    NS_WARNING("Failed to vertex shader constant buffer for Puppet");
205
    return;
206
  }
207
208
  cBufferDesc.ByteWidth = sizeof(layers::PixelShaderConstants);
209
  if (FAILED(mDevice->CreateBuffer(&cBufferDesc, nullptr, getter_AddRefs(mPSConstantBuffer)))) {
210
    NS_WARNING("Failed to pixel shader constant buffer for Puppet");
211
    return;
212
  }
213
214
  CD3D11_SAMPLER_DESC samplerDesc(D3D11_DEFAULT);
215
  if (FAILED(mDevice->CreateSamplerState(&samplerDesc, getter_AddRefs(mLinearSamplerState)))) {
216
    NS_WARNING("Failed to create sampler state for Puppet");
217
    return;
218
  }
219
220
  D3D11_INPUT_ELEMENT_DESC layout[] =
221
  {
222
    { "POSITION", 0, DXGI_FORMAT_R32G32_FLOAT, 0, 0, D3D11_INPUT_PER_VERTEX_DATA, 0 },
223
  };
224
225
  if (FAILED(mDevice->CreateInputLayout(layout,
226
                                        sizeof(layout) / sizeof(D3D11_INPUT_ELEMENT_DESC),
227
                                        sLayerQuadVS.mData,
228
                                        sLayerQuadVS.mLength,
229
                                        getter_AddRefs(mInputLayout)))) {
230
    NS_WARNING("Failed to create input layout for Puppet");
231
    return;
232
  }
233
234
  Vertex vertices[] = { { { 0.0, 0.0 } },{ { 1.0, 0.0 } },{ { 0.0, 1.0 } },{ { 1.0, 1.0 } } };
235
  CD3D11_BUFFER_DESC bufferDesc(sizeof(vertices), D3D11_BIND_VERTEX_BUFFER);
236
  D3D11_SUBRESOURCE_DATA data;
237
  data.pSysMem = (void*)vertices;
238
239
  if (FAILED(mDevice->CreateBuffer(&bufferDesc, &data, getter_AddRefs(mVertexBuffer)))) {
240
    NS_WARNING("Failed to create vertex buffer for Puppet");
241
    return;
242
  }
243
244
  memset(&mVSConstants, 0, sizeof(mVSConstants));
245
  memset(&mPSConstants, 0, sizeof(mPSConstants));
246
#endif // XP_WIN
247
}
248
249
void
250
VRDisplayPuppet::StopPresentation()
251
0
{
252
0
  if (!mIsPresenting) {
253
0
    return;
254
0
  }
255
0
256
0
  mIsPresenting = false;
257
0
}
258
259
#if defined(XP_WIN)
260
bool
261
VRDisplayPuppet::UpdateConstantBuffers()
262
{
263
  HRESULT hr;
264
  D3D11_MAPPED_SUBRESOURCE resource;
265
  resource.pData = nullptr;
266
267
  hr = mContext->Map(mVSConstantBuffer, 0, D3D11_MAP_WRITE_DISCARD, 0, &resource);
268
  if (FAILED(hr) || !resource.pData) {
269
    return false;
270
  }
271
  *(VertexShaderConstants*)resource.pData = mVSConstants;
272
  mContext->Unmap(mVSConstantBuffer, 0);
273
  resource.pData = nullptr;
274
275
  hr = mContext->Map(mPSConstantBuffer, 0, D3D11_MAP_WRITE_DISCARD, 0, &resource);
276
  if (FAILED(hr) || !resource.pData) {
277
    return false;
278
  }
279
  *(PixelShaderConstants*)resource.pData = mPSConstants;
280
  mContext->Unmap(mPSConstantBuffer, 0);
281
282
  ID3D11Buffer *buffer = mVSConstantBuffer;
283
  mContext->VSSetConstantBuffers(0, 1, &buffer);
284
  buffer = mPSConstantBuffer;
285
  mContext->PSSetConstantBuffers(0, 1, &buffer);
286
  return true;
287
}
288
289
bool
290
VRDisplayPuppet::SubmitFrame(ID3D11Texture2D* aSource,
291
                             const IntSize& aSize,
292
                             const gfx::Rect& aLeftEyeRect,
293
                             const gfx::Rect& aRightEyeRect)
294
{
295
  MOZ_ASSERT(mSubmitThread->GetThread() == NS_GetCurrentThread());
296
  if (!mIsPresenting) {
297
    return false;
298
  }
299
300
  if (!CreateD3DObjects()) {
301
    return false;
302
  }
303
  AutoRestoreRenderState restoreState(this);
304
  if (!restoreState.IsSuccess()) {
305
    return false;
306
  }
307
308
  VRManager *vm = VRManager::Get();
309
  MOZ_ASSERT(vm);
310
311
  switch (gfxPrefs::VRPuppetSubmitFrame()) {
312
    case 0:
313
      // The VR frame is not displayed.
314
      break;
315
    case 1:
316
    {
317
      // The frames are submitted to VR compositor are decoded
318
      // into a base64Image and dispatched to the DOM side.
319
      D3D11_TEXTURE2D_DESC desc;
320
      aSource->GetDesc(&desc);
321
      MOZ_ASSERT(desc.Format == DXGI_FORMAT_B8G8R8A8_UNORM,
322
                 "Only support B8G8R8A8_UNORM format.");
323
      // Map the staging resource
324
      ID3D11Texture2D* mappedTexture = nullptr;
325
      D3D11_MAPPED_SUBRESOURCE mapInfo;
326
      HRESULT hr = mContext->Map(aSource,
327
                                 0,  // Subsource
328
                                 D3D11_MAP_READ,
329
                                 0,  // MapFlags
330
                                 &mapInfo);
331
332
      if (FAILED(hr)) {
333
        // If we can't map this texture, copy it to a staging resource.
334
        if (hr == E_INVALIDARG) {
335
          D3D11_TEXTURE2D_DESC desc2;
336
          desc2.Width = desc.Width;
337
          desc2.Height = desc.Height;
338
          desc2.MipLevels = desc.MipLevels;
339
          desc2.ArraySize = desc.ArraySize;
340
          desc2.Format = desc.Format;
341
          desc2.SampleDesc = desc.SampleDesc;
342
          desc2.Usage = D3D11_USAGE_STAGING;
343
          desc2.BindFlags = 0;
344
          desc2.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
345
          desc2.MiscFlags = 0;
346
347
          ID3D11Texture2D* stagingTexture = nullptr;
348
          hr = mDevice->CreateTexture2D(&desc2, nullptr, &stagingTexture);
349
          if (FAILED(hr)) {
350
            MOZ_ASSERT(false, "Failed to create a staging texture");
351
            return false;
352
          }
353
          // Copy the texture to a staging resource
354
          mContext->CopyResource(stagingTexture, aSource);
355
          // Map the staging resource
356
          hr = mContext->Map(stagingTexture,
357
                             0,  // Subsource
358
                             D3D11_MAP_READ,
359
                             0,  // MapFlags
360
                             &mapInfo);
361
          if (FAILED(hr)) {
362
            MOZ_ASSERT(false, "Failed to map staging texture");
363
          }
364
          mappedTexture = stagingTexture;
365
        } else {
366
          MOZ_ASSERT(false, "Failed to map staging texture");
367
          return false;
368
        }
369
      } else {
370
        mappedTexture = aSource;
371
      }
372
      // Ideally, we should convert the srcData to a PNG image and decode it
373
      // to a Base64 string here, but the GPU process does not have the privilege to
374
      // access the image library. So, we have to convert the RAW image data
375
      // to a base64 string and forward it to let the content process to
376
      // do the image conversion.
377
      const char* srcData = static_cast<const char*>(mapInfo.pData);
378
      VRSubmitFrameResultInfo result;
379
      result.mFormat = SurfaceFormat::B8G8R8A8;
380
      result.mWidth = desc.Width;
381
      result.mHeight = desc.Height;
382
      result.mFrameNum = mDisplayInfo.mFrameId;
383
      // If the original texture size is not pow of 2, the data will not be tightly strided.
384
      // We have to copy the pixels by rows.
385
      nsCString rawString;
386
      for (uint32_t i = 0; i < desc.Height; i++) {
387
        rawString += Substring(srcData + i * mapInfo.RowPitch,
388
                               desc.Width * 4);
389
      }
390
      mContext->Unmap(mappedTexture, 0);
391
392
      if (Base64Encode(rawString, result.mBase64Image) != NS_OK) {
393
        MOZ_ASSERT(false, "Failed to encode base64 images.");
394
      }
395
      // Dispatch the base64 encoded string to the DOM side. Then, it will be decoded
396
      // and convert to a PNG image there.
397
      MessageLoop* loop = VRListenerThreadHolder::Loop();
398
      loop->PostTask(NewRunnableMethod<const uint32_t, VRSubmitFrameResultInfo>(
399
        "VRManager::DispatchSubmitFrameResult",
400
        vm, &VRManager::DispatchSubmitFrameResult, mDisplayInfo.mDisplayID, result
401
      ));
402
      break;
403
    }
404
    case 2:
405
    {
406
      // The VR compositor sumbmit frame to the screen window,
407
      // the current coordinate is at (0, 0, width, height).
408
      Matrix viewMatrix = Matrix::Translation(-1.0, 1.0);
409
      viewMatrix.PreScale(2.0f / float(aSize.width), 2.0f / float(aSize.height));
410
      viewMatrix.PreScale(1.0f, -1.0f);
411
      Matrix4x4 projection = Matrix4x4::From2D(viewMatrix);
412
      projection._33 = 0.0f;
413
414
      Matrix transform2d;
415
      gfx::Matrix4x4 transform = gfx::Matrix4x4::From2D(transform2d);
416
417
      const float posX = 0.0f, posY = 0.0f;
418
      D3D11_VIEWPORT viewport;
419
      viewport.MinDepth = 0.0f;
420
      viewport.MaxDepth = 1.0f;
421
      viewport.Width = aSize.width;
422
      viewport.Height = aSize.height;
423
      viewport.TopLeftX = posX;
424
      viewport.TopLeftY = posY;
425
426
      D3D11_RECT scissor;
427
      scissor.left = posX;
428
      scissor.right = aSize.width + posX;
429
      scissor.top = posY;
430
      scissor.bottom = aSize.height + posY;
431
432
      memcpy(&mVSConstants.layerTransform, &transform._11, sizeof(mVSConstants.layerTransform));
433
      memcpy(&mVSConstants.projection, &projection._11, sizeof(mVSConstants.projection));
434
      mVSConstants.renderTargetOffset[0] = 0.0f;
435
      mVSConstants.renderTargetOffset[1] = 0.0f;
436
      mVSConstants.layerQuad = Rect(0.0f, 0.0f, aSize.width, aSize.height);
437
      mVSConstants.textureCoords = Rect(0.0f, 1.0f, 1.0f, -1.0f);
438
439
      mPSConstants.layerOpacity[0] = 1.0f;
440
441
      ID3D11Buffer* vbuffer = mVertexBuffer;
442
      UINT vsize = sizeof(Vertex);
443
      UINT voffset = 0;
444
      mContext->IASetVertexBuffers(0, 1, &vbuffer, &vsize, &voffset);
445
      mContext->IASetIndexBuffer(nullptr, DXGI_FORMAT_R16_UINT, 0);
446
      mContext->IASetInputLayout(mInputLayout);
447
      mContext->RSSetViewports(1, &viewport);
448
      mContext->RSSetScissorRects(1, &scissor);
449
      mContext->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP);
450
      mContext->VSSetShader(mQuadVS, nullptr, 0);
451
      mContext->PSSetShader(mQuadPS, nullptr, 0);
452
453
      RefPtr<ID3D11ShaderResourceView> srView;
454
      HRESULT hr = mDevice->CreateShaderResourceView(aSource, nullptr, getter_AddRefs(srView));
455
      if (FAILED(hr) || !srView) {
456
        gfxWarning() << "Could not create shader resource view for Puppet: " << hexa(hr);
457
        return false;
458
      }
459
      ID3D11ShaderResourceView* viewPtr = srView.get();
460
      mContext->PSSetShaderResources(0 /* 0 == TexSlot::RGB */, 1, &viewPtr);
461
      // XXX Use Constant from TexSlot in CompositorD3D11.cpp?
462
463
      ID3D11SamplerState *sampler = mLinearSamplerState;
464
      mContext->PSSetSamplers(0, 1, &sampler);
465
466
      if (!UpdateConstantBuffers()) {
467
        NS_WARNING("Failed to update constant buffers for Puppet");
468
        return false;
469
      }
470
      mContext->Draw(4, 0);
471
      break;
472
    }
473
  }
474
475
  // We will always return false for gfxVRPuppet to ensure that the fallback "watchdog"
476
  // code in VRDisplayHost::NotifyVSync() throttles the render loop.  This "watchdog" will
477
  // result in a refresh rate that is quite low compared to real hardware, but should be
478
  // sufficient for non-performance oriented tests.  If we wish to simulate realistic frame
479
  // rates with VRDisplayPuppet, we should block here for the appropriate amount of time and
480
  // return true to indicate that we have blocked.
481
  return false;
482
}
483
484
#elif defined(XP_MACOSX)
485
486
bool
487
VRDisplayPuppet::SubmitFrame(MacIOSurface* aMacIOSurface,
488
                             const IntSize& aSize,
489
                             const gfx::Rect& aLeftEyeRect,
490
                             const gfx::Rect& aRightEyeRect)
491
{
492
  MOZ_ASSERT(mSubmitThread->GetThread() == NS_GetCurrentThread());
493
  if (!mIsPresenting || !aMacIOSurface) {
494
    return false;
495
  }
496
497
  VRManager* vm = VRManager::Get();
498
  MOZ_ASSERT(vm);
499
500
  switch (gfxPrefs::VRPuppetSubmitFrame()) {
501
    case 0:
502
      // The VR frame is not displayed.
503
      break;
504
    case 1:
505
    {
506
      // The frames are submitted to VR compositor are decoded
507
      // into a base64Image and dispatched to the DOM side.
508
      RefPtr<SourceSurface> surf = aMacIOSurface->GetAsSurface();
509
      RefPtr<DataSourceSurface> dataSurf = surf ? surf->GetDataSurface() :
510
                                           nullptr;
511
      if (dataSurf) {
512
        // Ideally, we should convert the srcData to a PNG image and decode it
513
        // to a Base64 string here, but the GPU process does not have the privilege to
514
        // access the image library. So, we have to convert the RAW image data
515
        // to a base64 string and forward it to let the content process to
516
        // do the image conversion.
517
        DataSourceSurface::MappedSurface map;
518
        if (!dataSurf->Map(gfx::DataSourceSurface::MapType::READ, &map)) {
519
          MOZ_ASSERT(false, "Read DataSourceSurface fail.");
520
          return false;
521
        }
522
        const uint8_t* srcData = map.mData;
523
        const auto& surfSize = dataSurf->GetSize();
524
        VRSubmitFrameResultInfo result;
525
        result.mFormat = SurfaceFormat::B8G8R8A8;
526
        result.mWidth = surfSize.width;
527
        result.mHeight = surfSize.height;
528
        result.mFrameNum = mDisplayInfo.mFrameId;
529
        // If the original texture size is not pow of 2, the data will not be tightly strided.
530
        // We have to copy the pixels by rows.
531
        nsCString rawString;
532
        for (int32_t i = 0; i < surfSize.height; i++) {
533
          rawString += Substring((const char*)(srcData) + i * map.mStride,
534
                                  surfSize.width * 4);
535
        }
536
        dataSurf->Unmap();
537
538
        if (Base64Encode(rawString, result.mBase64Image) != NS_OK) {
539
          MOZ_ASSERT(false, "Failed to encode base64 images.");
540
        }
541
        // Dispatch the base64 encoded string to the DOM side. Then, it will be decoded
542
        // and convert to a PNG image there.
543
        MessageLoop* loop = VRListenerThreadHolder::Loop();
544
        loop->PostTask(NewRunnableMethod<const uint32_t, VRSubmitFrameResultInfo>(
545
          "VRManager::DispatchSubmitFrameResult",
546
          vm, &VRManager::DispatchSubmitFrameResult, mDisplayInfo.mDisplayID, result
547
        ));
548
      }
549
      break;
550
    }
551
    case 2:
552
    {
553
      MOZ_ASSERT(false, "No support for showing VR frames on MacOSX yet.");
554
      break;
555
    }
556
  }
557
558
  return false;
559
}
560
561
#elif defined(MOZ_WIDGET_ANDROID)
562
563
bool
564
VRDisplayPuppet::SubmitFrame(const mozilla::layers::SurfaceTextureDescriptor& aDescriptor,
565
                             const gfx::Rect& aLeftEyeRect,
566
                             const gfx::Rect& aRightEyeRect)
567
{
568
  MOZ_ASSERT(mSubmitThread->GetThread() == NS_GetCurrentThread());
569
  return false;
570
}
571
572
#endif
573
574
void
575
VRDisplayPuppet::Refresh()
576
0
{
577
0
  // We update mIsConneced once per refresh.
578
0
  mDisplayInfo.mDisplayState.mIsConnected = true;
579
0
}
580
581
VRControllerPuppet::VRControllerPuppet(dom::GamepadHand aHand, uint32_t aDisplayID)
582
  : VRControllerHost(VRDeviceType::Puppet, aHand, aDisplayID)
583
  , mButtonPressState(0)
584
  , mButtonTouchState(0)
585
0
{
586
0
  MOZ_COUNT_CTOR_INHERITED(VRControllerPuppet, VRControllerHost);
587
0
  VRControllerState& state = mControllerInfo.mControllerState;
588
0
  strncpy(state.controllerName, "Puppet Gamepad", kVRControllerNameMaxLen);
589
0
  state.numButtons = kNumPuppetButtonMask;
590
0
  state.numAxes = kNumPuppetAxis;
591
0
  state.numHaptics = kNumPuppetHaptcs;
592
0
}
593
594
VRControllerPuppet::~VRControllerPuppet()
595
0
{
596
0
  MOZ_COUNT_DTOR_INHERITED(VRControllerPuppet, VRControllerHost);
597
0
}
598
599
void
600
VRControllerPuppet::SetButtonPressState(uint32_t aButton, bool aPressed)
601
0
{
602
0
  const uint64_t buttonMask = kPuppetButtonMask[aButton];
603
0
  uint64_t pressedBit = GetButtonPressed();
604
0
605
0
  if (aPressed) {
606
0
    pressedBit |= kPuppetButtonMask[aButton];
607
0
  } else if (pressedBit & buttonMask) {
608
0
    // this button was pressed but is released now.
609
0
    uint64_t mask = 0xff ^ buttonMask;
610
0
    pressedBit &= mask;
611
0
  }
612
0
613
0
  mButtonPressState = pressedBit;
614
0
}
615
616
uint64_t
617
VRControllerPuppet::GetButtonPressState()
618
0
{
619
0
  return mButtonPressState;
620
0
}
621
622
void
623
VRControllerPuppet::SetButtonTouchState(uint32_t aButton, bool aTouched)
624
0
{
625
0
  const uint64_t buttonMask = kPuppetButtonMask[aButton];
626
0
  uint64_t touchedBit = GetButtonTouched();
627
0
628
0
  if (aTouched) {
629
0
    touchedBit |= kPuppetButtonMask[aButton];
630
0
  } else if (touchedBit & buttonMask) {
631
0
    // this button was touched but is released now.
632
0
    uint64_t mask = 0xff ^ buttonMask;
633
0
    touchedBit &= mask;
634
0
  }
635
0
636
0
  mButtonTouchState = touchedBit;
637
0
}
638
639
uint64_t
640
VRControllerPuppet::GetButtonTouchState()
641
0
{
642
0
  return mButtonTouchState;
643
0
}
644
645
void
646
VRControllerPuppet::SetAxisMoveState(uint32_t aAxis, double aValue)
647
0
{
648
0
  MOZ_ASSERT((sizeof(mAxisMoveState) / sizeof(float)) == kNumPuppetAxis);
649
0
  MOZ_ASSERT(aAxis <= kNumPuppetAxis);
650
0
651
0
  mAxisMoveState[aAxis] = aValue;
652
0
}
653
654
double
655
VRControllerPuppet::GetAxisMoveState(uint32_t aAxis)
656
0
{
657
0
  return mAxisMoveState[aAxis];
658
0
}
659
660
void
661
VRControllerPuppet::SetPoseMoveState(const dom::GamepadPoseState& aPose)
662
0
{
663
0
  mPoseState = aPose;
664
0
}
665
666
const dom::GamepadPoseState&
667
VRControllerPuppet::GetPoseMoveState()
668
0
{
669
0
  return mPoseState;
670
0
}
671
672
float
673
VRControllerPuppet::GetAxisMove(uint32_t aAxis)
674
0
{
675
0
  return mControllerInfo.mControllerState.axisValue[aAxis];
676
0
}
677
678
void
679
VRControllerPuppet::SetAxisMove(uint32_t aAxis, float aValue)
680
0
{
681
0
  mControllerInfo.mControllerState.axisValue[aAxis] = aValue;
682
0
}
683
684
VRSystemManagerPuppet::VRSystemManagerPuppet()
685
  : mPuppetDisplayCount(0)
686
  , mPuppetDisplayInfo{}
687
  , mPuppetDisplaySensorState{}
688
0
{
689
0
}
690
691
/*static*/ already_AddRefed<VRSystemManagerPuppet>
692
VRSystemManagerPuppet::Create()
693
0
{
694
0
  if (!gfxPrefs::VREnabled() || !gfxPrefs::VRPuppetEnabled()) {
695
0
    return nullptr;
696
0
  }
697
0
698
0
  RefPtr<VRSystemManagerPuppet> manager = new VRSystemManagerPuppet();
699
0
  return manager.forget();
700
0
}
701
702
void
703
VRSystemManagerPuppet::Destroy()
704
0
{
705
0
  Shutdown();
706
0
}
707
708
void
709
VRSystemManagerPuppet::Shutdown()
710
0
{
711
0
  mPuppetHMDs.Clear();
712
0
}
713
714
void
715
VRSystemManagerPuppet::NotifyVSync()
716
0
{
717
0
  VRSystemManager::NotifyVSync();
718
0
719
0
  for (const auto& display: mPuppetHMDs) {
720
0
    display->Refresh();
721
0
  }
722
0
}
723
724
uint32_t
725
VRSystemManagerPuppet::CreateTestDisplay()
726
0
{
727
0
  if (mPuppetDisplayCount >= kMaxPuppetDisplays) {
728
0
    MOZ_ASSERT(false);
729
0
    return mPuppetDisplayCount;
730
0
  }
731
0
  return mPuppetDisplayCount++;
732
0
}
733
734
void
735
VRSystemManagerPuppet::ClearTestDisplays()
736
0
{
737
0
  mPuppetDisplayCount = 0;
738
0
}
739
740
void
741
VRSystemManagerPuppet::Enumerate()
742
0
{
743
0
  while (mPuppetHMDs.Length() < mPuppetDisplayCount) {
744
0
    VRDisplayPuppet* puppetDisplay = new VRDisplayPuppet();
745
0
    uint32_t deviceID = mPuppetHMDs.Length();
746
0
    puppetDisplay->SetDisplayInfo(mPuppetDisplayInfo[deviceID]);
747
0
    puppetDisplay->SetSensorState(mPuppetDisplaySensorState[deviceID]);
748
0
    mPuppetHMDs.AppendElement(puppetDisplay);
749
0
  }
750
0
  while (mPuppetHMDs.Length() > mPuppetDisplayCount) {
751
0
    mPuppetHMDs.RemoveLastElement();
752
0
  }
753
0
}
754
755
void
756
VRSystemManagerPuppet::SetPuppetDisplayInfo(const uint32_t& aDeviceID,
757
                                            const VRDisplayInfo& aDisplayInfo)
758
0
{
759
0
  if (aDeviceID >= mPuppetDisplayCount) {
760
0
    MOZ_ASSERT(false);
761
0
    return;
762
0
  }
763
0
  /**
764
0
   * Even if mPuppetHMDs.Length() <= aDeviceID, we need to
765
0
   * update mPuppetDisplayInfo[aDeviceID].  In the case that
766
0
   * a puppet display is added and SetPuppetDisplayInfo is
767
0
   * immediately called, mPuppetHMDs may not be populated yet.
768
0
   * VRSystemManagerPuppet::Enumerate() will initialize
769
0
   * the VRDisplayPuppet later using mPuppetDisplayInfo.
770
0
   */
771
0
  mPuppetDisplayInfo[aDeviceID] = aDisplayInfo;
772
0
  if (mPuppetHMDs.Length() > aDeviceID) {
773
0
    /**
774
0
     * In the event that the VRDisplayPuppet has already been
775
0
     * created, we update it directly.
776
0
     */
777
0
    mPuppetHMDs[aDeviceID]->SetDisplayInfo(aDisplayInfo);
778
0
  }
779
0
}
780
781
void
782
VRSystemManagerPuppet::SetPuppetDisplaySensorState(const uint32_t& aDeviceID,
783
                                                   const VRHMDSensorState& aSensorState)
784
0
{
785
0
  if (aDeviceID >= mPuppetDisplayCount) {
786
0
    MOZ_ASSERT(false);
787
0
    return;
788
0
  }
789
0
  /**
790
0
   * Even if mPuppetHMDs.Length() <= aDeviceID, we need to
791
0
   * update mPuppetDisplaySensorState[aDeviceID].  In the case that
792
0
   * a puppet display is added and SetPuppetDisplaySensorState is
793
0
   * immediately called, mPuppetHMDs may not be populated yet.
794
0
   * VRSystemManagerPuppet::Enumerate() will initialize
795
0
   * the VRDisplayPuppet later using mPuppetDisplaySensorState.
796
0
   */
797
0
  mPuppetDisplaySensorState[aDeviceID] = aSensorState;
798
0
  if (mPuppetHMDs.Length() > aDeviceID) {
799
0
    /**
800
0
     * In the event that the VRDisplayPuppet has already been
801
0
     * created, we update it directly.
802
0
     */
803
0
    mPuppetHMDs[aDeviceID]->SetSensorState(aSensorState);
804
0
  }
805
0
}
806
807
void
808
VRSystemManagerPuppet::GetHMDs(nsTArray<RefPtr<VRDisplayHost>>& aHMDResult)
809
0
{
810
0
  for (auto display: mPuppetHMDs) {
811
0
    aHMDResult.AppendElement(display);
812
0
  }
813
0
}
814
815
bool
816
VRSystemManagerPuppet::GetIsPresenting()
817
0
{
818
0
  for (const auto& display: mPuppetHMDs) {
819
0
    const VRDisplayInfo& displayInfo(display->GetDisplayInfo());
820
0
    if (displayInfo.GetPresentingGroups() != kVRGroupNone) {
821
0
      return true;
822
0
    }
823
0
  }
824
0
  return false;
825
0
}
826
827
void
828
VRSystemManagerPuppet::HandleInput()
829
0
{
830
0
  RefPtr<impl::VRControllerPuppet> controller;
831
0
  for (uint32_t i = 0; i < mPuppetController.Length(); ++i) {
832
0
    controller = mPuppetController[i];
833
0
    for (uint32_t j = 0; j < kNumPuppetButtonMask; ++j) {
834
0
      HandleButtonPress(i, j, kPuppetButtonMask[j], controller->GetButtonPressState(),
835
0
                        controller->GetButtonTouchState());
836
0
    }
837
0
    controller->SetButtonPressed(controller->GetButtonPressState());
838
0
    controller->SetButtonTouched(controller->GetButtonTouchState());
839
0
840
0
    for (uint32_t j = 0; j < kNumPuppetAxis; ++j) {
841
0
      HandleAxisMove(i, j, controller->GetAxisMoveState(j));
842
0
    }
843
0
    HandlePoseTracking(i, controller->GetPoseMoveState(), controller);
844
0
  }
845
0
}
846
847
void
848
VRSystemManagerPuppet::HandleButtonPress(uint32_t aControllerIdx,
849
                                         uint32_t aButton,
850
                                         uint64_t aButtonMask,
851
                                         uint64_t aButtonPressed,
852
                                         uint64_t aButtonTouched)
853
0
{
854
0
  RefPtr<impl::VRControllerPuppet> controller(mPuppetController[aControllerIdx]);
855
0
  MOZ_ASSERT(controller);
856
0
  const uint64_t pressedDiff = (controller->GetButtonPressed() ^ aButtonPressed);
857
0
  const uint64_t touchedDiff = (controller->GetButtonTouched() ^ aButtonTouched);
858
0
859
0
  if (!pressedDiff && !touchedDiff) {
860
0
    return;
861
0
  }
862
0
863
0
   if (pressedDiff & aButtonMask
864
0
      || touchedDiff & aButtonMask) {
865
0
    // diff & (aButtonPressed, aButtonTouched) would be true while a new button pressed or
866
0
    // touched event, otherwise it is an old event and needs to notify
867
0
    // the button has been released.
868
0
    NewButtonEvent(aControllerIdx, aButton, aButtonMask & aButtonPressed,
869
0
                   aButtonMask & aButtonPressed,
870
0
                   (aButtonMask & aButtonPressed) ? 1.0L : 0.0L);
871
0
  }
872
0
}
873
874
void
875
VRSystemManagerPuppet::HandleAxisMove(uint32_t aControllerIdx, uint32_t aAxis,
876
                                      float aValue)
877
0
{
878
0
  RefPtr<impl::VRControllerPuppet> controller(mPuppetController[aControllerIdx]);
879
0
  MOZ_ASSERT(controller);
880
0
881
0
  if (controller->GetAxisMove(aAxis) != aValue) {
882
0
    NewAxisMove(aControllerIdx, aAxis, aValue);
883
0
    controller->SetAxisMove(aAxis, aValue);
884
0
  }
885
0
}
886
887
void
888
VRSystemManagerPuppet::HandlePoseTracking(uint32_t aControllerIdx,
889
                                          const dom::GamepadPoseState& aPose,
890
                                          VRControllerHost* aController)
891
0
{
892
0
  MOZ_ASSERT(aController);
893
0
  if (aPose != aController->GetPose()) {
894
0
    aController->SetPose(aPose);
895
0
    NewPoseState(aControllerIdx, aPose);
896
0
  }
897
0
}
898
899
void
900
VRSystemManagerPuppet::VibrateHaptic(uint32_t aControllerIdx,
901
                                     uint32_t aHapticIndex,
902
                                     double aIntensity,
903
                                     double aDuration,
904
                                     const VRManagerPromise& aPromise)
905
0
{
906
0
}
907
908
void
909
VRSystemManagerPuppet::StopVibrateHaptic(uint32_t aControllerIdx)
910
0
{
911
0
}
912
913
void
914
VRSystemManagerPuppet::GetControllers(nsTArray<RefPtr<VRControllerHost>>& aControllerResult)
915
0
{
916
0
  aControllerResult.Clear();
917
0
  for (uint32_t i = 0; i < mPuppetController.Length(); ++i) {
918
0
    aControllerResult.AppendElement(mPuppetController[i]);
919
0
  }
920
0
}
921
922
void
923
VRSystemManagerPuppet::ScanForControllers()
924
0
{
925
0
  // We make sure VRSystemManagerPuppet has two controllers
926
0
  // for each display
927
0
  const uint32_t newControllerCount = mPuppetHMDs.Length() * 2;
928
0
929
0
  if (newControllerCount != mControllerCount) {
930
0
    RemoveControllers();
931
0
932
0
    // Re-adding controllers to VRControllerManager.
933
0
    for (const auto& display: mPuppetHMDs) {
934
0
      uint32_t displayID = display->GetDisplayInfo().GetDisplayID();
935
0
      for (uint32_t i = 0; i < 2; i++) {
936
0
        dom::GamepadHand hand = (i % 2) ? dom::GamepadHand::Right :
937
0
                                          dom::GamepadHand::Left;
938
0
        RefPtr<VRControllerPuppet> puppetController;
939
0
        puppetController = new VRControllerPuppet(hand, displayID);
940
0
        mPuppetController.AppendElement(puppetController);
941
0
942
0
        // Not already present, add it.
943
0
        AddGamepad(puppetController->GetControllerInfo());
944
0
        ++mControllerCount;
945
0
      }
946
0
    }
947
0
  }
948
0
}
949
950
void
951
VRSystemManagerPuppet::RemoveControllers()
952
0
{
953
0
  // controller count is changed, removing the existing gamepads first.
954
0
  for (uint32_t i = 0; i < mPuppetController.Length(); ++i) {
955
0
    RemoveGamepad(i);
956
0
  }
957
0
  mPuppetController.Clear();
958
0
  mControllerCount = 0;
959
0
}