/src/mozilla-central/gfx/vr/gfxVRPuppet.cpp
Line | Count | Source (jump to first uncovered line) |
1 | | /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ |
2 | | /* vim: set ts=8 sts=2 et sw=2 tw=80: */ |
3 | | /* This Source Code Form is subject to the terms of the Mozilla Public |
4 | | * License, v. 2.0. If a copy of the MPL was not distributed with this |
5 | | * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ |
6 | | |
7 | | #if defined(XP_WIN) |
8 | | #include "CompositorD3D11.h" |
9 | | #include "TextureD3D11.h" |
10 | | #include "mozilla/gfx/DeviceManagerDx.h" |
11 | | #elif defined(XP_MACOSX) |
12 | | #include "mozilla/gfx/MacIOSurface.h" |
13 | | #endif |
14 | | |
15 | | #include "mozilla/Base64.h" |
16 | | #include "mozilla/gfx/DataSurfaceHelpers.h" |
17 | | #include "gfxPrefs.h" |
18 | | #include "gfxUtils.h" |
19 | | #include "gfxVRPuppet.h" |
20 | | #include "VRManager.h" |
21 | | #include "VRThread.h" |
22 | | |
23 | | #include "mozilla/dom/GamepadEventTypes.h" |
24 | | #include "mozilla/dom/GamepadBinding.h" |
25 | | |
26 | | // See CompositorD3D11Shaders.h |
27 | | namespace mozilla { |
28 | | namespace layers { |
29 | | struct ShaderBytes { const void* mData; size_t mLength; }; |
30 | | extern ShaderBytes sRGBShader; |
31 | | extern ShaderBytes sLayerQuadVS; |
32 | | } // namespace layers |
33 | | } // namespace mozilla |
34 | | |
35 | | using namespace mozilla; |
36 | | using namespace mozilla::gfx; |
37 | | using namespace mozilla::gfx::impl; |
38 | | using namespace mozilla::layers; |
39 | | |
40 | | // Reminder: changing the order of these buttons may break web content |
41 | | static const uint64_t kPuppetButtonMask[] = { |
42 | | 1, |
43 | | 2, |
44 | | 4, |
45 | | 8 |
46 | | }; |
47 | | static const uint32_t kNumPuppetButtonMask = sizeof(kPuppetButtonMask) / |
48 | | sizeof(uint64_t); |
49 | | static const uint32_t kNumPuppetAxis = 3; |
50 | | static const uint32_t kNumPuppetHaptcs = 1; |
51 | | |
52 | | VRDisplayPuppet::VRDisplayPuppet() |
53 | | : VRDisplayLocal(VRDeviceType::Puppet) |
54 | | , mIsPresenting(false) |
55 | | , mSensorState{} |
56 | 0 | { |
57 | 0 | MOZ_COUNT_CTOR_INHERITED(VRDisplayPuppet, VRDisplayLocal); |
58 | 0 |
|
59 | 0 | VRDisplayState& state = mDisplayInfo.mDisplayState; |
60 | 0 | strncpy(state.mDisplayName, "Puppet HMD", kVRDisplayNameMaxLen); |
61 | 0 | state.mIsConnected = true; |
62 | 0 | state.mIsMounted = false; |
63 | 0 | state.mCapabilityFlags = VRDisplayCapabilityFlags::Cap_None | |
64 | 0 | VRDisplayCapabilityFlags::Cap_Orientation | |
65 | 0 | VRDisplayCapabilityFlags::Cap_Position | |
66 | 0 | VRDisplayCapabilityFlags::Cap_External | |
67 | 0 | VRDisplayCapabilityFlags::Cap_Present | |
68 | 0 | VRDisplayCapabilityFlags::Cap_StageParameters; |
69 | 0 | state.mEyeResolution.width = 1836; // 1080 * 1.7 |
70 | 0 | state.mEyeResolution.height = 2040; // 1200 * 1.7 |
71 | 0 |
|
72 | 0 | // SteamVR gives the application a single FOV to use; it's not configurable as with Oculus |
73 | 0 | for (uint32_t eye = 0; eye < 2; ++eye) { |
74 | 0 | state.mEyeTranslation[eye].x = 0.0f; |
75 | 0 | state.mEyeTranslation[eye].y = 0.0f; |
76 | 0 | state.mEyeTranslation[eye].z = 0.0f; |
77 | 0 | state.mEyeFOV[eye] = VRFieldOfView(45.0, 45.0, 45.0, 45.0); |
78 | 0 | } |
79 | 0 |
|
80 | 0 | // default: 1m x 1m space, 0.75m high in seated position |
81 | 0 | state.mStageSize.width = 1.0f; |
82 | 0 | state.mStageSize.height = 1.0f; |
83 | 0 |
|
84 | 0 | state.mSittingToStandingTransform[0] = 1.0f; |
85 | 0 | state.mSittingToStandingTransform[1] = 0.0f; |
86 | 0 | state.mSittingToStandingTransform[2] = 0.0f; |
87 | 0 | state.mSittingToStandingTransform[3] = 0.0f; |
88 | 0 |
|
89 | 0 | state.mSittingToStandingTransform[4] = 0.0f; |
90 | 0 | state.mSittingToStandingTransform[5] = 1.0f; |
91 | 0 | state.mSittingToStandingTransform[6] = 0.0f; |
92 | 0 | state.mSittingToStandingTransform[7] = 0.0f; |
93 | 0 |
|
94 | 0 | state.mSittingToStandingTransform[8] = 0.0f; |
95 | 0 | state.mSittingToStandingTransform[9] = 0.0f; |
96 | 0 | state.mSittingToStandingTransform[10] = 1.0f; |
97 | 0 | state.mSittingToStandingTransform[11] = 0.0f; |
98 | 0 |
|
99 | 0 | state.mSittingToStandingTransform[12] = 0.0f; |
100 | 0 | state.mSittingToStandingTransform[13] = 0.75f; |
101 | 0 | state.mSittingToStandingTransform[14] = 0.0f; |
102 | 0 | state.mSittingToStandingTransform[15] = 1.0f; |
103 | 0 |
|
104 | 0 | gfx::Quaternion rot; |
105 | 0 |
|
106 | 0 | mSensorState.flags |= VRDisplayCapabilityFlags::Cap_Orientation; |
107 | 0 | mSensorState.pose.orientation[0] = rot.x; |
108 | 0 | mSensorState.pose.orientation[1] = rot.y; |
109 | 0 | mSensorState.pose.orientation[2] = rot.z; |
110 | 0 | mSensorState.pose.orientation[3] = rot.w; |
111 | 0 | mSensorState.pose.angularVelocity[0] = 0.0f; |
112 | 0 | mSensorState.pose.angularVelocity[1] = 0.0f; |
113 | 0 | mSensorState.pose.angularVelocity[2] = 0.0f; |
114 | 0 |
|
115 | 0 | mSensorState.flags |= VRDisplayCapabilityFlags::Cap_Position; |
116 | 0 | mSensorState.pose.position[0] = 0.0f; |
117 | 0 | mSensorState.pose.position[1] = 0.0f; |
118 | 0 | mSensorState.pose.position[2] = 0.0f; |
119 | 0 | mSensorState.pose.linearVelocity[0] = 0.0f; |
120 | 0 | mSensorState.pose.linearVelocity[1] = 0.0f; |
121 | 0 | mSensorState.pose.linearVelocity[2] = 0.0f; |
122 | 0 | } |
123 | | |
124 | | VRDisplayPuppet::~VRDisplayPuppet() |
125 | 0 | { |
126 | 0 | MOZ_COUNT_DTOR_INHERITED(VRDisplayPuppet, VRDisplayLocal); |
127 | 0 | } |
128 | | |
129 | | void |
130 | | VRDisplayPuppet::SetDisplayInfo(const VRDisplayInfo& aDisplayInfo) |
131 | 0 | { |
132 | 0 | // We are only interested in the eye and mount info of the display info. |
133 | 0 | VRDisplayState& state = mDisplayInfo.mDisplayState; |
134 | 0 | state.mEyeResolution = aDisplayInfo.mDisplayState.mEyeResolution; |
135 | 0 | state.mIsMounted = aDisplayInfo.mDisplayState.mIsMounted; |
136 | 0 | memcpy(&state.mEyeFOV, &aDisplayInfo.mDisplayState.mEyeFOV, |
137 | 0 | sizeof(state.mEyeFOV[0]) * VRDisplayState::NumEyes); |
138 | 0 | memcpy(&state.mEyeTranslation, &aDisplayInfo.mDisplayState.mEyeTranslation, |
139 | 0 | sizeof(state.mEyeTranslation[0]) * VRDisplayState::NumEyes); |
140 | 0 | } |
141 | | |
142 | | void |
143 | | VRDisplayPuppet::Destroy() |
144 | 0 | { |
145 | 0 | StopPresentation(); |
146 | 0 | } |
147 | | |
148 | | void |
149 | | VRDisplayPuppet::ZeroSensor() |
150 | 0 | { |
151 | 0 | } |
152 | | |
153 | | VRHMDSensorState |
154 | | VRDisplayPuppet::GetSensorState() |
155 | 0 | { |
156 | 0 | mSensorState.inputFrameID = mDisplayInfo.mFrameId; |
157 | 0 |
|
158 | 0 | Matrix4x4 matHeadToEye[2]; |
159 | 0 | for (uint32_t eye = 0; eye < 2; ++eye) { |
160 | 0 | matHeadToEye[eye].PreTranslate(mDisplayInfo.GetEyeTranslation(eye)); |
161 | 0 | } |
162 | 0 | mSensorState.CalcViewMatrices(matHeadToEye); |
163 | 0 |
|
164 | 0 | return mSensorState; |
165 | 0 | } |
166 | | |
167 | | void |
168 | | VRDisplayPuppet::SetSensorState(const VRHMDSensorState& aSensorState) |
169 | 0 | { |
170 | 0 | memcpy(&mSensorState, &aSensorState, sizeof(mSensorState)); |
171 | 0 | } |
172 | | |
173 | | void |
174 | | VRDisplayPuppet::StartPresentation() |
175 | 0 | { |
176 | 0 | if (mIsPresenting) { |
177 | 0 | return; |
178 | 0 | } |
179 | 0 | mIsPresenting = true; |
180 | 0 |
|
181 | | #if defined(XP_WIN) |
182 | | if (!CreateD3DObjects()) { |
183 | | return; |
184 | | } |
185 | | |
186 | | if (FAILED(mDevice->CreateVertexShader(sLayerQuadVS.mData, |
187 | | sLayerQuadVS.mLength, nullptr, &mQuadVS))) { |
188 | | NS_WARNING("Failed to create vertex shader for Puppet"); |
189 | | return; |
190 | | } |
191 | | |
192 | | if (FAILED(mDevice->CreatePixelShader(sRGBShader.mData, |
193 | | sRGBShader.mLength, nullptr, &mQuadPS))) { |
194 | | NS_WARNING("Failed to create pixel shader for Puppet"); |
195 | | return; |
196 | | } |
197 | | |
198 | | CD3D11_BUFFER_DESC cBufferDesc(sizeof(layers::VertexShaderConstants), |
199 | | D3D11_BIND_CONSTANT_BUFFER, |
200 | | D3D11_USAGE_DYNAMIC, |
201 | | D3D11_CPU_ACCESS_WRITE); |
202 | | |
203 | | if (FAILED(mDevice->CreateBuffer(&cBufferDesc, nullptr, getter_AddRefs(mVSConstantBuffer)))) { |
204 | | NS_WARNING("Failed to vertex shader constant buffer for Puppet"); |
205 | | return; |
206 | | } |
207 | | |
208 | | cBufferDesc.ByteWidth = sizeof(layers::PixelShaderConstants); |
209 | | if (FAILED(mDevice->CreateBuffer(&cBufferDesc, nullptr, getter_AddRefs(mPSConstantBuffer)))) { |
210 | | NS_WARNING("Failed to pixel shader constant buffer for Puppet"); |
211 | | return; |
212 | | } |
213 | | |
214 | | CD3D11_SAMPLER_DESC samplerDesc(D3D11_DEFAULT); |
215 | | if (FAILED(mDevice->CreateSamplerState(&samplerDesc, getter_AddRefs(mLinearSamplerState)))) { |
216 | | NS_WARNING("Failed to create sampler state for Puppet"); |
217 | | return; |
218 | | } |
219 | | |
220 | | D3D11_INPUT_ELEMENT_DESC layout[] = |
221 | | { |
222 | | { "POSITION", 0, DXGI_FORMAT_R32G32_FLOAT, 0, 0, D3D11_INPUT_PER_VERTEX_DATA, 0 }, |
223 | | }; |
224 | | |
225 | | if (FAILED(mDevice->CreateInputLayout(layout, |
226 | | sizeof(layout) / sizeof(D3D11_INPUT_ELEMENT_DESC), |
227 | | sLayerQuadVS.mData, |
228 | | sLayerQuadVS.mLength, |
229 | | getter_AddRefs(mInputLayout)))) { |
230 | | NS_WARNING("Failed to create input layout for Puppet"); |
231 | | return; |
232 | | } |
233 | | |
234 | | Vertex vertices[] = { { { 0.0, 0.0 } },{ { 1.0, 0.0 } },{ { 0.0, 1.0 } },{ { 1.0, 1.0 } } }; |
235 | | CD3D11_BUFFER_DESC bufferDesc(sizeof(vertices), D3D11_BIND_VERTEX_BUFFER); |
236 | | D3D11_SUBRESOURCE_DATA data; |
237 | | data.pSysMem = (void*)vertices; |
238 | | |
239 | | if (FAILED(mDevice->CreateBuffer(&bufferDesc, &data, getter_AddRefs(mVertexBuffer)))) { |
240 | | NS_WARNING("Failed to create vertex buffer for Puppet"); |
241 | | return; |
242 | | } |
243 | | |
244 | | memset(&mVSConstants, 0, sizeof(mVSConstants)); |
245 | | memset(&mPSConstants, 0, sizeof(mPSConstants)); |
246 | | #endif // XP_WIN |
247 | | } |
248 | | |
249 | | void |
250 | | VRDisplayPuppet::StopPresentation() |
251 | 0 | { |
252 | 0 | if (!mIsPresenting) { |
253 | 0 | return; |
254 | 0 | } |
255 | 0 | |
256 | 0 | mIsPresenting = false; |
257 | 0 | } |
258 | | |
259 | | #if defined(XP_WIN) |
260 | | bool |
261 | | VRDisplayPuppet::UpdateConstantBuffers() |
262 | | { |
263 | | HRESULT hr; |
264 | | D3D11_MAPPED_SUBRESOURCE resource; |
265 | | resource.pData = nullptr; |
266 | | |
267 | | hr = mContext->Map(mVSConstantBuffer, 0, D3D11_MAP_WRITE_DISCARD, 0, &resource); |
268 | | if (FAILED(hr) || !resource.pData) { |
269 | | return false; |
270 | | } |
271 | | *(VertexShaderConstants*)resource.pData = mVSConstants; |
272 | | mContext->Unmap(mVSConstantBuffer, 0); |
273 | | resource.pData = nullptr; |
274 | | |
275 | | hr = mContext->Map(mPSConstantBuffer, 0, D3D11_MAP_WRITE_DISCARD, 0, &resource); |
276 | | if (FAILED(hr) || !resource.pData) { |
277 | | return false; |
278 | | } |
279 | | *(PixelShaderConstants*)resource.pData = mPSConstants; |
280 | | mContext->Unmap(mPSConstantBuffer, 0); |
281 | | |
282 | | ID3D11Buffer *buffer = mVSConstantBuffer; |
283 | | mContext->VSSetConstantBuffers(0, 1, &buffer); |
284 | | buffer = mPSConstantBuffer; |
285 | | mContext->PSSetConstantBuffers(0, 1, &buffer); |
286 | | return true; |
287 | | } |
288 | | |
289 | | bool |
290 | | VRDisplayPuppet::SubmitFrame(ID3D11Texture2D* aSource, |
291 | | const IntSize& aSize, |
292 | | const gfx::Rect& aLeftEyeRect, |
293 | | const gfx::Rect& aRightEyeRect) |
294 | | { |
295 | | MOZ_ASSERT(mSubmitThread->GetThread() == NS_GetCurrentThread()); |
296 | | if (!mIsPresenting) { |
297 | | return false; |
298 | | } |
299 | | |
300 | | if (!CreateD3DObjects()) { |
301 | | return false; |
302 | | } |
303 | | AutoRestoreRenderState restoreState(this); |
304 | | if (!restoreState.IsSuccess()) { |
305 | | return false; |
306 | | } |
307 | | |
308 | | VRManager *vm = VRManager::Get(); |
309 | | MOZ_ASSERT(vm); |
310 | | |
311 | | switch (gfxPrefs::VRPuppetSubmitFrame()) { |
312 | | case 0: |
313 | | // The VR frame is not displayed. |
314 | | break; |
315 | | case 1: |
316 | | { |
317 | | // The frames are submitted to VR compositor are decoded |
318 | | // into a base64Image and dispatched to the DOM side. |
319 | | D3D11_TEXTURE2D_DESC desc; |
320 | | aSource->GetDesc(&desc); |
321 | | MOZ_ASSERT(desc.Format == DXGI_FORMAT_B8G8R8A8_UNORM, |
322 | | "Only support B8G8R8A8_UNORM format."); |
323 | | // Map the staging resource |
324 | | ID3D11Texture2D* mappedTexture = nullptr; |
325 | | D3D11_MAPPED_SUBRESOURCE mapInfo; |
326 | | HRESULT hr = mContext->Map(aSource, |
327 | | 0, // Subsource |
328 | | D3D11_MAP_READ, |
329 | | 0, // MapFlags |
330 | | &mapInfo); |
331 | | |
332 | | if (FAILED(hr)) { |
333 | | // If we can't map this texture, copy it to a staging resource. |
334 | | if (hr == E_INVALIDARG) { |
335 | | D3D11_TEXTURE2D_DESC desc2; |
336 | | desc2.Width = desc.Width; |
337 | | desc2.Height = desc.Height; |
338 | | desc2.MipLevels = desc.MipLevels; |
339 | | desc2.ArraySize = desc.ArraySize; |
340 | | desc2.Format = desc.Format; |
341 | | desc2.SampleDesc = desc.SampleDesc; |
342 | | desc2.Usage = D3D11_USAGE_STAGING; |
343 | | desc2.BindFlags = 0; |
344 | | desc2.CPUAccessFlags = D3D11_CPU_ACCESS_READ; |
345 | | desc2.MiscFlags = 0; |
346 | | |
347 | | ID3D11Texture2D* stagingTexture = nullptr; |
348 | | hr = mDevice->CreateTexture2D(&desc2, nullptr, &stagingTexture); |
349 | | if (FAILED(hr)) { |
350 | | MOZ_ASSERT(false, "Failed to create a staging texture"); |
351 | | return false; |
352 | | } |
353 | | // Copy the texture to a staging resource |
354 | | mContext->CopyResource(stagingTexture, aSource); |
355 | | // Map the staging resource |
356 | | hr = mContext->Map(stagingTexture, |
357 | | 0, // Subsource |
358 | | D3D11_MAP_READ, |
359 | | 0, // MapFlags |
360 | | &mapInfo); |
361 | | if (FAILED(hr)) { |
362 | | MOZ_ASSERT(false, "Failed to map staging texture"); |
363 | | } |
364 | | mappedTexture = stagingTexture; |
365 | | } else { |
366 | | MOZ_ASSERT(false, "Failed to map staging texture"); |
367 | | return false; |
368 | | } |
369 | | } else { |
370 | | mappedTexture = aSource; |
371 | | } |
372 | | // Ideally, we should convert the srcData to a PNG image and decode it |
373 | | // to a Base64 string here, but the GPU process does not have the privilege to |
374 | | // access the image library. So, we have to convert the RAW image data |
375 | | // to a base64 string and forward it to let the content process to |
376 | | // do the image conversion. |
377 | | const char* srcData = static_cast<const char*>(mapInfo.pData); |
378 | | VRSubmitFrameResultInfo result; |
379 | | result.mFormat = SurfaceFormat::B8G8R8A8; |
380 | | result.mWidth = desc.Width; |
381 | | result.mHeight = desc.Height; |
382 | | result.mFrameNum = mDisplayInfo.mFrameId; |
383 | | // If the original texture size is not pow of 2, the data will not be tightly strided. |
384 | | // We have to copy the pixels by rows. |
385 | | nsCString rawString; |
386 | | for (uint32_t i = 0; i < desc.Height; i++) { |
387 | | rawString += Substring(srcData + i * mapInfo.RowPitch, |
388 | | desc.Width * 4); |
389 | | } |
390 | | mContext->Unmap(mappedTexture, 0); |
391 | | |
392 | | if (Base64Encode(rawString, result.mBase64Image) != NS_OK) { |
393 | | MOZ_ASSERT(false, "Failed to encode base64 images."); |
394 | | } |
395 | | // Dispatch the base64 encoded string to the DOM side. Then, it will be decoded |
396 | | // and convert to a PNG image there. |
397 | | MessageLoop* loop = VRListenerThreadHolder::Loop(); |
398 | | loop->PostTask(NewRunnableMethod<const uint32_t, VRSubmitFrameResultInfo>( |
399 | | "VRManager::DispatchSubmitFrameResult", |
400 | | vm, &VRManager::DispatchSubmitFrameResult, mDisplayInfo.mDisplayID, result |
401 | | )); |
402 | | break; |
403 | | } |
404 | | case 2: |
405 | | { |
406 | | // The VR compositor sumbmit frame to the screen window, |
407 | | // the current coordinate is at (0, 0, width, height). |
408 | | Matrix viewMatrix = Matrix::Translation(-1.0, 1.0); |
409 | | viewMatrix.PreScale(2.0f / float(aSize.width), 2.0f / float(aSize.height)); |
410 | | viewMatrix.PreScale(1.0f, -1.0f); |
411 | | Matrix4x4 projection = Matrix4x4::From2D(viewMatrix); |
412 | | projection._33 = 0.0f; |
413 | | |
414 | | Matrix transform2d; |
415 | | gfx::Matrix4x4 transform = gfx::Matrix4x4::From2D(transform2d); |
416 | | |
417 | | const float posX = 0.0f, posY = 0.0f; |
418 | | D3D11_VIEWPORT viewport; |
419 | | viewport.MinDepth = 0.0f; |
420 | | viewport.MaxDepth = 1.0f; |
421 | | viewport.Width = aSize.width; |
422 | | viewport.Height = aSize.height; |
423 | | viewport.TopLeftX = posX; |
424 | | viewport.TopLeftY = posY; |
425 | | |
426 | | D3D11_RECT scissor; |
427 | | scissor.left = posX; |
428 | | scissor.right = aSize.width + posX; |
429 | | scissor.top = posY; |
430 | | scissor.bottom = aSize.height + posY; |
431 | | |
432 | | memcpy(&mVSConstants.layerTransform, &transform._11, sizeof(mVSConstants.layerTransform)); |
433 | | memcpy(&mVSConstants.projection, &projection._11, sizeof(mVSConstants.projection)); |
434 | | mVSConstants.renderTargetOffset[0] = 0.0f; |
435 | | mVSConstants.renderTargetOffset[1] = 0.0f; |
436 | | mVSConstants.layerQuad = Rect(0.0f, 0.0f, aSize.width, aSize.height); |
437 | | mVSConstants.textureCoords = Rect(0.0f, 1.0f, 1.0f, -1.0f); |
438 | | |
439 | | mPSConstants.layerOpacity[0] = 1.0f; |
440 | | |
441 | | ID3D11Buffer* vbuffer = mVertexBuffer; |
442 | | UINT vsize = sizeof(Vertex); |
443 | | UINT voffset = 0; |
444 | | mContext->IASetVertexBuffers(0, 1, &vbuffer, &vsize, &voffset); |
445 | | mContext->IASetIndexBuffer(nullptr, DXGI_FORMAT_R16_UINT, 0); |
446 | | mContext->IASetInputLayout(mInputLayout); |
447 | | mContext->RSSetViewports(1, &viewport); |
448 | | mContext->RSSetScissorRects(1, &scissor); |
449 | | mContext->IASetPrimitiveTopology(D3D11_PRIMITIVE_TOPOLOGY_TRIANGLESTRIP); |
450 | | mContext->VSSetShader(mQuadVS, nullptr, 0); |
451 | | mContext->PSSetShader(mQuadPS, nullptr, 0); |
452 | | |
453 | | RefPtr<ID3D11ShaderResourceView> srView; |
454 | | HRESULT hr = mDevice->CreateShaderResourceView(aSource, nullptr, getter_AddRefs(srView)); |
455 | | if (FAILED(hr) || !srView) { |
456 | | gfxWarning() << "Could not create shader resource view for Puppet: " << hexa(hr); |
457 | | return false; |
458 | | } |
459 | | ID3D11ShaderResourceView* viewPtr = srView.get(); |
460 | | mContext->PSSetShaderResources(0 /* 0 == TexSlot::RGB */, 1, &viewPtr); |
461 | | // XXX Use Constant from TexSlot in CompositorD3D11.cpp? |
462 | | |
463 | | ID3D11SamplerState *sampler = mLinearSamplerState; |
464 | | mContext->PSSetSamplers(0, 1, &sampler); |
465 | | |
466 | | if (!UpdateConstantBuffers()) { |
467 | | NS_WARNING("Failed to update constant buffers for Puppet"); |
468 | | return false; |
469 | | } |
470 | | mContext->Draw(4, 0); |
471 | | break; |
472 | | } |
473 | | } |
474 | | |
475 | | // We will always return false for gfxVRPuppet to ensure that the fallback "watchdog" |
476 | | // code in VRDisplayHost::NotifyVSync() throttles the render loop. This "watchdog" will |
477 | | // result in a refresh rate that is quite low compared to real hardware, but should be |
478 | | // sufficient for non-performance oriented tests. If we wish to simulate realistic frame |
479 | | // rates with VRDisplayPuppet, we should block here for the appropriate amount of time and |
480 | | // return true to indicate that we have blocked. |
481 | | return false; |
482 | | } |
483 | | |
484 | | #elif defined(XP_MACOSX) |
485 | | |
486 | | bool |
487 | | VRDisplayPuppet::SubmitFrame(MacIOSurface* aMacIOSurface, |
488 | | const IntSize& aSize, |
489 | | const gfx::Rect& aLeftEyeRect, |
490 | | const gfx::Rect& aRightEyeRect) |
491 | | { |
492 | | MOZ_ASSERT(mSubmitThread->GetThread() == NS_GetCurrentThread()); |
493 | | if (!mIsPresenting || !aMacIOSurface) { |
494 | | return false; |
495 | | } |
496 | | |
497 | | VRManager* vm = VRManager::Get(); |
498 | | MOZ_ASSERT(vm); |
499 | | |
500 | | switch (gfxPrefs::VRPuppetSubmitFrame()) { |
501 | | case 0: |
502 | | // The VR frame is not displayed. |
503 | | break; |
504 | | case 1: |
505 | | { |
506 | | // The frames are submitted to VR compositor are decoded |
507 | | // into a base64Image and dispatched to the DOM side. |
508 | | RefPtr<SourceSurface> surf = aMacIOSurface->GetAsSurface(); |
509 | | RefPtr<DataSourceSurface> dataSurf = surf ? surf->GetDataSurface() : |
510 | | nullptr; |
511 | | if (dataSurf) { |
512 | | // Ideally, we should convert the srcData to a PNG image and decode it |
513 | | // to a Base64 string here, but the GPU process does not have the privilege to |
514 | | // access the image library. So, we have to convert the RAW image data |
515 | | // to a base64 string and forward it to let the content process to |
516 | | // do the image conversion. |
517 | | DataSourceSurface::MappedSurface map; |
518 | | if (!dataSurf->Map(gfx::DataSourceSurface::MapType::READ, &map)) { |
519 | | MOZ_ASSERT(false, "Read DataSourceSurface fail."); |
520 | | return false; |
521 | | } |
522 | | const uint8_t* srcData = map.mData; |
523 | | const auto& surfSize = dataSurf->GetSize(); |
524 | | VRSubmitFrameResultInfo result; |
525 | | result.mFormat = SurfaceFormat::B8G8R8A8; |
526 | | result.mWidth = surfSize.width; |
527 | | result.mHeight = surfSize.height; |
528 | | result.mFrameNum = mDisplayInfo.mFrameId; |
529 | | // If the original texture size is not pow of 2, the data will not be tightly strided. |
530 | | // We have to copy the pixels by rows. |
531 | | nsCString rawString; |
532 | | for (int32_t i = 0; i < surfSize.height; i++) { |
533 | | rawString += Substring((const char*)(srcData) + i * map.mStride, |
534 | | surfSize.width * 4); |
535 | | } |
536 | | dataSurf->Unmap(); |
537 | | |
538 | | if (Base64Encode(rawString, result.mBase64Image) != NS_OK) { |
539 | | MOZ_ASSERT(false, "Failed to encode base64 images."); |
540 | | } |
541 | | // Dispatch the base64 encoded string to the DOM side. Then, it will be decoded |
542 | | // and convert to a PNG image there. |
543 | | MessageLoop* loop = VRListenerThreadHolder::Loop(); |
544 | | loop->PostTask(NewRunnableMethod<const uint32_t, VRSubmitFrameResultInfo>( |
545 | | "VRManager::DispatchSubmitFrameResult", |
546 | | vm, &VRManager::DispatchSubmitFrameResult, mDisplayInfo.mDisplayID, result |
547 | | )); |
548 | | } |
549 | | break; |
550 | | } |
551 | | case 2: |
552 | | { |
553 | | MOZ_ASSERT(false, "No support for showing VR frames on MacOSX yet."); |
554 | | break; |
555 | | } |
556 | | } |
557 | | |
558 | | return false; |
559 | | } |
560 | | |
561 | | #elif defined(MOZ_WIDGET_ANDROID) |
562 | | |
563 | | bool |
564 | | VRDisplayPuppet::SubmitFrame(const mozilla::layers::SurfaceTextureDescriptor& aDescriptor, |
565 | | const gfx::Rect& aLeftEyeRect, |
566 | | const gfx::Rect& aRightEyeRect) |
567 | | { |
568 | | MOZ_ASSERT(mSubmitThread->GetThread() == NS_GetCurrentThread()); |
569 | | return false; |
570 | | } |
571 | | |
572 | | #endif |
573 | | |
574 | | void |
575 | | VRDisplayPuppet::Refresh() |
576 | 0 | { |
577 | 0 | // We update mIsConneced once per refresh. |
578 | 0 | mDisplayInfo.mDisplayState.mIsConnected = true; |
579 | 0 | } |
580 | | |
581 | | VRControllerPuppet::VRControllerPuppet(dom::GamepadHand aHand, uint32_t aDisplayID) |
582 | | : VRControllerHost(VRDeviceType::Puppet, aHand, aDisplayID) |
583 | | , mButtonPressState(0) |
584 | | , mButtonTouchState(0) |
585 | 0 | { |
586 | 0 | MOZ_COUNT_CTOR_INHERITED(VRControllerPuppet, VRControllerHost); |
587 | 0 | VRControllerState& state = mControllerInfo.mControllerState; |
588 | 0 | strncpy(state.controllerName, "Puppet Gamepad", kVRControllerNameMaxLen); |
589 | 0 | state.numButtons = kNumPuppetButtonMask; |
590 | 0 | state.numAxes = kNumPuppetAxis; |
591 | 0 | state.numHaptics = kNumPuppetHaptcs; |
592 | 0 | } |
593 | | |
594 | | VRControllerPuppet::~VRControllerPuppet() |
595 | 0 | { |
596 | 0 | MOZ_COUNT_DTOR_INHERITED(VRControllerPuppet, VRControllerHost); |
597 | 0 | } |
598 | | |
599 | | void |
600 | | VRControllerPuppet::SetButtonPressState(uint32_t aButton, bool aPressed) |
601 | 0 | { |
602 | 0 | const uint64_t buttonMask = kPuppetButtonMask[aButton]; |
603 | 0 | uint64_t pressedBit = GetButtonPressed(); |
604 | 0 |
|
605 | 0 | if (aPressed) { |
606 | 0 | pressedBit |= kPuppetButtonMask[aButton]; |
607 | 0 | } else if (pressedBit & buttonMask) { |
608 | 0 | // this button was pressed but is released now. |
609 | 0 | uint64_t mask = 0xff ^ buttonMask; |
610 | 0 | pressedBit &= mask; |
611 | 0 | } |
612 | 0 |
|
613 | 0 | mButtonPressState = pressedBit; |
614 | 0 | } |
615 | | |
616 | | uint64_t |
617 | | VRControllerPuppet::GetButtonPressState() |
618 | 0 | { |
619 | 0 | return mButtonPressState; |
620 | 0 | } |
621 | | |
622 | | void |
623 | | VRControllerPuppet::SetButtonTouchState(uint32_t aButton, bool aTouched) |
624 | 0 | { |
625 | 0 | const uint64_t buttonMask = kPuppetButtonMask[aButton]; |
626 | 0 | uint64_t touchedBit = GetButtonTouched(); |
627 | 0 |
|
628 | 0 | if (aTouched) { |
629 | 0 | touchedBit |= kPuppetButtonMask[aButton]; |
630 | 0 | } else if (touchedBit & buttonMask) { |
631 | 0 | // this button was touched but is released now. |
632 | 0 | uint64_t mask = 0xff ^ buttonMask; |
633 | 0 | touchedBit &= mask; |
634 | 0 | } |
635 | 0 |
|
636 | 0 | mButtonTouchState = touchedBit; |
637 | 0 | } |
638 | | |
639 | | uint64_t |
640 | | VRControllerPuppet::GetButtonTouchState() |
641 | 0 | { |
642 | 0 | return mButtonTouchState; |
643 | 0 | } |
644 | | |
645 | | void |
646 | | VRControllerPuppet::SetAxisMoveState(uint32_t aAxis, double aValue) |
647 | 0 | { |
648 | 0 | MOZ_ASSERT((sizeof(mAxisMoveState) / sizeof(float)) == kNumPuppetAxis); |
649 | 0 | MOZ_ASSERT(aAxis <= kNumPuppetAxis); |
650 | 0 |
|
651 | 0 | mAxisMoveState[aAxis] = aValue; |
652 | 0 | } |
653 | | |
654 | | double |
655 | | VRControllerPuppet::GetAxisMoveState(uint32_t aAxis) |
656 | 0 | { |
657 | 0 | return mAxisMoveState[aAxis]; |
658 | 0 | } |
659 | | |
660 | | void |
661 | | VRControllerPuppet::SetPoseMoveState(const dom::GamepadPoseState& aPose) |
662 | 0 | { |
663 | 0 | mPoseState = aPose; |
664 | 0 | } |
665 | | |
666 | | const dom::GamepadPoseState& |
667 | | VRControllerPuppet::GetPoseMoveState() |
668 | 0 | { |
669 | 0 | return mPoseState; |
670 | 0 | } |
671 | | |
672 | | float |
673 | | VRControllerPuppet::GetAxisMove(uint32_t aAxis) |
674 | 0 | { |
675 | 0 | return mControllerInfo.mControllerState.axisValue[aAxis]; |
676 | 0 | } |
677 | | |
678 | | void |
679 | | VRControllerPuppet::SetAxisMove(uint32_t aAxis, float aValue) |
680 | 0 | { |
681 | 0 | mControllerInfo.mControllerState.axisValue[aAxis] = aValue; |
682 | 0 | } |
683 | | |
684 | | VRSystemManagerPuppet::VRSystemManagerPuppet() |
685 | | : mPuppetDisplayCount(0) |
686 | | , mPuppetDisplayInfo{} |
687 | | , mPuppetDisplaySensorState{} |
688 | 0 | { |
689 | 0 | } |
690 | | |
691 | | /*static*/ already_AddRefed<VRSystemManagerPuppet> |
692 | | VRSystemManagerPuppet::Create() |
693 | 0 | { |
694 | 0 | if (!gfxPrefs::VREnabled() || !gfxPrefs::VRPuppetEnabled()) { |
695 | 0 | return nullptr; |
696 | 0 | } |
697 | 0 | |
698 | 0 | RefPtr<VRSystemManagerPuppet> manager = new VRSystemManagerPuppet(); |
699 | 0 | return manager.forget(); |
700 | 0 | } |
701 | | |
702 | | void |
703 | | VRSystemManagerPuppet::Destroy() |
704 | 0 | { |
705 | 0 | Shutdown(); |
706 | 0 | } |
707 | | |
708 | | void |
709 | | VRSystemManagerPuppet::Shutdown() |
710 | 0 | { |
711 | 0 | mPuppetHMDs.Clear(); |
712 | 0 | } |
713 | | |
714 | | void |
715 | | VRSystemManagerPuppet::NotifyVSync() |
716 | 0 | { |
717 | 0 | VRSystemManager::NotifyVSync(); |
718 | 0 |
|
719 | 0 | for (const auto& display: mPuppetHMDs) { |
720 | 0 | display->Refresh(); |
721 | 0 | } |
722 | 0 | } |
723 | | |
724 | | uint32_t |
725 | | VRSystemManagerPuppet::CreateTestDisplay() |
726 | 0 | { |
727 | 0 | if (mPuppetDisplayCount >= kMaxPuppetDisplays) { |
728 | 0 | MOZ_ASSERT(false); |
729 | 0 | return mPuppetDisplayCount; |
730 | 0 | } |
731 | 0 | return mPuppetDisplayCount++; |
732 | 0 | } |
733 | | |
734 | | void |
735 | | VRSystemManagerPuppet::ClearTestDisplays() |
736 | 0 | { |
737 | 0 | mPuppetDisplayCount = 0; |
738 | 0 | } |
739 | | |
740 | | void |
741 | | VRSystemManagerPuppet::Enumerate() |
742 | 0 | { |
743 | 0 | while (mPuppetHMDs.Length() < mPuppetDisplayCount) { |
744 | 0 | VRDisplayPuppet* puppetDisplay = new VRDisplayPuppet(); |
745 | 0 | uint32_t deviceID = mPuppetHMDs.Length(); |
746 | 0 | puppetDisplay->SetDisplayInfo(mPuppetDisplayInfo[deviceID]); |
747 | 0 | puppetDisplay->SetSensorState(mPuppetDisplaySensorState[deviceID]); |
748 | 0 | mPuppetHMDs.AppendElement(puppetDisplay); |
749 | 0 | } |
750 | 0 | while (mPuppetHMDs.Length() > mPuppetDisplayCount) { |
751 | 0 | mPuppetHMDs.RemoveLastElement(); |
752 | 0 | } |
753 | 0 | } |
754 | | |
755 | | void |
756 | | VRSystemManagerPuppet::SetPuppetDisplayInfo(const uint32_t& aDeviceID, |
757 | | const VRDisplayInfo& aDisplayInfo) |
758 | 0 | { |
759 | 0 | if (aDeviceID >= mPuppetDisplayCount) { |
760 | 0 | MOZ_ASSERT(false); |
761 | 0 | return; |
762 | 0 | } |
763 | 0 | /** |
764 | 0 | * Even if mPuppetHMDs.Length() <= aDeviceID, we need to |
765 | 0 | * update mPuppetDisplayInfo[aDeviceID]. In the case that |
766 | 0 | * a puppet display is added and SetPuppetDisplayInfo is |
767 | 0 | * immediately called, mPuppetHMDs may not be populated yet. |
768 | 0 | * VRSystemManagerPuppet::Enumerate() will initialize |
769 | 0 | * the VRDisplayPuppet later using mPuppetDisplayInfo. |
770 | 0 | */ |
771 | 0 | mPuppetDisplayInfo[aDeviceID] = aDisplayInfo; |
772 | 0 | if (mPuppetHMDs.Length() > aDeviceID) { |
773 | 0 | /** |
774 | 0 | * In the event that the VRDisplayPuppet has already been |
775 | 0 | * created, we update it directly. |
776 | 0 | */ |
777 | 0 | mPuppetHMDs[aDeviceID]->SetDisplayInfo(aDisplayInfo); |
778 | 0 | } |
779 | 0 | } |
780 | | |
781 | | void |
782 | | VRSystemManagerPuppet::SetPuppetDisplaySensorState(const uint32_t& aDeviceID, |
783 | | const VRHMDSensorState& aSensorState) |
784 | 0 | { |
785 | 0 | if (aDeviceID >= mPuppetDisplayCount) { |
786 | 0 | MOZ_ASSERT(false); |
787 | 0 | return; |
788 | 0 | } |
789 | 0 | /** |
790 | 0 | * Even if mPuppetHMDs.Length() <= aDeviceID, we need to |
791 | 0 | * update mPuppetDisplaySensorState[aDeviceID]. In the case that |
792 | 0 | * a puppet display is added and SetPuppetDisplaySensorState is |
793 | 0 | * immediately called, mPuppetHMDs may not be populated yet. |
794 | 0 | * VRSystemManagerPuppet::Enumerate() will initialize |
795 | 0 | * the VRDisplayPuppet later using mPuppetDisplaySensorState. |
796 | 0 | */ |
797 | 0 | mPuppetDisplaySensorState[aDeviceID] = aSensorState; |
798 | 0 | if (mPuppetHMDs.Length() > aDeviceID) { |
799 | 0 | /** |
800 | 0 | * In the event that the VRDisplayPuppet has already been |
801 | 0 | * created, we update it directly. |
802 | 0 | */ |
803 | 0 | mPuppetHMDs[aDeviceID]->SetSensorState(aSensorState); |
804 | 0 | } |
805 | 0 | } |
806 | | |
807 | | void |
808 | | VRSystemManagerPuppet::GetHMDs(nsTArray<RefPtr<VRDisplayHost>>& aHMDResult) |
809 | 0 | { |
810 | 0 | for (auto display: mPuppetHMDs) { |
811 | 0 | aHMDResult.AppendElement(display); |
812 | 0 | } |
813 | 0 | } |
814 | | |
815 | | bool |
816 | | VRSystemManagerPuppet::GetIsPresenting() |
817 | 0 | { |
818 | 0 | for (const auto& display: mPuppetHMDs) { |
819 | 0 | const VRDisplayInfo& displayInfo(display->GetDisplayInfo()); |
820 | 0 | if (displayInfo.GetPresentingGroups() != kVRGroupNone) { |
821 | 0 | return true; |
822 | 0 | } |
823 | 0 | } |
824 | 0 | return false; |
825 | 0 | } |
826 | | |
827 | | void |
828 | | VRSystemManagerPuppet::HandleInput() |
829 | 0 | { |
830 | 0 | RefPtr<impl::VRControllerPuppet> controller; |
831 | 0 | for (uint32_t i = 0; i < mPuppetController.Length(); ++i) { |
832 | 0 | controller = mPuppetController[i]; |
833 | 0 | for (uint32_t j = 0; j < kNumPuppetButtonMask; ++j) { |
834 | 0 | HandleButtonPress(i, j, kPuppetButtonMask[j], controller->GetButtonPressState(), |
835 | 0 | controller->GetButtonTouchState()); |
836 | 0 | } |
837 | 0 | controller->SetButtonPressed(controller->GetButtonPressState()); |
838 | 0 | controller->SetButtonTouched(controller->GetButtonTouchState()); |
839 | 0 |
|
840 | 0 | for (uint32_t j = 0; j < kNumPuppetAxis; ++j) { |
841 | 0 | HandleAxisMove(i, j, controller->GetAxisMoveState(j)); |
842 | 0 | } |
843 | 0 | HandlePoseTracking(i, controller->GetPoseMoveState(), controller); |
844 | 0 | } |
845 | 0 | } |
846 | | |
847 | | void |
848 | | VRSystemManagerPuppet::HandleButtonPress(uint32_t aControllerIdx, |
849 | | uint32_t aButton, |
850 | | uint64_t aButtonMask, |
851 | | uint64_t aButtonPressed, |
852 | | uint64_t aButtonTouched) |
853 | 0 | { |
854 | 0 | RefPtr<impl::VRControllerPuppet> controller(mPuppetController[aControllerIdx]); |
855 | 0 | MOZ_ASSERT(controller); |
856 | 0 | const uint64_t pressedDiff = (controller->GetButtonPressed() ^ aButtonPressed); |
857 | 0 | const uint64_t touchedDiff = (controller->GetButtonTouched() ^ aButtonTouched); |
858 | 0 |
|
859 | 0 | if (!pressedDiff && !touchedDiff) { |
860 | 0 | return; |
861 | 0 | } |
862 | 0 | |
863 | 0 | if (pressedDiff & aButtonMask |
864 | 0 | || touchedDiff & aButtonMask) { |
865 | 0 | // diff & (aButtonPressed, aButtonTouched) would be true while a new button pressed or |
866 | 0 | // touched event, otherwise it is an old event and needs to notify |
867 | 0 | // the button has been released. |
868 | 0 | NewButtonEvent(aControllerIdx, aButton, aButtonMask & aButtonPressed, |
869 | 0 | aButtonMask & aButtonPressed, |
870 | 0 | (aButtonMask & aButtonPressed) ? 1.0L : 0.0L); |
871 | 0 | } |
872 | 0 | } |
873 | | |
874 | | void |
875 | | VRSystemManagerPuppet::HandleAxisMove(uint32_t aControllerIdx, uint32_t aAxis, |
876 | | float aValue) |
877 | 0 | { |
878 | 0 | RefPtr<impl::VRControllerPuppet> controller(mPuppetController[aControllerIdx]); |
879 | 0 | MOZ_ASSERT(controller); |
880 | 0 |
|
881 | 0 | if (controller->GetAxisMove(aAxis) != aValue) { |
882 | 0 | NewAxisMove(aControllerIdx, aAxis, aValue); |
883 | 0 | controller->SetAxisMove(aAxis, aValue); |
884 | 0 | } |
885 | 0 | } |
886 | | |
887 | | void |
888 | | VRSystemManagerPuppet::HandlePoseTracking(uint32_t aControllerIdx, |
889 | | const dom::GamepadPoseState& aPose, |
890 | | VRControllerHost* aController) |
891 | 0 | { |
892 | 0 | MOZ_ASSERT(aController); |
893 | 0 | if (aPose != aController->GetPose()) { |
894 | 0 | aController->SetPose(aPose); |
895 | 0 | NewPoseState(aControllerIdx, aPose); |
896 | 0 | } |
897 | 0 | } |
898 | | |
899 | | void |
900 | | VRSystemManagerPuppet::VibrateHaptic(uint32_t aControllerIdx, |
901 | | uint32_t aHapticIndex, |
902 | | double aIntensity, |
903 | | double aDuration, |
904 | | const VRManagerPromise& aPromise) |
905 | 0 | { |
906 | 0 | } |
907 | | |
908 | | void |
909 | | VRSystemManagerPuppet::StopVibrateHaptic(uint32_t aControllerIdx) |
910 | 0 | { |
911 | 0 | } |
912 | | |
913 | | void |
914 | | VRSystemManagerPuppet::GetControllers(nsTArray<RefPtr<VRControllerHost>>& aControllerResult) |
915 | 0 | { |
916 | 0 | aControllerResult.Clear(); |
917 | 0 | for (uint32_t i = 0; i < mPuppetController.Length(); ++i) { |
918 | 0 | aControllerResult.AppendElement(mPuppetController[i]); |
919 | 0 | } |
920 | 0 | } |
921 | | |
922 | | void |
923 | | VRSystemManagerPuppet::ScanForControllers() |
924 | 0 | { |
925 | 0 | // We make sure VRSystemManagerPuppet has two controllers |
926 | 0 | // for each display |
927 | 0 | const uint32_t newControllerCount = mPuppetHMDs.Length() * 2; |
928 | 0 |
|
929 | 0 | if (newControllerCount != mControllerCount) { |
930 | 0 | RemoveControllers(); |
931 | 0 |
|
932 | 0 | // Re-adding controllers to VRControllerManager. |
933 | 0 | for (const auto& display: mPuppetHMDs) { |
934 | 0 | uint32_t displayID = display->GetDisplayInfo().GetDisplayID(); |
935 | 0 | for (uint32_t i = 0; i < 2; i++) { |
936 | 0 | dom::GamepadHand hand = (i % 2) ? dom::GamepadHand::Right : |
937 | 0 | dom::GamepadHand::Left; |
938 | 0 | RefPtr<VRControllerPuppet> puppetController; |
939 | 0 | puppetController = new VRControllerPuppet(hand, displayID); |
940 | 0 | mPuppetController.AppendElement(puppetController); |
941 | 0 |
|
942 | 0 | // Not already present, add it. |
943 | 0 | AddGamepad(puppetController->GetControllerInfo()); |
944 | 0 | ++mControllerCount; |
945 | 0 | } |
946 | 0 | } |
947 | 0 | } |
948 | 0 | } |
949 | | |
950 | | void |
951 | | VRSystemManagerPuppet::RemoveControllers() |
952 | 0 | { |
953 | 0 | // controller count is changed, removing the existing gamepads first. |
954 | 0 | for (uint32_t i = 0; i < mPuppetController.Length(); ++i) { |
955 | 0 | RemoveGamepad(i); |
956 | 0 | } |
957 | 0 | mPuppetController.Clear(); |
958 | 0 | mControllerCount = 0; |
959 | 0 | } |