1 | // Copyright 2013 The Chromium Authors. All rights reserved. |
2 | // Use of this source code is governed by a BSD-style license that can be |
3 | // found in the LICENSE file. |
4 | |
5 | package org.chromium.chromoting; |
6 | |
7 | import android.app.ActionBar; |
8 | import android.app.Activity; |
9 | import android.graphics.Bitmap; |
10 | import android.graphics.Canvas; |
11 | import android.graphics.Color; |
12 | import android.graphics.Matrix; |
13 | import android.graphics.Paint; |
14 | import android.os.Bundle; |
15 | import android.os.Looper; |
16 | import android.text.InputType; |
17 | import android.util.Log; |
18 | import android.view.GestureDetector; |
19 | import android.view.MotionEvent; |
20 | import android.view.ScaleGestureDetector; |
21 | import android.view.SurfaceHolder; |
22 | import android.view.SurfaceView; |
23 | import android.view.inputmethod.EditorInfo; |
24 | import android.view.inputmethod.InputConnection; |
25 | |
26 | import org.chromium.chromoting.jni.JniInterface; |
27 | |
28 | /** |
29 | * The user interface for viewing and interacting with a specific remote host. |
30 | * It provides a canvas onto which the video feed is rendered, handles |
31 | * multitouch pan and zoom gestures, and collects and forwards input events. |
32 | */ |
33 | /** GUI element that holds the drawing canvas. */ |
34 | public class DesktopView extends SurfaceView implements Runnable, SurfaceHolder.Callback { |
35 | /** |
36 | * *Square* of the minimum displacement (in pixels) to be recognized as a scroll gesture. |
37 | * Setting this to a lower value forces more frequent canvas redraws during scrolling. |
38 | */ |
39 | private static final int MIN_SCROLL_DISTANCE = 8 * 8; |
40 | |
41 | /** |
42 | * Minimum change to the scaling factor to be recognized as a zoom gesture. Setting lower |
43 | * values here will result in more frequent canvas redraws during zooming. |
44 | */ |
45 | private static final double MIN_ZOOM_FACTOR = 0.05; |
46 | |
47 | /* |
48 | * These constants must match those in the generated struct protoc::MouseEvent_MouseButton. |
49 | */ |
50 | private static final int BUTTON_UNDEFINED = 0; |
51 | private static final int BUTTON_LEFT = 1; |
52 | private static final int BUTTON_RIGHT = 3; |
53 | |
54 | /** Specifies one dimension of an image. */ |
55 | private static enum Constraint { |
56 | UNDEFINED, WIDTH, HEIGHT |
57 | } |
58 | |
59 | private ActionBar mActionBar; |
60 | |
61 | private GestureDetector mScroller; |
62 | private ScaleGestureDetector mZoomer; |
63 | |
64 | /** Stores pan and zoom configuration and converts image coordinates to screen coordinates. */ |
65 | private Matrix mTransform; |
66 | |
67 | private int mScreenWidth; |
68 | private int mScreenHeight; |
69 | |
70 | /** Specifies the dimension by which the zoom level is being lower-bounded. */ |
71 | private Constraint mConstraint; |
72 | |
73 | /** Whether the dimension of constraint should be reckecked on the next aspect ratio change. */ |
74 | private boolean mRecheckConstraint; |
75 | |
76 | /** Whether the right edge of the image was visible on-screen during the last render. */ |
77 | private boolean mRightUsedToBeOut; |
78 | |
79 | /** Whether the bottom edge of the image was visible on-screen during the last render. */ |
80 | private boolean mBottomUsedToBeOut; |
81 | |
82 | private int mMouseButton; |
83 | private boolean mMousePressed; |
84 | |
85 | public DesktopView(Activity context) { |
86 | super(context); |
87 | |
88 | // Give this view keyboard focus, allowing us to customize the soft keyboard's settings. |
89 | setFocusableInTouchMode(true); |
90 | |
91 | mActionBar = context.getActionBar(); |
92 | |
93 | getHolder().addCallback(this); |
94 | DesktopListener listener = new DesktopListener(); |
95 | mScroller = new GestureDetector(context, listener, null, false); |
96 | mZoomer = new ScaleGestureDetector(context, listener); |
97 | |
98 | mTransform = new Matrix(); |
99 | mScreenWidth = 0; |
100 | mScreenHeight = 0; |
101 | |
102 | mConstraint = Constraint.UNDEFINED; |
103 | mRecheckConstraint = false; |
104 | |
105 | mRightUsedToBeOut = false; |
106 | mBottomUsedToBeOut = false; |
107 | |
108 | mMouseButton = BUTTON_UNDEFINED; |
109 | mMousePressed = false; |
110 | } |
111 | |
112 | /** |
113 | * Redraws the canvas. This should be done on a non-UI thread or it could |
114 | * cause the UI to lag. Specifically, it is currently invoked on the native |
115 | * graphics thread using a JNI. |
116 | */ |
117 | @Override |
118 | public void run() { |
119 | if (Looper.myLooper() == Looper.getMainLooper()) { |
120 | Log.w("deskview", "Canvas being redrawn on UI thread"); |
121 | } |
122 | |
123 | Bitmap image = JniInterface.retrieveVideoFrame(); |
124 | Canvas canvas = getHolder().lockCanvas(); |
125 | synchronized (mTransform) { |
126 | canvas.setMatrix(mTransform); |
127 | |
128 | // Internal parameters of the transformation matrix. |
129 | float[] values = new float[9]; |
130 | mTransform.getValues(values); |
131 | |
132 | // Screen coordinates of two defining points of the image. |
133 | float[] topleft = {0, 0}; |
134 | mTransform.mapPoints(topleft); |
135 | float[] bottomright = {image.getWidth(), image.getHeight()}; |
136 | mTransform.mapPoints(bottomright); |
137 | |
138 | // Whether to rescale and recenter the view. |
139 | boolean recenter = false; |
140 | |
141 | if (mConstraint == Constraint.UNDEFINED) { |
142 | mConstraint = (double)image.getWidth()/image.getHeight() > |
143 | (double)mScreenWidth/mScreenHeight ? Constraint.WIDTH : Constraint.HEIGHT; |
144 | recenter = true; // We always rescale and recenter after a rotation. |
145 | } |
146 | |
147 | if (mConstraint == Constraint.WIDTH && |
148 | ((int)(bottomright[0] - topleft[0] + 0.5) < mScreenWidth || recenter)) { |
149 | // The vertical edges of the image are flush against the device's screen edges |
150 | // when the entire host screen is visible, and the user has zoomed out too far. |
151 | float imageMiddle = (float)image.getHeight() / 2; |
152 | float screenMiddle = (float)mScreenHeight / 2; |
153 | mTransform.setPolyToPoly( |
154 | new float[] {0, imageMiddle, image.getWidth(), imageMiddle}, 0, |
155 | new float[] {0, screenMiddle, mScreenWidth, screenMiddle}, 0, 2); |
156 | } else if (mConstraint == Constraint.HEIGHT && |
157 | ((int)(bottomright[1] - topleft[1] + 0.5) < mScreenHeight || recenter)) { |
158 | // The horizontal image edges are flush against the device's screen edges when |
159 | // the entire host screen is visible, and the user has zoomed out too far. |
160 | float imageCenter = (float)image.getWidth() / 2; |
161 | float screenCenter = (float)mScreenWidth / 2; |
162 | mTransform.setPolyToPoly( |
163 | new float[] {imageCenter, 0, imageCenter, image.getHeight()}, 0, |
164 | new float[] {screenCenter, 0, screenCenter, mScreenHeight}, 0, 2); |
165 | } else { |
166 | // It's fine for both members of a pair of image edges to be within the screen |
167 | // edges (or "out of bounds"); that simply means that the image is zoomed out as |
168 | // far as permissible. And both members of a pair can obviously be outside the |
169 | // screen's edges, which indicates that the image is zoomed in to far to see the |
170 | // whole host screen. However, if only one of a pair of edges has entered the |
171 | // screen, the user is attempting to scroll into a blank area of the canvas. |
172 | |
173 | // A value of true means the corresponding edge has entered the screen's borders. |
174 | boolean leftEdgeOutOfBounds = values[Matrix.MTRANS_X] > 0; |
175 | boolean topEdgeOutOfBounds = values[Matrix.MTRANS_Y] > 0; |
176 | boolean rightEdgeOutOfBounds = bottomright[0] < mScreenWidth; |
177 | boolean bottomEdgeOutOfBounds = bottomright[1] < mScreenHeight; |
178 | |
179 | // Prevent the user from scrolling past the left or right edge of the image. |
180 | if (leftEdgeOutOfBounds != rightEdgeOutOfBounds) { |
181 | if (leftEdgeOutOfBounds != mRightUsedToBeOut) { |
182 | // Make the left edge of the image flush with the left screen edge. |
183 | values[Matrix.MTRANS_X] = 0; |
184 | } |
185 | else { |
186 | // Make the right edge of the image flush with the right screen edge. |
187 | values[Matrix.MTRANS_X] += mScreenWidth - bottomright[0]; |
188 | } |
189 | } else { |
190 | // The else prevents this from being updated during the repositioning process, |
191 | // in which case the view would begin to oscillate. |
192 | mRightUsedToBeOut = rightEdgeOutOfBounds; |
193 | } |
194 | |
195 | // Prevent the user from scrolling past the top or bottom edge of the image. |
196 | if (topEdgeOutOfBounds != bottomEdgeOutOfBounds) { |
197 | if (topEdgeOutOfBounds != mBottomUsedToBeOut) { |
198 | // Make the top edge of the image flush with the top screen edge. |
199 | values[Matrix.MTRANS_Y] = 0; |
200 | } else { |
201 | // Make the bottom edge of the image flush with the bottom screen edge. |
202 | values[Matrix.MTRANS_Y] += mScreenHeight - bottomright[1]; |
203 | } |
204 | } |
205 | else { |
206 | // The else prevents this from being updated during the repositioning process, |
207 | // in which case the view would begin to oscillate. |
208 | mBottomUsedToBeOut = bottomEdgeOutOfBounds; |
209 | } |
210 | |
211 | mTransform.setValues(values); |
212 | } |
213 | |
214 | canvas.setMatrix(mTransform); |
215 | } |
216 | |
217 | canvas.drawColor(Color.BLACK); |
218 | canvas.drawBitmap(image, 0, 0, new Paint()); |
219 | getHolder().unlockCanvasAndPost(canvas); |
220 | } |
221 | |
222 | /** |
223 | * Causes the next canvas redraw to perform a check for which screen dimension more tightly |
224 | * constrains the view of the image. This should be called between the time that a screen size |
225 | * change is requested and the time it actually occurs. If it is not called in such a case, the |
226 | * screen will not be rearranged as aggressively (which is desirable when the software keyboard |
227 | * appears in order to allow it to cover the image without forcing a resize). |
228 | */ |
229 | public void requestRecheckConstrainingDimension() { |
230 | mRecheckConstraint = true; |
231 | } |
232 | |
233 | /** |
234 | * Called after the canvas is initially created, then after every |
235 | * subsequent resize, as when the display is rotated. |
236 | */ |
237 | @Override |
238 | public void surfaceChanged( |
239 | SurfaceHolder holder, int format, int width, int height) { |
240 | mActionBar.hide(); |
241 | |
242 | synchronized (mTransform) { |
243 | mScreenWidth = width; |
244 | mScreenHeight = height; |
245 | |
246 | if (mRecheckConstraint) { |
247 | mConstraint = Constraint.UNDEFINED; |
248 | mRecheckConstraint = false; |
249 | } |
250 | } |
251 | |
252 | if (!JniInterface.redrawGraphics()) { |
253 | JniInterface.provideRedrawCallback(this); |
254 | } |
255 | } |
256 | |
257 | /** Called when the canvas is first created. */ |
258 | @Override |
259 | public void surfaceCreated(SurfaceHolder holder) { |
260 | Log.i("deskview", "DesktopView.surfaceCreated(...)"); |
261 | } |
262 | |
263 | /** |
264 | * Called when the canvas is finally destroyed. Marks the canvas as needing a redraw so that it |
265 | * will not be blank if the user later switches back to our window. |
266 | */ |
267 | @Override |
268 | public void surfaceDestroyed(SurfaceHolder holder) { |
269 | Log.i("deskview", "DesktopView.surfaceDestroyed(...)"); |
270 | |
271 | // Stop this canvas from being redrawn. |
272 | JniInterface.provideRedrawCallback(null); |
273 | } |
274 | |
275 | /** Called when a software keyboard is requested, and specifies its options. */ |
276 | @Override |
277 | public InputConnection onCreateInputConnection(EditorInfo outAttrs) { |
278 | // Disables rich input support and instead requests simple key events. |
279 | outAttrs.inputType = InputType.TYPE_NULL; |
280 | |
281 | // Prevents most third-party IMEs from ignoring our Activity's adjustResize preference. |
282 | outAttrs.imeOptions |= EditorInfo.IME_FLAG_NO_FULLSCREEN; |
283 | |
284 | // Ensures that keyboards will not decide to hide the remote desktop on small displays. |
285 | outAttrs.imeOptions |= EditorInfo.IME_FLAG_NO_EXTRACT_UI; |
286 | |
287 | // Stops software keyboards from closing as soon as the enter key is pressed. |
288 | outAttrs.imeOptions |= EditorInfo.IME_MASK_ACTION | EditorInfo.IME_FLAG_NO_ENTER_ACTION; |
289 | |
290 | return null; |
291 | } |
292 | |
293 | /** Called when a mouse action is made. */ |
294 | private void handleMouseMovement(float x, float y, int button, boolean pressed) { |
295 | float[] coordinates = {x, y}; |
296 | |
297 | // Coordinates are relative to the canvas, but we need image coordinates. |
298 | Matrix canvasToImage = new Matrix(); |
299 | mTransform.invert(canvasToImage); |
300 | canvasToImage.mapPoints(coordinates); |
301 | |
302 | // Coordinates are now relative to the image, so transmit them to the host. |
303 | JniInterface.mouseAction((int)coordinates[0], (int)coordinates[1], button, pressed); |
304 | } |
305 | |
306 | /** |
307 | * Called whenever the user attempts to touch the canvas. Forwards such |
308 | * events to the appropriate gesture detector until one accepts them. |
309 | */ |
310 | @Override |
311 | public boolean onTouchEvent(MotionEvent event) { |
312 | if (event.getPointerCount() == 3) { |
313 | mActionBar.show(); |
314 | } |
315 | |
316 | boolean handled = mScroller.onTouchEvent(event) || mZoomer.onTouchEvent(event); |
317 | |
318 | if (event.getPointerCount() == 1) { |
319 | float x = event.getRawX(); |
320 | float y = event.getY(); |
321 | |
322 | switch (event.getActionMasked()) { |
323 | case MotionEvent.ACTION_DOWN: |
324 | Log.i("mouse", "Found a finger"); |
325 | mMouseButton = BUTTON_UNDEFINED; |
326 | mMousePressed = false; |
327 | break; |
328 | |
329 | case MotionEvent.ACTION_MOVE: |
330 | Log.i("mouse", "Finger is dragging"); |
331 | if (mMouseButton == BUTTON_UNDEFINED) { |
332 | Log.i("mouse", "\tStarting left click"); |
333 | mMouseButton = BUTTON_LEFT; |
334 | mMousePressed = true; |
335 | } |
336 | break; |
337 | |
338 | case MotionEvent.ACTION_UP: |
339 | Log.i("mouse", "Lost the finger"); |
340 | if (mMouseButton == BUTTON_UNDEFINED) { |
341 | // The user pressed and released without moving: do left click and release. |
342 | Log.i("mouse", "\tStarting and finishing left click"); |
343 | handleMouseMovement(x, y, BUTTON_LEFT, true); |
344 | mMouseButton = BUTTON_LEFT; |
345 | mMousePressed = false; |
346 | } |
347 | else if (mMousePressed) { |
348 | Log.i("mouse", "\tReleasing the currently-pressed button"); |
349 | mMousePressed = false; |
350 | } |
351 | else { |
352 | Log.w("mouse", "Button already in released state before gesture ended"); |
353 | } |
354 | break; |
355 | |
356 | default: |
357 | return handled; |
358 | } |
359 | handleMouseMovement(x, y, mMouseButton, mMousePressed); |
360 | |
361 | return true; |
362 | } |
363 | |
364 | return handled; |
365 | } |
366 | |
367 | /** Responds to touch events filtered by the gesture detectors. */ |
368 | private class DesktopListener extends GestureDetector.SimpleOnGestureListener |
369 | implements ScaleGestureDetector.OnScaleGestureListener { |
370 | /** |
371 | * Called when the user is scrolling. We refuse to accept or process the event unless it |
372 | * is being performed with 2 or more touch points, in order to reserve single-point touch |
373 | * events for emulating mouse input. |
374 | */ |
375 | @Override |
376 | public boolean onScroll( |
377 | MotionEvent e1, MotionEvent e2, float distanceX, float distanceY) { |
378 | if (e2.getPointerCount() < 2 || |
379 | Math.pow(distanceX, 2) + Math.pow(distanceY, 2) < MIN_SCROLL_DISTANCE) { |
380 | return false; |
381 | } |
382 | |
383 | synchronized (mTransform) { |
384 | mTransform.postTranslate(-distanceX, -distanceY); |
385 | } |
386 | JniInterface.redrawGraphics(); |
387 | return true; |
388 | } |
389 | |
390 | /** Called when the user is in the process of pinch-zooming. */ |
391 | @Override |
392 | public boolean onScale(ScaleGestureDetector detector) { |
393 | if (Math.abs(detector.getScaleFactor() - 1) < MIN_ZOOM_FACTOR) { |
394 | return false; |
395 | } |
396 | |
397 | synchronized (mTransform) { |
398 | float scaleFactor = detector.getScaleFactor(); |
399 | mTransform.postScale( |
400 | scaleFactor, scaleFactor, detector.getFocusX(), detector.getFocusY()); |
401 | } |
402 | JniInterface.redrawGraphics(); |
403 | return true; |
404 | } |
405 | |
406 | /** Called whenever a gesture starts. Always accepts the gesture so it isn't ignored. */ |
407 | @Override |
408 | public boolean onDown(MotionEvent e) { |
409 | return true; |
410 | } |
411 | |
412 | /** |
413 | * Called when the user starts to zoom. Always accepts the zoom so that |
414 | * onScale() can decide whether to respond to it. |
415 | */ |
416 | @Override |
417 | public boolean onScaleBegin(ScaleGestureDetector detector) { |
418 | return true; |
419 | } |
420 | |
421 | /** Called when the user is done zooming. Defers to onScale()'s judgement. */ |
422 | @Override |
423 | public void onScaleEnd(ScaleGestureDetector detector) { |
424 | onScale(detector); |
425 | } |
426 | |
427 | /** Called when the user holds down on the screen. Starts a right-click. */ |
428 | @Override |
429 | public void onLongPress(MotionEvent e) { |
430 | if (e.getPointerCount() > 1) { |
431 | return; |
432 | } |
433 | |
434 | float x = e.getRawX(); |
435 | float y = e.getY(); |
436 | |
437 | Log.i("mouse", "Finger held down"); |
438 | if (mMousePressed) { |
439 | Log.i("mouse", "\tReleasing the currently-pressed button"); |
440 | handleMouseMovement(x, y, mMouseButton, false); |
441 | } |
442 | |
443 | Log.i("mouse", "\tStarting right click"); |
444 | mMouseButton = BUTTON_RIGHT; |
445 | mMousePressed = true; |
446 | handleMouseMovement(x, y, mMouseButton, mMousePressed); |
447 | } |
448 | } |
449 | } |