From 8960584b788b0a0741f5211f6a5115f26d0ed797 Mon Sep 17 00:00:00 2001 From: Joel Einbinder Date: Mon, 7 Jun 2021 10:27:34 -0700 Subject: [PATCH] fix(chromium): drag and drop works in chromium (#6207) Waiting for #6203 to percolate to the cdn. But this all works locally. Fixes #1094 --- src/server/chromium/crDragDrop.ts | 137 ++++++++++++++++++++++++ src/server/chromium/crInput.ts | 45 ++++---- src/server/chromium/crPage.ts | 6 +- src/server/chromium/crProtocolHelper.ts | 13 +++ src/server/frames.ts | 20 ++++ tests/page/page-drag.spec.ts | 88 +++++++++++++-- 6 files changed, 281 insertions(+), 28 deletions(-) create mode 100644 src/server/chromium/crDragDrop.ts diff --git a/src/server/chromium/crDragDrop.ts b/src/server/chromium/crDragDrop.ts new file mode 100644 index 0000000000000..7e0e615c159e4 --- /dev/null +++ b/src/server/chromium/crDragDrop.ts @@ -0,0 +1,137 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +import { CRPage } from './crPage'; +import * as types from '../types'; +import { assert } from '../../utils/utils'; +import { Protocol } from './protocol'; +import { toModifiersMask } from './crProtocolHelper'; + +declare global { + interface Window { + __cleanupDrag?: () => Promise; + } +} + +export class DragManager { + private _crPage: CRPage; + private _dragState: Protocol.Input.DragData | null = null; + private _lastPosition = {x: 0, y: 0}; + constructor(page: CRPage) { + this._crPage = page; + } + + async cancelDrag() { + if (!this._dragState) + return false; + await this._crPage._mainFrameSession._client.send('Input.dispatchDragEvent', { + type: 'dragCancel', + x: this._lastPosition.x, + y: this._lastPosition.y, + data: { + items: [], + dragOperationsMask: 0xFFFF, + } + }); + this._dragState = null; + return true; + } + + async interceptDragCausedByMove(x: number, y: number, button: types.MouseButton | 'none', buttons: Set, modifiers: Set, moveCallback: () => Promise): Promise { + this._lastPosition = {x, y}; + if (this._dragState) { + await this._crPage._mainFrameSession._client.send('Input.dispatchDragEvent', { + type: 'dragOver', + x, + y, + data: this._dragState, + modifiers: toModifiersMask(modifiers), + }); + return; + } + if (button !== 'left') + return moveCallback(); + + const client = this._crPage._mainFrameSession._client; + let onDragIntercepted: (payload: Protocol.Input.dragInterceptedPayload) => void; + const dragInterceptedPromise = new Promise(x => onDragIntercepted = x); + + await Promise.all(this._crPage._page.frames().map(async frame => { + await frame.nonStallingEvaluateInExistingContext((function() { + let didStartDrag = Promise.resolve(false); + let dragEvent: Event|null = null; + const dragListener = (event: Event) => dragEvent = event; + const mouseListener = () => { + didStartDrag = new Promise(callback => { + window.addEventListener('dragstart', dragListener, {once: true, capture: true}); + setTimeout(() => callback(dragEvent ? !dragEvent.defaultPrevented : false), 0); + }); + }; + window.addEventListener('mousemove', mouseListener, {once: true, capture: true}); + window.__cleanupDrag = async () => { + const val = await didStartDrag; + window.removeEventListener('mousemove', mouseListener, {capture: true}); + window.removeEventListener('dragstart', dragListener, {capture: true}); + return val; + }; + }).toString(), true, 'utility').catch(() => {}); + })); + + client.on('Input.dragIntercepted', onDragIntercepted!); + try { + await client.send('Input.setInterceptDrags', {enabled: true}); + } catch { + // If Input.setInterceptDrags is not supported, just do a regular move. + // This can be removed once we stop supporting old Electron. + client.off('Input.dragIntercepted', onDragIntercepted!); + return moveCallback(); + } + await moveCallback(); + + const expectingDrag = (await Promise.all(this._crPage._page.frames().map(async frame => { + return frame.nonStallingEvaluateInExistingContext('window.__cleanupDrag && window.__cleanupDrag()', false, 'utility').catch(() => false); + }))).some(x => x); + this._dragState = expectingDrag ? (await dragInterceptedPromise).data : null; + client.off('Input.dragIntercepted', onDragIntercepted!); + await client.send('Input.setInterceptDrags', {enabled: false}); + + + if (this._dragState) { + await this._crPage._mainFrameSession._client.send('Input.dispatchDragEvent', { + type: 'dragEnter', + x, + y, + data: this._dragState, + modifiers: toModifiersMask(modifiers), + }); + } + } + + isDragging() { + return !!this._dragState; + } + + async drop(x: number, y: number, modifiers: Set) { + assert(this._dragState, 'missing drag state'); + await this._crPage._mainFrameSession._client.send('Input.dispatchDragEvent', { + type: 'drop', + x, + y, + data: this._dragState, + modifiers: toModifiersMask(modifiers), + }); + this._dragState = null; + } +} diff --git a/src/server/chromium/crInput.ts b/src/server/chromium/crInput.ts index 44d39bb8e6523..48b6dd37f6cda 100644 --- a/src/server/chromium/crInput.ts +++ b/src/server/chromium/crInput.ts @@ -20,24 +20,15 @@ import * as types from '../types'; import { CRSession } from './crConnection'; import { macEditingCommands } from '../macEditingCommands'; import { isString } from '../../utils/utils'; - -function toModifiersMask(modifiers: Set): number { - let mask = 0; - if (modifiers.has('Alt')) - mask |= 1; - if (modifiers.has('Control')) - mask |= 2; - if (modifiers.has('Meta')) - mask |= 4; - if (modifiers.has('Shift')) - mask |= 8; - return mask; -} +import { DragManager } from './crDragDrop'; +import { CRPage } from './crPage'; +import { toModifiersMask } from './crProtocolHelper'; export class RawKeyboardImpl implements input.RawKeyboard { constructor( private _client: CRSession, private _isMac: boolean, + private _dragManger: DragManager, ) { } _commandsForCode(code: string, modifiers: Set) { @@ -60,6 +51,8 @@ export class RawKeyboardImpl implements input.RawKeyboard { } async keydown(modifiers: Set, code: string, keyCode: number, keyCodeWithoutLocation: number, key: string, location: number, autoRepeat: boolean, text: string | undefined): Promise { + if (code === 'Escape' && await this._dragManger.cancelDrag()) + return; const commands = this._commandsForCode(code, modifiers); await this._client.send('Input.dispatchKeyEvent', { type: text ? 'keyDown' : 'rawKeyDown', @@ -94,22 +87,30 @@ export class RawKeyboardImpl implements input.RawKeyboard { export class RawMouseImpl implements input.RawMouse { private _client: CRSession; + private _page: CRPage; + private _dragManager: DragManager; - constructor(client: CRSession) { + constructor(page: CRPage, client: CRSession, dragManager: DragManager) { + this._page = page; this._client = client; + this._dragManager = dragManager; } async move(x: number, y: number, button: types.MouseButton | 'none', buttons: Set, modifiers: Set): Promise { - await this._client.send('Input.dispatchMouseEvent', { - type: 'mouseMoved', - button, - x, - y, - modifiers: toModifiersMask(modifiers) + await this._dragManager.interceptDragCausedByMove(x, y, button, buttons, modifiers, async () => { + await this._client.send('Input.dispatchMouseEvent', { + type: 'mouseMoved', + button, + x, + y, + modifiers: toModifiersMask(modifiers) + }); }); } async down(x: number, y: number, button: types.MouseButton, buttons: Set, modifiers: Set, clickCount: number): Promise { + if (this._dragManager.isDragging()) + return; await this._client.send('Input.dispatchMouseEvent', { type: 'mousePressed', button, @@ -121,6 +122,10 @@ export class RawMouseImpl implements input.RawMouse { } async up(x: number, y: number, button: types.MouseButton, buttons: Set, modifiers: Set, clickCount: number): Promise { + if (this._dragManager.isDragging()) { + await this._dragManager.drop(x, y, modifiers); + return; + } await this._client.send('Input.dispatchMouseEvent', { type: 'mouseReleased', button, diff --git a/src/server/chromium/crPage.ts b/src/server/chromium/crPage.ts index 652c77c0c0f94..f54167df30895 100644 --- a/src/server/chromium/crPage.ts +++ b/src/server/chromium/crPage.ts @@ -39,6 +39,7 @@ import { rewriteErrorMessage } from '../../utils/stackTrace'; import { assert, headersArrayToObject, createGuid, canAccessFile } from '../../utils/utils'; import { VideoRecorder } from './videoRecorder'; import { Progress } from '../progress'; +import { DragManager } from './crDragDrop'; const UTILITY_WORLD_NAME = '__playwright_utility_world__'; @@ -76,8 +77,9 @@ export class CRPage implements PageDelegate { this._targetId = targetId; this._opener = opener; this._isBackgroundPage = isBackgroundPage; - this.rawKeyboard = new RawKeyboardImpl(client, browserContext._browser._isMac); - this.rawMouse = new RawMouseImpl(client); + const dragManager = new DragManager(this); + this.rawKeyboard = new RawKeyboardImpl(client, browserContext._browser._isMac, dragManager); + this.rawMouse = new RawMouseImpl(this, client, dragManager); this.rawTouchscreen = new RawTouchscreenImpl(client); this._pdf = new CRPDF(client); this._coverage = new CRCoverage(client); diff --git a/src/server/chromium/crProtocolHelper.ts b/src/server/chromium/crProtocolHelper.ts index 4c76569a855b6..61b97d0be228c 100644 --- a/src/server/chromium/crProtocolHelper.ts +++ b/src/server/chromium/crProtocolHelper.ts @@ -89,3 +89,16 @@ export function exceptionToError(exceptionDetails: Protocol.Runtime.ExceptionDet err.name = name; return err; } + +export function toModifiersMask(modifiers: Set): number { + let mask = 0; + if (modifiers.has('Alt')) + mask |= 1; + if (modifiers.has('Control')) + mask |= 2; + if (modifiers.has('Meta')) + mask |= 4; + if (modifiers.has('Shift')) + mask |= 8; + return mask; +} diff --git a/src/server/frames.ts b/src/server/frames.ts index cdf60ec890832..2c9240063e2dc 100644 --- a/src/server/frames.ts +++ b/src/server/frames.ts @@ -495,6 +495,26 @@ export class Frame extends SdkObject { } } + async nonStallingEvaluateInExistingContext(expression: string, isFunction: boolean|undefined, world: types.World): Promise { + if (this._pendingDocument) + throw new Error('Frame is currently attempting a navigation'); + const context = this._contextData.get(world)?.context; + if (!context) + throw new Error('Frame does not yet have the execution context'); + + let callback = () => {}; + const frameInvalidated = new Promise((f, r) => callback = r); + this._nonStallingEvaluations.add(callback); + try { + return await Promise.race([ + context.evaluateExpression(expression, isFunction), + frameInvalidated + ]); + } finally { + this._nonStallingEvaluations.delete(callback); + } + } + private _recalculateLifecycle() { const events = new Set(this._firedLifecycleEvents); for (const child of this._childFrames) { diff --git a/tests/page/page-drag.spec.ts b/tests/page/page-drag.spec.ts index 45a89ffe6a24a..430af6a7edfe8 100644 --- a/tests/page/page-drag.spec.ts +++ b/tests/page/page-drag.spec.ts @@ -14,13 +14,13 @@ * limitations under the License. */ -import type { ElementHandle } from '../../index'; +import type { ElementHandle, Route } from '../../index'; import { test as it, expect } from './pageTest'; import { attachFrame } from '../config/utils'; it.describe('Drag and drop', () => { - it.skip(({ isAndroid }) => isAndroid); - it.fixme(({ browserName }) => browserName === 'chromium'); + it.skip(({isAndroid}) => isAndroid); + it.skip(({browserName, browserMajorVersion}) => browserName === 'chromium' && browserMajorVersion < 91); it('should work', async ({page, server}) => { await page.goto(server.PREFIX + '/drag-n-drop.html'); @@ -65,10 +65,10 @@ it.describe('Drag and drop', () => { browserName === 'firefox' ? 'dragstart' : 'mousemove', browserName === 'firefox' ? 'mousemove' : 'dragstart', 'dragenter', - 'dragover', + browserName !== 'chromium' ? 'dragover' : null, 'dragend', 'mouseup', - ]); + ].filter(Boolean)); }); it.describe('iframe', () => { @@ -122,7 +122,6 @@ it.describe('Drag and drop', () => { }); it('should respect the drop effect', async ({page, browserName, platform}) => { - it.fixme(browserName === 'chromium', 'Chromium doesn\'t let users set dropEffect on our fake data transfer'); it.fixme(browserName === 'webkit' && platform !== 'linux', 'WebKit doesn\'t handle the drop effect correctly outside of linux.'); it.fixme(browserName === 'firefox'); @@ -174,6 +173,60 @@ it.describe('Drag and drop', () => { return await page.evaluate('dropped'); } }); + it('should work if the drag is canceled', async ({page, server}) => { + await page.goto(server.PREFIX + '/drag-n-drop.html'); + await page.evaluate(() => { + document.body.addEventListener('dragstart', event => { + event.preventDefault(); + }, false); + }); + await page.hover('#source'); + await page.mouse.down(); + await page.hover('#target'); + await page.mouse.up(); + expect(await page.$eval('#target', target => target.contains(document.querySelector('#source')))).toBe(false); + }); + + it('should work if the drag event is captured but not canceled', async ({page, server}) => { + await page.goto(server.PREFIX + '/drag-n-drop.html'); + await page.evaluate(() => { + document.body.addEventListener('dragstart', event => { + event.stopImmediatePropagation(); + }, false); + }); + await page.hover('#source'); + await page.mouse.down(); + await page.hover('#target'); + await page.mouse.up(); + expect(await page.$eval('#target', target => target.contains(document.querySelector('#source')))).toBe(true); + }); + + it('should be able to drag the mouse in a frame', async ({page, server}) => { + await page.goto(server.PREFIX + '/frames/one-frame.html'); + const eventsHandle = await trackEvents(await page.frames()[1].$('html')); + await page.mouse.move(30, 30); + await page.mouse.down(); + await page.mouse.move(60, 60); + await page.mouse.up(); + expect(await eventsHandle.jsonValue()).toEqual(['mousemove', 'mousedown', 'mousemove', 'mouseup']); + }); + + it('should work if a frame is stalled', async ({page, server, toImpl}) => { + await page.goto(server.PREFIX + '/drag-n-drop.html'); + let madeRequest; + const routePromise = new Promise(x => madeRequest = x); + await page.route('**/empty.html', async (route, request) => { + madeRequest(route); + }); + attachFrame(page, 'frame', server.EMPTY_PAGE).catch(() => {}); + const route = await routePromise; + await page.hover('#source'); + await page.mouse.down(); + await page.hover('#target'); + await page.mouse.up(); + route.abort(); + expect(await page.$eval('#target', target => target.contains(document.querySelector('#source')))).toBe(true); // could not find source in target + }) async function trackEvents(target: ElementHandle) { const eventsHandle = await target.evaluateHandle(target => { @@ -189,3 +242,26 @@ it.describe('Drag and drop', () => { return eventsHandle; } }); + +it('should work if not doing a drag', async ({page}) => { + const eventsHandle = await trackEvents(await page.$('html')); + await page.mouse.move(50, 50); + await page.mouse.down(); + await page.mouse.move(100, 100); + await page.mouse.up(); + expect(await eventsHandle.jsonValue()).toEqual(['mousemove', 'mousedown', 'mousemove', 'mouseup']); +}); + +async function trackEvents(target: ElementHandle) { + const eventsHandle = await target.evaluateHandle(target => { + const events: string[] = []; + for (const event of [ + 'mousedown', 'mousemove', 'mouseup', + 'dragstart', 'dragend', 'dragover', 'dragenter', 'dragleave', 'dragexit', + 'drop' + ]) + target.addEventListener(event, () => events.push(event), false); + return events; + }); + return eventsHandle; +}