fix(chromium): drag and drop works in chromium (#6207)

Waiting for #6203 to percolate to the cdn. But this all works locally.

Fixes #1094

src/server/chromium/crDragDrop.ts

@@ -0,0 +1,137 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+import { CRPage } from './crPage';
+import * as types from '../types';
+import { assert } from '../../utils/utils';
+import { Protocol } from './protocol';
+import { toModifiersMask } from './crProtocolHelper';
+
+declare global {
+  interface Window {
+    __cleanupDrag?: () => Promise<boolean>;
+  }
+}
+
+export class DragManager {
+  private _crPage: CRPage;
+  private _dragState: Protocol.Input.DragData | null = null;
+  private _lastPosition = {x: 0, y: 0};
+  constructor(page: CRPage) {
+    this._crPage = page;
+  }
+
+  async cancelDrag() {
+    if (!this._dragState)
+      return false;
+    await this._crPage._mainFrameSession._client.send('Input.dispatchDragEvent', {
+      type: 'dragCancel',
+      x: this._lastPosition.x,
+      y: this._lastPosition.y,
+      data: {
+        items: [],
+        dragOperationsMask: 0xFFFF,
+      }
+    });
+    this._dragState = null;
+    return true;
+  }
+
+  async interceptDragCausedByMove(x: number, y: number, button: types.MouseButton | 'none', buttons: Set<types.MouseButton>, modifiers: Set<types.KeyboardModifier>, moveCallback: () => Promise<void>): Promise<void> {
+    this._lastPosition = {x, y};
+    if (this._dragState) {
+      await this._crPage._mainFrameSession._client.send('Input.dispatchDragEvent', {
+        type: 'dragOver',
+        x,
+        y,
+        data: this._dragState,
+        modifiers: toModifiersMask(modifiers),
+      });
+      return;
+    }
+    if (button !== 'left')
+      return moveCallback();
+
+    const client = this._crPage._mainFrameSession._client;
+    let onDragIntercepted: (payload: Protocol.Input.dragInterceptedPayload) => void;
+    const dragInterceptedPromise = new Promise<Protocol.Input.dragInterceptedPayload>(x => onDragIntercepted = x);
+
+    await Promise.all(this._crPage._page.frames().map(async frame => {
+      await frame.nonStallingEvaluateInExistingContext((function() {
+        let didStartDrag = Promise.resolve(false);
+        let dragEvent: Event|null = null;
+        const dragListener = (event: Event) => dragEvent = event;
+        const mouseListener = () => {
+          didStartDrag = new Promise<boolean>(callback => {
+            window.addEventListener('dragstart', dragListener, {once: true, capture: true});
+            setTimeout(() => callback(dragEvent ? !dragEvent.defaultPrevented : false), 0);
+          });
+        };
+        window.addEventListener('mousemove', mouseListener, {once: true, capture: true});
+        window.__cleanupDrag = async () => {
+          const val = await didStartDrag;
+          window.removeEventListener('mousemove', mouseListener, {capture: true});
+          window.removeEventListener('dragstart', dragListener, {capture: true});
+          return val;
+        };
+      }).toString(), true, 'utility').catch(() => {});
+    }));
+
+    client.on('Input.dragIntercepted', onDragIntercepted!);
+    try {
+      await client.send('Input.setInterceptDrags', {enabled: true});
+    } catch {
+      // If Input.setInterceptDrags is not supported, just do a regular move.
+      // This can be removed once we stop supporting old Electron.
+      client.off('Input.dragIntercepted', onDragIntercepted!);
+      return moveCallback();
+    }
+    await moveCallback();
+
+    const expectingDrag = (await Promise.all(this._crPage._page.frames().map(async frame => {
+      return frame.nonStallingEvaluateInExistingContext('window.__cleanupDrag && window.__cleanupDrag()', false, 'utility').catch(() => false);
+    }))).some(x => x);
+    this._dragState = expectingDrag ? (await dragInterceptedPromise).data : null;
+    client.off('Input.dragIntercepted', onDragIntercepted!);
+    await client.send('Input.setInterceptDrags', {enabled: false});
+
+
+    if (this._dragState) {
+      await this._crPage._mainFrameSession._client.send('Input.dispatchDragEvent', {
+        type: 'dragEnter',
+        x,
+        y,
+        data: this._dragState,
+        modifiers: toModifiersMask(modifiers),
+      });
+    }
+  }
+
+  isDragging() {
+    return !!this._dragState;
+  }
+
+  async drop(x: number, y: number, modifiers: Set<types.KeyboardModifier>) {
+    assert(this._dragState, 'missing drag state');
+    await this._crPage._mainFrameSession._client.send('Input.dispatchDragEvent', {
+      type: 'drop',
+      x,
+      y,
+      data: this._dragState,
+      modifiers: toModifiersMask(modifiers),
+    });
+    this._dragState = null;
+  }
+}

src/server/chromium/crInput.ts

@@ -20,24 +20,15 @@ import * as types from '../types';
 import { CRSession } from './crConnection';
 import { macEditingCommands } from '../macEditingCommands';
 import { isString } from '../../utils/utils';
-
-function toModifiersMask(modifiers: Set<types.KeyboardModifier>): number {
-  let mask = 0;
-  if (modifiers.has('Alt'))
-    mask |= 1;
-  if (modifiers.has('Control'))
-    mask |= 2;
-  if (modifiers.has('Meta'))
-    mask |= 4;
-  if (modifiers.has('Shift'))
-    mask |= 8;
-  return mask;
-}
+import { DragManager } from './crDragDrop';
+import { CRPage } from './crPage';
+import { toModifiersMask } from './crProtocolHelper';
 
 export class RawKeyboardImpl implements input.RawKeyboard {
   constructor(
     private _client: CRSession,
     private _isMac: boolean,
+    private _dragManger: DragManager,
   ) { }
 
   _commandsForCode(code: string, modifiers: Set<types.KeyboardModifier>) {
@@ -60,6 +51,8 @@ export class RawKeyboardImpl implements input.RawKeyboard {
   }
 
   async keydown(modifiers: Set<types.KeyboardModifier>, code: string, keyCode: number, keyCodeWithoutLocation: number, key: string, location: number, autoRepeat: boolean, text: string | undefined): Promise<void> {
+    if (code === 'Escape' && await this._dragManger.cancelDrag())
+      return;
     const commands = this._commandsForCode(code, modifiers);
     await this._client.send('Input.dispatchKeyEvent', {
       type: text ? 'keyDown' : 'rawKeyDown',
@@ -94,22 +87,30 @@ export class RawKeyboardImpl implements input.RawKeyboard {
 
 export class RawMouseImpl implements input.RawMouse {
   private _client: CRSession;
+  private _page: CRPage;
+  private _dragManager: DragManager;
 
-  constructor(client: CRSession) {
+  constructor(page: CRPage, client: CRSession, dragManager: DragManager) {
+    this._page = page;
     this._client = client;
+    this._dragManager = dragManager;
   }
 
   async move(x: number, y: number, button: types.MouseButton | 'none', buttons: Set<types.MouseButton>, modifiers: Set<types.KeyboardModifier>): Promise<void> {
-    await this._client.send('Input.dispatchMouseEvent', {
-      type: 'mouseMoved',
-      button,
-      x,
-      y,
-      modifiers: toModifiersMask(modifiers)
+    await this._dragManager.interceptDragCausedByMove(x, y, button, buttons, modifiers, async () => {
+      await this._client.send('Input.dispatchMouseEvent', {
+        type: 'mouseMoved',
+        button,
+        x,
+        y,
+        modifiers: toModifiersMask(modifiers)
+      });
     });
   }
 
   async down(x: number, y: number, button: types.MouseButton, buttons: Set<types.MouseButton>, modifiers: Set<types.KeyboardModifier>, clickCount: number): Promise<void> {
+    if (this._dragManager.isDragging())
+      return;
     await this._client.send('Input.dispatchMouseEvent', {
       type: 'mousePressed',
       button,
@@ -121,6 +122,10 @@ export class RawMouseImpl implements input.RawMouse {
   }
 
   async up(x: number, y: number, button: types.MouseButton, buttons: Set<types.MouseButton>, modifiers: Set<types.KeyboardModifier>, clickCount: number): Promise<void> {
+    if (this._dragManager.isDragging()) {
+      await this._dragManager.drop(x, y, modifiers);
+      return;
+    }
     await this._client.send('Input.dispatchMouseEvent', {
       type: 'mouseReleased',
       button,

src/server/chromium/crPage.ts

@@ -39,6 +39,7 @@ import { rewriteErrorMessage } from '../../utils/stackTrace';
 import { assert, headersArrayToObject, createGuid, canAccessFile } from '../../utils/utils';
 import { VideoRecorder } from './videoRecorder';
 import { Progress } from '../progress';
+import { DragManager } from './crDragDrop';
 
 
 const UTILITY_WORLD_NAME = '__playwright_utility_world__';
@@ -76,8 +77,9 @@ export class CRPage implements PageDelegate {
     this._targetId = targetId;
     this._opener = opener;
     this._isBackgroundPage = isBackgroundPage;
-    this.rawKeyboard = new RawKeyboardImpl(client, browserContext._browser._isMac);
-    this.rawMouse = new RawMouseImpl(client);
+    const dragManager = new DragManager(this);
+    this.rawKeyboard = new RawKeyboardImpl(client, browserContext._browser._isMac, dragManager);
+    this.rawMouse = new RawMouseImpl(this, client, dragManager);
     this.rawTouchscreen = new RawTouchscreenImpl(client);
     this._pdf = new CRPDF(client);
     this._coverage = new CRCoverage(client);

src/server/chromium/crProtocolHelper.ts

@@ -89,3 +89,16 @@ export function exceptionToError(exceptionDetails: Protocol.Runtime.ExceptionDet
   err.name = name;
   return err;
 }
+
+export function toModifiersMask(modifiers: Set<types.KeyboardModifier>): number {
+  let mask = 0;
+  if (modifiers.has('Alt'))
+    mask |= 1;
+  if (modifiers.has('Control'))
+    mask |= 2;
+  if (modifiers.has('Meta'))
+    mask |= 4;
+  if (modifiers.has('Shift'))
+    mask |= 8;
+  return mask;
+}

src/server/frames.ts

@@ -495,6 +495,26 @@ export class Frame extends SdkObject {
     }
   }
 
+  async nonStallingEvaluateInExistingContext(expression: string, isFunction: boolean|undefined, world: types.World): Promise<any> {
+    if (this._pendingDocument)
+      throw new Error('Frame is currently attempting a navigation');
+    const context = this._contextData.get(world)?.context;
+    if (!context)
+      throw new Error('Frame does not yet have the execution context');
+
+    let callback = () => {};
+    const frameInvalidated = new Promise<void>((f, r) => callback = r);
+    this._nonStallingEvaluations.add(callback);
+    try {
+      return await Promise.race([
+        context.evaluateExpression(expression, isFunction),
+        frameInvalidated
+      ]);
+    } finally {
+      this._nonStallingEvaluations.delete(callback);
+    }
+  }
+
   private _recalculateLifecycle() {
     const events = new Set<types.LifecycleEvent>(this._firedLifecycleEvents);
     for (const child of this._childFrames) {