From d88b0348e7e9c0ff32bfa7a3104018a48e965b07 Mon Sep 17 00:00:00 2001 From: Makoto Kato Date: Sun, 28 Jan 2024 07:49:53 +0000 Subject: [PATCH] Bug 1586471 - Part 2. Implement nsDragService on GeckoView. r=geckoview-reviewers,amejiamarmol This implementation supports - HTML drag & drop API. - Drop and drop for text/plain or text/html from/to external application. Differential Revision: https://phabricator.services.mozilla.com/D197330 --- mobile/android/geckoview/api.txt | 2 + .../src/androidTest/assets/www/dnd.html | 26 ++ .../androidTest/assets/www/mouseToReload.html | 2 +- .../mozilla/geckoview/test/BaseSessionTest.kt | 4 + .../mozilla/geckoview/test/DragAndDropTest.kt | 113 ++++++++ .../test/GeckoSessionTestRuleTest.kt | 19 ++ .../test/rule/GeckoSessionTestRule.java | 35 ++- .../org/mozilla/gecko/GeckoDragAndDrop.java | 231 +++++++++++++++ .../org/mozilla/geckoview/GeckoSession.java | 39 +++ .../java/org/mozilla/geckoview/GeckoView.java | 10 + .../mozilla/geckoview/PanZoomController.java | 33 +++ .../mozilla/geckoview/doc-files/CHANGELOG.md | 5 +- widget/android/AndroidWidgetUtils.cpp | 48 ++++ widget/android/AndroidWidgetUtils.h | 37 +++ widget/android/WebExecutorSupport.cpp | 5 +- .../bindings/AndroidDragEvent-classes.txt | 3 + widget/android/bindings/moz.build | 1 + widget/android/components.conf | 9 + widget/android/moz.build | 4 + widget/android/nsAppShell.cpp | 2 + widget/android/nsClipboard.cpp | 47 +-- widget/android/nsClipboard.h | 3 + widget/android/nsDragService.cpp | 268 ++++++++++++++++++ widget/android/nsDragService.h | 56 ++++ widget/android/nsWindow.cpp | 173 +++++++++-- widget/android/nsWindow.h | 7 + 26 files changed, 1131 insertions(+), 51 deletions(-) create mode 100644 mobile/android/geckoview/src/androidTest/assets/www/dnd.html create mode 100644 mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/DragAndDropTest.kt create mode 100644 mobile/android/geckoview/src/main/java/org/mozilla/gecko/GeckoDragAndDrop.java create mode 100644 widget/android/AndroidWidgetUtils.cpp create mode 100644 widget/android/AndroidWidgetUtils.h create mode 100644 widget/android/bindings/AndroidDragEvent-classes.txt create mode 100644 widget/android/nsDragService.cpp create mode 100644 widget/android/nsDragService.h diff --git a/mobile/android/geckoview/api.txt b/mobile/android/geckoview/api.txt index d12fa6dd8c22..1eefb57bdc1f 100644 --- a/mobile/android/geckoview/api.txt +++ b/mobile/android/geckoview/api.txt @@ -24,6 +24,7 @@ import android.print.PrintDocumentAdapter; import android.util.AttributeSet; import android.util.SparseArray; import android.view.ActionMode; +import android.view.DragEvent; import android.view.KeyEvent; import android.view.Menu; import android.view.MenuItem; @@ -2089,6 +2090,7 @@ package org.mozilla.geckoview { @UiThread public class PanZoomController { ctor protected PanZoomController(GeckoSession); method public float getScrollFactor(); + method public boolean onDragEvent(@NonNull DragEvent); method public void onMotionEvent(@NonNull MotionEvent); method public void onMouseEvent(@NonNull MotionEvent); method public void onTouchEvent(@NonNull MotionEvent); diff --git a/mobile/android/geckoview/src/androidTest/assets/www/dnd.html b/mobile/android/geckoview/src/androidTest/assets/www/dnd.html new file mode 100644 index 000000000000..1a1c9b3d43c4 --- /dev/null +++ b/mobile/android/geckoview/src/androidTest/assets/www/dnd.html @@ -0,0 +1,26 @@ + + + + + + + + +
+
+ drop +
+ + diff --git a/mobile/android/geckoview/src/androidTest/assets/www/mouseToReload.html b/mobile/android/geckoview/src/androidTest/assets/www/mouseToReload.html index fef911a92642..4ef36261198a 100644 --- a/mobile/android/geckoview/src/androidTest/assets/www/mouseToReload.html +++ b/mobile/android/geckoview/src/androidTest/assets/www/mouseToReload.html @@ -4,7 +4,7 @@ Hello, world! - +

Hello, world!

diff --git a/mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/BaseSessionTest.kt b/mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/BaseSessionTest.kt index b1f22645ff8c..655db7248f5a 100644 --- a/mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/BaseSessionTest.kt +++ b/mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/BaseSessionTest.kt @@ -38,6 +38,7 @@ open class BaseSessionTest( const val CLICK_TO_RELOAD_HTML_PATH = "/assets/www/clickToReload.html" const val CLIPBOARD_READ_HTML_PATH = "/assets/www/clipboard_read.html" const val CONTENT_CRASH_URL = "about:crashcontent" + const val DND_HTML_PATH = "/assets/www/dnd.html" const val DOWNLOAD_HTML_PATH = "/assets/www/download.html" const val FORM_BLANK_HTML_PATH = "/assets/www/form_blank.html" const val FORMS_HTML_PATH = "/assets/www/forms.html" @@ -233,6 +234,9 @@ open class BaseSessionTest( fun GeckoSession.synthesizeTap(x: Int, y: Int) = sessionRule.synthesizeTap(this, x, y) + fun GeckoSession.synthesizeMouse(downTime: Long, action: Int, x: Int, y: Int, buttonState: Int) = + sessionRule.synthesizeMouse(this, downTime, action, x, y, buttonState) + fun GeckoSession.synthesizeMouseMove(x: Int, y: Int) = sessionRule.synthesizeMouseMove(this, x, y) diff --git a/mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/DragAndDropTest.kt b/mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/DragAndDropTest.kt new file mode 100644 index 000000000000..27dd8bf67608 --- /dev/null +++ b/mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/DragAndDropTest.kt @@ -0,0 +1,113 @@ +/* -*- Mode: Java; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil; -*- + * Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +package org.mozilla.geckoview.test + +import android.content.ClipData +import android.os.Build +import android.os.SystemClock +import android.view.DragEvent +import android.view.MotionEvent +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.filters.MediumTest +import androidx.test.filters.SdkSuppress +import org.hamcrest.Matchers.equalTo +import org.junit.Test +import org.junit.runner.RunWith +import org.mozilla.geckoview.test.rule.GeckoSessionTestRule.WithDisplay + +@RunWith(AndroidJUnit4::class) +@SdkSuppress(minSdkVersion = Build.VERSION_CODES.N) +@MediumTest +class DragAndDropTest : BaseSessionTest() { + // DragEvent has no constructor, so we create it via Java reflection. + fun createDragEvent(action: Int, x: Float = 0.0F, y: Float = 0.0F): DragEvent { + val method = DragEvent::class.java.getDeclaredMethod("obtain") + method.setAccessible(true) + val dragEvent = method.invoke(null) as DragEvent + + val fieldAction = DragEvent::class.java.getDeclaredField("mAction") + fieldAction.setAccessible(true) + fieldAction.set(dragEvent, action) + + if (listOf(DragEvent.ACTION_DRAG_STARTED, DragEvent.ACTION_DRAG_LOCATION, DragEvent.ACTION_DROP).contains(action)) { + val fieldX = DragEvent::class.java.getDeclaredField("mX") + fieldX.setAccessible(true) + fieldX.set(dragEvent, x) + + val fieldY = DragEvent::class.java.getDeclaredField("mY") + fieldY.setAccessible(true) + fieldY.set(dragEvent, y) + } + + if (action == DragEvent.ACTION_DROP) { + val clipData = ClipData.newPlainText("label", "foo") + val fieldClipData = DragEvent::class.java.getDeclaredField("mClipData") + fieldClipData.setAccessible(true) + fieldClipData.set(dragEvent, clipData) + + var clipDescription = clipData.getDescription() + val fieldClipDescription = DragEvent::class.java.getDeclaredField("mClipDescription") + fieldClipDescription.setAccessible(true) + fieldClipDescription.set(dragEvent, clipDescription) + } + + return dragEvent + } + + @WithDisplay(width = 300, height = 300) + @Test + fun dragStartTest() { + mainSession.loadTestPath(DND_HTML_PATH) + sessionRule.waitForPageStop() + + val promise = mainSession.evaluatePromiseJS( + """ + new Promise(r => document.querySelector('#drag').addEventListener('dragstart', r, { once: true })) + """.trimIndent(), + ) + val downTime = SystemClock.uptimeMillis() + mainSession.synthesizeMouse(downTime, MotionEvent.ACTION_DOWN, 50, 20, MotionEvent.BUTTON_PRIMARY) + for (y in 30..50) { + mainSession.synthesizeMouse(downTime, MotionEvent.ACTION_MOVE, 50, y, MotionEvent.BUTTON_PRIMARY) + } + mainSession.synthesizeMouse(downTime, MotionEvent.ACTION_UP, 50, 50, 0) + promise.value + + assertThat("drag event is started correctly", true, equalTo(true)) + } + + @WithDisplay(width = 300, height = 300) + @Test + fun dropFromExternalTest() { + mainSession.loadTestPath(DND_HTML_PATH) + sessionRule.waitForPageStop() + + val promise = mainSession.evaluatePromiseJS( + """ + new Promise( + r => document.querySelector('#drop').addEventListener( + 'drop', + e => r(e.dataTransfer.getData('text/plain')), + { once: true })) + """.trimIndent(), + ) + + // Android doesn't fire MotionEvent during drag and drop. + val dragStartEvent = createDragEvent(DragEvent.ACTION_DRAG_STARTED) + mainSession.panZoomController.onDragEvent(dragStartEvent) + val dragEnteredEvent = createDragEvent(DragEvent.ACTION_DRAG_ENTERED) + mainSession.panZoomController.onDragEvent(dragEnteredEvent) + listOf(150.0F, 250.0F).forEach { + val dragLocationEvent = createDragEvent(DragEvent.ACTION_DRAG_LOCATION, 100.0F, it) + mainSession.panZoomController.onDragEvent(dragLocationEvent) + } + val dropEvent = createDragEvent(DragEvent.ACTION_DROP, 100.0F, 250.0F) + mainSession.panZoomController.onDragEvent(dropEvent) + val dragEndedEvent = createDragEvent(DragEvent.ACTION_DRAG_ENDED) + mainSession.panZoomController.onDragEvent(dragEndedEvent) + + assertThat("drop event is fired correctly", promise.value as String, equalTo("foo")) + } +} diff --git a/mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/GeckoSessionTestRuleTest.kt b/mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/GeckoSessionTestRuleTest.kt index d7169c0266b6..d6380bf5bfcb 100644 --- a/mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/GeckoSessionTestRuleTest.kt +++ b/mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/GeckoSessionTestRuleTest.kt @@ -6,6 +6,8 @@ package org.mozilla.geckoview.test import android.os.Handler import android.os.Looper +import android.os.SystemClock +import android.view.MotionEvent import androidx.test.ext.junit.runners.AndroidJUnit4 import androidx.test.filters.MediumTest import org.hamcrest.Matchers.* // ktlint-disable no-wildcard-imports @@ -1462,12 +1464,29 @@ class GeckoSessionTestRuleTest : BaseSessionTest(noErrorCollector = true) { mainSession.waitForPageStop() } + @WithDisplay(width = 100, height = 100) + @Test + fun synthesizeMouse() { + mainSession.loadTestPath(MOUSE_TO_RELOAD_HTML_PATH) + mainSession.waitForPageStop() + + val time = SystemClock.uptimeMillis() + mainSession.evaluateJS("document.body.addEventListener('mousedown', () => { window.location.reload() })") + mainSession.synthesizeMouse(time, MotionEvent.ACTION_DOWN, 50, 50, MotionEvent.BUTTON_PRIMARY) + mainSession.waitForPageStop() + + mainSession.evaluateJS("document.body.addEventListener('mouseup', () => { window.location.reload() })") + mainSession.synthesizeMouse(time, MotionEvent.ACTION_UP, 50, 50, 0) + mainSession.waitForPageStop() + } + @WithDisplay(width = 100, height = 100) @Test fun synthesizeMouseMove() { mainSession.loadTestPath(MOUSE_TO_RELOAD_HTML_PATH) mainSession.waitForPageStop() + mainSession.evaluateJS("document.body.addEventListener('mousemove', () => { window.location.reload() })") mainSession.synthesizeMouseMove(50, 50) mainSession.waitForPageStop() } diff --git a/mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/rule/GeckoSessionTestRule.java b/mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/rule/GeckoSessionTestRule.java index 8b3435a67bdf..b443fee5c981 100644 --- a/mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/rule/GeckoSessionTestRule.java +++ b/mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/rule/GeckoSessionTestRule.java @@ -2075,14 +2075,23 @@ public class GeckoSessionTestRule implements TestRule { } /** - * Synthesize a mouse move event at the specified location using the main session. The session - * must have been created with a display. + * Synthesize a mouse event at the specified location using the main session. The session must + * have been created with a display. * * @param session Target session + * @param downTime A time when any buttons are down + * @param action An action such as MotionEvent.ACTION_DOWN * @param x X coordinate * @param y Y coordinate + * @param buttonState A button stats such as MotionEvent.BUTTON_PRIMARY */ - public void synthesizeMouseMove(final @NonNull GeckoSession session, final int x, final int y) { + public void synthesizeMouse( + final @NonNull GeckoSession session, + final long downTime, + final int action, + final int x, + final int y, + final int buttonState) { final MotionEvent.PointerProperties pointerProperty = new MotionEvent.PointerProperties(); pointerProperty.id = 0; pointerProperty.toolType = MotionEvent.TOOL_TYPE_MOUSE; @@ -2096,17 +2105,16 @@ public class GeckoSessionTestRule implements TestRule { final MotionEvent.PointerCoords[] pointerCoords = new MotionEvent.PointerCoords[] {pointerCoord}; - final long moveTime = SystemClock.uptimeMillis(); final MotionEvent moveEvent = MotionEvent.obtain( - moveTime, + downTime, SystemClock.uptimeMillis(), - MotionEvent.ACTION_HOVER_MOVE, + action, 1, pointerProperties, pointerCoords, 0, - 0, + buttonState, 1.0f, 1.0f, 0, @@ -2116,6 +2124,19 @@ public class GeckoSessionTestRule implements TestRule { session.getPanZoomController().onTouchEvent(moveEvent); } + /** + * Synthesize a mouse move event at the specified location using the main session. The session + * must have been created with a display. + * + * @param session Target session + * @param x X coordinate + * @param y Y coordinate + */ + public void synthesizeMouseMove(final @NonNull GeckoSession session, final int x, final int y) { + final long moveTime = SystemClock.uptimeMillis(); + synthesizeMouse(session, moveTime, MotionEvent.ACTION_HOVER_MOVE, x, y, 0); + } + /** * Simulates a press to the Home button, causing the application to go to onPause. NB: Some time * must elapse for the event to fully occur. diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/gecko/GeckoDragAndDrop.java b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/GeckoDragAndDrop.java new file mode 100644 index 000000000000..e48fb0961b8f --- /dev/null +++ b/mobile/android/geckoview/src/main/java/org/mozilla/gecko/GeckoDragAndDrop.java @@ -0,0 +1,231 @@ +/* -*- Mode: Java; c-basic-offset: 2; tab-width: 20; indent-tabs-mode: nil; -*- */ +/* vim: set ts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko; + +import android.annotation.TargetApi; +import android.content.ClipData; +import android.content.ClipDescription; +import android.graphics.Bitmap; +import android.graphics.Canvas; +import android.graphics.Point; +import android.os.Build; +import android.text.TextUtils; +import android.util.Log; +import android.view.DragEvent; +import android.view.View; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import org.mozilla.gecko.annotation.WrapForJNI; + +@TargetApi(Build.VERSION_CODES.N) +public class GeckoDragAndDrop { + private static final String LOGTAG = "GeckoDragAndDrop"; + private static final boolean DEBUG = false; + + /** The drag/drop data is nsITransferable and stored into nsDragService. */ + private static final String MIMETYPE_NATIVE = "application/x-moz-draganddrop"; + + private static ClipData sDragClipData; + private static float sX; + private static float sY; + private static boolean mEndingSession; + + private static class DrawDragImage extends View.DragShadowBuilder { + private final Bitmap mBitmap; + + public DrawDragImage(final Bitmap bitmap) { + mBitmap = bitmap; + } + + @Override + public void onProvideShadowMetrics(final Point outShadowSize, final Point outShadowTouchPoint) { + if (mBitmap == null) { + super.onProvideShadowMetrics(outShadowSize, outShadowTouchPoint); + return; + } + outShadowSize.set(mBitmap.getWidth(), mBitmap.getHeight()); + } + + @Override + public void onDrawShadow(final Canvas canvas) { + if (mBitmap == null) { + super.onDrawShadow(canvas); + return; + } + canvas.drawBitmap(mBitmap, 0.0f, 0.0f, null); + } + } + + @WrapForJNI + public static class DropData { + public final String mimeType; + public final String text; + + @WrapForJNI(skip = true) + public DropData() { + this.mimeType = MIMETYPE_NATIVE; + this.text = null; + } + + @WrapForJNI(skip = true) + public DropData(final String mimeType, final String text) { + this.mimeType = mimeType; + this.text = text; + } + } + + public static void startDragAndDrop(final View view, final Bitmap bitmap) { + view.startDragAndDrop(sDragClipData, new DrawDragImage(bitmap), null, View.DRAG_FLAG_GLOBAL); + sDragClipData = null; + } + + public static void updateDragImage(final View view, final Bitmap bitmap) { + view.updateDragShadow(new DrawDragImage(bitmap)); + } + + public static boolean onDragEvent(@NonNull final DragEvent event) { + if (DEBUG) { + final StringBuilder sb = new StringBuilder("onDragEvent: action="); + sb.append(event.getAction()) + .append(", x=") + .append(event.getX()) + .append(", y=") + .append(event.getY()); + Log.d(LOGTAG, sb.toString()); + } + + switch (event.getAction()) { + case DragEvent.ACTION_DRAG_STARTED: + mEndingSession = false; + sX = event.getX(); + sY = event.getY(); + break; + case DragEvent.ACTION_DRAG_LOCATION: + sX = event.getX(); + sY = event.getY(); + break; + case DragEvent.ACTION_DROP: + sX = event.getX(); + sY = event.getY(); + break; + case DragEvent.ACTION_DRAG_ENDED: + mEndingSession = true; + return true; + default: + break; + } + if (mEndingSession) { + return false; + } + return true; + } + + public static float getLocationX() { + return sX; + } + + public static float getLocationY() { + return sY; + } + + /** + * Create drop data by DragEvent.ACTION_DROP. This ClipData will be stored into nsDragService as + * nsITransferable. If this type has MIMETYPE_NATIVE, this is already stored into nsDragService. + * So do nothing. + * + * @param event A DragEvent + * @return DropData that is from ClipData. If null, no data that we can convert to Gecko's type. + */ + public static DropData createDropData(final DragEvent event) { + if (event.getAction() != DragEvent.ACTION_DROP) { + return null; + } + final ClipData clip = event.getClipData(); + if (clip == null || clip.getItemCount() == 0) { + return null; + } + + final ClipDescription description = event.getClipDescription(); + if (description.hasMimeType(MIMETYPE_NATIVE)) { + if (DEBUG) { + Log.d(LOGTAG, "Drop data is native nsITransferable. Do nothing"); + } + return new DropData(); + } + if (description.hasMimeType(ClipDescription.MIMETYPE_TEXT_HTML)) { + final CharSequence data = clip.getItemAt(0).getHtmlText(); + if (data == null) { + return null; + } + if (DEBUG) { + Log.d(LOGTAG, "Drop data is text/html"); + } + return new DropData(ClipDescription.MIMETYPE_TEXT_HTML, data.toString()); + } + + final CharSequence text = clip.getItemAt(0).coerceToText(GeckoAppShell.getApplicationContext()); + if (!TextUtils.isEmpty(text)) { + if (DEBUG) { + Log.d(LOGTAG, "Drop data is text/plain"); + } + return new DropData(ClipDescription.MIMETYPE_TEXT_PLAIN, text.toString()); + } + return null; + } + + private static void setDragClipData(final ClipData clipData) { + sDragClipData = clipData; + } + + private static @Nullable ClipData getDragClipData() { + return sDragClipData; + } + + /** + * Set drag item before calling View.startDragAndDrop. This is set from nsITransferable, so it + * marks as native data. + */ + @WrapForJNI + private static void setDragData(final CharSequence text, final String htmlText) { + if (TextUtils.isEmpty(text)) { + final ClipDescription description = + new ClipDescription("drag item", new String[] {MIMETYPE_NATIVE}); + final ClipData.Item item = new ClipData.Item(""); + final ClipData clipData = new ClipData(description, item); + setDragClipData(clipData); + return; + } + + if (TextUtils.isEmpty(htmlText)) { + final ClipDescription description = + new ClipDescription( + "drag item", new String[] {MIMETYPE_NATIVE, ClipDescription.MIMETYPE_TEXT_PLAIN}); + final ClipData.Item item = new ClipData.Item(text); + final ClipData clipData = new ClipData(description, item); + setDragClipData(clipData); + return; + } + + final ClipDescription description = + new ClipDescription( + "drag item", + new String[] { + MIMETYPE_NATIVE, + ClipDescription.MIMETYPE_TEXT_HTML, + ClipDescription.MIMETYPE_TEXT_PLAIN + }); + final ClipData.Item item = new ClipData.Item(text, htmlText); + final ClipData clipData = new ClipData(description, item); + setDragClipData(clipData); + return; + } + + @WrapForJNI + private static void endDragSession() { + mEndingSession = true; + } +} diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/GeckoSession.java b/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/GeckoSession.java index 5d62e1806598..f8f7f858e334 100644 --- a/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/GeckoSession.java +++ b/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/GeckoSession.java @@ -71,6 +71,7 @@ import org.json.JSONException; import org.json.JSONObject; import org.mozilla.gecko.EventDispatcher; import org.mozilla.gecko.GeckoAppShell; +import org.mozilla.gecko.GeckoDragAndDrop; import org.mozilla.gecko.GeckoThread; import org.mozilla.gecko.IGeckoEditableParent; import org.mozilla.gecko.MagnifiableSurfaceView; @@ -414,6 +415,16 @@ public class GeckoSession { GeckoSession.this.setPointerIcon(defaultCursor, customCursor, x, y); } + @WrapForJNI(calledFrom = "ui") + private void startDragAndDrop(final Bitmap bitmap) { + GeckoSession.this.startDragAndDrop(bitmap); + } + + @WrapForJNI(calledFrom = "ui") + private void updateDragImage(final Bitmap bitmap) { + GeckoSession.this.updateDragImage(bitmap); + } + @Override protected void finalize() throws Throwable { disposeNative(); @@ -7889,6 +7900,34 @@ public class GeckoSession { } } + /* package */ void startDragAndDrop(final Bitmap bitmap) { + ThreadUtils.assertOnUiThread(); + + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) { + return; + } + final View view = getTextInput().getView(); + if (view == null) { + return; + } + + GeckoDragAndDrop.startDragAndDrop(view, bitmap); + } + + /* package */ void updateDragImage(final Bitmap bitmap) { + ThreadUtils.assertOnUiThread(); + + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) { + return; + } + final View view = getTextInput().getView(); + if (view == null) { + return; + } + + GeckoDragAndDrop.updateDragImage(view, bitmap); + } + /** GeckoSession applications implement this interface to handle media events. */ public interface MediaDelegate { diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/GeckoView.java b/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/GeckoView.java index 31d4e3daf18b..74eccaeb1594 100644 --- a/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/GeckoView.java +++ b/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/GeckoView.java @@ -32,6 +32,7 @@ import android.util.Log; import android.util.SparseArray; import android.util.TypedValue; import android.view.DisplayCutout; +import android.view.DragEvent; import android.view.KeyEvent; import android.view.MotionEvent; import android.view.Surface; @@ -1233,4 +1234,13 @@ public class GeckoView extends FrameLayout implements GeckoDisplay.NewSurfacePro } }); } + + /** Handle drag and drop event */ + @Override + public boolean onDragEvent(final DragEvent event) { + if (mSession == null) { + return false; + } + return mSession.getPanZoomController().onDragEvent(event); + } } diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/PanZoomController.java b/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/PanZoomController.java index 0731e4e09560..877e0e34a69c 100644 --- a/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/PanZoomController.java +++ b/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/PanZoomController.java @@ -9,9 +9,11 @@ import android.app.UiModeManager; import android.content.Context; import android.content.res.Configuration; import android.graphics.Rect; +import android.os.Build; import android.os.SystemClock; import android.util.Log; import android.util.Pair; +import android.view.DragEvent; import android.view.InputDevice; import android.view.MotionEvent; import androidx.annotation.AnyThread; @@ -22,6 +24,7 @@ import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.util.ArrayList; import org.mozilla.gecko.GeckoAppShell; +import org.mozilla.gecko.GeckoDragAndDrop; import org.mozilla.gecko.annotation.WrapForJNI; import org.mozilla.gecko.mozglue.JNIObject; import org.mozilla.gecko.util.GeckoBundle; @@ -298,6 +301,10 @@ public class PanZoomController { private native @InputResult int handleMouseEvent( int action, long time, int metaState, float x, float y, int buttons); + @WrapForJNI(calledFrom = "ui", dispatchTo = "gecko") + private native void handleDragEvent( + int action, long time, float x, float y, GeckoDragAndDrop.DropData data); + @WrapForJNI(stubName = "SetIsLongpressEnabled") // Called from test thread. private native void nativeSetIsLongpressEnabled(boolean isLongpressEnabled); @@ -597,6 +604,32 @@ public class PanZoomController { } } + /** + * Process a drag event. + * + * @param event DragEvent to process. + * @return true if this event is accepted. + */ + public boolean onDragEvent(@NonNull final DragEvent event) { + ThreadUtils.assertOnUiThread(); + + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) { + return false; + } + + if (!GeckoDragAndDrop.onDragEvent(event)) { + return false; + } + + mNative.handleDragEvent( + event.getAction(), + SystemClock.uptimeMillis(), + GeckoDragAndDrop.getLocationX(), + GeckoDragAndDrop.getLocationY(), + GeckoDragAndDrop.createDropData(event)); + return true; + } + private void enableEventQueue() { if (mQueuedEvents != null) { throw new IllegalStateException("Already have an event queue"); diff --git a/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/doc-files/CHANGELOG.md b/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/doc-files/CHANGELOG.md index 0cb23e62c447..3d53329a0586 100644 --- a/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/doc-files/CHANGELOG.md +++ b/mobile/android/geckoview/src/main/java/org/mozilla/geckoview/doc-files/CHANGELOG.md @@ -18,10 +18,13 @@ exclude: true - Added [`GeckoRuntimeSettings#setTrustedRecursiveResolverMode`][124.1] to enable DNS-over-HTTPS using different resolver modes ([bug 1591533]({{bugzilla}}1591533)). - Added [`GeckoRuntimeSettings#setTrustedRecursiveResolverUri`][124.2] to specify the DNS-over-HTTPS server to be used if DoH is enabled ([bug 1591533]({{bugzilla}}1591533)). - Added [`GeckoRuntimeSettings#setLargeKeepaliveFactor`][124.3] to increase the keepalive timeout used for a connection ([bug 1591533]({{bugzilla}}1591533)). +- Added [`PanZoomController.onDragEvent`][124.4] to support drag and drop. + ([bug 1586471]({{bugzilla}}1586471)) [124.1]: {{javadoc_uri}}/GeckoRuntimeSettings.html#setTrustedRecursiveResolverMode-int- [124.2]: {{javadoc_uri}}/GeckoRuntimeSettings.html#setTrustedRecursiveResolverUri-java.lang.String- [124.3]: {{javadoc_uri}}/GeckoRuntimeSettings.html#setLargeKeepaliveFactor-int- +[124.4]: {{javadoc_uri}}/PanZoomController.html#onDragEvent(android.view.DragEvent) ## v123 - For Translations, added [`checkPairDownloadSize`][123.1] and [`TranslationsException.ERROR_MODEL_LANGUAGE_REQUIRED`][123.2] as an error state. @@ -1514,4 +1517,4 @@ to allow adding gecko profiler markers. [65.24]: {{javadoc_uri}}/CrashReporter.html#sendCrashReport(android.content.Context,android.os.Bundle,java.lang.String) [65.25]: {{javadoc_uri}}/GeckoResult.html -[api-version]: 757616f5eabb19164140a9f303277496fd1a4993 +[api-version]: 6ba1de66c7ab46b0ac52910c9fb76f94c2bcc5b9 diff --git a/widget/android/AndroidWidgetUtils.cpp b/widget/android/AndroidWidgetUtils.cpp new file mode 100644 index 000000000000..c013a5b4f86e --- /dev/null +++ b/widget/android/AndroidWidgetUtils.cpp @@ -0,0 +1,48 @@ +/* -*- Mode: c++; c-basic-offset: 2; tab-width: 20; indent-tabs-mode: nil; -*- + * vim: set sw=2 ts=4 expandtab: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "AndroidWidgetUtils.h" + +#include "mozilla/gfx/2D.h" +#include "mozilla/gfx/Swizzle.h" + +using namespace mozilla::gfx; + +namespace mozilla::widget { + +// static +already_AddRefed +AndroidWidgetUtils::GetDataSourceSurfaceForAndroidBitmap( + gfx::SourceSurface* aSurface, const LayoutDeviceIntRect* aRect, + uint32_t aStride) { + RefPtr srcDataSurface = aSurface->GetDataSurface(); + if (NS_WARN_IF(!srcDataSurface)) { + return nullptr; + } + + DataSourceSurface::ScopedMap sourceMap(srcDataSurface, + DataSourceSurface::READ); + + RefPtr destDataSurface = + gfx::Factory::CreateDataSourceSurfaceWithStride( + aRect ? IntSize(aRect->width, aRect->height) + : srcDataSurface->GetSize(), + SurfaceFormat::R8G8B8A8, aStride ? aStride : sourceMap.GetStride()); + if (NS_WARN_IF(!destDataSurface)) { + return nullptr; + } + + DataSourceSurface::ScopedMap destMap(destDataSurface, + DataSourceSurface::READ_WRITE); + + SwizzleData(sourceMap.GetData(), sourceMap.GetStride(), aSurface->GetFormat(), + destMap.GetData(), destMap.GetStride(), SurfaceFormat::R8G8B8A8, + destDataSurface->GetSize()); + + return destDataSurface.forget(); +} + +} // namespace mozilla::widget diff --git a/widget/android/AndroidWidgetUtils.h b/widget/android/AndroidWidgetUtils.h new file mode 100644 index 000000000000..ccf831206d6b --- /dev/null +++ b/widget/android/AndroidWidgetUtils.h @@ -0,0 +1,37 @@ +/* -*- Mode: c++; c-basic-offset: 2; tab-width: 20; indent-tabs-mode: nil; -*- + * vim: set sw=2 ts=4 expandtab: + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_widget_AndroidWidgetUtils_h__ +#define mozilla_widget_AndroidWidgetUtils_h__ + +#include "Units.h" + +namespace mozilla { + +namespace gfx { +class SourceSurface; +class DataSourceSurface; +} // namespace gfx + +namespace widget { + +class AndroidWidgetUtils final { + public: + typedef mozilla::LayoutDeviceIntRect LayoutDeviceIntRect; + + /** + * Return Android's bitmap object compatible data surface. + */ + static already_AddRefed + GetDataSourceSurfaceForAndroidBitmap( + gfx::SourceSurface* aSurface, const LayoutDeviceIntRect* aRect = nullptr, + uint32_t aStride = 0); +}; + +} // namespace widget +} // namespace mozilla + +#endif diff --git a/widget/android/WebExecutorSupport.cpp b/widget/android/WebExecutorSupport.cpp index 352476e1eced..44b9a4696eb8 100644 --- a/widget/android/WebExecutorSupport.cpp +++ b/widget/android/WebExecutorSupport.cpp @@ -11,13 +11,16 @@ #include "WebExecutorSupport.h" #include "nsIAsyncVerifyRedirectCallback.h" +#include "nsICancelable.h" #include "nsIHttpChannel.h" #include "nsIHttpChannelInternal.h" #include "nsIHttpHeaderVisitor.h" +#include "nsIInputStream.h" #include "nsIDNSService.h" #include "nsIDNSListener.h" #include "nsIDNSRecord.h" #include "nsINSSErrorsService.h" +#include "nsContentUtils.h" #include "nsNetUtil.h" // for NS_NewURI, NS_NewChannel, NS_NewStreamLoader #include "nsIPrivateBrowsingChannel.h" #include "nsIUploadChannel2.h" @@ -178,7 +181,7 @@ class LoaderListener final : public GeckoViewStreamListener { } void CompleteWithError(nsresult aStatus, nsIChannel* aChannel) override { - ::CompleteWithError(mResult, aStatus, aChannel); + mozilla::widget::CompleteWithError(mResult, aStatus, aChannel); } virtual ~LoaderListener() {} diff --git a/widget/android/bindings/AndroidDragEvent-classes.txt b/widget/android/bindings/AndroidDragEvent-classes.txt new file mode 100644 index 000000000000..368f50013912 --- /dev/null +++ b/widget/android/bindings/AndroidDragEvent-classes.txt @@ -0,0 +1,3 @@ +# We only use constants from DragEvent +[android.view.DragEvent = skip:true] + = skip:false diff --git a/widget/android/bindings/moz.build b/widget/android/bindings/moz.build index 49a7dbd9e0f8..f3f4d138ff1f 100644 --- a/widget/android/bindings/moz.build +++ b/widget/android/bindings/moz.build @@ -12,6 +12,7 @@ with Files("**"): generated = [ "AccessibilityEvent", "AndroidBuild", + "AndroidDragEvent", "AndroidGraphics", "AndroidInputType", "AndroidProcess", diff --git a/widget/android/components.conf b/widget/android/components.conf index c678ca8b32ee..1c7b49cd384b 100644 --- a/widget/android/components.conf +++ b/widget/android/components.conf @@ -97,4 +97,13 @@ Classes = [ 'type': 'mozilla::widget::AndroidAlerts', 'headers': ['/widget/android/AndroidAlerts.h'], }, + { + 'cid': '{b1abaf0e-52b2-4e65-aee1-299ea9a74230}', + 'contract_ids': ['@mozilla.org/widget/parent/dragservice;1'], + 'singleton': True, + 'type': 'nsDragService', + 'headers': ['/widget/android/nsDragService.h'], + 'constructor': 'nsDragService::GetInstance', + 'processes': ProcessSelector.MAIN_PROCESS_ONLY, + }, ] diff --git a/widget/android/moz.build b/widget/android/moz.build index 1f2ae1eba592..e4525bb74767 100644 --- a/widget/android/moz.build +++ b/widget/android/moz.build @@ -41,6 +41,7 @@ classes_with_WrapForJNI = [ "GeckoAudioInfo", "GeckoBatteryManager", "GeckoBundle", + "GeckoDragAndDrop", "GeckoEditableChild", "GeckoHLSDemuxerWrapper", "GeckoHLSResourceWrapper", @@ -108,6 +109,7 @@ EXPORTS.mozilla.widget += [ "AndroidUiThread.h", "AndroidView.h", "AndroidVsync.h", + "AndroidWidgetUtils.h", "CompositorWidgetChild.h", "CompositorWidgetParent.h", "EventDispatcher.h", @@ -134,6 +136,7 @@ UNIFIED_SOURCES += [ "AndroidContentController.cpp", "AndroidUiThread.cpp", "AndroidVsync.cpp", + "AndroidWidgetUtils.cpp", "CompositorWidgetChild.cpp", "CompositorWidgetParent.cpp", "EventDispatcher.cpp", @@ -145,6 +148,7 @@ UNIFIED_SOURCES += [ "nsAppShell.cpp", "nsClipboard.cpp", "nsDeviceContextAndroid.cpp", + "nsDragService.cpp", "nsLookAndFeel.cpp", "nsPrintSettingsServiceAndroid.cpp", "nsUserIdleServiceAndroid.cpp", diff --git a/widget/android/nsAppShell.cpp b/widget/android/nsAppShell.cpp index 28389184a078..4d396945c4ab 100644 --- a/widget/android/nsAppShell.cpp +++ b/widget/android/nsAppShell.cpp @@ -10,6 +10,7 @@ #include "base/task.h" #include "mozilla/Hal.h" #include "gfxConfig.h" +#include "nsDragService.h" #include "nsExceptionHandler.h" #include "nsIScreen.h" #include "nsWindow.h" @@ -36,6 +37,7 @@ #include "mozilla/intl/OSPreferences.h" #include "mozilla/ipc/GeckoChildProcessHost.h" #include "mozilla/java/GeckoAppShellNatives.h" +#include "mozilla/java/GeckoDragAndDropNatives.h" #include "mozilla/java/GeckoResultWrappers.h" #include "mozilla/java/GeckoThreadNatives.h" #include "mozilla/java/XPCOMEventTargetNatives.h" diff --git a/widget/android/nsClipboard.cpp b/widget/android/nsClipboard.cpp index 3f748a2fa4db..c2a03dd5408e 100644 --- a/widget/android/nsClipboard.cpp +++ b/widget/android/nsClipboard.cpp @@ -36,22 +36,15 @@ nsClipboard::~nsClipboard() { java::GeckoAppShell::GetApplicationContext()); } -NS_IMETHODIMP -nsClipboard::SetNativeClipboardData(nsITransferable* aTransferable, - int32_t aWhichClipboard) { - MOZ_DIAGNOSTIC_ASSERT(aTransferable); - MOZ_DIAGNOSTIC_ASSERT( - nsIClipboard::IsClipboardTypeSupported(aWhichClipboard)); - - if (!jni::IsAvailable()) { - return NS_ERROR_NOT_AVAILABLE; - } - +// static +nsresult nsClipboard::GetTextFromTransferable(nsITransferable* aTransferable, + nsString& aText, + nsString& aHTML) { nsTArray flavors; - aTransferable->FlavorsTransferableCanImport(flavors); - - nsAutoString html; - nsAutoString text; + nsresult rv = aTransferable->FlavorsTransferableCanImport(flavors); + if (NS_FAILED(rv)) { + return rv; + } for (auto& flavorStr : flavors) { if (flavorStr.EqualsLiteral(kTextMime)) { @@ -63,7 +56,7 @@ nsClipboard::SetNativeClipboardData(nsITransferable* aTransferable, } nsCOMPtr supportsString = do_QueryInterface(item); if (supportsString) { - supportsString->GetData(text); + supportsString->GetData(aText); } } else if (flavorStr.EqualsLiteral(kHTMLMime)) { nsCOMPtr item; @@ -74,10 +67,30 @@ nsClipboard::SetNativeClipboardData(nsITransferable* aTransferable, } nsCOMPtr supportsString = do_QueryInterface(item); if (supportsString) { - supportsString->GetData(html); + supportsString->GetData(aHTML); } } } + return NS_OK; +} + +NS_IMETHODIMP +nsClipboard::SetNativeClipboardData(nsITransferable* aTransferable, + int32_t aWhichClipboard) { + MOZ_DIAGNOSTIC_ASSERT(aTransferable); + MOZ_DIAGNOSTIC_ASSERT( + nsIClipboard::IsClipboardTypeSupported(aWhichClipboard)); + + if (!jni::IsAvailable()) { + return NS_ERROR_NOT_AVAILABLE; + } + + nsString text; + nsString html; + nsresult rv = GetTextFromTransferable(aTransferable, text, html); + if (NS_FAILED(rv)) { + return rv; + } if (!html.IsEmpty() && java::Clipboard::SetHTML(java::GeckoAppShell::GetApplicationContext(), diff --git a/widget/android/nsClipboard.h b/widget/android/nsClipboard.h index 528a5554e526..d3a45d47e58e 100644 --- a/widget/android/nsClipboard.h +++ b/widget/android/nsClipboard.h @@ -17,6 +17,9 @@ class nsClipboard final : public nsBaseClipboard { NS_DECL_ISUPPORTS_INHERITED + static nsresult GetTextFromTransferable(nsITransferable* aTransferable, + nsString& aText, nsString& aHTML); + protected: // Implement the native clipboard behavior. NS_IMETHOD SetNativeClipboardData(nsITransferable* aTransferable, diff --git a/widget/android/nsDragService.cpp b/widget/android/nsDragService.cpp new file mode 100644 index 000000000000..9e162b109196 --- /dev/null +++ b/widget/android/nsDragService.cpp @@ -0,0 +1,268 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsDragService.h" + +#include "AndroidGraphics.h" +#include "AndroidWidgetUtils.h" +#include "mozilla/dom/Document.h" +#include "mozilla/java/GeckoDragAndDropWrappers.h" +#include "mozilla/PresShell.h" +#include "mozilla/ScopeExit.h" +#include "nsArrayUtils.h" +#include "nsClipboard.h" +#include "nsComponentManagerUtils.h" +#include "nsIArray.h" +#include "nsITransferable.h" +#include "nsPrimitiveHelpers.h" +#include "nsViewManager.h" +#include "nsWindow.h" + +NS_IMPL_ISUPPORTS_INHERITED0(nsDragService, nsBaseDragService) + +using namespace mozilla; +using namespace mozilla::widget; + +StaticRefPtr sDragServiceInstance; + +/* static */ +already_AddRefed nsDragService::GetInstance() { + if (!sDragServiceInstance) { + sDragServiceInstance = new nsDragService(); + ClearOnShutdown(&sDragServiceInstance); + } + + RefPtr service = sDragServiceInstance.get(); + return service.forget(); +} + +static nsWindow* GetWindow(dom::Document* aDocument) { + if (!aDocument) { + return nullptr; + } + + PresShell* presShell = aDocument->GetPresShell(); + if (!presShell) { + return nullptr; + } + + RefPtr vm = presShell->GetViewManager(); + if (!vm) { + return nullptr; + } + + nsCOMPtr widget = vm->GetRootWidget(); + if (!widget) { + return nullptr; + } + + RefPtr window = nsWindow::From(widget); + return window.get(); +} + +nsresult nsDragService::InvokeDragSessionImpl( + nsIArray* aTransferableArray, const Maybe& aRegion, + uint32_t aActionType) { + if (jni::GetAPIVersion() < 24) { + return NS_ERROR_NOT_AVAILABLE; + } + + uint32_t count = 0; + aTransferableArray->GetLength(&count); + if (count != 1) { + return NS_ERROR_FAILURE; + } + + nsCOMPtr transferable = + do_QueryElementAt(aTransferableArray, 0); + + nsAutoString html; + nsAutoString text; + nsresult rv = nsClipboard::GetTextFromTransferable(transferable, text, html); + if (NS_FAILED(rv)) { + return rv; + } + java::GeckoDragAndDrop::SetDragData(text, html); + + if (nsWindow* window = GetWindow(mSourceDocument)) { + mTransferable = transferable; + + nsBaseDragService::StartDragSession(); + nsBaseDragService::OpenDragPopup(); + + auto bitmap = CreateDragImage(mSourceNode, aRegion); + window->StartDragAndDrop(bitmap); + + return NS_OK; + } + + return NS_ERROR_FAILURE; +} + +NS_IMETHODIMP +nsDragService::GetData(nsITransferable* aTransferable, uint32_t aItem) { + if (!aTransferable) { + return NS_ERROR_INVALID_ARG; + } + + nsTArray flavors; + nsresult rv = aTransferable->FlavorsTransferableCanImport(flavors); + if (NS_FAILED(rv)) { + return NS_ERROR_FAILURE; + } + + for (const auto& flavor : flavors) { + nsCOMPtr data; + rv = mTransferable->GetTransferData(flavor.get(), getter_AddRefs(data)); + if (NS_FAILED(rv)) { + continue; + } + rv = aTransferable->SetTransferData(flavor.get(), data); + if (NS_SUCCEEDED(rv)) { + return rv; + } + } + + return NS_ERROR_FAILURE; +} + +NS_IMETHODIMP +nsDragService::GetNumDropItems(uint32_t* aNumItems) { + if (mTransferable) { + *aNumItems = 1; + return NS_OK; + } + *aNumItems = 0; + return NS_OK; +} + +NS_IMETHODIMP +nsDragService::IsDataFlavorSupported(const char* aDataFlavor, bool* _retval) { + *_retval = false; + + nsDependentCString dataFlavor(aDataFlavor); + auto logging = MakeScopeExit([&] { + MOZ_DRAGSERVICE_LOG("IsDataFlavorSupported: %s is%s found", aDataFlavor, + *_retval ? "" : " not"); + }); + + nsTArray flavors; + nsresult rv = mTransferable->FlavorsTransferableCanImport(flavors); + if (NS_FAILED(rv)) { + return NS_OK; + } + + for (const auto& flavor : flavors) { + if (dataFlavor.Equals(flavor)) { + *_retval = true; + return NS_OK; + } + } + + return NS_OK; +} + +NS_IMETHODIMP +nsDragService::EndDragSession(bool aDoneDrag, uint32_t aKeyModifiers) { + java::GeckoDragAndDrop::EndDragSession(); + + nsresult rv = nsBaseDragService::EndDragSession(aDoneDrag, aKeyModifiers); + mTransferable = nullptr; + return rv; +} + +NS_IMETHODIMP +nsDragService::UpdateDragImage(nsINode* aImage, int32_t aImageX, + int32_t aImageY) { + nsBaseDragService::UpdateDragImage(aImage, aImageX, aImageY); + auto bitmap = CreateDragImage(mSourceNode, Nothing()); + + if (nsWindow* window = GetWindow(mSourceDocument)) { + window->UpdateDragImage(bitmap); + } + + return NS_OK; +} + +java::sdk::Bitmap::LocalRef nsDragService::CreateDragImage( + nsINode* aNode, const Maybe& aRegion) { + LayoutDeviceIntRect dragRect; + RefPtr surface; + nsPresContext* pc; + DrawDrag(aNode, aRegion, mScreenPosition, &dragRect, &surface, &pc); + if (!surface) { + return nullptr; + } + + RefPtr destDataSurface = + AndroidWidgetUtils::GetDataSourceSurfaceForAndroidBitmap( + surface, &dragRect, dragRect.width * 4); + if (!destDataSurface) { + return nullptr; + } + + DataSourceSurface::ScopedMap destMap(destDataSurface, + DataSourceSurface::READ); + + java::sdk::Bitmap::LocalRef bitmap; + auto pixels = mozilla::jni::ByteBuffer::New( + reinterpret_cast(destMap.GetData()), + destMap.GetStride() * destDataSurface->GetSize().height); + bitmap = java::sdk::Bitmap::CreateBitmap( + dragRect.width, dragRect.height, java::sdk::Bitmap::Config::ARGB_8888()); + bitmap->CopyPixelsFromBuffer(pixels); + return bitmap; +} + +void nsDragService::SetData(nsITransferable* aTransferable) { + mTransferable = aTransferable; +} + +// static +void nsDragService::SetDropData( + mozilla::java::GeckoDragAndDrop::DropData::Param aDropData) { + MOZ_ASSERT(NS_IsMainThread()); + + RefPtr dragService = nsDragService::GetInstance(); + if (!dragService) { + return; + } + + if (!aDropData) { + dragService->SetData(nullptr); + return; + } + + nsCString mime(aDropData->MimeType()->ToCString()); + + if (mime.EqualsLiteral("application/x-moz-draganddrop")) { + // The drop data isn't changed. + return; + } + + if (!mime.EqualsLiteral("text/plain") && !mime.EqualsLiteral("text/html")) { + // Not supported data. + dragService->SetData(nullptr); + return; + } + + nsString buffer(aDropData->Text()->ToString()); + if (buffer.IsEmpty()) { + dragService->SetData(nullptr); + return; + } + nsCOMPtr wrapper; + nsPrimitiveHelpers::CreatePrimitiveForData( + mime, buffer.get(), buffer.Length() * 2, getter_AddRefs(wrapper)); + if (!wrapper) { + dragService->SetData(nullptr); + return; + } + nsCOMPtr transferable = + do_CreateInstance("@mozilla.org/widget/transferable;1"); + transferable->Init(nullptr); + transferable->SetTransferData(mime.get(), wrapper); + dragService->SetData(transferable); +} diff --git a/widget/android/nsDragService.h b/widget/android/nsDragService.h new file mode 100644 index 000000000000..839f177b1d57 --- /dev/null +++ b/widget/android/nsDragService.h @@ -0,0 +1,56 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsDragService_h__ +#define nsDragService_h__ + +#include "nsBaseDragService.h" + +#include "AndroidGraphics.h" +#include "mozilla/java/GeckoDragAndDropNatives.h" + +class nsITransferable; + +class nsDragService final : public nsBaseDragService { + public: + nsDragService() = default; + + NS_DECL_ISUPPORTS_INHERITED + + static already_AddRefed GetInstance(); + + // nsIDragSession + NS_IMETHOD GetData(nsITransferable* aTransferable, uint32_t anItem) override; + NS_IMETHOD GetNumDropItems(uint32_t* aNumItems) override; + NS_IMETHOD IsDataFlavorSupported(const char* aDataFlavor, + bool* _retval) override; + MOZ_CAN_RUN_SCRIPT NS_IMETHOD EndDragSession(bool aDoneDrag, + uint32_t aKeyModifiers) override; + NS_IMETHOD + UpdateDragImage(nsINode* aImage, int32_t aImageX, int32_t aImageY) override; + + void SetData(nsITransferable* aTransferable); + + static void SetDropData( + mozilla::java::GeckoDragAndDrop::DropData::Param aDropData); + + protected: + virtual ~nsDragService() = default; + + // nsBaseDragService + MOZ_CAN_RUN_SCRIPT nsresult + InvokeDragSessionImpl(nsIArray* anArrayTransferables, + const mozilla::Maybe& aRegion, + uint32_t aActionType) override; + + private: + mozilla::java::sdk::Bitmap::LocalRef CreateDragImage( + nsINode* aNode, const mozilla::Maybe& aRegion); + + // our source data items + nsCOMPtr mTransferable; +}; + +#endif // nsDragService_h__ diff --git a/widget/android/nsWindow.cpp b/widget/android/nsWindow.cpp index f89bde832796..016033338ce9 100644 --- a/widget/android/nsWindow.cpp +++ b/widget/android/nsWindow.cpp @@ -14,13 +14,14 @@ #include #include -#include "AndroidGraphics.h" #include "AndroidBridge.h" #include "AndroidBridgeUtilities.h" #include "AndroidCompositorWidget.h" #include "AndroidContentController.h" +#include "AndroidDragEvent.h" #include "AndroidUiThread.h" #include "AndroidView.h" +#include "AndroidWidgetUtils.h" #include "gfxContext.h" #include "GeckoEditableSupport.h" #include "GeckoViewOutputStream.h" @@ -824,6 +825,19 @@ class NPZCSupport final mListeningToVsync = aNeedVsync; } + void HandleDragEvent(int32_t aAction, int64_t aTime, float aX, float aY, + jni::Object::Param aDropData) { + // APZ handles some drag event type on APZ thread, but it cannot handle all + // types. + MOZ_ASSERT(NS_IsMainThread()); + + if (auto window = mWindow.Access()) { + if (nsWindow* gkWindow = window->GetNsWindow()) { + gkWindow->OnDragEvent(aAction, aTime, aX, aY, aDropData); + } + } + } + void ConsumeMotionEventsFromResampler() { auto outgoing = mTouchResampler.ConsumeOutgoingEvents(); while (!outgoing.empty()) { @@ -2582,6 +2596,139 @@ void GeckoViewSupport::OnUpdateSessionStore( window->OnUpdateSessionStore(aBundle); } +static EventMessage convertDragEventActionToGeckoEvent(int32_t aAction) { + switch (aAction) { + case java::sdk::DragEvent::ACTION_DRAG_ENTERED: + return eDragEnter; + case java::sdk::DragEvent::ACTION_DRAG_EXITED: + return eDragExit; + case java::sdk::DragEvent::ACTION_DRAG_LOCATION: + return eDragOver; + case java::sdk::DragEvent::ACTION_DROP: + return eDrop; + } + return eVoidEvent; +} + +void nsWindow::OnDragEvent(int32_t aAction, int64_t aTime, float aX, float aY, + jni::Object::Param aDropData) { + MOZ_ASSERT(NS_IsMainThread()); + + RefPtr dragService = nsDragService::GetInstance(); + if (!dragService) { + return; + } + + LayoutDeviceIntPoint point = + LayoutDeviceIntPoint(int32_t(floorf(aX)), int32_t(floorf(aY))); + + if (aAction == java::sdk::DragEvent::ACTION_DRAG_STARTED) { + dragService->SetDragEndPoint(point); + return; + } + + if (aAction == java::sdk::DragEvent::ACTION_DRAG_ENDED) { + dragService->EndDragSession(false, 0); + return; + } + + EventMessage message = convertDragEventActionToGeckoEvent(aAction); + + if (message == eDragEnter) { + dragService->StartDragSession(); + } + + nsCOMPtr dragSession; + dragService->GetCurrentSession(getter_AddRefs(dragSession)); + if (dragSession) { + switch (message) { + case eDragOver: + dragService->SetDragEndPoint(point); + dragService->FireDragEventAtSource(eDrag, 0); + break; + case eDrop: { + bool canDrop = false; + dragSession->GetCanDrop(&canDrop); + if (!canDrop) { + nsCOMPtr sourceNode; + dragSession->GetSourceNode(getter_AddRefs(sourceNode)); + if (!sourceNode) { + dragService->EndDragSession(false, 0); + } + return; + } + auto dropData = + mozilla::java::GeckoDragAndDrop::DropData::Ref::From(aDropData); + nsDragService::SetDropData(dropData); + dragService->SetDragEndPoint(point); + break; + } + default: + break; + } + + dragSession->SetDragAction(nsIDragService::DRAGDROP_ACTION_MOVE); + } + + WidgetDragEvent geckoEvent(true, message, this); + geckoEvent.mRefPoint = point; + geckoEvent.mTimeStamp = nsWindow::GetEventTimeStamp(aTime); + geckoEvent.mModifiers = 0; // DragEvent has no modifiers + DispatchInputEvent(&geckoEvent); + + if (!dragSession) { + return; + } + + switch (message) { + case eDragExit: { + nsCOMPtr sourceNode; + dragSession->GetSourceNode(getter_AddRefs(sourceNode)); + if (!sourceNode) { + // We're leaving a window while doing a drag that was + // initiated in a different app. End the drag session, + // since we're done with it for now (until the user + // drags back into mozilla). + dragService->EndDragSession(false, 0); + } + break; + } + case eDrop: + dragService->EndDragSession(true, 0); + break; + default: + break; + } +} + +void nsWindow::StartDragAndDrop(java::sdk::Bitmap::LocalRef aBitmap) { + if (mozilla::jni::NativeWeakPtr::Accessor lvs{ + mLayerViewSupport.Access()}) { + const auto& compositor = lvs->GetJavaCompositor(); + + DispatchToUiThread( + "nsWindow::StartDragAndDrop", + [compositor = GeckoSession::Compositor::GlobalRef(compositor), + bitmap = java::sdk::Bitmap::GlobalRef(aBitmap)] { + compositor->StartDragAndDrop(bitmap); + }); + } +} + +void nsWindow::UpdateDragImage(java::sdk::Bitmap::LocalRef aBitmap) { + if (mozilla::jni::NativeWeakPtr::Accessor lvs{ + mLayerViewSupport.Access()}) { + const auto& compositor = lvs->GetJavaCompositor(); + + DispatchToUiThread( + "nsWindow::UpdateDragImage", + [compositor = GeckoSession::Compositor::GlobalRef(compositor), + bitmap = java::sdk::Bitmap::GlobalRef(aBitmap)] { + compositor->UpdateDragImage(bitmap); + }); + } +} + void nsWindow::OnSizeChanged(const gfx::IntSize& aSize) { ALOG("nsWindow: %p OnSizeChanged [%d %d]", (void*)this, aSize.width, aSize.height); @@ -3083,29 +3230,7 @@ static already_AddRefed GetCursorImage( return nullptr; } - RefPtr srcDataSurface = surface->GetDataSurface(); - if (NS_WARN_IF(!srcDataSurface)) { - return nullptr; - } - - DataSourceSurface::ScopedMap sourceMap(srcDataSurface, - DataSourceSurface::READ); - - destDataSurface = gfx::Factory::CreateDataSourceSurfaceWithStride( - srcDataSurface->GetSize(), SurfaceFormat::R8G8B8A8, - sourceMap.GetStride()); - if (NS_WARN_IF(!destDataSurface)) { - return nullptr; - } - - DataSourceSurface::ScopedMap destMap(destDataSurface, - DataSourceSurface::READ_WRITE); - - SwizzleData(sourceMap.GetData(), sourceMap.GetStride(), surface->GetFormat(), - destMap.GetData(), destMap.GetStride(), SurfaceFormat::R8G8B8A8, - destDataSurface->GetSize()); - - return destDataSurface.forget(); + return AndroidWidgetUtils::GetDataSourceSurfaceForAndroidBitmap(surface); } static int32_t GetCursorType(nsCursor aCursor) { diff --git a/widget/android/nsWindow.h b/widget/android/nsWindow.h index 7c6f8ba9bb1d..a2c2a798feb5 100644 --- a/widget/android/nsWindow.h +++ b/widget/android/nsWindow.h @@ -7,6 +7,7 @@ #ifndef NSWINDOW_H_ #define NSWINDOW_H_ +#include "AndroidGraphics.h" #include "nsBaseWidget.h" #include "gfxPoint.h" #include "nsIUserIdleServiceInternal.h" @@ -127,6 +128,12 @@ class nsWindow final : public nsBaseWidget { void ShowDynamicToolbar(); + MOZ_CAN_RUN_SCRIPT_BOUNDARY void OnDragEvent( + int32_t aAction, int64_t aTime, float aX, float aY, + mozilla::jni::Object::Param aDropData); + void StartDragAndDrop(mozilla::java::sdk::Bitmap::LocalRef aBitmap); + void UpdateDragImage(mozilla::java::sdk::Bitmap::LocalRef aBitmap); + void DetachNatives(); mozilla::Mutex& GetDestroyMutex() { return mDestroyMutex; }