From 733053b5487a383efdc3eaae4c66a33160cb9d3a Mon Sep 17 00:00:00 2001 From: David Perez Date: Thu, 5 Sep 2024 11:20:11 -0500 Subject: [PATCH] PM-11484: Add logic to parse URI from AccessibilityNodeInfo (#3864) --- .../accessibility/di/AccessibilityModule.kt | 6 + .../autofill/accessibility/model/Browser.kt | 20 ++ .../accessibility/model/FillableFields.kt | 11 + .../parser/AccessibilityParser.kt | 20 ++ .../parser/AccessibilityParserImpl.kt | 38 +++ .../accessibility/util/BrowserUtil.kt | 219 ++++++++++++++++++ .../xml/autofill_service_configuration.xml | 5 +- .../parser/AccessibilityParserTest.kt | 84 +++++++ 8 files changed, 402 insertions(+), 1 deletion(-) create mode 100644 app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/model/Browser.kt create mode 100644 app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/model/FillableFields.kt create mode 100644 app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/parser/AccessibilityParser.kt create mode 100644 app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/parser/AccessibilityParserImpl.kt create mode 100644 app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/util/BrowserUtil.kt create mode 100644 app/src/test/java/com/x8bit/bitwarden/data/autofill/accessibility/parser/AccessibilityParserTest.kt diff --git a/app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/di/AccessibilityModule.kt b/app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/di/AccessibilityModule.kt index df7f7a184..79fcf8132 100644 --- a/app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/di/AccessibilityModule.kt +++ b/app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/di/AccessibilityModule.kt @@ -5,6 +5,8 @@ import com.x8bit.bitwarden.data.autofill.accessibility.manager.AccessibilityAuto import com.x8bit.bitwarden.data.autofill.accessibility.manager.AccessibilityAutofillManagerImpl import com.x8bit.bitwarden.data.autofill.accessibility.manager.LauncherPackageNameManager import com.x8bit.bitwarden.data.autofill.accessibility.manager.LauncherPackageNameManagerImpl +import com.x8bit.bitwarden.data.autofill.accessibility.parser.AccessibilityParser +import com.x8bit.bitwarden.data.autofill.accessibility.parser.AccessibilityParserImpl import dagger.Module import dagger.Provides import dagger.hilt.InstallIn @@ -23,6 +25,10 @@ object AccessibilityModule { fun providesAccessibilityInvokeManager(): AccessibilityAutofillManager = AccessibilityAutofillManagerImpl() + @Singleton + @Provides + fun providesAccessibilityParser(): AccessibilityParser = AccessibilityParserImpl() + @Singleton @Provides fun providesLauncherPackageNameManager( diff --git a/app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/model/Browser.kt b/app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/model/Browser.kt new file mode 100644 index 000000000..6d5c150ab --- /dev/null +++ b/app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/model/Browser.kt @@ -0,0 +1,20 @@ +package com.x8bit.bitwarden.data.autofill.accessibility.model + +/** + * A model representing a supported browser. + */ +data class Browser( + val packageName: String, + val possibleUrlFieldIds: List, + val urlExtractor: (String) -> String? = { it }, +) { + constructor( + packageName: String, + urlFieldId: String, + urlExtractor: (String) -> String? = { it }, + ) : this( + packageName = packageName, + possibleUrlFieldIds = listOf(urlFieldId), + urlExtractor = urlExtractor, + ) +} diff --git a/app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/model/FillableFields.kt b/app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/model/FillableFields.kt new file mode 100644 index 000000000..1bd8c8ab9 --- /dev/null +++ b/app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/model/FillableFields.kt @@ -0,0 +1,11 @@ +package com.x8bit.bitwarden.data.autofill.accessibility.model + +import android.view.accessibility.AccessibilityNodeInfo + +/** + * Represents the fillable fields for accessibility based autofill. + */ +data class FillableFields( + val usernameFields: List, + val passwordFields: List, +) diff --git a/app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/parser/AccessibilityParser.kt b/app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/parser/AccessibilityParser.kt new file mode 100644 index 000000000..a95ec3f51 --- /dev/null +++ b/app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/parser/AccessibilityParser.kt @@ -0,0 +1,20 @@ +package com.x8bit.bitwarden.data.autofill.accessibility.parser + +import android.net.Uri +import android.view.accessibility.AccessibilityNodeInfo +import com.x8bit.bitwarden.data.autofill.accessibility.model.FillableFields + +/** + * A tool for parsing accessibility data from the OS into domain models. + */ +interface AccessibilityParser { + /** + * Parses the fillable fields from [rootNode]. + */ + fun parseForFillableFields(rootNode: AccessibilityNodeInfo): FillableFields + + /** + * Parses the [Uri] from [rootNode] and returns a url, package name. + */ + fun parseForUriOrPackageName(rootNode: AccessibilityNodeInfo): Uri? +} diff --git a/app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/parser/AccessibilityParserImpl.kt b/app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/parser/AccessibilityParserImpl.kt new file mode 100644 index 000000000..6bd403ab2 --- /dev/null +++ b/app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/parser/AccessibilityParserImpl.kt @@ -0,0 +1,38 @@ +package com.x8bit.bitwarden.data.autofill.accessibility.parser + +import android.net.Uri +import android.view.accessibility.AccessibilityNodeInfo +import androidx.core.net.toUri +import com.x8bit.bitwarden.data.autofill.accessibility.model.FillableFields +import com.x8bit.bitwarden.data.autofill.accessibility.util.getSupportedBrowserOrNull + +/** + * The default implementation for the [AccessibilityParser]. + */ +class AccessibilityParserImpl : AccessibilityParser { + override fun parseForFillableFields(rootNode: AccessibilityNodeInfo): FillableFields { + // TODO: Parse for username and password fields (PM-11486) + return FillableFields( + usernameFields = listOf(), + passwordFields = listOf(), + ) + } + + override fun parseForUriOrPackageName(rootNode: AccessibilityNodeInfo): Uri? { + val packageName = rootNode.packageName.toString() + val browser = packageName + .getSupportedBrowserOrNull() + ?: return "androidapp://$packageName".toUri() + return browser + .possibleUrlFieldIds + .flatMap { viewId -> + rootNode + .findAccessibilityNodeInfosByViewId("$packageName:id/$viewId") + .map { accessibilityNodeInfo -> + browser.urlExtractor(accessibilityNodeInfo.text.toString()) + } + } + .firstOrNull() + ?.toUri() + } +} diff --git a/app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/util/BrowserUtil.kt b/app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/util/BrowserUtil.kt new file mode 100644 index 000000000..70440ea8f --- /dev/null +++ b/app/src/main/java/com/x8bit/bitwarden/data/autofill/accessibility/util/BrowserUtil.kt @@ -0,0 +1,219 @@ +package com.x8bit.bitwarden.data.autofill.accessibility.util + +import com.x8bit.bitwarden.data.autofill.accessibility.model.Browser + +/** + * Determines if the [String] receiver is a package name for a supported browser and returns that + * [Browser] if it is a match. + */ +fun String.getSupportedBrowserOrNull(): Browser? = + ACCESSIBILITY_SUPPORTED_BROWSERS.find { it.packageName == this@getSupportedBrowserOrNull } + +/** + * A list of supported browsers and the field ID used to find the url bar. + * + * This list should be kept in order and match the list of compatibility browsers in the + * autofill_service_configuration.xml. + */ +private val ACCESSIBILITY_SUPPORTED_BROWSERS = listOf( + Browser(packageName = "alook.browser", urlFieldId = "search_fragment_input_view"), + Browser(packageName = "alook.browser.google", urlFieldId = "search_fragment_input_view"), + Browser(packageName = "app.vanadium.browser", urlFieldId = "url_bar"), + Browser(packageName = "com.amazon.cloud9", urlFieldId = "url"), + Browser(packageName = "com.android.browser", urlFieldId = "url"), + Browser(packageName = "com.android.chrome", urlFieldId = "url_bar"), + // "com.android.htmlviewer": Doesn't have a URL bar + Browser(packageName = "com.avast.android.secure.browser", urlFieldId = "editor"), + Browser(packageName = "com.avg.android.secure.browser", urlFieldId = "editor"), + Browser(packageName = "com.brave.browser", urlFieldId = "url_bar"), + Browser(packageName = "com.brave.browser_beta", urlFieldId = "url_bar"), + Browser(packageName = "com.brave.browser_default", urlFieldId = "url_bar"), + Browser(packageName = "com.brave.browser_dev", urlFieldId = "url_bar"), + Browser(packageName = "com.brave.browser_nightly", urlFieldId = "url_bar"), + Browser(packageName = "com.chrome.beta", urlFieldId = "url_bar"), + Browser(packageName = "com.chrome.canary", urlFieldId = "url_bar"), + Browser(packageName = "com.chrome.dev", urlFieldId = "url_bar"), + Browser(packageName = "com.cookiegames.smartcookie", urlFieldId = "search"), + Browser( + packageName = "com.cookiejarapps.android.smartcookieweb", + urlFieldId = "mozac_browser_toolbar_url_view", + ), + Browser(packageName = "com.duckduckgo.mobile.android", urlFieldId = "omnibarTextInput"), + Browser(packageName = "com.ecosia.android", urlFieldId = "url_bar"), + Browser(packageName = "com.google.android.apps.chrome", urlFieldId = "url_bar"), + Browser(packageName = "com.google.android.apps.chrome_dev", urlFieldId = "url_bar"), + // "com.google.android.captiveportallogin": URL displayed in ActionBar subtitle without viewId + Browser(packageName = "com.iode.firefox", urlFieldId = "mozac_browser_toolbar_url_view"), + Browser(packageName = "com.jamal2367.styx", urlFieldId = "search"), + Browser(packageName = "com.kiwibrowser.browser", urlFieldId = "url_bar"), + Browser(packageName = "com.kiwibrowser.browser.dev", urlFieldId = "url_bar"), + Browser(packageName = "com.microsoft.emmx", urlFieldId = "url_bar"), + Browser(packageName = "com.microsoft.emmx.beta", urlFieldId = "url_bar"), + Browser(packageName = "com.microsoft.emmx.canary", urlFieldId = "url_bar"), + Browser(packageName = "com.microsoft.emmx.dev", urlFieldId = "url_bar"), + Browser(packageName = "com.mmbox.browser", urlFieldId = "search_box"), + Browser(packageName = "com.mmbox.xbrowser", urlFieldId = "search_box"), + Browser(packageName = "com.mycompany.app.soulbrowser", urlFieldId = "edit_text"), + Browser(packageName = "com.naver.whale", urlFieldId = "url_bar"), + Browser(packageName = "com.neeva.app", urlFieldId = "full_url_text_view"), + Browser(packageName = "com.opera.browser", urlFieldId = "url_field"), + Browser(packageName = "com.opera.browser.beta", urlFieldId = "url_field"), + Browser(packageName = "com.opera.gx", urlFieldId = "addressbarEdit"), + Browser(packageName = "com.opera.mini.native", urlFieldId = "url_field"), + Browser(packageName = "com.opera.mini.native.beta", urlFieldId = "url_field"), + Browser(packageName = "com.opera.touch", urlFieldId = "addressbarEdit"), + Browser(packageName = "com.qflair.browserq", urlFieldId = "url"), + Browser( + packageName = "com.qwant.liberty", + // 2nd = Legacy (before v4) + possibleUrlFieldIds = listOf("mozac_browser_toolbar_url_view", "url_bar_title"), + ), + Browser(packageName = "com.rainsee.create", urlFieldId = "search_box"), + Browser(packageName = "com.sec.android.app.sbrowser", urlFieldId = "location_bar_edit_text"), + Browser( + packageName = "com.sec.android.app.sbrowser.beta", + urlFieldId = "location_bar_edit_text", + ), + Browser(packageName = "com.stoutner.privacybrowser.free", urlFieldId = "url_edittext"), + Browser(packageName = "com.stoutner.privacybrowser.standard", urlFieldId = "url_edittext"), + Browser(packageName = "com.vivaldi.browser", urlFieldId = "url_bar"), + Browser(packageName = "com.vivaldi.browser.snapshot", urlFieldId = "url_bar"), + Browser(packageName = "com.vivaldi.browser.sopranos", urlFieldId = "url_bar"), + Browser( + packageName = "com.yandex.browser", + possibleUrlFieldIds = listOf( + "bro_omnibar_address_title_text", + "bro_omnibox_collapsed_title", + ), + urlExtractor = { + // 0 = Regular Space, 1 = No-break space (00A0) + it.split(' ', ' ').firstOrNull() + }, + ), + Browser(packageName = "com.yjllq.internet", urlFieldId = "search_box"), + Browser(packageName = "com.yjllq.kito", urlFieldId = "search_box"), + Browser(packageName = "com.yujian.ResideMenuDemo", urlFieldId = "search_box"), + Browser(packageName = "com.z28j.feel", urlFieldId = "g2"), + Browser(packageName = "idm.internet.download.manager", urlFieldId = "search"), + Browser(packageName = "idm.internet.download.manager.adm.lite", urlFieldId = "search"), + Browser(packageName = "idm.internet.download.manager.plus", urlFieldId = "search"), + Browser( + packageName = "io.github.forkmaintainers.iceraven", + urlFieldId = "mozac_browser_toolbar_url_view", + ), + Browser(packageName = "mark.via", urlFieldId = "am,an"), + Browser(packageName = "mark.via.gp", urlFieldId = "as"), + Browser(packageName = "net.dezor.browser", urlFieldId = "url_bar"), + Browser(packageName = "net.slions.fulguris.full.download", urlFieldId = "search"), + Browser(packageName = "net.slions.fulguris.full.download.debug", urlFieldId = "search"), + Browser(packageName = "net.slions.fulguris.full.playstore", urlFieldId = "search"), + Browser(packageName = "net.slions.fulguris.full.playstore.debug", urlFieldId = "search"), + Browser( + packageName = "org.adblockplus.browser", + // 2nd = Legacy (before v2) + possibleUrlFieldIds = listOf("url_bar", "url_bar_title"), + ), + Browser( + packageName = "org.adblockplus.browser.beta", + // 2nd = Legacy (before v2) + possibleUrlFieldIds = listOf("url_bar", "url_bar_title"), + ), + Browser(packageName = "org.bromite.bromite", urlFieldId = "url_bar"), + Browser(packageName = "org.bromite.chromium", urlFieldId = "url_bar"), + Browser(packageName = "org.chromium.chrome", urlFieldId = "url_bar"), + Browser(packageName = "org.codeaurora.swe.browser", urlFieldId = "url_bar"), + Browser(packageName = "org.cromite.cromite", urlFieldId = "url_bar"), + Browser( + packageName = "org.gnu.icecat", + // 2nd = Anticipation + possibleUrlFieldIds = listOf("url_bar_title", "mozac_browser_toolbar_url_view"), + ), + Browser(packageName = "org.mozilla.fenix", urlFieldId = "mozac_browser_toolbar_url_view"), + // [DEPRECATED ENTRY] + Browser( + packageName = "org.mozilla.fenix.nightly", + urlFieldId = "mozac_browser_toolbar_url_view", + ), + // [DEPRECATED ENTRY] + Browser( + packageName = "org.mozilla.fennec_aurora", + possibleUrlFieldIds = listOf("mozac_browser_toolbar_url_view", "url_bar_title"), + ), + Browser( + packageName = "org.mozilla.fennec_fdroid", + // 2nd = Legacy + possibleUrlFieldIds = listOf("mozac_browser_toolbar_url_view", "url_bar_title"), + ), + Browser( + packageName = "org.mozilla.firefox", + // 2nd = Legacy + possibleUrlFieldIds = listOf("mozac_browser_toolbar_url_view", "url_bar_title"), + ), + Browser( + packageName = "org.mozilla.firefox_beta", + // 2nd = Legacy + possibleUrlFieldIds = listOf("mozac_browser_toolbar_url_view", "url_bar_title"), + ), + Browser( + packageName = "org.mozilla.focus", + // 2nd = Legacy + possibleUrlFieldIds = listOf("mozac_browser_toolbar_url_view", "display_url"), + ), + Browser( + packageName = "org.mozilla.focus.beta", + // 2nd = Legacy + possibleUrlFieldIds = listOf("mozac_browser_toolbar_url_view", "display_url"), + ), + Browser( + packageName = "org.mozilla.focus.nightly", + // 2nd = Legacy + possibleUrlFieldIds = listOf("mozac_browser_toolbar_url_view", "display_url"), + ), + Browser( + packageName = "org.mozilla.klar", + // 2nd = Legacy + possibleUrlFieldIds = listOf("mozac_browser_toolbar_url_view", "display_url"), + ), + Browser( + packageName = "org.mozilla.reference.browser", + urlFieldId = "mozac_browser_toolbar_url_view", + ), + Browser(packageName = "org.mozilla.rocket", urlFieldId = "display_url"), + Browser( + packageName = "org.torproject.torbrowser", + // 2nd = Legacy (before v10.0.3) + possibleUrlFieldIds = listOf("mozac_browser_toolbar_url_view", "url_bar_title"), + ), + Browser( + packageName = "org.torproject.torbrowser_alpha", + // 2nd = Legacy (before v10.0a8) + possibleUrlFieldIds = listOf("mozac_browser_toolbar_url_view", "url_bar_title"), + ), + Browser(packageName = "org.ungoogled.chromium.extensions.stable", urlFieldId = "url_bar"), + Browser(packageName = "org.ungoogled.chromium.stable", urlFieldId = "url_bar"), + Browser( + packageName = "us.spotco.fennec_dos", + // 2nd = Legacy + possibleUrlFieldIds = listOf("mozac_browser_toolbar_url_view", "url_bar_title"), + ), + + // [Section B] Entries only present here + // TODO: Test the compatibility of these with Autofill Framework + Browser(packageName = "acr.browser.barebones", urlFieldId = "search"), + Browser(packageName = "acr.browser.lightning", urlFieldId = "search"), + Browser(packageName = "com.feedback.browser.wjbrowser", urlFieldId = "addressbar_url"), + Browser(packageName = "com.ghostery.android.ghostery", urlFieldId = "search_field"), + Browser(packageName = "com.htc.sense.browser", urlFieldId = "title"), + Browser(packageName = "com.jerky.browser2", urlFieldId = "enterUrl"), + Browser(packageName = "com.ksmobile.cb", urlFieldId = "address_bar_edit_text"), + Browser(packageName = "com.lemurbrowser.exts", urlFieldId = "url_bar"), + Browser(packageName = "com.linkbubble.playstore", urlFieldId = "url_text"), + Browser(packageName = "com.mx.browser", urlFieldId = "address_editor_with_progress"), + Browser(packageName = "com.mx.browser.tablet", urlFieldId = "address_editor_with_progress"), + Browser(packageName = "com.nubelacorp.javelin", urlFieldId = "enterUrl"), + Browser(packageName = "jp.co.fenrir.android.sleipnir", urlFieldId = "url_text"), + Browser(packageName = "jp.co.fenrir.android.sleipnir_black", urlFieldId = "url_text"), + Browser(packageName = "jp.co.fenrir.android.sleipnir_test", urlFieldId = "url_text"), + Browser(packageName = "mobi.mgeek.TunnyBrowser", urlFieldId = "title"), + Browser(packageName = "org.iron.srware", urlFieldId = "url_bar"), +) diff --git a/app/src/main/res/xml/autofill_service_configuration.xml b/app/src/main/res/xml/autofill_service_configuration.xml index e745505b2..4f0e636cb 100644 --- a/app/src/main/res/xml/autofill_service_configuration.xml +++ b/app/src/main/res/xml/autofill_service_configuration.xml @@ -3,7 +3,10 @@ xmlns:tools="http://schemas.android.com/tools" android:supportsInlineSuggestions="true" tools:ignore="UnusedAttribute"> - + diff --git a/app/src/test/java/com/x8bit/bitwarden/data/autofill/accessibility/parser/AccessibilityParserTest.kt b/app/src/test/java/com/x8bit/bitwarden/data/autofill/accessibility/parser/AccessibilityParserTest.kt new file mode 100644 index 000000000..163df4482 --- /dev/null +++ b/app/src/test/java/com/x8bit/bitwarden/data/autofill/accessibility/parser/AccessibilityParserTest.kt @@ -0,0 +1,84 @@ +package com.x8bit.bitwarden.data.autofill.accessibility.parser + +import android.view.accessibility.AccessibilityNodeInfo +import androidx.core.net.toUri +import com.x8bit.bitwarden.data.autofill.accessibility.model.Browser +import com.x8bit.bitwarden.data.autofill.accessibility.model.FillableFields +import io.mockk.every +import io.mockk.mockk +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.junit.jupiter.api.Test + +class AccessibilityParserTest { + + private val accessibilityParser: AccessibilityParser = AccessibilityParserImpl() + + @Test + fun `parseForFillableFields should return empty data`() { + val rootNode = mockk() + val expectedResult = FillableFields( + usernameFields = emptyList(), + passwordFields = emptyList(), + ) + + val result = accessibilityParser.parseForFillableFields(rootNode = rootNode) + + assertEquals(expectedResult, result) + } + + @Suppress("MaxLineLength") + @Test + fun `parseForUriOrPackageName should return the package name as a URI when not a supported browser`() { + val testPackageName = "testPackageName" + val rootNode = mockk { + every { packageName } returns testPackageName + } + val expectedResult = "androidapp://$testPackageName".toUri() + + val result = accessibilityParser.parseForUriOrPackageName(rootNode = rootNode) + + assertEquals(expectedResult, result) + } + + @Suppress("MaxLineLength") + @Test + fun `parseForUriOrPackageName should return null when package is a supported browser and URL bar is not found`() { + val testBrowser = Browser(packageName = "com.android.chrome", urlFieldId = "url_bar") + val rootNode = mockk { + every { packageName } returns testBrowser.packageName + every { + findAccessibilityNodeInfosByViewId( + "$packageName:id/${testBrowser.possibleUrlFieldIds.first()}", + ) + } returns emptyList() + } + + val result = accessibilityParser.parseForUriOrPackageName(rootNode = rootNode) + + assertNull(result) + } + + @Suppress("MaxLineLength") + @Test + fun `parseForUriOrPackageName should return the site url as a URI when package is a supported browser and URL is found`() { + val testBrowser = Browser(packageName = "com.android.chrome", urlFieldId = "url_bar") + val url = "https://www.google.com" + val urlNode = mockk { + every { text } returns url + } + val rootNode = mockk { + every { packageName } returns testBrowser.packageName + every { + findAccessibilityNodeInfosByViewId( + "$packageName:id/${testBrowser.possibleUrlFieldIds.first()}", + ) + } returns listOf(urlNode) + } + val expectedResult = url.toUri() + + val result = accessibilityParser.parseForUriOrPackageName(rootNode = rootNode) + + assertEquals(expectedResult, result) + } +}