From a5961357d0285291fafeabd5be53daf8d19627b5 Mon Sep 17 00:00:00 2001 From: Drew Willcoxon Date: Wed, 7 Feb 2024 20:07:24 +0000 Subject: [PATCH] Bug 1878987 - Part 1: Vendor application-services into mozilla-central to pick up Suggest changes. r=lina Differential Revision: https://phabricator.services.mozilla.com/D200891 --- .cargo/config.in | 4 +- Cargo.lock | 26 +- Cargo.toml | 14 +- .../rust/error-support/.cargo-checksum.json | 2 +- third_party/rust/error-support/Cargo.toml | 1 + .../rust/error-support/android/build.gradle | 10 - .../android/src/main/AndroidManifest.xml | 1 - .../rust/remote_settings/.cargo-checksum.json | 2 +- third_party/rust/remote_settings/Cargo.toml | 5 +- .../remote_settings/src/remote_settings.udl | 6 +- .../rust/sql-support/.cargo-checksum.json | 2 +- third_party/rust/sql-support/Cargo.toml | 2 +- third_party/rust/suggest/.cargo-checksum.json | 2 +- third_party/rust/suggest/Cargo.toml | 8 +- third_party/rust/suggest/src/config.rs | 31 + third_party/rust/suggest/src/db.rs | 719 ++++++-- third_party/rust/suggest/src/error.rs | 48 +- third_party/rust/suggest/src/lib.rs | 4 +- third_party/rust/suggest/src/provider.rs | 4 + third_party/rust/suggest/src/rs.rs | 61 +- third_party/rust/suggest/src/schema.rs | 19 +- third_party/rust/suggest/src/store.rs | 1507 +++++++++++++++-- third_party/rust/suggest/src/suggest.udl | 57 +- third_party/rust/suggest/src/suggestion.rs | 40 + third_party/rust/suggest/src/yelp.rs | 122 +- third_party/rust/sync15/.cargo-checksum.json | 2 +- third_party/rust/sync15/Cargo.toml | 2 +- third_party/rust/tabs/.cargo-checksum.json | 2 +- third_party/rust/tabs/Cargo.toml | 8 +- .../rust/webext-storage/.cargo-checksum.json | 2 +- third_party/rust/webext-storage/Cargo.toml | 3 +- .../rust/webext-storage/android/build.gradle | 22 - .../android/src/main/AndroidManifest.xml | 2 - .../webextstorage/WebExtStorageTest.kt | 76 - 34 files changed, 2258 insertions(+), 558 deletions(-) delete mode 100644 third_party/rust/error-support/android/build.gradle delete mode 100644 third_party/rust/error-support/android/src/main/AndroidManifest.xml create mode 100644 third_party/rust/suggest/src/config.rs delete mode 100644 third_party/rust/webext-storage/android/build.gradle delete mode 100644 third_party/rust/webext-storage/android/src/main/AndroidManifest.xml delete mode 100644 third_party/rust/webext-storage/android/src/test/java/mozilla/appservices/webextstorage/WebExtStorageTest.kt diff --git a/.cargo/config.in b/.cargo/config.in index f3d21abdc2f4..9807a486aa24 100644 --- a/.cargo/config.in +++ b/.cargo/config.in @@ -60,9 +60,9 @@ git = "https://github.com/mozilla-spidermonkey/jsparagus" rev = "61f399c53a641ebd3077c1f39f054f6d396a633c" replace-with = "vendored-sources" -[source."git+https://github.com/mozilla/application-services?rev=9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253"] +[source."git+https://github.com/mozilla/application-services?rev=41367fda038268843a87c459698691f33f0d9063"] git = "https://github.com/mozilla/application-services" -rev = "9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253" +rev = "41367fda038268843a87c459698691f33f0d9063" replace-with = "vendored-sources" [source."git+https://github.com/mozilla/audioipc?rev=596bdb7fbb5745ea415726e16bd497e6c850a540"] diff --git a/Cargo.lock b/Cargo.lock index 5de8b97fc54d..29a3e3a3949b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1585,7 +1585,7 @@ dependencies = [ [[package]] name = "error-support" version = "0.1.0" -source = "git+https://github.com/mozilla/application-services?rev=9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253#9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253" +source = "git+https://github.com/mozilla/application-services?rev=41367fda038268843a87c459698691f33f0d9063#41367fda038268843a87c459698691f33f0d9063" dependencies = [ "error-support-macros", "lazy_static", @@ -1597,7 +1597,7 @@ dependencies = [ [[package]] name = "error-support-macros" version = "0.1.0" -source = "git+https://github.com/mozilla/application-services?rev=9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253#9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253" +source = "git+https://github.com/mozilla/application-services?rev=41367fda038268843a87c459698691f33f0d9063#41367fda038268843a87c459698691f33f0d9063" dependencies = [ "proc-macro2", "quote", @@ -2855,7 +2855,7 @@ dependencies = [ [[package]] name = "interrupt-support" version = "0.1.0" -source = "git+https://github.com/mozilla/application-services?rev=9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253#9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253" +source = "git+https://github.com/mozilla/application-services?rev=41367fda038268843a87c459698691f33f0d9063#41367fda038268843a87c459698691f33f0d9063" dependencies = [ "lazy_static", "parking_lot", @@ -4041,7 +4041,7 @@ dependencies = [ [[package]] name = "nss_build_common" version = "0.1.0" -source = "git+https://github.com/mozilla/application-services?rev=9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253#9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253" +source = "git+https://github.com/mozilla/application-services?rev=41367fda038268843a87c459698691f33f0d9063#41367fda038268843a87c459698691f33f0d9063" [[package]] name = "nsstring" @@ -4717,7 +4717,7 @@ checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "remote_settings" version = "0.1.0" -source = "git+https://github.com/mozilla/application-services?rev=9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253#9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253" +source = "git+https://github.com/mozilla/application-services?rev=41367fda038268843a87c459698691f33f0d9063#41367fda038268843a87c459698691f33f0d9063" dependencies = [ "parking_lot", "serde", @@ -5240,7 +5240,7 @@ dependencies = [ [[package]] name = "sql-support" version = "0.1.0" -source = "git+https://github.com/mozilla/application-services?rev=9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253#9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253" +source = "git+https://github.com/mozilla/application-services?rev=41367fda038268843a87c459698691f33f0d9063#41367fda038268843a87c459698691f33f0d9063" dependencies = [ "ffi-support", "interrupt-support", @@ -5421,10 +5421,11 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "suggest" version = "0.1.0" -source = "git+https://github.com/mozilla/application-services?rev=9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253#9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253" +source = "git+https://github.com/mozilla/application-services?rev=41367fda038268843a87c459698691f33f0d9063#41367fda038268843a87c459698691f33f0d9063" dependencies = [ "anyhow", "chrono", + "error-support", "interrupt-support", "once_cell", "parking_lot", @@ -5436,6 +5437,7 @@ dependencies = [ "thiserror", "uniffi", "url", + "viaduct", ] [[package]] @@ -5468,7 +5470,7 @@ dependencies = [ [[package]] name = "sync-guid" version = "0.1.0" -source = "git+https://github.com/mozilla/application-services?rev=9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253#9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253" +source = "git+https://github.com/mozilla/application-services?rev=41367fda038268843a87c459698691f33f0d9063#41367fda038268843a87c459698691f33f0d9063" dependencies = [ "base64 0.21.3", "rand", @@ -5479,7 +5481,7 @@ dependencies = [ [[package]] name = "sync15" version = "0.1.0" -source = "git+https://github.com/mozilla/application-services?rev=9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253#9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253" +source = "git+https://github.com/mozilla/application-services?rev=41367fda038268843a87c459698691f33f0d9063#41367fda038268843a87c459698691f33f0d9063" dependencies = [ "anyhow", "error-support", @@ -5511,7 +5513,7 @@ dependencies = [ [[package]] name = "tabs" version = "0.1.0" -source = "git+https://github.com/mozilla/application-services?rev=9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253#9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253" +source = "git+https://github.com/mozilla/application-services?rev=41367fda038268843a87c459698691f33f0d9063#41367fda038268843a87c459698691f33f0d9063" dependencies = [ "anyhow", "error-support", @@ -6161,7 +6163,7 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "viaduct" version = "0.1.0" -source = "git+https://github.com/mozilla/application-services?rev=9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253#9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253" +source = "git+https://github.com/mozilla/application-services?rev=41367fda038268843a87c459698691f33f0d9063#41367fda038268843a87c459698691f33f0d9063" dependencies = [ "ffi-support", "log", @@ -6307,7 +6309,7 @@ dependencies = [ [[package]] name = "webext-storage" version = "0.1.0" -source = "git+https://github.com/mozilla/application-services?rev=9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253#9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253" +source = "git+https://github.com/mozilla/application-services?rev=41367fda038268843a87c459698691f33f0d9063#41367fda038268843a87c459698691f33f0d9063" dependencies = [ "anyhow", "error-support", diff --git a/Cargo.toml b/Cargo.toml index 9eb36091413b..16af5d84843c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -206,13 +206,13 @@ warp = { git = "https://github.com/seanmonstar/warp", rev = "9d081461ae1167eb321 malloc_size_of_derive = { path = "xpcom/rust/malloc_size_of_derive" } # application-services overrides to make updating them all simpler. -interrupt-support = { git = "https://github.com/mozilla/application-services", rev = "9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253" } -sql-support = { git = "https://github.com/mozilla/application-services", rev = "9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253" } -suggest = { git = "https://github.com/mozilla/application-services", rev = "9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253" } -sync15 = { git = "https://github.com/mozilla/application-services", rev = "9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253" } -tabs = { git = "https://github.com/mozilla/application-services", rev = "9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253" } -viaduct = { git = "https://github.com/mozilla/application-services", rev = "9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253" } -webext-storage = { git = "https://github.com/mozilla/application-services", rev = "9d1ab279d2aa31e4c6c80dd1ee0baf9345e26253" } +interrupt-support = { git = "https://github.com/mozilla/application-services", rev = "41367fda038268843a87c459698691f33f0d9063" } +sql-support = { git = "https://github.com/mozilla/application-services", rev = "41367fda038268843a87c459698691f33f0d9063" } +suggest = { git = "https://github.com/mozilla/application-services", rev = "41367fda038268843a87c459698691f33f0d9063" } +sync15 = { git = "https://github.com/mozilla/application-services", rev = "41367fda038268843a87c459698691f33f0d9063" } +tabs = { git = "https://github.com/mozilla/application-services", rev = "41367fda038268843a87c459698691f33f0d9063" } +viaduct = { git = "https://github.com/mozilla/application-services", rev = "41367fda038268843a87c459698691f33f0d9063" } +webext-storage = { git = "https://github.com/mozilla/application-services", rev = "41367fda038268843a87c459698691f33f0d9063" } # Patch mio 0.8.8 to use windows-sys 0.52 (backport https://github.com/tokio-rs/mio/commit/eea9e3e0c469480e5c59c01e6c3c7e5fd88f0848) mio_0_8 = { package = "mio", git = "https://github.com/glandium/mio", rev = "9a2ef335c366044ffe73b1c4acabe50a1daefe05" } diff --git a/third_party/rust/error-support/.cargo-checksum.json b/third_party/rust/error-support/.cargo-checksum.json index bace7441572f..475bff76658f 100644 --- a/third_party/rust/error-support/.cargo-checksum.json +++ b/third_party/rust/error-support/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"cf73fe7b6066cb2ccccb0939f19b14d4e27d9cfdc20c9e527de3210029f2ee6f","README.md":"8030b4a314b1be31ba018ac12c3b586bb736db5307c3c395f2857fffe0130322","android/build.gradle":"d4ecda8eebf9c1b3c7542ca86652a3e8c0d2dfc0ad7426e78447a35e4fb39eab","android/src/main/AndroidManifest.xml":"108cabbbdc93da70e1da3e60b74171580872017d996c20e37946c27aaa078031","build.rs":"c8d3c38c1208eea36224662b284d8daf3e7ad1b07d22d750524f3da1cc66ccca","src/errorsupport.udl":"e793034d01a2608298528051757f38405e006ee1abc4cf65dc6f18c53590ace8","src/handling.rs":"6e0568b18d426531cb2ae9967c8dd0d51ece5a065f68b15eeb308b995edaa167","src/lib.rs":"96ae3cc2c1077ae45442ace6b5b5311b86267d0b9067f3ff58396af30ccbbc07","src/macros.rs":"0d03f82fab20c96a182f941baf3fcf2a286b00fea871ee7fd8e339abc14f9522","src/redact.rs":"c9a4df1a87be68b15d583587bda941d4c60a1d0449e2d43ff99f3611a290a863","src/reporting.rs":"38efd24d86ba8facfb181cb27e8b698d2831db0afab85691ffda034a4dc68dfa","uniffi.toml":"644fe81c12fe3c01ee81e017ca3c00d0e611f014b7eade51aadaf208179a3450"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"49ef90bd388b59229db34b35fe06eb769183431c88b5712e6e9992851aef605d","README.md":"8030b4a314b1be31ba018ac12c3b586bb736db5307c3c395f2857fffe0130322","build.rs":"c8d3c38c1208eea36224662b284d8daf3e7ad1b07d22d750524f3da1cc66ccca","src/errorsupport.udl":"e793034d01a2608298528051757f38405e006ee1abc4cf65dc6f18c53590ace8","src/handling.rs":"6e0568b18d426531cb2ae9967c8dd0d51ece5a065f68b15eeb308b995edaa167","src/lib.rs":"96ae3cc2c1077ae45442ace6b5b5311b86267d0b9067f3ff58396af30ccbbc07","src/macros.rs":"0d03f82fab20c96a182f941baf3fcf2a286b00fea871ee7fd8e339abc14f9522","src/redact.rs":"c9a4df1a87be68b15d583587bda941d4c60a1d0449e2d43ff99f3611a290a863","src/reporting.rs":"38efd24d86ba8facfb181cb27e8b698d2831db0afab85691ffda034a4dc68dfa","uniffi.toml":"644fe81c12fe3c01ee81e017ca3c00d0e611f014b7eade51aadaf208179a3450"},"package":null} \ No newline at end of file diff --git a/third_party/rust/error-support/Cargo.toml b/third_party/rust/error-support/Cargo.toml index e6d62999218b..e4c39618a64a 100644 --- a/third_party/rust/error-support/Cargo.toml +++ b/third_party/rust/error-support/Cargo.toml @@ -14,6 +14,7 @@ edition = "2021" name = "error-support" version = "0.1.0" authors = ["Thom Chiovoloni "] +exclude = ["/android"] autotests = false readme = "README.md" license = "MPL-2.0" diff --git a/third_party/rust/error-support/android/build.gradle b/third_party/rust/error-support/android/build.gradle deleted file mode 100644 index 7d42f2e79fef..000000000000 --- a/third_party/rust/error-support/android/build.gradle +++ /dev/null @@ -1,10 +0,0 @@ -apply from: "$rootDir/build-scripts/component-common.gradle" -apply from: "$rootDir/publish.gradle" - -android { - namespace 'org.mozilla.appservices.errorsupport' -} - -ext.configureUniFFIBindgen("../src/errorsupport.udl") -ext.dependsOnTheMegazord() -ext.configurePublish() diff --git a/third_party/rust/error-support/android/src/main/AndroidManifest.xml b/third_party/rust/error-support/android/src/main/AndroidManifest.xml deleted file mode 100644 index c4e6c98d775b..000000000000 --- a/third_party/rust/error-support/android/src/main/AndroidManifest.xml +++ /dev/null @@ -1 +0,0 @@ - diff --git a/third_party/rust/remote_settings/.cargo-checksum.json b/third_party/rust/remote_settings/.cargo-checksum.json index 28ac46a2c25b..e4294a63b949 100644 --- a/third_party/rust/remote_settings/.cargo-checksum.json +++ b/third_party/rust/remote_settings/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"cf0a08d6b0d6285a459b78115aa24818a04b5987652655e64b80ffd8c8ae0813","build.rs":"4326f03729cf8f1673e4228e6dc111de1ea4d8bcc06351f7ae563efb2613f866","src/client.rs":"3d87162e6913a81cc6f5178a7ca791e262d0d029e7dedf3df4fe2f66e5501185","src/config.rs":"7bb678addfae3b4ed5f2892d32263e5b33cc05e5a12a250f664150e78211f94a","src/error.rs":"192ca42af7c6b882f3129378c23b45dab8a0d2b179e23a8813a335ffd56b21dc","src/lib.rs":"416e99894e152f6cea7418ad2fabfd94bc3d907efd9f33fbd2a83fb99452b2df","src/remote_settings.udl":"e38758592ca75adbebb8fe688b10520d9931a5f3292d94f229cba05310756a43","uniffi.toml":"f8ec8dc593e0d501c2e9e40368ec93ec33b1edd8608e29495e0a54b63144e880"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"4fa89b0606fe8ec8ac8c479b8b9adf33d0c936b09fa5af108ded74139ace37fb","build.rs":"4326f03729cf8f1673e4228e6dc111de1ea4d8bcc06351f7ae563efb2613f866","src/client.rs":"3d87162e6913a81cc6f5178a7ca791e262d0d029e7dedf3df4fe2f66e5501185","src/config.rs":"7bb678addfae3b4ed5f2892d32263e5b33cc05e5a12a250f664150e78211f94a","src/error.rs":"192ca42af7c6b882f3129378c23b45dab8a0d2b179e23a8813a335ffd56b21dc","src/lib.rs":"416e99894e152f6cea7418ad2fabfd94bc3d907efd9f33fbd2a83fb99452b2df","src/remote_settings.udl":"2e71491ad3894d17e5bde0663d9490bfea6294d99cdbe9d67a36137faeedc593","uniffi.toml":"f8ec8dc593e0d501c2e9e40368ec93ec33b1edd8608e29495e0a54b63144e880"},"package":null} \ No newline at end of file diff --git a/third_party/rust/remote_settings/Cargo.toml b/third_party/rust/remote_settings/Cargo.toml index cb6aaea30702..b04e6ed6c60a 100644 --- a/third_party/rust/remote_settings/Cargo.toml +++ b/third_party/rust/remote_settings/Cargo.toml @@ -17,7 +17,10 @@ authors = [ "The Android Mobile Team ", "The Glean Team ", ] -exclude = ["/android"] +exclude = [ + "/android", + "/ios", +] description = "A Remote Settings client intended for application layer platforms." license = "MPL-2.0" diff --git a/third_party/rust/remote_settings/src/remote_settings.udl b/third_party/rust/remote_settings/src/remote_settings.udl index bb52874211aa..d830b6778fa3 100644 --- a/third_party/rust/remote_settings/src/remote_settings.udl +++ b/third_party/rust/remote_settings/src/remote_settings.udl @@ -8,9 +8,9 @@ typedef string RsJsonObject; namespace remote_settings {}; dictionary RemoteSettingsConfig { - string? server_url = null; - string? bucket_name = null; string collection_name; + string? bucket_name = null; + string? server_url = null; }; dictionary RemoteSettingsResponse { @@ -54,7 +54,7 @@ interface RemoteSettings { [Throws=RemoteSettingsError] RemoteSettingsResponse get_records(); - // Fetch all records added to the server since the provided timestamp, + // Fetch all records added to the server since the provided timestamp, // using the configuration this client was initialized with. [Throws=RemoteSettingsError] RemoteSettingsResponse get_records_since(u64 timestamp); diff --git a/third_party/rust/sql-support/.cargo-checksum.json b/third_party/rust/sql-support/.cargo-checksum.json index b0f5018d7060..bbfb487f7fab 100644 --- a/third_party/rust/sql-support/.cargo-checksum.json +++ b/third_party/rust/sql-support/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"691ec27ed968518a05aa650230c80abe07282c1feb483d40aae4ae95cbc046ef","src/conn_ext.rs":"e48e862e47c000c545dcc766fc1889498a8709bee00e240ed68d247b0fbef577","src/debug_tools.rs":"bece2bc3d35379b81ea2f942a0a3e909e0ab0553656505904745548eacaf402a","src/each_chunk.rs":"8aaba842e43b002fbc0fee95d14ce08faa7187b1979c765b2e270cd4802607a5","src/lib.rs":"af704ec04beb6c2c388d4566710e1167b18fb64acb248ccf37a67679daffddb6","src/maybe_cached.rs":"0b18425595055883a98807fbd62ff27a79c18af34e7cb3439f8c3438463ef2dd","src/open_database.rs":"40ad2da7d5559f0e5180e35d403c307ce230fe9d0d2a3fec7c9481ce13acda64","src/repeat.rs":"b4c5ff5d083afba7f9f153f54aba2e6859b78b85c82d48dbd6bd58f67da9e6b9"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"812811e5a8e00abe3ec345cd8fd435e27fec7cb8f2e45a0e93e5becf564c46ad","src/conn_ext.rs":"e48e862e47c000c545dcc766fc1889498a8709bee00e240ed68d247b0fbef577","src/debug_tools.rs":"bece2bc3d35379b81ea2f942a0a3e909e0ab0553656505904745548eacaf402a","src/each_chunk.rs":"8aaba842e43b002fbc0fee95d14ce08faa7187b1979c765b2e270cd4802607a5","src/lib.rs":"af704ec04beb6c2c388d4566710e1167b18fb64acb248ccf37a67679daffddb6","src/maybe_cached.rs":"0b18425595055883a98807fbd62ff27a79c18af34e7cb3439f8c3438463ef2dd","src/open_database.rs":"40ad2da7d5559f0e5180e35d403c307ce230fe9d0d2a3fec7c9481ce13acda64","src/repeat.rs":"b4c5ff5d083afba7f9f153f54aba2e6859b78b85c82d48dbd6bd58f67da9e6b9"},"package":null} \ No newline at end of file diff --git a/third_party/rust/sql-support/Cargo.toml b/third_party/rust/sql-support/Cargo.toml index 541383dd3904..0e6137ddbf05 100644 --- a/third_party/rust/sql-support/Cargo.toml +++ b/third_party/rust/sql-support/Cargo.toml @@ -40,7 +40,7 @@ features = [ ] [dev-dependencies.env_logger] -version = "0.7" +version = "0.10" default-features = false [build-dependencies.nss_build_common] diff --git a/third_party/rust/suggest/.cargo-checksum.json b/third_party/rust/suggest/.cargo-checksum.json index 941ee1bd4f04..4827ba5a2b05 100644 --- a/third_party/rust/suggest/.cargo-checksum.json +++ b/third_party/rust/suggest/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"7e5e03ee92f01a28fa398d77167619e5a55c73db083cc27b74c4d63ab44db173","README.md":"8d7457893194e255b87e5a2667ee25c87bd470f5338d7078506f866a67a3fdbd","build.rs":"78780c5cccfe22c3ff4198624b9e188559c437c3e6fa1c8bb66548eee6aa66bf","src/db.rs":"6d39e96bcdaa2b01dc662b3ed17b3a022246534b919b13be637e4646c3b9c1dd","src/error.rs":"f47763a1a5d228b446eb8f718433e49fdb1c7b304de1255891215144dddd7a43","src/keyword.rs":"988d0ab021c0df19cfd3c519df7d37f606bf984cd14d0efca4e5a7aff88344dd","src/lib.rs":"95ac0da8585ddfe712884e05ae53fd5fdf79564675a1070222e4a310a79959f4","src/pocket.rs":"c4dda43390d1c39dc795933596b3c1e4e282932cac6c69da53c6e05d39e9ef29","src/provider.rs":"8cecefb01d3c09c164d7334647e586c5407e14d0ee7ef99eea6e5095ec59586c","src/rs.rs":"1388ae7473d1d87f7eb731b1e34bd1d2f8ef844b7e0f09d7e3442cfc7d56286a","src/schema.rs":"3135d28652e3f0df6670a3afa3bdf03d95039383720f1f0d77523c0f4faf1797","src/store.rs":"6743136b02bcc3caec89ff1386a8ae5a5e89bab2bb095c60eb905f5c19735be9","src/suggest.udl":"e129cc04665a8484572c57c87b4a1713a7600254bf4110e06a5fe0d4eb746bb5","src/suggestion.rs":"dfe7c02e11ea9a01d3fdd71eb2c83b1c1d1b830c34f76829a91a01cda44c38b8","src/yelp.rs":"5d788854c3e1f9e5a2e55a8ecc4c5a10c1ad93ab84791535e4d88ec94602d615","uniffi.toml":"f26317442ddb5b3281245bef6e60ffcb78bb95d29fe4a351a56dbb88d4ec8aab"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"4aa81cff67e67b08ba3348c1acddaa5aee887df3c35006754c9cda4273a94458","README.md":"8d7457893194e255b87e5a2667ee25c87bd470f5338d7078506f866a67a3fdbd","build.rs":"78780c5cccfe22c3ff4198624b9e188559c437c3e6fa1c8bb66548eee6aa66bf","src/config.rs":"03630b2219b6674e332a1f96f44db74def17f985c850a800299b815fa72241c2","src/db.rs":"208916c915f29fa2fd8725f635822d63ac86023ea0eb2050a92d4c7fbc9f6697","src/error.rs":"f563210a6c050d98ec85e0f6d9401e7373bfb816e865e8edabbabb23d848ba13","src/keyword.rs":"988d0ab021c0df19cfd3c519df7d37f606bf984cd14d0efca4e5a7aff88344dd","src/lib.rs":"65a035dbfb17e2d2d9f237ad52dc03982ae28c70e3dcf3d96cc9f2d7af79efe3","src/pocket.rs":"c4dda43390d1c39dc795933596b3c1e4e282932cac6c69da53c6e05d39e9ef29","src/provider.rs":"3fe8f90d77586f5ff683374f24df026110bfaaab128fd4503d2c695eed71d4fe","src/rs.rs":"038e954eaeaa6898a2edb4d29728233a6286ac818b42028da1ba5c1b03a3efc0","src/schema.rs":"f7995c1cdd98c642e4497b5f3d0881a3ab6ac4f086172142c195adb0934b93de","src/store.rs":"b868b2853beba800cf43183f2b629a4024e435fa9617dcd751480ee89f8fe12f","src/suggest.udl":"9b82f97afb49d94a82cad3a9d44a25e8cab3b998b018ea9c3467b170b795e65b","src/suggestion.rs":"355f01b8a82a55a506e0a4c7d26dcf6d059802f84df3789309f5e0f42b80d793","src/yelp.rs":"2844e639bdf163fc23ba6e463495f47ec8d9c617de4754b49a58aa16d6ea39f5","uniffi.toml":"f26317442ddb5b3281245bef6e60ffcb78bb95d29fe4a351a56dbb88d4ec8aab"},"package":null} \ No newline at end of file diff --git a/third_party/rust/suggest/Cargo.toml b/third_party/rust/suggest/Cargo.toml index adeb0abc49a0..17ce1af26d08 100644 --- a/third_party/rust/suggest/Cargo.toml +++ b/third_party/rust/suggest/Cargo.toml @@ -30,6 +30,9 @@ serde_json = "1" thiserror = "1" uniffi = "0.25.2" +[dependencies.error-support] +path = "../support/error" + [dependencies.interrupt-support] path = "../support/interrupt" @@ -54,12 +57,15 @@ path = "../support/sql" version = "2.1" features = ["serde"] +[dependencies.viaduct] +path = "../viaduct" + [dev-dependencies] expect-test = "1.4" hex = "0.4" [dev-dependencies.env_logger] -version = "0.7" +version = "0.10" default-features = false [dev-dependencies.rc_crypto] diff --git a/third_party/rust/suggest/src/config.rs b/third_party/rust/suggest/src/config.rs new file mode 100644 index 000000000000..fcb3c2e256a4 --- /dev/null +++ b/third_party/rust/suggest/src/config.rs @@ -0,0 +1,31 @@ +use serde::{Deserialize, Serialize}; + +use crate::rs::{DownloadedGlobalConfig, DownloadedWeatherData}; + +/// Global Suggest configuration data. +#[derive(Clone, Default, Debug, Deserialize, Serialize)] +pub struct SuggestGlobalConfig { + pub show_less_frequently_cap: i32, +} + +impl From<&DownloadedGlobalConfig> for SuggestGlobalConfig { + fn from(config: &DownloadedGlobalConfig) -> Self { + Self { + show_less_frequently_cap: config.configuration.show_less_frequently_cap, + } + } +} + +/// Per-provider configuration data. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub enum SuggestProviderConfig { + Weather { min_keyword_length: i32 }, +} + +impl From<&DownloadedWeatherData> for SuggestProviderConfig { + fn from(data: &DownloadedWeatherData) -> Self { + Self::Weather { + min_keyword_length: data.weather.min_keyword_length, + } + } +} diff --git a/third_party/rust/suggest/src/db.rs b/third_party/rust/suggest/src/db.rs index 6344b76a4846..52f721167978 100644 --- a/third_party/rust/suggest/src/db.rs +++ b/third_party/rust/suggest/src/db.rs @@ -3,10 +3,11 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -use std::{path::Path, sync::Arc}; +use std::{collections::HashSet, path::Path, sync::Arc}; use interrupt_support::{SqlInterruptHandle, SqlInterruptScope}; use parking_lot::Mutex; +use remote_settings::RemoteSettingsRecord; use rusqlite::{ named_params, types::{FromSql, ToSql}, @@ -14,12 +15,15 @@ use rusqlite::{ }; use sql_support::{open_database::open_database_with_flags, ConnExt}; -use crate::rs::{DownloadedAmoSuggestion, DownloadedPocketSuggestion}; use crate::{ + config::{SuggestGlobalConfig, SuggestProviderConfig}, keyword::full_keyword, pocket::{split_keyword, KeywordConfidence}, provider::SuggestionProvider, - rs::{DownloadedAmpWikipediaSuggestion, SuggestRecordId}, + rs::{ + DownloadedAmoSuggestion, DownloadedAmpWikipediaSuggestion, DownloadedMdnSuggestion, + DownloadedPocketSuggestion, DownloadedWeatherData, SuggestRecordId, + }, schema::{SuggestConnectionInitializer, VERSION}, store::{UnparsableRecord, UnparsableRecords}, suggestion::{cook_raw_suggestion_url, Suggestion}, @@ -32,6 +36,16 @@ pub const LAST_INGEST_META_KEY: &str = "last_quicksuggest_ingest"; /// The metadata key whose value keeps track of records of suggestions /// that aren't parsable and which schema version it was first seen in. pub const UNPARSABLE_RECORDS_META_KEY: &str = "unparsable_records"; +/// The metadata key whose value is a JSON string encoding a +/// `SuggestGlobalConfig`, which contains global Suggest configuration data. +pub const GLOBAL_CONFIG_META_KEY: &str = "global_config"; +/// Prefix of metadata keys whose values are JSON strings encoding +/// `SuggestProviderConfig`, which contains per-provider configuration data. The +/// full key is this prefix plus the `SuggestionProvider` value as a u8. +pub const PROVIDER_CONFIG_META_KEY_PREFIX: &str = "provider_config_"; + +// Default value when Suggestion does not have a value for score +pub const DEFAULT_SUGGESTION_SCORE: f64 = 0.2; /// The database connection type. #[derive(Clone, Copy)] @@ -123,65 +137,93 @@ impl<'a> SuggestDao<'a> { Self { conn, scope } } + // =============== High level API =============== + // + // These methods combine several low-level calls into one logical operation. + + pub fn handle_unparsable_record(&mut self, record: &RemoteSettingsRecord) -> Result<()> { + let record_id = SuggestRecordId::from(&record.id); + // Remember this record's ID so that we will try again later + self.put_unparsable_record_id(&record_id)?; + // Advance the last fetch time, so that we can resume + // fetching after this record if we're interrupted. + self.put_last_ingest_if_newer(record.last_modified) + } + + pub fn handle_ingested_record(&mut self, record: &RemoteSettingsRecord) -> Result<()> { + let record_id = SuggestRecordId::from(&record.id); + // Remove this record's ID from the list of unparsable + // records, since we understand it now. + self.drop_unparsable_record_id(&record_id)?; + // Advance the last fetch time, so that we can resume + // fetching after this record if we're interrupted. + self.put_last_ingest_if_newer(record.last_modified) + } + + pub fn handle_deleted_record(&mut self, record: &RemoteSettingsRecord) -> Result<()> { + let record_id = SuggestRecordId::from(&record.id); + // Drop either the icon or suggestions, records only contain one or the other + match record_id.as_icon_id() { + Some(icon_id) => self.drop_icon(icon_id)?, + None => self.drop_suggestions(&record_id)?, + }; + // Remove this record's ID from the list of unparsable + // records, since we understand it now. + self.drop_unparsable_record_id(&record_id)?; + // Advance the last fetch time, so that we can resume + // fetching after this record if we're interrupted. + self.put_last_ingest_if_newer(record.last_modified) + } + + // =============== Low level API =============== + // + // These methods implement CRUD operations + /// Fetches suggestions that match the given query from the database. pub fn fetch_suggestions(&self, query: &SuggestionQuery) -> Result> { - if let Some(suggestion) = self.fetch_yelp_suggestion(query)? { - return Ok(vec![suggestion]); - } - - let keyword_lowercased = &query.keyword.to_lowercase(); - let (keyword_prefix, keyword_suffix) = split_keyword(keyword_lowercased); - let suggestions_limit = query.limit.unwrap_or(-1); - - let (mut statement, params) = if query - .providers + let unique_providers = query.providers.iter().collect::>(); + unique_providers .iter() - .any(|p| matches!(p, SuggestionProvider::Pocket | SuggestionProvider::Amo)) - { - (self.conn.prepare_cached( - &format!( - "SELECT s.id, k.rank, s.title, s.url, s.provider, NULL as confidence, NULL as keyword_suffix - FROM suggestions s - JOIN keywords k ON k.suggestion_id = s.id - WHERE s.provider IN ({}) AND - k.keyword = :keyword - UNION ALL - SELECT s.id, k.rank, s.title, s.url, s.provider, k.confidence, k.keyword_suffix - FROM suggestions s - JOIN prefix_keywords k ON k.suggestion_id = s.id - WHERE k.keyword_prefix = :keyword_prefix - ORDER BY s.provider - LIMIT :suggestions_limit", - providers_to_sql_list(&query.providers), - ), - )?, vec![ - (":keyword", keyword_lowercased as &dyn ToSql), - (":keyword_prefix", &keyword_prefix as &dyn ToSql), - (":suggestions_limit", &suggestions_limit as &dyn ToSql), - ]) - } else { - (self.conn.prepare_cached( - &format!( - "SELECT s.id, k.rank, s.title, s.url, s.provider, NULL as confidence, NULL as keyword_suffix - FROM suggestions s - JOIN keywords k ON k.suggestion_id = s.id - WHERE s.provider IN ({}) AND - k.keyword = :keyword - ORDER BY s.provider - LIMIT :suggestions_limit", - providers_to_sql_list(&query.providers), - ), - )?, vec![ - (":keyword", keyword_lowercased as &dyn ToSql), - (":suggestions_limit", &suggestions_limit as &dyn ToSql), - ]) - }; + .try_fold(vec![], |mut acc, provider| { + let suggestions = match provider { + SuggestionProvider::Amp => self.fetch_amp_suggestions(query), + SuggestionProvider::Wikipedia => self.fetch_wikipedia_suggestions(query), + SuggestionProvider::Amo => self.fetch_amo_suggestions(query), + SuggestionProvider::Pocket => self.fetch_pocket_suggestions(query), + SuggestionProvider::Yelp => self.fetch_yelp_suggestions(query), + SuggestionProvider::Mdn => self.fetch_mdn_suggestions(query), + SuggestionProvider::Weather => self.fetch_weather_suggestions(query), + }?; + acc.extend(suggestions); + Ok(acc) + }) + .map(|mut suggestions| { + suggestions.sort(); + if let Some(limit) = query.limit.and_then(|limit| usize::try_from(limit).ok()) { + suggestions.truncate(limit); + } + suggestions + }) + } - let suggestions = statement.query_and_then(&*params, |row| -> Result> { + /// Fetches Suggestions of type Amp provider that match the given query + pub fn fetch_amp_suggestions(&self, query: &SuggestionQuery) -> Result> { + let keyword_lowercased = &query.keyword.to_lowercase(); + let suggestions = self.conn.query_rows_and_then_cached( + "SELECT s.id, k.rank, s.title, s.url, s.provider, s.score + FROM suggestions s + JOIN keywords k ON k.suggestion_id = s.id + WHERE s.provider = :provider AND + k.keyword = :keyword", + named_params! { + ":keyword": keyword_lowercased, + ":provider": SuggestionProvider::Amp + }, + |row| -> Result{ let suggestion_id: i64 = row.get("id")?; let title = row.get("title")?; let raw_url = row.get::<_, String>("url")?; - let provider = row.get("provider")?; + let score = row.get::<_, f64>("score")?; let keywords: Vec = self.conn.query_rows_and_then_cached( "SELECT keyword FROM keywords @@ -193,120 +235,280 @@ impl<'a> SuggestDao<'a> { }, |row| row.get(0), )?; - - match provider { - SuggestionProvider::Amp => { - self.conn.query_row_and_then( - "SELECT amp.advertiser, amp.block_id, amp.iab_category, amp.impression_url, amp.click_url, - (SELECT i.data FROM icons i WHERE i.id = amp.icon_id) AS icon - FROM amp_custom_details amp - WHERE amp.suggestion_id = :suggestion_id", - named_params! { - ":suggestion_id": suggestion_id - }, - |row| { - let cooked_url = cook_raw_suggestion_url(&raw_url); - let raw_click_url = row.get::<_, String>("click_url")?; - let cooked_click_url = cook_raw_suggestion_url(&raw_click_url); - Ok(Some(Suggestion::Amp { - block_id: row.get("block_id")?, - advertiser: row.get("advertiser")?, - iab_category: row.get("iab_category")?, - title, - url: cooked_url, - raw_url, - full_keyword: full_keyword(keyword_lowercased, &keywords), - icon: row.get("icon")?, - impression_url: row.get("impression_url")?, - click_url: cooked_click_url, - raw_click_url, - })) - } - ) + self.conn.query_row_and_then( + "SELECT amp.advertiser, amp.block_id, amp.iab_category, amp.impression_url, amp.click_url, + (SELECT i.data FROM icons i WHERE i.id = amp.icon_id) AS icon + FROM amp_custom_details amp + WHERE amp.suggestion_id = :suggestion_id", + named_params! { + ":suggestion_id": suggestion_id }, - SuggestionProvider::Wikipedia => { - let icon = self.conn.try_query_one( - "SELECT i.data - FROM icons i - JOIN wikipedia_custom_details s ON s.icon_id = i.id - WHERE s.suggestion_id = :suggestion_id", - named_params! { - ":suggestion_id": suggestion_id - }, - true, - )?; - Ok(Some(Suggestion::Wikipedia { + |row| { + let cooked_url = cook_raw_suggestion_url(&raw_url); + let raw_click_url = row.get::<_, String>("click_url")?; + let cooked_click_url = cook_raw_suggestion_url(&raw_click_url); + Ok(Suggestion::Amp { + block_id: row.get("block_id")?, + advertiser: row.get("advertiser")?, + iab_category: row.get("iab_category")?, + title, + url: cooked_url, + raw_url, + full_keyword: full_keyword(keyword_lowercased, &keywords), + icon: row.get("icon")?, + impression_url: row.get("impression_url")?, + click_url: cooked_click_url, + raw_click_url, + score, + }) + } + ) + } + )?; + Ok(suggestions) + } + + /// Fetches Suggestions of type Wikipedia provider that match the given query + pub fn fetch_wikipedia_suggestions(&self, query: &SuggestionQuery) -> Result> { + let keyword_lowercased = &query.keyword.to_lowercase(); + let suggestions = self.conn.query_rows_and_then_cached( + "SELECT s.id, k.rank, s.title, s.url + FROM suggestions s + JOIN keywords k ON k.suggestion_id = s.id + WHERE s.provider = :provider AND + k.keyword = :keyword", + named_params! { + ":keyword": keyword_lowercased, + ":provider": SuggestionProvider::Wikipedia + }, + |row| -> Result { + let suggestion_id: i64 = row.get("id")?; + let title = row.get("title")?; + let raw_url = row.get::<_, String>("url")?; + + let keywords: Vec = self.conn.query_rows_and_then_cached( + "SELECT keyword FROM keywords + WHERE suggestion_id = :suggestion_id AND rank >= :rank + ORDER BY rank ASC", + named_params! { + ":suggestion_id": suggestion_id, + ":rank": row.get::<_, i64>("rank")?, + }, + |row| row.get(0), + )?; + let icon = self.conn.try_query_one( + "SELECT i.data + FROM icons i + JOIN wikipedia_custom_details s ON s.icon_id = i.id + WHERE s.suggestion_id = :suggestion_id", + named_params! { + ":suggestion_id": suggestion_id + }, + true, + )?; + Ok(Suggestion::Wikipedia { + title, + url: raw_url, + full_keyword: full_keyword(keyword_lowercased, &keywords), + icon, + }) + }, + )?; + Ok(suggestions) + } + + /// Fetches Suggestions of type Amo provider that match the given query + pub fn fetch_amo_suggestions(&self, query: &SuggestionQuery) -> Result> { + let keyword_lowercased = &query.keyword.to_lowercase(); + let (keyword_prefix, keyword_suffix) = split_keyword(keyword_lowercased); + let suggestions_limit = &query.limit.unwrap_or(-1); + let suggestions = self.conn.query_rows_and_then_cached( + "SELECT s.id, k.rank, s.title, s.url, s.provider, s.score, k.confidence, k.keyword_suffix + FROM suggestions s + JOIN prefix_keywords k ON k.suggestion_id = s.id + WHERE k.keyword_prefix = :keyword_prefix AND s.provider = :provider + ORDER by s.score DESC + LIMIT :suggestions_limit", + named_params! { + ":keyword_prefix": keyword_prefix, + ":provider": SuggestionProvider::Amo, + ":suggestions_limit": suggestions_limit, + }, + |row| -> Result>{ + let suggestion_id: i64 = row.get("id")?; + let title = row.get("title")?; + let raw_url = row.get::<_, String>("url")?; + let score = row.get::<_, f64>("score")?; + + let full_suffix = row.get::<_, String>("keyword_suffix")?; + full_suffix.starts_with(keyword_suffix).then(|| + self.conn.query_row_and_then( + "SELECT amo.description, amo.guid, amo.rating, amo.icon_url, amo.number_of_ratings + FROM amo_custom_details amo + WHERE amo.suggestion_id = :suggestion_id", + named_params! { + ":suggestion_id": suggestion_id + }, + |row| { + Ok(Suggestion::Amo { title, url: raw_url, - full_keyword: full_keyword(keyword_lowercased, &keywords), - icon, - })) - } - SuggestionProvider::Amo => { - let full_suffix = row.get::<_, String>("keyword_suffix")?; - self.conn.query_row_and_then( - "SELECT amo.description, amo.guid, amo.rating, amo.icon_url, amo.number_of_ratings, amo.score - FROM amo_custom_details amo - WHERE amo.suggestion_id = :suggestion_id", - named_params! { - ":suggestion_id": suggestion_id - }, - |row| { - if full_suffix.starts_with(keyword_suffix) { - Ok(Some(Suggestion::Amo{ - title, - url: raw_url, - icon_url: row.get("icon_url")?, - description: row.get("description")?, - rating: row.get("rating")?, - number_of_ratings: row.get("number_of_ratings")?, - guid: row.get("guid")?, - score: row.get("score")?, - })) - } else { - Ok(None) - } - }) - }, - SuggestionProvider::Pocket => { - let confidence = row.get("confidence")?; - let full_suffix = row.get::<_, String>("keyword_suffix")?; - let suffixes_match = match confidence { - KeywordConfidence::Low => full_suffix.starts_with(keyword_suffix), - KeywordConfidence::High => full_suffix == keyword_suffix, - }; - if suffixes_match { + icon_url: row.get("icon_url")?, + description: row.get("description")?, + rating: row.get("rating")?, + number_of_ratings: row.get("number_of_ratings")?, + guid: row.get("guid")?, + score, + }) + })).transpose() + } + )?.into_iter().flatten().collect(); + Ok(suggestions) + } + + /// Fetches Suggestions of type pocket provider that match the given query + pub fn fetch_pocket_suggestions(&self, query: &SuggestionQuery) -> Result> { + let keyword_lowercased = &query.keyword.to_lowercase(); + let (keyword_prefix, keyword_suffix) = split_keyword(keyword_lowercased); + let suggestions_limit = &query.limit.unwrap_or(-1); + let suggestions = self.conn.query_rows_and_then_cached( + "SELECT s.id, k.rank, s.title, s.url, s.provider, s.score, k.confidence, k.keyword_suffix + FROM suggestions s + JOIN prefix_keywords k ON k.suggestion_id = s.id + WHERE k.keyword_prefix = :keyword_prefix AND s.provider = :provider + ORDER BY s.score DESC + LIMIT :suggestions_limit", + named_params! { + ":keyword_prefix": keyword_prefix, + ":provider": SuggestionProvider::Pocket, + ":suggestions_limit": suggestions_limit, + + }, + |row| -> Result>{ + let title = row.get("title")?; + let raw_url = row.get::<_, String>("url")?; + let score = row.get::<_, f64>("score")?; + let confidence = row.get("confidence")?; + let full_suffix = row.get::<_, String>("keyword_suffix")?; + let suffixes_match = match confidence { + KeywordConfidence::Low => full_suffix.starts_with(keyword_suffix), + KeywordConfidence::High => full_suffix == keyword_suffix, + }; + if suffixes_match { + Ok(Some(Suggestion::Pocket { + title, + url: raw_url, + score, + is_top_pick: matches!( + confidence, + KeywordConfidence::High) + })) + } else { + Ok(None) + } + } + )?.into_iter().flatten().collect(); + Ok(suggestions) + } + + /// Fetches suggestions for MDN + pub fn fetch_mdn_suggestions(&self, query: &SuggestionQuery) -> Result> { + let keyword_lowercased = &query.keyword.to_lowercase(); + let (keyword_prefix, keyword_suffix) = split_keyword(keyword_lowercased); + let suggestions_limit = &query.limit.unwrap_or(-1); + let suggestions = self + .conn + .query_rows_and_then_cached( + r#" + SELECT + s.id, s.title, s.url, s.provider, s.score, k.keyword_suffix + FROM + suggestions s + JOIN + prefix_keywords k ON k.suggestion_id = s.id + WHERE + k.keyword_prefix = :keyword_prefix + AND + s.provider = :provider + ORDER BY + s.score DESC + LIMIT :suggestions_limit + "#, + named_params! { + ":keyword_prefix": keyword_prefix, + ":provider": SuggestionProvider::Mdn, + ":suggestions_limit": suggestions_limit, + }, + |row| -> Result> { + let suggestion_id: i64 = row.get("id")?; + let title = row.get("title")?; + let raw_url = row.get::<_, String>("url")?; + let score = row.get::<_, f64>("score")?; + + let full_suffix = row.get::<_, String>("keyword_suffix")?; + full_suffix + .starts_with(keyword_suffix) + .then(|| { self.conn.query_row_and_then( - "SELECT p.score - FROM pocket_custom_details p - WHERE p.suggestion_id = :suggestion_id", + r#" + SELECT + description + FROM + mdn_custom_details + WHERE + suggestion_id = :suggestion_id + "#, named_params! { ":suggestion_id": suggestion_id }, |row| { - Ok(Some(Suggestion::Pocket { + Ok(Suggestion::Mdn { title, url: raw_url, - score: row.get("score")?, - is_top_pick: matches!( - confidence, - KeywordConfidence::High - ) - })) - } + description: row.get("description")?, + score, + }) + }, ) - } else { - Ok(None) - } - }, - _ => Ok(None), - } - } - )?.flat_map(Result::transpose).collect::>()?; + }) + .transpose() + }, + )? + .into_iter() + .flatten() + .collect(); Ok(suggestions) } + /// Fetches weather suggestions + pub fn fetch_weather_suggestions(&self, query: &SuggestionQuery) -> Result> { + // Weather keywords are matched by prefix but the query must be at least + // three chars long. Unlike the prefix matching of other suggestion + // types, the query doesn't need to contain the first full word. + if query.keyword.len() < 3 { + return Ok(vec![]); + } + + let keyword_lowercased = &query.keyword.trim().to_lowercase(); + let suggestions = self.conn.query_rows_and_then_cached( + "SELECT s.score + FROM suggestions s + JOIN keywords k ON k.suggestion_id = s.id + WHERE s.provider = :provider AND (k.keyword BETWEEN :keyword AND :keyword || X'FFFF')", + named_params! { + ":keyword": keyword_lowercased, + ":provider": SuggestionProvider::Weather + }, + |row| -> Result { + Ok(Suggestion::Weather { + score: row.get::<_, f64>("score")?, + }) + }, + )?; + Ok(suggestions) + } + /// Inserts all suggestions from a downloaded AMO attachment into /// the database. pub fn insert_amo_suggestions( @@ -322,13 +524,15 @@ impl<'a> SuggestDao<'a> { record_id, provider, title, - url + url, + score ) VALUES( :record_id, {}, :title, - :url + :url, + :score ) RETURNING id", SuggestionProvider::Amo as u8 @@ -337,6 +541,7 @@ impl<'a> SuggestDao<'a> { ":record_id": record_id.as_str(), ":title": suggestion.title, ":url": suggestion.url, + ":score": suggestion.score, }, |row| row.get(0), true, @@ -348,8 +553,7 @@ impl<'a> SuggestDao<'a> { guid, icon_url, rating, - number_of_ratings, - score + number_of_ratings ) VALUES( :suggestion_id, @@ -357,8 +561,7 @@ impl<'a> SuggestDao<'a> { :guid, :icon_url, :rating, - :number_of_ratings, - :score + :number_of_ratings )", named_params! { ":suggestion_id": suggestion_id, @@ -366,8 +569,7 @@ impl<'a> SuggestDao<'a> { ":guid": suggestion.guid, ":icon_url": suggestion.icon_url, ":rating": suggestion.rating, - ":number_of_ratings": suggestion.number_of_ratings, - ":score": suggestion.score, + ":number_of_ratings": suggestion.number_of_ratings }, )?; for (index, keyword) in suggestion.keywords.iter().enumerate() { @@ -414,13 +616,15 @@ impl<'a> SuggestDao<'a> { record_id, provider, title, - url + url, + score ) VALUES( :record_id, {}, :title, - :url + :url, + :score ) RETURNING id", provider as u8 @@ -429,7 +633,7 @@ impl<'a> SuggestDao<'a> { ":record_id": record_id.as_str(), ":title": common_details.title, ":url": common_details.url, - + ":score": common_details.score.unwrap_or(DEFAULT_SUGGESTION_SCORE) }, |row| row.get(0), true, @@ -521,13 +725,15 @@ impl<'a> SuggestDao<'a> { record_id, provider, title, - url + url, + score ) VALUES( :record_id, {}, :title, - :url + :url, + :score ) RETURNING id", SuggestionProvider::Pocket as u8 @@ -536,24 +742,12 @@ impl<'a> SuggestDao<'a> { ":record_id": record_id.as_str(), ":title": suggestion.title, ":url": suggestion.url, + ":score": suggestion.score, }, |row| row.get(0), true, )?; - self.conn.execute( - "INSERT INTO pocket_custom_details( - suggestion_id, - score - ) - VALUES( - :suggestion_id, - :score - )", - named_params! { - ":suggestion_id": suggestion_id, - ":score": suggestion.score, - }, - )?; + for ((rank, keyword), confidence) in suggestion .high_confidence_keywords .iter() @@ -596,6 +790,123 @@ impl<'a> SuggestDao<'a> { Ok(()) } + /// Inserts all suggestions from a downloaded MDN attachment into + /// the database. + pub fn insert_mdn_suggestions( + &mut self, + record_id: &SuggestRecordId, + suggestions: &[DownloadedMdnSuggestion], + ) -> Result<()> { + for suggestion in suggestions { + self.scope.err_if_interrupted()?; + let suggestion_id: i64 = self.conn.query_row_and_then_cachable( + &format!( + "INSERT INTO suggestions( + record_id, + provider, + title, + url, + score + ) + VALUES( + :record_id, + {}, + :title, + :url, + :score + ) + RETURNING id", + SuggestionProvider::Mdn as u8 + ), + named_params! { + ":record_id": record_id.as_str(), + ":title": suggestion.title, + ":url": suggestion.url, + ":score": suggestion.score, + }, + |row| row.get(0), + true, + )?; + self.conn.execute_cached( + "INSERT INTO mdn_custom_details( + suggestion_id, + description + ) + VALUES( + :suggestion_id, + :description + )", + named_params! { + ":suggestion_id": suggestion_id, + ":description": suggestion.description, + }, + )?; + for (index, keyword) in suggestion.keywords.iter().enumerate() { + let (keyword_prefix, keyword_suffix) = split_keyword(keyword); + self.conn.execute_cached( + "INSERT INTO prefix_keywords( + keyword_prefix, + keyword_suffix, + suggestion_id, + rank + ) + VALUES( + :keyword_prefix, + :keyword_suffix, + :suggestion_id, + :rank + )", + named_params! { + ":keyword_prefix": keyword_prefix, + ":keyword_suffix": keyword_suffix, + ":rank": index, + ":suggestion_id": suggestion_id, + }, + )?; + } + } + Ok(()) + } + + /// Inserts weather record data into the database. + pub fn insert_weather_data( + &mut self, + record_id: &SuggestRecordId, + data: &DownloadedWeatherData, + ) -> Result<()> { + self.scope.err_if_interrupted()?; + let suggestion_id: i64 = self.conn.query_row_and_then_cachable( + &format!( + "INSERT INTO suggestions(record_id, provider, title, url, score) + VALUES(:record_id, {}, '', '', :score) + RETURNING id", + SuggestionProvider::Weather as u8 + ), + named_params! { + ":record_id": record_id.as_str(), + ":score": data.weather.score.unwrap_or(DEFAULT_SUGGESTION_SCORE), + }, + |row| row.get(0), + true, + )?; + for (index, keyword) in data.weather.keywords.iter().enumerate() { + self.conn.execute( + "INSERT INTO keywords(keyword, suggestion_id, rank) + VALUES(:keyword, :suggestion_id, :rank)", + named_params! { + ":keyword": keyword, + ":suggestion_id": suggestion_id, + ":rank": index, + }, + )?; + } + self.put_provider_config( + SuggestionProvider::Weather, + &SuggestProviderConfig::from(data), + )?; + Ok(()) + } + /// Inserts or replaces an icon for a suggestion into the database. pub fn put_icon(&mut self, icon_id: &str, data: &[u8]) -> Result<()> { self.conn.execute( @@ -723,13 +1034,45 @@ impl<'a> SuggestDao<'a> { }; self.put_meta(UNPARSABLE_RECORDS_META_KEY, unparsable_records) } + + /// Stores global Suggest configuration data. + pub fn put_global_config(&mut self, config: &SuggestGlobalConfig) -> Result<()> { + self.put_meta(GLOBAL_CONFIG_META_KEY, serde_json::to_string(config)?) + } + + /// Gets the stored global Suggest configuration data or a default config if + /// none is stored. + pub fn get_global_config(&self) -> Result { + self.get_meta::(GLOBAL_CONFIG_META_KEY)? + .map_or_else( + || Ok(SuggestGlobalConfig::default()), + |json| Ok(serde_json::from_str(&json)?), + ) + } + + /// Stores configuration data for a given provider. + pub fn put_provider_config( + &mut self, + provider: SuggestionProvider, + config: &SuggestProviderConfig, + ) -> Result<()> { + self.put_meta( + &provider_config_meta_key(provider), + serde_json::to_string(config)?, + ) + } + + /// Gets the stored configuration data for a given provider or None if none + /// is stored. + pub fn get_provider_config( + &self, + provider: SuggestionProvider, + ) -> Result> { + self.get_meta::(&provider_config_meta_key(provider))? + .map_or_else(|| Ok(None), |json| Ok(serde_json::from_str(&json)?)) + } } -/// Formats a slice of [`SuggestionProvider`]s as a SQL list. -fn providers_to_sql_list(providers: &[SuggestionProvider]) -> String { - providers - .iter() - .map(|&provider| (provider as u8).to_string()) - .collect::>() - .join(",") +fn provider_config_meta_key(provider: SuggestionProvider) -> String { + format!("{}{}", PROVIDER_CONFIG_META_KEY_PREFIX, provider as u8) } diff --git a/third_party/rust/suggest/src/error.rs b/third_party/rust/suggest/src/error.rs index 25ab2d5a6623..cd07c3591cc6 100644 --- a/third_party/rust/suggest/src/error.rs +++ b/third_party/rust/suggest/src/error.rs @@ -3,6 +3,9 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +use error_support::{ErrorHandling, GetErrorHandling}; +use remote_settings::RemoteSettingsError; + /// A list of errors that are internal to the component. This is the error /// type for private and crate-internal methods, and is never returned to the /// application. @@ -18,10 +21,13 @@ pub(crate) enum Error { Json(#[from] serde_json::Error), #[error("Error from Remote Settings: {0}")] - RemoteSettings(#[from] remote_settings::RemoteSettingsError), + RemoteSettings(#[from] RemoteSettingsError), #[error("Operation interrupted")] Interrupted(#[from] interrupt_support::Interrupted), + + #[error("SuggestStoreBuilder {0}")] + SuggestStoreBuilder(String), } /// The error type for all Suggest component operations. These errors are @@ -29,15 +35,45 @@ pub(crate) enum Error { #[derive(Debug, thiserror::Error)] #[non_exhaustive] pub enum SuggestApiError { + #[error("Network error: {reason}")] + Network { reason: String }, + // The server requested a backoff after too many requests + #[error("Backoff")] + Backoff { seconds: u64 }, + // The application interrupted a request + #[error("Interrupted")] + Interrupted, #[error("Other error: {reason}")] Other { reason: String }, } -impl From for SuggestApiError { - /// Converts an internal component error to a public application error. - fn from(error: Error) -> Self { - Self::Other { - reason: error.to_string(), +// Define how our internal errors are handled and converted to external errors +// See `support/error/README.md` for how this works, especially the warning about PII. +impl GetErrorHandling for Error { + type ExternalError = SuggestApiError; + + fn get_error_handling(&self) -> ErrorHandling { + match self { + // Do nothing for interrupted errors, this is just normal operation. + Self::Interrupted(_) => ErrorHandling::convert(SuggestApiError::Interrupted), + // Network errors are expected to happen in practice. Let's log, but not report them. + Self::RemoteSettings(RemoteSettingsError::RequestError( + viaduct::Error::NetworkError(e), + )) => ErrorHandling::convert(SuggestApiError::Network { + reason: e.to_string(), + }) + .log_warning(), + // Backoff error shouldn't happen in practice, so let's report them for now. + // If these do happen in practice and we decide that there is a valid reason for them, + // then consider switching from reporting to Sentry to counting in Glean. + Self::RemoteSettings(RemoteSettingsError::BackoffError(seconds)) => { + ErrorHandling::convert(SuggestApiError::Backoff { seconds: *seconds }) + .report_error("suggest-backoff") + } + _ => ErrorHandling::convert(SuggestApiError::Other { + reason: self.to_string(), + }) + .report_error("suggest-unexpected"), } } } diff --git a/third_party/rust/suggest/src/lib.rs b/third_party/rust/suggest/src/lib.rs index e45d5e93957b..23775b7decbd 100644 --- a/third_party/rust/suggest/src/lib.rs +++ b/third_party/rust/suggest/src/lib.rs @@ -4,6 +4,7 @@ */ use remote_settings::RemoteSettingsConfig; +mod config; mod db; mod error; mod keyword; @@ -15,9 +16,10 @@ mod store; mod suggestion; mod yelp; +pub use config::{SuggestGlobalConfig, SuggestProviderConfig}; pub use error::SuggestApiError; pub use provider::SuggestionProvider; -pub use store::{SuggestIngestionConstraints, SuggestStore}; +pub use store::{SuggestIngestionConstraints, SuggestStore, SuggestStoreBuilder}; pub use suggestion::{raw_suggestion_url_matches, Suggestion}; pub(crate) type Result = std::result::Result; diff --git a/third_party/rust/suggest/src/provider.rs b/third_party/rust/suggest/src/provider.rs index a36c65c74c67..cbcc314444a4 100644 --- a/third_party/rust/suggest/src/provider.rs +++ b/third_party/rust/suggest/src/provider.rs @@ -17,6 +17,8 @@ pub enum SuggestionProvider { Amo = 3, Pocket = 4, Yelp = 5, + Mdn = 6, + Weather = 7, } impl FromSql for SuggestionProvider { @@ -38,6 +40,8 @@ impl SuggestionProvider { 3 => Some(SuggestionProvider::Amo), 4 => Some(SuggestionProvider::Pocket), 5 => Some(SuggestionProvider::Yelp), + 6 => Some(SuggestionProvider::Mdn), + 7 => Some(SuggestionProvider::Weather), _ => None, } } diff --git a/third_party/rust/suggest/src/rs.rs b/third_party/rust/suggest/src/rs.rs index 9ffa43200528..a40295e2c76d 100644 --- a/third_party/rust/suggest/src/rs.rs +++ b/third_party/rust/suggest/src/rs.rs @@ -79,7 +79,7 @@ impl SuggestRemoteSettingsClient for remote_settings::Client { /// /// Except for the type, Suggest records don't carry additional fields. All /// suggestions are stored in each record's attachment. -#[derive(Clone, Debug, Deserialize, PartialEq)] +#[derive(Clone, Debug, Deserialize)] #[serde(tag = "type")] pub(crate) enum SuggestRecord { #[serde(rename = "icon")] @@ -92,6 +92,12 @@ pub(crate) enum SuggestRecord { Pocket, #[serde(rename = "yelp-suggestions")] Yelp, + #[serde(rename = "mdn-suggestions")] + Mdn, + #[serde(rename = "weather")] + Weather(DownloadedWeatherData), + #[serde(rename = "configuration")] + GlobalConfig(DownloadedGlobalConfig), } /// Represents either a single value, or a list of values. This is used to @@ -148,15 +154,16 @@ where } /// Fields that are common to all downloaded suggestions. -#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +#[derive(Clone, Debug, Deserialize)] pub(crate) struct DownloadedSuggestionCommonDetails { pub keywords: Vec, pub title: String, pub url: String, + pub score: Option, } /// An AMP suggestion to ingest from an AMP-Wikipedia attachment. -#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +#[derive(Clone, Debug, Deserialize)] pub(crate) struct DownloadedAmpSuggestion { #[serde(flatten)] pub common_details: DownloadedSuggestionCommonDetails, @@ -171,7 +178,7 @@ pub(crate) struct DownloadedAmpSuggestion { } /// A Wikipedia suggestion to ingest from an AMP-Wikipedia attachment. -#[derive(Clone, Debug, Deserialize, Eq, PartialEq)] +#[derive(Clone, Debug, Deserialize)] pub(crate) struct DownloadedWikipediaSuggestion { #[serde(flatten)] pub common_details: DownloadedSuggestionCommonDetails, @@ -181,7 +188,7 @@ pub(crate) struct DownloadedWikipediaSuggestion { /// A suggestion to ingest from an AMP-Wikipedia attachment downloaded from /// Remote Settings. -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug)] pub(crate) enum DownloadedAmpWikipediaSuggestion { Amp(DownloadedAmpSuggestion), Wikipedia(DownloadedWikipediaSuggestion), @@ -288,3 +295,47 @@ pub(crate) struct DownloadedYelpSuggestion { #[serde(rename = "yelpModifiers")] pub yelp_modifiers: Vec, } + +/// An MDN suggestion to ingest from an attachment +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct DownloadedMdnSuggestion { + pub url: String, + pub title: String, + pub description: String, + pub keywords: Vec, + pub score: f64, +} + +/// Weather data to ingest from a weather record +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct DownloadedWeatherData { + pub weather: DownloadedWeatherDataInner, +} +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct DownloadedWeatherDataInner { + pub min_keyword_length: i32, + pub keywords: Vec, + // Remote settings doesn't support floats in record JSON so we use a + // stringified float instead. If a float can't be parsed, this will be None. + #[serde(default, deserialize_with = "de_stringified_f64")] + pub score: Option, +} + +/// Global Suggest configuration data to ingest from a configuration record +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct DownloadedGlobalConfig { + pub configuration: DownloadedGlobalConfigInner, +} +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct DownloadedGlobalConfigInner { + /// The maximum number of times the user can click "Show less frequently" + /// for a suggestion in the UI. + pub show_less_frequently_cap: i32, +} + +fn de_stringified_f64<'de, D>(deserializer: D) -> std::result::Result, D::Error> +where + D: Deserializer<'de>, +{ + String::deserialize(deserializer).map(|s| s.parse().ok()) +} diff --git a/third_party/rust/suggest/src/schema.rs b/third_party/rust/suggest/src/schema.rs index 39ea3b912c55..4c79c837598c 100644 --- a/third_party/rust/suggest/src/schema.rs +++ b/third_party/rust/suggest/src/schema.rs @@ -6,7 +6,7 @@ use rusqlite::{Connection, Transaction}; use sql_support::open_database::{self, ConnectionInitializer}; -pub const VERSION: u32 = 10; +pub const VERSION: u32 = 12; pub const SQL: &str = " CREATE TABLE meta( @@ -37,7 +37,8 @@ pub const SQL: &str = " record_id TEXT NOT NULL, provider INTEGER NOT NULL, title TEXT NOT NULL, - url TEXT NOT NULL + url TEXT NOT NULL, + score REAL NOT NULL ); CREATE TABLE amp_custom_details( @@ -51,11 +52,6 @@ pub const SQL: &str = " FOREIGN KEY(suggestion_id) REFERENCES suggestions(id) ON DELETE CASCADE ); - CREATE TABLE pocket_custom_details( - suggestion_id INTEGER PRIMARY KEY REFERENCES suggestions(id) ON DELETE CASCADE, - score REAL NOT NULL - ); - CREATE TABLE wikipedia_custom_details( suggestion_id INTEGER PRIMARY KEY REFERENCES suggestions(id) ON DELETE CASCADE, icon_id TEXT NOT NULL @@ -68,7 +64,6 @@ pub const SQL: &str = " icon_url TEXT NOT NULL, rating TEXT, number_of_ratings INTEGER NOT NULL, - score REAL NOT NULL, FOREIGN KEY(suggestion_id) REFERENCES suggestions(id) ON DELETE CASCADE ); @@ -96,6 +91,12 @@ pub const SQL: &str = " need_location INTEGER NOT NULL, record_id TEXT NOT NULL ) WITHOUT ROWID; + + CREATE TABLE mdn_custom_details( + suggestion_id INTEGER PRIMARY KEY, + description TEXT NOT NULL, + FOREIGN KEY(suggestion_id) REFERENCES suggestions(id) ON DELETE CASCADE + ); "; /// Initializes an SQLite connection to the Suggest database, performing @@ -126,7 +127,7 @@ impl ConnectionInitializer for SuggestConnectionInitializer { fn upgrade_from(&self, _db: &Transaction<'_>, version: u32) -> open_database::Result<()> { match version { - 1..=9 => { + 1..=11 => { // These schema versions were used during development, and never // shipped in any applications. Treat these databases as // corrupt, so that they'll be replaced. diff --git a/third_party/rust/suggest/src/store.rs b/third_party/rust/suggest/src/store.rs index 7c5fb556083a..72c959050a28 100644 --- a/third_party/rust/suggest/src/store.rs +++ b/third_party/rust/suggest/src/store.rs @@ -6,9 +6,12 @@ use std::{ collections::BTreeMap, path::{Path, PathBuf}, + sync::Arc, }; +use error_support::handle_error; use once_cell::sync::OnceCell; +use parking_lot::Mutex; use remote_settings::{ self, GetItemsOptions, RemoteSettingsConfig, RemoteSettingsRecord, SortOrder, }; @@ -19,9 +22,12 @@ use rusqlite::{ use serde::{de::DeserializeOwned, Deserialize, Serialize}; use crate::{ + config::{SuggestGlobalConfig, SuggestProviderConfig}, db::{ ConnectionType, SuggestDao, SuggestDb, LAST_INGEST_META_KEY, UNPARSABLE_RECORDS_META_KEY, }, + error::Error, + provider::SuggestionProvider, rs::{ SuggestAttachment, SuggestRecord, SuggestRecordId, SuggestRemoteSettingsClient, REMOTE_SETTINGS_COLLECTION, SUGGESTIONS_PER_ATTACHMENT, @@ -33,6 +39,70 @@ use crate::{ /// The chunk size used to request unparsable records. pub const UNPARSABLE_IDS_PER_REQUEST: usize = 150; +/// Builder for [SuggestStore] +/// +/// Using a builder is preferred to calling the constructor directly since it's harder to confuse +/// the data_path and cache_path strings. +pub struct SuggestStoreBuilder(Mutex); + +#[derive(Default)] +struct SuggestStoreBuilderInner { + data_path: Option, + cache_path: Option, + remote_settings_config: Option, +} + +impl Default for SuggestStoreBuilder { + fn default() -> Self { + Self::new() + } +} + +impl SuggestStoreBuilder { + pub fn new() -> SuggestStoreBuilder { + Self(Mutex::new(SuggestStoreBuilderInner::default())) + } + + pub fn data_path(self: Arc, path: String) -> Arc { + self.0.lock().data_path = Some(path); + self + } + + pub fn cache_path(self: Arc, path: String) -> Arc { + self.0.lock().cache_path = Some(path); + self + } + + pub fn remote_settings_config(self: Arc, config: RemoteSettingsConfig) -> Arc { + self.0.lock().remote_settings_config = Some(config); + self + } + + #[handle_error(Error)] + pub fn build(&self) -> SuggestApiResult> { + let inner = self.0.lock(); + let data_path = inner + .data_path + .clone() + .ok_or_else(|| Error::SuggestStoreBuilder("data_path not specified".to_owned()))?; + let cache_path = inner + .cache_path + .clone() + .ok_or_else(|| Error::SuggestStoreBuilder("cache_path not specified".to_owned()))?; + let settings_client = + remote_settings::Client::new(inner.remote_settings_config.clone().unwrap_or_else( + || RemoteSettingsConfig { + server_url: None, + bucket_name: None, + collection_name: REMOTE_SETTINGS_COLLECTION.into(), + }, + ))?; + Ok(Arc::new(SuggestStore { + inner: SuggestStoreInner::new(data_path, cache_path, settings_client), + })) + } +} + /// The store is the entry point to the Suggest component. It incrementally /// downloads suggestions from the Remote Settings service, stores them in a /// local database, and returns them in response to user queries. @@ -97,6 +167,7 @@ pub(crate) struct UnparsableRecord { impl SuggestStore { /// Creates a Suggest store. + #[handle_error(Error)] pub fn new( path: &str, settings_config: Option, @@ -111,13 +182,14 @@ impl SuggestStore { )?) }()?; Ok(Self { - inner: SuggestStoreInner::new(path, settings_client), + inner: SuggestStoreInner::new("".to_owned(), path.to_owned(), settings_client), }) } /// Queries the database for suggestions. + #[handle_error(Error)] pub fn query(&self, query: SuggestionQuery) -> SuggestApiResult> { - Ok(self.inner.query(query)?) + self.inner.query(query) } /// Interrupts any ongoing queries. @@ -130,13 +202,30 @@ impl SuggestStore { } /// Ingests new suggestions from Remote Settings. + #[handle_error(Error)] pub fn ingest(&self, constraints: SuggestIngestionConstraints) -> SuggestApiResult<()> { - Ok(self.inner.ingest(constraints)?) + self.inner.ingest(constraints) } /// Removes all content from the database. + #[handle_error(Error)] pub fn clear(&self) -> SuggestApiResult<()> { - Ok(self.inner.clear()?) + self.inner.clear() + } + + // Returns global Suggest configuration data. + #[handle_error(Error)] + pub fn fetch_global_config(&self) -> SuggestApiResult { + self.inner.fetch_global_config() + } + + // Returns per-provider Suggest configuration data. + #[handle_error(Error)] + pub fn fetch_provider_config( + &self, + provider: SuggestionProvider, + ) -> SuggestApiResult> { + self.inner.fetch_provider_config(provider) } } @@ -155,15 +244,29 @@ pub struct SuggestIngestionConstraints { /// client, and is split out from the concrete [`SuggestStore`] for testing /// with a mock client. pub(crate) struct SuggestStoreInner { - path: PathBuf, + /// Path to the persistent SQL database. + /// + /// This stores things that should persist when the user clears their cache. + /// It's not currently used because not all consumers pass this in yet. + #[allow(unused)] + data_path: PathBuf, + /// Path to the temporary SQL database. + /// + /// This stores things that should be deleted when the user clears their cache. + cache_path: PathBuf, dbs: OnceCell, settings_client: S, } impl SuggestStoreInner { - fn new(path: impl AsRef, settings_client: S) -> Self { + fn new( + data_path: impl Into, + cache_path: impl Into, + settings_client: S, + ) -> Self { Self { - path: path.as_ref().into(), + data_path: data_path.into(), + cache_path: cache_path.into(), dbs: OnceCell::new(), settings_client, } @@ -173,7 +276,7 @@ impl SuggestStoreInner { /// they're not already open. fn dbs(&self) -> Result<&SuggestStoreDbs> { self.dbs - .get_or_try_init(|| SuggestStoreDbs::open(&self.path)) + .get_or_try_init(|| SuggestStoreDbs::open(&self.cache_path)) } fn query(&self, query: SuggestionQuery) -> Result> { @@ -193,6 +296,19 @@ impl SuggestStoreInner { fn clear(&self) -> Result<()> { self.dbs()?.writer.write(|dao| dao.clear()) } + + pub fn fetch_global_config(&self) -> Result { + self.dbs()?.reader.read(|dao| dao.get_global_config()) + } + + pub fn fetch_provider_config( + &self, + provider: SuggestionProvider, + ) -> Result> { + self.dbs()? + .reader + .read(|dao| dao.get_provider_config(provider)) + } } impl SuggestStoreInner @@ -258,16 +374,7 @@ where if record.deleted { // If the entire record was deleted, drop all its suggestions // and advance the last ingest time. - writer.write(|dao| { - match record_id.as_icon_id() { - Some(icon_id) => dao.drop_icon(icon_id)?, - None => dao.drop_suggestions(&record_id)?, - }; - dao.drop_unparsable_record_id(&record_id)?; - dao.put_last_ingest_if_newer(record.last_modified)?; - - Ok(()) - })?; + writer.write(|dao| dao.handle_deleted_record(record))?; continue; } let Ok(fields) = @@ -275,23 +382,15 @@ where else { // We don't recognize this record's type, so we don't know how // to ingest its suggestions. Record this in the meta table. - writer.write(|dao| { - dao.put_unparsable_record_id(&record_id)?; - dao.put_last_ingest_if_newer(record.last_modified)?; - Ok(()) - })?; + writer.write(|dao| dao.handle_unparsable_record(record))?; continue; }; match fields { SuggestRecord::AmpWikipedia => { - self.ingest_suggestions_from_record( - writer, - record, - |dao, record_id, suggestions| { - dao.insert_amp_wikipedia_suggestions(record_id, suggestions) - }, - )?; + self.ingest_attachment(writer, record, |dao, record_id, suggestions| { + dao.insert_amp_wikipedia_suggestions(record_id, suggestions) + })?; } SuggestRecord::Icon => { let (Some(icon_id), Some(attachment)) = @@ -306,48 +405,70 @@ where let data = self.settings_client.get_attachment(&attachment.location)?; writer.write(|dao| { dao.put_icon(icon_id, &data)?; - dao.put_last_ingest_if_newer(record.last_modified)?; - // Remove this record's ID from the list of unparsable - // records, since we understand it now. - dao.drop_unparsable_record_id(&record_id)?; - - Ok(()) + dao.handle_ingested_record(record) })?; } SuggestRecord::Amo => { - self.ingest_suggestions_from_record( - writer, - record, - |dao, record_id, suggestions| { - dao.insert_amo_suggestions(record_id, suggestions) - }, - )?; + self.ingest_attachment(writer, record, |dao, record_id, suggestions| { + dao.insert_amo_suggestions(record_id, suggestions) + })?; } SuggestRecord::Pocket => { - self.ingest_suggestions_from_record( - writer, - record, - |dao, record_id, suggestions| { - dao.insert_pocket_suggestions(record_id, suggestions) - }, - )?; + self.ingest_attachment(writer, record, |dao, record_id, suggestions| { + dao.insert_pocket_suggestions(record_id, suggestions) + })?; } SuggestRecord::Yelp => { - self.ingest_suggestions_from_record( - writer, - record, - |dao, record_id, suggestions| match suggestions.first() { + self.ingest_attachment(writer, record, |dao, record_id, suggestions| { + match suggestions.first() { Some(suggestion) => dao.insert_yelp_suggestions(record_id, suggestion), None => Ok(()), - }, - )?; + } + })?; + } + SuggestRecord::Mdn => { + self.ingest_attachment(writer, record, |dao, record_id, suggestions| { + dao.insert_mdn_suggestions(record_id, suggestions) + })?; + } + SuggestRecord::Weather(data) => { + self.ingest_record(writer, record, |dao, record_id| { + dao.insert_weather_data(record_id, &data) + })?; + } + SuggestRecord::GlobalConfig(config) => { + self.ingest_record(writer, record, |dao, _| { + dao.put_global_config(&SuggestGlobalConfig::from(&config)) + })?; } } } Ok(()) } - fn ingest_suggestions_from_record( + fn ingest_record( + &self, + writer: &SuggestDb, + record: &RemoteSettingsRecord, + ingestion_handler: impl FnOnce(&mut SuggestDao<'_>, &SuggestRecordId) -> Result<()>, + ) -> Result<()> { + let record_id = SuggestRecordId::from(&record.id); + + writer.write(|dao| { + // Drop any data that we previously ingested from this record. + // Suggestions in particular don't have a stable identifier, and + // determining which suggestions in the record actually changed is + // more complicated than dropping and re-ingesting all of them. + dao.drop_suggestions(&record_id)?; + + // Ingest (or re-ingest) all data in the record. + ingestion_handler(dao, &record_id)?; + + dao.handle_ingested_record(record) + }) + } + + fn ingest_attachment( &self, writer: &SuggestDb, record: &RemoteSettingsRecord, @@ -356,41 +477,21 @@ where where T: DeserializeOwned, { - let record_id = SuggestRecordId::from(&record.id); - let Some(attachment) = record.attachment.as_ref() else { - // A record should always have an - // attachment with suggestions. If it doesn't, it's - // malformed, so skip to the next record. + // This method should be called only when a record is expected to + // have an attachment. If it doesn't have one, it's malformed, so + // skip to the next record. writer.write(|dao| dao.put_last_ingest_if_newer(record.last_modified))?; return Ok(()); }; - let attachment: SuggestAttachment = - serde_json::from_slice(&self.settings_client.get_attachment(&attachment.location)?)?; - - writer.write(|dao| { - // Drop any suggestions that we previously ingested from - // this record's attachment. Suggestions don't have a - // stable identifier, and determining which suggestions in - // the attachment actually changed is more complicated than - // dropping and re-ingesting all of them. - dao.drop_suggestions(&record_id)?; - - // Ingest (or re-ingest) all suggestions in the - // attachment. - ingestion_handler(dao, &record_id, attachment.suggestions())?; - - // Remove this record's ID from the list of unparsable - // records, since we understand it now. - dao.drop_unparsable_record_id(&record_id)?; - - // Advance the last fetch time, so that we can resume - // fetching after this record if we're interrupted. - dao.put_last_ingest_if_newer(record.last_modified)?; - - Ok(()) - }) + let attachment_data = self.settings_client.get_attachment(&attachment.location)?; + match serde_json::from_slice::>(&attachment_data) { + Ok(attachment) => self.ingest_record(writer, record, |dao, record_id| { + ingestion_handler(dao, record_id, attachment.suggestions()) + }), + Err(_) => writer.write(|dao| dao.handle_unparsable_record(record)), + } } } @@ -440,7 +541,11 @@ mod tests { // it in shared-cache mode so that both connections can access it. SuggestStoreInner::new( format!( - "file:test_store_{}?mode=memory&cache=shared", + "file:test_store_data_{}?mode=memory&cache=shared", + hex::encode(unique_suffix), + ), + format!( + "file:test_store_cache_{}?mode=memory&cache=shared", hex::encode(unique_suffix), ), settings_client, @@ -602,6 +707,7 @@ mod tests { "icon": "5678", "impression_url": "https://example.com/impression_url", "click_url": "https://example.com/click_url", + "score": 0.3 }]), )?; @@ -625,6 +731,7 @@ mod tests { impression_url: "https://example.com/impression_url", click_url: "https://example.com/click_url", raw_click_url: "https://example.com/click_url", + score: 0.3, }, ] "#]] @@ -689,7 +796,8 @@ mod tests { "url": "https://penne.biz", "icon": "2", "impression_url": "https://example.com/impression_url", - "click_url": "https://example.com/click_url" + "click_url": "https://example.com/click_url", + "score": 0.3 }]), )? .with_icon("icon-2.png", "i-am-an-icon".as_bytes().into()); @@ -728,6 +836,7 @@ mod tests { impression_url: "https://example.com/impression_url", click_url: "https://example.com/click_url", raw_click_url: "https://example.com/click_url", + score: 0.2, }, ] "#]] @@ -765,6 +874,7 @@ mod tests { impression_url: "https://example.com/impression_url", click_url: "https://example.com/click_url", raw_click_url: "https://example.com/click_url", + score: 0.3, }, ] "#]] @@ -809,7 +919,8 @@ mod tests { "url": "https://www.lasagna.restaurant", "icon": "2", "impression_url": "https://example.com/impression_url", - "click_url": "https://example.com/click_url" + "click_url": "https://example.com/click_url", + "score": 0.3 }), )?; @@ -832,6 +943,7 @@ mod tests { impression_url: "https://example.com/impression_url", click_url: "https://example.com/click_url", raw_click_url: "https://example.com/click_url", + score: 0.3, }, ] "#]] @@ -876,7 +988,8 @@ mod tests { "url": "https://www.lasagna.restaurant", "icon": "1", "impression_url": "https://example.com/impression_url", - "click_url": "https://example.com/click_url" + "click_url": "https://example.com/click_url", + "score": 0.3 }, { "id": 0, "advertiser": "Los Pollos Hermanos", @@ -886,7 +999,8 @@ mod tests { "url": "https://www.lph-nm.biz", "icon": "2", "impression_url": "https://example.com/impression_url", - "click_url": "https://example.com/click_url" + "click_url": "https://example.com/click_url", + "score": 0.3 }]), )?; @@ -910,6 +1024,7 @@ mod tests { impression_url: "https://example.com/impression_url", click_url: "https://example.com/click_url", raw_click_url: "https://example.com/click_url", + score: 0.3, }, ] "#]] @@ -946,7 +1061,8 @@ mod tests { "url": "https://www.lph-nm.biz", "icon": "2", "impression_url": "https://example.com/impression_url", - "click_url": "https://example.com/click_url" + "click_url": "https://example.com/click_url", + "score": 0.3 }, { "id": 0, "advertiser": "Good Place Eats", @@ -956,7 +1072,8 @@ mod tests { "url": "https://penne.biz", "icon": "2", "impression_url": "https://example.com/impression_url", - "click_url": "https://example.com/click_url" + "click_url": "https://example.com/click_url", + "score": 0.3 }]), )?; @@ -985,6 +1102,7 @@ mod tests { impression_url: "https://example.com/impression_url", click_url: "https://example.com/click_url", raw_click_url: "https://example.com/click_url", + score: 0.3, }, ] "#]] @@ -1007,6 +1125,7 @@ mod tests { impression_url: "https://example.com/impression_url", click_url: "https://example.com/click_url", raw_click_url: "https://example.com/click_url", + score: 0.3, }, ] "#]] @@ -1072,7 +1191,8 @@ mod tests { "url": "https://www.lasagna.restaurant", "icon": "2", "impression_url": "https://example.com/impression_url", - "click_url": "https://example.com/click_url" + "click_url": "https://example.com/click_url", + "score": 0.3 }, { "id": 0, "advertiser": "Los Pollos Hermanos", @@ -1082,7 +1202,8 @@ mod tests { "url": "https://www.lph-nm.biz", "icon": "3", "impression_url": "https://example.com/impression_url", - "click_url": "https://example.com/click_url" + "click_url": "https://example.com/click_url", + "score": 0.3 }]), )? .with_icon("icon-2.png", "lasagna-icon".as_bytes().into()) @@ -1167,6 +1288,7 @@ mod tests { impression_url: "https://example.com/impression_url", click_url: "https://example.com/click_url", raw_click_url: "https://example.com/click_url", + score: 0.3, }, ] "#]] @@ -1207,6 +1329,7 @@ mod tests { impression_url: "https://example.com/impression_url", click_url: "https://example.com/click_url", raw_click_url: "https://example.com/click_url", + score: 0.3, }, ] "#]] @@ -1509,7 +1632,8 @@ mod tests { "url": "https://www.lasagna.restaurant", "icon": "2", "impression_url": "https://example.com/impression_url", - "click_url": "https://example.com/click_url" + "click_url": "https://example.com/click_url", + "score": 0.3 }]), )? .with_icon("icon-2.png", "i-am-an-icon".as_bytes().into()); @@ -1634,6 +1758,7 @@ mod tests { "icon": "2", "impression_url": "https://example.com", "click_url": "https://example.com", + "score": 0.3 }]), )?; @@ -1726,6 +1851,17 @@ mod tests { "hash": "", "size": 0, }, + }, { + "id": "data-5", + "type": "mdn-suggestions", + "last_modified": 15, + "attachment": { + "filename": "data-5.json", + "mimetype": "application/json", + "location": "data-5.json", + "hash": "", + "size": 0, + }, }, { "id": "icon-2", "type": "icon", @@ -1761,6 +1897,7 @@ mod tests { "icon": "2", "impression_url": "https://example.com/impression_url", "click_url": "https://example.com/click_url", + "score": 0.3 }, { "id": 0, "advertiser": "Wikipedia", @@ -1831,14 +1968,14 @@ mod tests { "lowConfidenceKeywords": [], "highConfidenceKeywords": ["multimatch"], "title": "Multimatching", - "score": 0.25 + "score": 0.88 }, ]), )? .with_data( "data-4.json", json!({ - "subjects": ["ramen", "spicy ramen", "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789Z"], + "subjects": ["ramen", "spicy ramen", "spicy random ramen", "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789Z"], "preModifiers": ["best", "super best", "same_modifier"], "postModifiers": ["delivery", "super delivery", "same_modifier"], "locationSigns": [ @@ -1850,6 +1987,18 @@ mod tests { "yelpModifiers": ["yelp", "yelp keyword"], }), )? + .with_data( + "data-5.json", + json!([ + { + "description": "Javascript Array", + "url": "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array", + "keywords": ["array javascript", "javascript array", "wildcard"], + "title": "Array", + "score": 0.24 + }, + ]), + )? .with_icon("icon-2.png", "i-am-an-icon".as_bytes().into()) .with_icon("icon-3.png", "also-an-icon".as_bytes().into()); @@ -1868,6 +2017,7 @@ mod tests { SuggestionProvider::Amo, SuggestionProvider::Pocket, SuggestionProvider::Yelp, + SuggestionProvider::Weather, ], limit: None, }, @@ -1885,6 +2035,7 @@ mod tests { SuggestionProvider::Amo, SuggestionProvider::Pocket, SuggestionProvider::Yelp, + SuggestionProvider::Weather, ], limit: None, }, @@ -1917,6 +2068,7 @@ mod tests { impression_url: "https://example.com/impression_url", click_url: "https://example.com/click_url", raw_click_url: "https://example.com/click_url", + score: 0.3, }, ] "#]], @@ -1935,6 +2087,24 @@ mod tests { }, expect![[r#" [ + Pocket { + title: "Multimatching", + url: "https://getpocket.com/collections/multimatch", + score: 0.88, + is_top_pick: true, + }, + Amo { + title: "Firefox Multimatch", + url: "https://addons.mozilla.org/en-US/firefox/addon/multimatch", + icon_url: "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b", + description: "amo suggestion multi-match", + rating: Some( + "4.9", + ), + number_of_ratings: 888, + guid: "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}", + score: 0.25, + }, Wikipedia { title: "Multimatch", url: "https://wikipedia.org/Multimatch", @@ -1956,24 +2126,6 @@ mod tests { ), full_keyword: "multimatch", }, - Amo { - title: "Firefox Multimatch", - url: "https://addons.mozilla.org/en-US/firefox/addon/multimatch", - icon_url: "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b", - description: "amo suggestion multi-match", - rating: Some( - "4.9", - ), - number_of_ratings: 888, - guid: "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}", - score: 0.25, - }, - Pocket { - title: "Multimatching", - url: "https://getpocket.com/collections/multimatch", - score: 0.25, - is_top_pick: true, - }, ] "#]], ), @@ -1991,6 +2143,24 @@ mod tests { }, expect![[r#" [ + Pocket { + title: "Multimatching", + url: "https://getpocket.com/collections/multimatch", + score: 0.88, + is_top_pick: true, + }, + Amo { + title: "Firefox Multimatch", + url: "https://addons.mozilla.org/en-US/firefox/addon/multimatch", + icon_url: "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b", + description: "amo suggestion multi-match", + rating: Some( + "4.9", + ), + number_of_ratings: 888, + guid: "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}", + score: 0.25, + }, Wikipedia { title: "Multimatch", url: "https://wikipedia.org/Multimatch", @@ -2012,24 +2182,6 @@ mod tests { ), full_keyword: "multimatch", }, - Amo { - title: "Firefox Multimatch", - url: "https://addons.mozilla.org/en-US/firefox/addon/multimatch", - icon_url: "https://addons.mozilla.org/user-media/addon_icons/2633/2633704-64.png?modified=2c11a80b", - description: "amo suggestion multi-match", - rating: Some( - "4.9", - ), - number_of_ratings: 888, - guid: "{b9db16a4-6edc-47ec-a1f4-b86292ed211d}", - score: 0.25, - }, - Pocket { - title: "Multimatching", - url: "https://getpocket.com/collections/multimatch", - score: 0.25, - is_top_pick: true, - }, ] "#]], ), @@ -2047,26 +2199,11 @@ mod tests { }, expect![[r#" [ - Wikipedia { - title: "Multimatch", - url: "https://wikipedia.org/Multimatch", - icon: Some( - [ - 97, - 108, - 115, - 111, - 45, - 97, - 110, - 45, - 105, - 99, - 111, - 110, - ], - ), - full_keyword: "multimatch", + Pocket { + title: "Multimatching", + url: "https://getpocket.com/collections/multimatch", + score: 0.88, + is_top_pick: true, }, Amo { title: "Firefox Multimatch", @@ -2119,6 +2256,7 @@ mod tests { impression_url: "https://example.com/impression_url", click_url: "https://example.com/click_url", raw_click_url: "https://example.com/click_url", + score: 0.3, }, ] "#]], @@ -2444,6 +2582,7 @@ mod tests { Yelp { url: "https://www.yelp.com/search?find_desc=best+spicy+ramen+delivery&find_loc=tokyo", title: "best spicy ramen delivery in tokyo", + is_top_pick: true, }, ] "#]], @@ -2460,6 +2599,7 @@ mod tests { Yelp { url: "https://www.yelp.com/search?find_desc=BeSt+SpIcY+rAmEn+DeLiVeRy&find_loc=ToKyO", title: "BeSt SpIcY rAmEn DeLiVeRy In ToKyO", + is_top_pick: true, }, ] "#]], @@ -2476,6 +2616,7 @@ mod tests { Yelp { url: "https://www.yelp.com/search?find_desc=best+ramen+delivery&find_loc=tokyo", title: "best ramen delivery in tokyo", + is_top_pick: true, }, ] "#]], @@ -2514,6 +2655,7 @@ mod tests { Yelp { url: "https://www.yelp.com/search?find_desc=super+best+ramen+delivery&find_loc=tokyo", title: "super best ramen delivery in tokyo", + is_top_pick: true, }, ] "#]], @@ -2541,6 +2683,7 @@ mod tests { Yelp { url: "https://www.yelp.com/search?find_desc=ramen+delivery&find_loc=tokyo", title: "ramen delivery in tokyo", + is_top_pick: true, }, ] "#]], @@ -2557,6 +2700,7 @@ mod tests { Yelp { url: "https://www.yelp.com/search?find_desc=ramen+super+delivery&find_loc=tokyo", title: "ramen super delivery in tokyo", + is_top_pick: true, }, ] "#]], @@ -2584,6 +2728,7 @@ mod tests { Yelp { url: "https://www.yelp.com/search?find_desc=ramen&find_loc=tokyo", title: "ramen in tokyo", + is_top_pick: true, }, ] "#]], @@ -2600,6 +2745,7 @@ mod tests { Yelp { url: "https://www.yelp.com/search?find_desc=ramen&find_loc=tokyo", title: "ramen near tokyo", + is_top_pick: true, }, ] "#]], @@ -2627,6 +2773,7 @@ mod tests { Yelp { url: "https://www.yelp.com/search?find_desc=ramen&find_loc=San+Francisco", title: "ramen in San Francisco", + is_top_pick: true, }, ] "#]], @@ -2643,6 +2790,7 @@ mod tests { Yelp { url: "https://www.yelp.com/search?find_desc=ramen", title: "ramen in", + is_top_pick: true, }, ] "#]], @@ -2659,6 +2807,7 @@ mod tests { Yelp { url: "https://www.yelp.com/search?find_desc=ramen+near+by", title: "ramen near by", + is_top_pick: true, }, ] "#]], @@ -2675,6 +2824,7 @@ mod tests { Yelp { url: "https://www.yelp.com/search?find_desc=ramen+near+me", title: "ramen near me", + is_top_pick: true, }, ] "#]], @@ -2702,6 +2852,7 @@ mod tests { Yelp { url: "https://www.yelp.com/search?find_desc=ramen", title: "ramen", + is_top_pick: true, }, ] "#]], @@ -2718,6 +2869,7 @@ mod tests { Yelp { url: "https://www.yelp.com/search?find_desc=012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", title: "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789", + is_top_pick: true, }, ] "#]], @@ -2778,6 +2930,7 @@ mod tests { Yelp { url: "https://www.yelp.com/search?find_desc=ramen", title: "yelp ramen", + is_top_pick: true, }, ] "#]], @@ -2794,6 +2947,7 @@ mod tests { Yelp { url: "https://www.yelp.com/search?find_desc=ramen", title: "yelp keyword ramen", + is_top_pick: true, }, ] "#]], @@ -2810,6 +2964,7 @@ mod tests { Yelp { url: "https://www.yelp.com/search?find_desc=ramen&find_loc=tokyo", title: "ramen in tokyo yelp", + is_top_pick: true, }, ] "#]], @@ -2826,6 +2981,7 @@ mod tests { Yelp { url: "https://www.yelp.com/search?find_desc=ramen&find_loc=tokyo", title: "ramen in tokyo yelp keyword", + is_top_pick: true, }, ] "#]], @@ -2842,6 +2998,7 @@ mod tests { Yelp { url: "https://www.yelp.com/search?find_desc=ramen", title: "yelp ramen yelp", + is_top_pick: true, }, ] "#]], @@ -2857,6 +3014,68 @@ mod tests { [] "#]], ), + ( + "keyword = `Spicy R`; Yelp only", + SuggestionQuery { + keyword: "Spicy R".into(), + providers: vec![SuggestionProvider::Yelp], + limit: None, + }, + expect![[r#" + [ + Yelp { + url: "https://www.yelp.com/search?find_desc=Spicy+Ramen", + title: "Spicy Ramen", + is_top_pick: false, + }, + ] + "#]], + ), + ( + "keyword = `BeSt Ramen`; Yelp only", + SuggestionQuery { + keyword: "BeSt Ramen".into(), + providers: vec![SuggestionProvider::Yelp], + limit: None, + }, + expect![[r#" + [ + Yelp { + url: "https://www.yelp.com/search?find_desc=BeSt+Ramen", + title: "BeSt Ramen", + is_top_pick: true, + }, + ] + "#]], + ), + ( + "keyword = `BeSt Spicy R`; Yelp only", + SuggestionQuery { + keyword: "BeSt Spicy R".into(), + providers: vec![SuggestionProvider::Yelp], + limit: None, + }, + expect![[r#" + [ + Yelp { + url: "https://www.yelp.com/search?find_desc=BeSt+Spicy+Ramen", + title: "BeSt Spicy Ramen", + is_top_pick: false, + }, + ] + "#]], + ), + ( + "keyword = `BeSt R`; Yelp only", + SuggestionQuery { + keyword: "BeSt R".into(), + providers: vec![SuggestionProvider::Yelp], + limit: None, + }, + expect![[r#" + [] + "#]], + ), ]; for (what, query, expect) in table { expect.assert_debug_eq( @@ -2869,6 +3088,329 @@ mod tests { Ok(()) } + // Tests querying amp wikipedia + #[test] + fn query_with_multiple_providers_and_diff_scores() -> anyhow::Result<()> { + before_each(); + + let snapshot = Snapshot::with_records(json!([{ + "id": "data-1", + "type": "data", + "last_modified": 15, + "attachment": { + "filename": "data-1.json", + "mimetype": "application/json", + "location": "data-1.json", + "hash": "", + "size": 0, + }, + }, { + "id": "data-2", + "type": "pocket-suggestions", + "last_modified": 15, + "attachment": { + "filename": "data-2.json", + "mimetype": "application/json", + "location": "data-2.json", + "hash": "", + "size": 0, + }, + }, { + "id": "icon-3", + "type": "icon", + "last_modified": 25, + "attachment": { + "filename": "icon-3.png", + "mimetype": "image/png", + "location": "icon-3.png", + "hash": "", + "size": 0, + }, + }]))? + .with_data( + "data-1.json", + json!([{ + "id": 0, + "advertiser": "Good Place Eats", + "iab_category": "8 - Food & Drink", + "keywords": ["la", "las", "lasa", "lasagna", "lasagna come out tomorrow", "amp wiki match"], + "title": "Lasagna Come Out Tomorrow", + "url": "https://www.lasagna.restaurant", + "icon": "2", + "impression_url": "https://example.com/impression_url", + "click_url": "https://example.com/click_url", + "score": 0.3 + }, { + "id": 0, + "advertiser": "Good Place Eats", + "iab_category": "8 - Food & Drink", + "keywords": ["pe", "pen", "penne", "penne for your thoughts", "amp wiki match"], + "title": "Penne for Your Thoughts", + "url": "https://penne.biz", + "icon": "2", + "impression_url": "https://example.com/impression_url", + "click_url": "https://example.com/click_url", + "score": 0.1 + }, { + "id": 0, + "advertiser": "Wikipedia", + "iab_category": "5 - Education", + "keywords": ["amp wiki match", "pocket wiki match"], + "title": "Multimatch", + "url": "https://wikipedia.org/Multimatch", + "icon": "3" + }]), + )? + .with_data( + "data-2.json", + json!([ + { + "description": "pocket suggestion", + "url": "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women", + "lowConfidenceKeywords": ["soft life", "workaholism", "toxic work culture", "work-life balance", "pocket wiki match"], + "highConfidenceKeywords": ["burnout women", "grind culture", "women burnout"], + "title": "‘It’s Not Just Burnout:’ How Grind Culture Fails Women", + "score": 0.05 + }, + { + "description": "pocket suggestion multi-match", + "url": "https://getpocket.com/collections/multimatch", + "lowConfidenceKeywords": [], + "highConfidenceKeywords": ["pocket wiki match"], + "title": "Pocket wiki match", + "score": 0.88 + }, + ]), + )? + .with_icon("icon-3.png", "also-an-icon".as_bytes().into()); + + let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot)); + + store.ingest(SuggestIngestionConstraints::default())?; + + let table = [ + ( + "keyword = `amp wiki match`; all providers", + SuggestionQuery { + keyword: "amp wiki match".into(), + providers: vec![ + SuggestionProvider::Amp, + SuggestionProvider::Wikipedia, + SuggestionProvider::Amo, + SuggestionProvider::Pocket, + SuggestionProvider::Yelp, + ], + limit: None, + }, + expect![[r#" + [ + Amp { + title: "Lasagna Come Out Tomorrow", + url: "https://www.lasagna.restaurant", + raw_url: "https://www.lasagna.restaurant", + icon: None, + full_keyword: "amp wiki match", + block_id: 0, + advertiser: "Good Place Eats", + iab_category: "8 - Food & Drink", + impression_url: "https://example.com/impression_url", + click_url: "https://example.com/click_url", + raw_click_url: "https://example.com/click_url", + score: 0.3, + }, + Wikipedia { + title: "Multimatch", + url: "https://wikipedia.org/Multimatch", + icon: Some( + [ + 97, + 108, + 115, + 111, + 45, + 97, + 110, + 45, + 105, + 99, + 111, + 110, + ], + ), + full_keyword: "amp wiki match", + }, + Amp { + title: "Penne for Your Thoughts", + url: "https://penne.biz", + raw_url: "https://penne.biz", + icon: None, + full_keyword: "amp wiki match", + block_id: 0, + advertiser: "Good Place Eats", + iab_category: "8 - Food & Drink", + impression_url: "https://example.com/impression_url", + click_url: "https://example.com/click_url", + raw_click_url: "https://example.com/click_url", + score: 0.1, + }, + ] + "#]], + ), + ( + "keyword = `amp wiki match`; all providers, limit 2", + SuggestionQuery { + keyword: "amp wiki match".into(), + providers: vec![ + SuggestionProvider::Amp, + SuggestionProvider::Wikipedia, + SuggestionProvider::Amo, + SuggestionProvider::Pocket, + SuggestionProvider::Yelp, + ], + limit: Some(2), + }, + expect![[r#" + [ + Amp { + title: "Lasagna Come Out Tomorrow", + url: "https://www.lasagna.restaurant", + raw_url: "https://www.lasagna.restaurant", + icon: None, + full_keyword: "amp wiki match", + block_id: 0, + advertiser: "Good Place Eats", + iab_category: "8 - Food & Drink", + impression_url: "https://example.com/impression_url", + click_url: "https://example.com/click_url", + raw_click_url: "https://example.com/click_url", + score: 0.3, + }, + Wikipedia { + title: "Multimatch", + url: "https://wikipedia.org/Multimatch", + icon: Some( + [ + 97, + 108, + 115, + 111, + 45, + 97, + 110, + 45, + 105, + 99, + 111, + 110, + ], + ), + full_keyword: "amp wiki match", + }, + ] + "#]], + ), + ( + "pocket wiki match; all providers", + SuggestionQuery { + keyword: "pocket wiki match".into(), + providers: vec![ + SuggestionProvider::Amp, + SuggestionProvider::Wikipedia, + SuggestionProvider::Amo, + SuggestionProvider::Pocket, + ], + limit: None, + }, + expect![[r#" + [ + Pocket { + title: "Pocket wiki match", + url: "https://getpocket.com/collections/multimatch", + score: 0.88, + is_top_pick: true, + }, + Wikipedia { + title: "Multimatch", + url: "https://wikipedia.org/Multimatch", + icon: Some( + [ + 97, + 108, + 115, + 111, + 45, + 97, + 110, + 45, + 105, + 99, + 111, + 110, + ], + ), + full_keyword: "pocket wiki match", + }, + Pocket { + title: "‘It’s Not Just Burnout:’ How Grind Culture Fails Women", + url: "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women", + score: 0.05, + is_top_pick: false, + }, + ] + "#]], + ), + ( + "pocket wiki match; all providers limit 1", + SuggestionQuery { + keyword: "pocket wiki match".into(), + providers: vec![ + SuggestionProvider::Amp, + SuggestionProvider::Wikipedia, + SuggestionProvider::Amo, + SuggestionProvider::Pocket, + ], + limit: Some(1), + }, + expect![[r#" + [ + Pocket { + title: "Pocket wiki match", + url: "https://getpocket.com/collections/multimatch", + score: 0.88, + is_top_pick: true, + }, + ] + "#]], + ), + ( + "work-life balance; duplicate providers", + SuggestionQuery { + keyword: "work-life balance".into(), + providers: vec![SuggestionProvider::Pocket, SuggestionProvider::Pocket], + limit: Some(-1), + }, + expect![[r#" + [ + Pocket { + title: "‘It’s Not Just Burnout:’ How Grind Culture Fails Women", + url: "https://getpocket.com/collections/its-not-just-burnout-how-grind-culture-failed-women", + score: 0.05, + is_top_pick: false, + }, + ] + "#]], + ), + ]; + for (what, query, expect) in table { + expect.assert_debug_eq( + &store + .query(query) + .with_context(|| format!("Couldn't query store for {}", what))?, + ); + } + + Ok(()) + } /// Tests ingesting malformed Remote Settings records that we understand, /// but that are missing fields, or aren't in the format we expect. #[test] @@ -2947,10 +3489,10 @@ mod tests { UnparsableRecords( { "clippy-2": UnparsableRecord { - schema_version: 10, + schema_version: 12, }, "fancy-new-suggestions-1": UnparsableRecord { - schema_version: 10, + schema_version: 12, }, }, ), @@ -3001,6 +3543,7 @@ mod tests { "icon": "5678", "impression_url": "https://example.com/impression_url", "click_url": "https://example.com/click_url", + "score": 0.3, }]), )?; @@ -3015,10 +3558,10 @@ mod tests { UnparsableRecords( { "clippy-2": UnparsableRecord { - schema_version: 10, + schema_version: 12, }, "fancy-new-suggestions-1": UnparsableRecord { - schema_version: 10, + schema_version: 12, }, }, ), @@ -3096,6 +3639,7 @@ mod tests { "icon": "5678", "impression_url": "https://example.com/impression_url", "click_url": "https://example.com/click_url", + "score": 0.3 }]), )?; @@ -3121,10 +3665,10 @@ mod tests { UnparsableRecords( { "clippy-2": UnparsableRecord { - schema_version: 10, + schema_version: 12, }, "fancy-new-suggestions-1": UnparsableRecord { - schema_version: 10, + schema_version: 12, }, }, ), @@ -3137,10 +3681,651 @@ mod tests { Ok(()) } + /// Tests that records with invalid attachments are ignored and marked as unparsable. + #[test] + fn skip_over_invalid_records() -> anyhow::Result<()> { + before_each(); + + let snapshot = Snapshot::with_records(json!([ + { + "id": "invalid-attachment", + "type": "data", + "last_modified": 15, + "attachment": { + "filename": "data-2.json", + "mimetype": "application/json", + "location": "data-2.json", + "hash": "", + "size": 0, + }, + }, + { + "id": "valid-record", + "type": "data", + "last_modified": 15, + "attachment": { + "filename": "data-1.json", + "mimetype": "application/json", + "location": "data-1.json", + "hash": "", + "size": 0, + }, + }, + ]))? + .with_data( + "data-1.json", + json!([{ + "id": 0, + "advertiser": "Los Pollos Hermanos", + "iab_category": "8 - Food & Drink", + "keywords": ["lo", "los", "los p", "los pollos", "los pollos h", "los pollos hermanos"], + "title": "Los Pollos Hermanos - Albuquerque", + "url": "https://www.lph-nm.biz", + "icon": "5678", + "impression_url": "https://example.com/impression_url", + "click_url": "https://example.com/click_url", + "score": 0.3 + }]), + )? + // This attachment is missing the `keywords` field and is invalid + .with_data( + "data-2.json", + json!([{ + "id": 1, + "advertiser": "Los Pollos Hermanos", + "iab_category": "8 - Food & Drink", + "title": "Los Pollos Hermanos - Albuquerque", + "url": "https://www.lph-nm.biz", + "icon": "5678", + "impression_url": "https://example.com/impression_url", + "click_url": "https://example.com/click_url", + "score": 0.3 + }]), + )?; + + let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot)); + + store.ingest(SuggestIngestionConstraints::default())?; + + // Test that the invalid record marked as unparsable + store.dbs()?.reader.read(|dao| { + expect![[r#" + Some( + UnparsableRecords( + { + "invalid-attachment": UnparsableRecord { + schema_version: 12, + }, + }, + ), + ) + "#]] + .assert_debug_eq(&dao.get_meta::(UNPARSABLE_RECORDS_META_KEY)?); + Ok(()) + })?; + + // Test that the valid record was read + store.dbs()?.reader.read(|dao| { + assert_eq!(dao.get_meta::(LAST_INGEST_META_KEY)?, Some(15)); + expect![[r#" + [ + Amp { + title: "Los Pollos Hermanos - Albuquerque", + url: "https://www.lph-nm.biz", + raw_url: "https://www.lph-nm.biz", + icon: None, + full_keyword: "los", + block_id: 0, + advertiser: "Los Pollos Hermanos", + iab_category: "8 - Food & Drink", + impression_url: "https://example.com/impression_url", + click_url: "https://example.com/click_url", + raw_click_url: "https://example.com/click_url", + score: 0.3, + }, + ] + "#]] + .assert_debug_eq(&dao.fetch_suggestions(&SuggestionQuery { + keyword: "lo".into(), + providers: vec![SuggestionProvider::Amp], + limit: None, + })?); + + Ok(()) + })?; + + Ok(()) + } + #[test] fn unparsable_record_serialized_correctly() -> anyhow::Result<()> { let unparseable_record = UnparsableRecord { schema_version: 1 }; assert_eq!(serde_json::to_value(unparseable_record)?, json!({ "v": 1 }),); Ok(()) } + + #[test] + fn query_mdn() -> anyhow::Result<()> { + before_each(); + + let snapshot = Snapshot::with_records(json!([{ + "id": "data-1", + "type": "mdn-suggestions", + "last_modified": 15, + "attachment": { + "filename": "data-1.json", + "mimetype": "application/json", + "location": "data-1.json", + "hash": "", + "size": 0, + }, + }]))? + .with_data( + "data-1.json", + json!([ + { + "description": "Javascript Array", + "url": "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array", + "keywords": ["array javascript", "javascript array", "wildcard"], + "title": "Array", + "score": 0.24 + }, + ]), + )?; + + let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot)); + + store.ingest(SuggestIngestionConstraints::default())?; + + let table = [ + ( + "keyword = prefix; MDN only", + SuggestionQuery { + keyword: "array".into(), + providers: vec![SuggestionProvider::Mdn], + limit: None, + }, + expect![[r#" + [ + Mdn { + title: "Array", + url: "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array", + description: "Javascript Array", + score: 0.24, + }, + ] + "#]], + ), + ( + "keyword = prefix + partial suffix; MDN only", + SuggestionQuery { + keyword: "array java".into(), + providers: vec![SuggestionProvider::Mdn], + limit: None, + }, + expect![[r#" + [ + Mdn { + title: "Array", + url: "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array", + description: "Javascript Array", + score: 0.24, + }, + ] + "#]], + ), + ( + "keyword = prefix + entire suffix; MDN only", + SuggestionQuery { + keyword: "javascript array".into(), + providers: vec![SuggestionProvider::Mdn], + limit: None, + }, + expect![[r#" + [ + Mdn { + title: "Array", + url: "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array", + description: "Javascript Array", + score: 0.24, + }, + ] + "#]], + ), + ( + "keyword = `partial prefix word`; MDN only", + SuggestionQuery { + keyword: "wild".into(), + providers: vec![SuggestionProvider::Mdn], + limit: None, + }, + expect![[r#" + [] + "#]], + ), + ( + "keyword = single word; MDN only", + SuggestionQuery { + keyword: "wildcard".into(), + providers: vec![SuggestionProvider::Mdn], + limit: None, + }, + expect![[r#" + [ + Mdn { + title: "Array", + url: "https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array", + description: "Javascript Array", + score: 0.24, + }, + ] + "#]], + ), + ]; + + for (what, query, expect) in table { + expect.assert_debug_eq( + &store + .query(query) + .with_context(|| format!("Couldn't query store for {}", what))?, + ); + } + + Ok(()) + } + + #[test] + fn weather() -> anyhow::Result<()> { + before_each(); + + let snapshot = Snapshot::with_records(json!([{ + "id": "data-1", + "type": "weather", + "last_modified": 15, + "weather": { + "min_keyword_length": 3, + "keywords": ["ab", "xyz", "weather"], + "score": "0.24" + } + }]))?; + + let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot)); + store.ingest(SuggestIngestionConstraints::default())?; + + let table = [ + ( + "keyword = 'ab'; Weather only, no match since query is too short", + SuggestionQuery { + keyword: "ab".into(), + providers: vec![SuggestionProvider::Weather], + limit: None, + }, + expect![[r#" + [] + "#]], + ), + ( + "keyword = 'xab'; Weather only, no matching keyword", + SuggestionQuery { + keyword: "xab".into(), + providers: vec![SuggestionProvider::Weather], + limit: None, + }, + expect![[r#" + [] + "#]], + ), + ( + "keyword = 'abx'; Weather only, no matching keyword", + SuggestionQuery { + keyword: "abx".into(), + providers: vec![SuggestionProvider::Weather], + limit: None, + }, + expect![[r#" + [] + "#]], + ), + ( + "keyword = 'xy'; Weather only, no match since query is too short", + SuggestionQuery { + keyword: "xy".into(), + providers: vec![SuggestionProvider::Weather], + limit: None, + }, + expect![[r#" + [] + "#]], + ), + ( + "keyword = 'xyz'; Weather only, match", + SuggestionQuery { + keyword: "xyz".into(), + providers: vec![SuggestionProvider::Weather], + limit: None, + }, + expect![[r#" + [ + Weather { + score: 0.24, + }, + ] + "#]], + ), + ( + "keyword = 'xxyz'; Weather only, no matching keyword", + SuggestionQuery { + keyword: "xxyz".into(), + providers: vec![SuggestionProvider::Weather], + limit: None, + }, + expect![[r#" + [] + "#]], + ), + ( + "keyword = 'xyzx'; Weather only, no matching keyword", + SuggestionQuery { + keyword: "xyzx".into(), + providers: vec![SuggestionProvider::Weather], + limit: None, + }, + expect![[r#" + [] + "#]], + ), + ( + "keyword = 'we'; Weather only, no match since query is too short", + SuggestionQuery { + keyword: "we".into(), + providers: vec![SuggestionProvider::Weather], + limit: None, + }, + expect![[r#" + [] + "#]], + ), + ( + "keyword = 'wea'; Weather only, match", + SuggestionQuery { + keyword: "wea".into(), + providers: vec![SuggestionProvider::Weather], + limit: None, + }, + expect![[r#" + [ + Weather { + score: 0.24, + }, + ] + "#]], + ), + ( + "keyword = 'weat'; Weather only, match", + SuggestionQuery { + keyword: "weat".into(), + providers: vec![SuggestionProvider::Weather], + limit: None, + }, + expect![[r#" + [ + Weather { + score: 0.24, + }, + ] + "#]], + ), + ( + "keyword = 'weath'; Weather only, match", + SuggestionQuery { + keyword: "weath".into(), + providers: vec![SuggestionProvider::Weather], + limit: None, + }, + expect![[r#" + [ + Weather { + score: 0.24, + }, + ] + "#]], + ), + ( + "keyword = 'weathe'; Weather only, match", + SuggestionQuery { + keyword: "weathe".into(), + providers: vec![SuggestionProvider::Weather], + limit: None, + }, + expect![[r#" + [ + Weather { + score: 0.24, + }, + ] + "#]], + ), + ( + "keyword = 'weather'; Weather only, match", + SuggestionQuery { + keyword: "weather".into(), + providers: vec![SuggestionProvider::Weather], + limit: None, + }, + expect![[r#" + [ + Weather { + score: 0.24, + }, + ] + "#]], + ), + ( + "keyword = 'weatherx'; Weather only, no matching keyword", + SuggestionQuery { + keyword: "weatherx".into(), + providers: vec![SuggestionProvider::Weather], + limit: None, + }, + expect![[r#" + [] + "#]], + ), + ( + "keyword = 'xweather'; Weather only, no matching keyword", + SuggestionQuery { + keyword: "xweather".into(), + providers: vec![SuggestionProvider::Weather], + limit: None, + }, + expect![[r#" + [] + "#]], + ), + ( + "keyword = 'xwea'; Weather only, no matching keyword", + SuggestionQuery { + keyword: "xwea".into(), + providers: vec![SuggestionProvider::Weather], + limit: None, + }, + expect![[r#" + [] + "#]], + ), + ( + "keyword = ' weather '; Weather only, match", + SuggestionQuery { + keyword: " weather ".into(), + providers: vec![SuggestionProvider::Weather], + limit: None, + }, + expect![[r#" + [ + Weather { + score: 0.24, + }, + ] + "#]], + ), + ( + "keyword = 'x weather '; Weather only, no matching keyword", + SuggestionQuery { + keyword: "x weather ".into(), + providers: vec![SuggestionProvider::Weather], + limit: None, + }, + expect![[r#" + [] + "#]], + ), + ( + "keyword = ' weather x'; Weather only, no matching keyword", + SuggestionQuery { + keyword: " weather x".into(), + providers: vec![SuggestionProvider::Weather], + limit: None, + }, + expect![[r#" + [] + "#]], + ), + ]; + + for (what, query, expect) in table { + expect.assert_debug_eq( + &store + .query(query) + .with_context(|| format!("Couldn't query store for {}", what))?, + ); + } + + expect![[r#" + Some( + Weather { + min_keyword_length: 3, + }, + ) + "#]] + .assert_debug_eq( + &store + .fetch_provider_config(SuggestionProvider::Weather) + .with_context(|| "Couldn't fetch provider config")?, + ); + + Ok(()) + } + + #[test] + fn fetch_global_config() -> anyhow::Result<()> { + before_each(); + + let snapshot = Snapshot::with_records(json!([{ + "id": "data-1", + "type": "configuration", + "last_modified": 15, + "configuration": { + "show_less_frequently_cap": 3, + } + }]))?; + + let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot)); + store.ingest(SuggestIngestionConstraints::default())?; + + expect![[r#" + SuggestGlobalConfig { + show_less_frequently_cap: 3, + } + "#]] + .assert_debug_eq( + &store + .fetch_global_config() + .with_context(|| "fetch_global_config failed")?, + ); + + Ok(()) + } + + #[test] + fn fetch_global_config_default() -> anyhow::Result<()> { + before_each(); + + let snapshot = Snapshot::with_records(json!([]))?; + let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot)); + store.ingest(SuggestIngestionConstraints::default())?; + + expect![[r#" + SuggestGlobalConfig { + show_less_frequently_cap: 0, + } + "#]] + .assert_debug_eq( + &store + .fetch_global_config() + .with_context(|| "fetch_global_config failed")?, + ); + + Ok(()) + } + + #[test] + fn fetch_provider_config_none() -> anyhow::Result<()> { + before_each(); + + let snapshot = Snapshot::with_records(json!([]))?; + let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot)); + store.ingest(SuggestIngestionConstraints::default())?; + + expect![[r#" + None + "#]] + .assert_debug_eq( + &store + .fetch_provider_config(SuggestionProvider::Amp) + .with_context(|| "fetch_provider_config failed for Amp")?, + ); + + expect![[r#" + None + "#]] + .assert_debug_eq( + &store + .fetch_provider_config(SuggestionProvider::Weather) + .with_context(|| "fetch_provider_config failed for Weather")?, + ); + + Ok(()) + } + + #[test] + fn fetch_provider_config_other() -> anyhow::Result<()> { + before_each(); + + // Add some weather config. + let snapshot = Snapshot::with_records(json!([{ + "id": "data-1", + "type": "weather", + "last_modified": 15, + "weather": { + "min_keyword_length": 3, + "keywords": ["weather"], + "score": "0.24" + } + }]))?; + + let store = unique_test_store(SnapshotSettingsClient::with_snapshot(snapshot)); + store.ingest(SuggestIngestionConstraints::default())?; + + // Getting the config for a different provider should return None. + expect![[r#" + None + "#]] + .assert_debug_eq( + &store + .fetch_provider_config(SuggestionProvider::Amp) + .with_context(|| "fetch_provider_config failed for Amp")?, + ); + + Ok(()) + } } diff --git a/third_party/rust/suggest/src/suggest.udl b/third_party/rust/suggest/src/suggest.udl index e7c85df26aca..b63645dac3d9 100644 --- a/third_party/rust/suggest/src/suggest.udl +++ b/third_party/rust/suggest/src/suggest.udl @@ -14,6 +14,11 @@ boolean raw_suggestion_url_matches([ByRef] string raw_url, [ByRef] string url); [Error] interface SuggestApiError { + // An operation was interrupted by calling `SuggestStore.interrupt()` + Interrupted(); + // The server requested a backoff after too many requests + Backoff(u64 seconds); + Network(string reason); Other(string reason); }; @@ -22,7 +27,9 @@ enum SuggestionProvider { "Pocket", "Wikipedia", "Amo", - "Yelp" + "Yelp", + "Mdn", + "Weather", }; [Enum] @@ -38,7 +45,8 @@ interface Suggestion { string iab_category, string impression_url, string click_url, - string raw_click_url + string raw_click_url, + f64 score ); Pocket( string title, @@ -64,7 +72,17 @@ interface Suggestion { ); Yelp( string url, - string title + string title, + boolean is_top_pick + ); + Mdn( + string title, + string url, + string description, + f64 score + ); + Weather( + f64 score ); }; @@ -78,6 +96,17 @@ dictionary SuggestIngestionConstraints { u64? max_suggestions = null; }; +dictionary SuggestGlobalConfig { + i32 show_less_frequently_cap; +}; + +[Enum] +interface SuggestProviderConfig { + Weather( + i32 min_keyword_length + ); +}; + interface SuggestStore { [Throws=SuggestApiError] constructor([ByRef] string path, optional RemoteSettingsConfig? settings_config = null); @@ -92,4 +121,26 @@ interface SuggestStore { [Throws=SuggestApiError] void clear(); + + [Throws=SuggestApiError] + SuggestGlobalConfig fetch_global_config(); + + [Throws=SuggestApiError] + SuggestProviderConfig? fetch_provider_config(SuggestionProvider provider); +}; + +interface SuggestStoreBuilder { + constructor(); + + [Self=ByArc] + SuggestStoreBuilder data_path(string path); + + [Self=ByArc] + SuggestStoreBuilder cache_path(string path); + + [Self=ByArc] + SuggestStoreBuilder remote_settings_config(RemoteSettingsConfig config); + + [Throws=SuggestApiError] + SuggestStore build(); }; diff --git a/third_party/rust/suggest/src/suggestion.rs b/third_party/rust/suggest/src/suggestion.rs index 314c4210a409..68135d65d896 100644 --- a/third_party/rust/suggest/src/suggestion.rs +++ b/third_party/rust/suggest/src/suggestion.rs @@ -5,6 +5,8 @@ use chrono::Local; +use crate::db::DEFAULT_SUGGESTION_SCORE; + /// The template parameter for a timestamp in a "raw" sponsored suggestion URL. const TIMESTAMP_TEMPLATE: &str = "%YYYYMMDDHH%"; @@ -29,6 +31,7 @@ pub enum Suggestion { impression_url: String, click_url: String, raw_click_url: String, + score: f64, }, Pocket { title: String, @@ -55,9 +58,46 @@ pub enum Suggestion { Yelp { url: String, title: String, + is_top_pick: bool, + }, + Mdn { + title: String, + url: String, + description: String, + score: f64, + }, + Weather { + score: f64, }, } +impl PartialOrd for Suggestion { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for Suggestion { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + let a_score = match self { + Suggestion::Amp { score, .. } + | Suggestion::Pocket { score, .. } + | Suggestion::Amo { score, .. } => score, + _ => &DEFAULT_SUGGESTION_SCORE, + }; + let b_score = match other { + Suggestion::Amp { score, .. } + | Suggestion::Pocket { score, .. } + | Suggestion::Amo { score, .. } => score, + _ => &DEFAULT_SUGGESTION_SCORE, + }; + b_score + .partial_cmp(a_score) + .unwrap_or(std::cmp::Ordering::Equal) + } +} + +impl Eq for Suggestion {} /// Replaces all template parameters in a "raw" sponsored suggestion URL, /// producing a "cooked" URL with real values. pub(crate) fn cook_raw_suggestion_url(raw_url: &str) -> String { diff --git a/third_party/rust/suggest/src/yelp.rs b/third_party/rust/suggest/src/yelp.rs index 1e6cb976699a..f3c385250c29 100644 --- a/third_party/rust/suggest/src/yelp.rs +++ b/third_party/rust/suggest/src/yelp.rs @@ -47,6 +47,9 @@ const MAX_QUERY_LENGTH: usize = 150; /// "keyword=:modifier" (please see is_modifier()), define this how many words we should check. const MAX_MODIFIER_WORDS_NUMBER: usize = 2; +/// The threshold that enables prefix-match. +const PREFIX_MATCH_THRESHOLD: usize = 6; + impl<'a> SuggestDao<'a> { /// Inserts the suggestions for Yelp attachment into the database. pub fn insert_yelp_suggestions( @@ -117,35 +120,36 @@ impl<'a> SuggestDao<'a> { } /// Fetch Yelp suggestion from given user's query. - pub fn fetch_yelp_suggestion(&self, query: &SuggestionQuery) -> Result> { + pub fn fetch_yelp_suggestions(&self, query: &SuggestionQuery) -> Result> { if !query.providers.contains(&SuggestionProvider::Yelp) { - return Ok(None); + return Ok(vec![]); } if query.keyword.len() > MAX_QUERY_LENGTH { - return Ok(None); + return Ok(vec![]); } let query_string = &query.keyword.trim(); if !query_string.contains(' ') { - if !self.is_subject(query_string)? { - return Ok(None); - } - + let Some((subject, subject_exact_match)) = self.find_subject(query_string)? else { + return Ok(vec![]); + }; let builder = SuggestionBuilder { - query, - subject: query_string, + subject: &subject, + subject_exact_match, pre_modifier: None, post_modifier: None, location_sign: None, location: None, need_location: false, + pre_yelp_modifier: None, + post_yelp_modifier: None, }; - return Ok(Some(builder.into())); + return Ok(vec![builder.into()]); } // Find the yelp keyword modifier and remove them from the query. - let (query_without_yelp_modifiers, _, _) = + let (query_without_yelp_modifiers, pre_yelp_modifier, post_yelp_modifier) = self.find_modifiers(query_string, Modifier::Yelp, Modifier::Yelp)?; // Find the location sign and the location. @@ -154,32 +158,33 @@ impl<'a> SuggestDao<'a> { if let (Some(_), false) = (&location, need_location) { // The location sign does not need the specific location, but user is setting something. - return Ok(None); + return Ok(vec![]); } if query_without_location.is_empty() { // No remained query. - return Ok(None); + return Ok(vec![]); } // Find the modifiers. let (subject_candidate, pre_modifier, post_modifier) = self.find_modifiers(&query_without_location, Modifier::Pre, Modifier::Post)?; - if !self.is_subject(&subject_candidate)? { - return Ok(None); - } - + let Some((subject, subject_exact_match)) = self.find_subject(&subject_candidate)? else { + return Ok(vec![]); + }; let builder = SuggestionBuilder { - query, - subject: &subject_candidate, + subject: &subject, + subject_exact_match, pre_modifier, post_modifier, location_sign, location, need_location, + pre_yelp_modifier, + post_yelp_modifier, }; - Ok(Some(builder.into())) + Ok(vec![builder.into()]) } /// Find the location information from the given query string. @@ -296,6 +301,46 @@ impl<'a> SuggestDao<'a> { )) } + /// Find the subject from the given string. + /// It returns the Option. If it is not none, it contains the tuple as follows: + /// ( + /// String: Subject. + /// bool: Whether the subject matched exactly with the paramter. + /// ) + fn find_subject(&self, candidate: &str) -> Result> { + if candidate.is_empty() { + return Ok(None); + } + + // If the length of subject candidate is less than PREFIX_MATCH_THRESHOLD, + // should exact match. + if candidate.len() < PREFIX_MATCH_THRESHOLD { + return Ok(if self.is_subject(candidate)? { + Some((candidate.to_string(), true)) + } else { + None + }); + } + + // Otherwise, apply prefix-match. + Ok( + match self.conn.query_row_and_then_cachable( + "SELECT keyword FROM yelp_subjects WHERE keyword BETWEEN :candidate AND :candidate || x'FFFF' ORDER BY LENGTH(keyword) ASC LIMIT 1", + named_params! { + ":candidate": candidate.to_lowercase(), + }, + |row| row.get::<_, String>(0), + true, + ) { + Ok(keyword) => { + debug_assert!(candidate.len() <= keyword.len()); + Some((format!("{}{}", candidate, &keyword[candidate.len()..]), candidate.len() == keyword.len())) + }, + Err(_) => None + } + ) + } + fn is_modifier(&self, word: &str, modifier_type: Modifier) -> Result { let result = self.conn.query_row_and_then_cachable( " @@ -315,10 +360,6 @@ impl<'a> SuggestDao<'a> { } fn is_subject(&self, word: &str) -> Result { - if word.is_empty() { - return Ok(false); - } - let result = self.conn.query_row_and_then_cachable( " SELECT EXISTS ( @@ -337,32 +378,34 @@ impl<'a> SuggestDao<'a> { } struct SuggestionBuilder<'a> { - query: &'a SuggestionQuery, subject: &'a str, + subject_exact_match: bool, pre_modifier: Option, post_modifier: Option, location_sign: Option, location: Option, need_location: bool, + pre_yelp_modifier: Option, + post_yelp_modifier: Option, } impl<'a> From> for Suggestion { fn from(builder: SuggestionBuilder<'a>) -> Suggestion { // This location sign such the 'near by' needs to add as a description parameter. let location_modifier = if !builder.need_location { - builder.location_sign + builder.location_sign.as_deref() } else { None }; let description = [ - builder.pre_modifier, - Some(builder.subject.to_string()), - builder.post_modifier, + builder.pre_modifier.as_deref(), + Some(builder.subject), + builder.post_modifier.as_deref(), location_modifier, ] .iter() .flatten() - .map(|s| s.as_str()) + .copied() .collect::>() .join(" "); @@ -375,10 +418,25 @@ impl<'a> From> for Suggestion { } url.push_str(¶meters.finish()); + let title = [ + builder.pre_yelp_modifier.as_deref(), + builder.pre_modifier.as_deref(), + Some(builder.subject), + builder.post_modifier.as_deref(), + builder.location_sign.as_deref(), + builder.location.as_deref(), + builder.post_yelp_modifier.as_deref(), + ] + .iter() + .flatten() + .copied() + .collect::>() + .join(" "); + Suggestion::Yelp { url, - // Use user’s query as title as it is. - title: builder.query.keyword.clone(), + title, + is_top_pick: builder.subject_exact_match, } } } diff --git a/third_party/rust/sync15/.cargo-checksum.json b/third_party/rust/sync15/.cargo-checksum.json index 0605b231c870..5f995892d5da 100644 --- a/third_party/rust/sync15/.cargo-checksum.json +++ b/third_party/rust/sync15/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"f1c393a0b8b62e476bdde3e01e69087ec1a25597ec1c4a2d996d4f5514a39768","README.md":"6d4ff5b079ac5340d18fa127f583e7ad793c5a2328b8ecd12c3fc723939804f2","build.rs":"aa971160d67ce8626b26e15c04c34b730f594c45c817aae34cfc9f3ea14ae284","src/bso/content.rs":"92935258745bdf0c3915a555cb6884a7fa69faa1290ec2c1815f6e2f3c0f0562","src/bso/crypto.rs":"27602dcccb37d3a55620ee4e16b705da455d49af575de115c7c79c0178eb1d6d","src/bso/mod.rs":"09e723dc7e99295ecafdcadffaf604d66ea27cf2b7f1fd9ab3cac4f4698ff6a7","src/bso/test_utils.rs":"4ec5a2df5e1c0ec14dc770681e959bdcef6ef04f6fde435999197f46a8ae4831","src/client/coll_state.rs":"13e6ef55273baf5536acc369be522e34a803a32cabf19cce43e426aea9b6223e","src/client/coll_update.rs":"dac04a90c29dd969f8b4250414609c9b6d61daf2dfa4ae77d1c4a165ba970b05","src/client/collection_keys.rs":"c27b2277a3a52033b58ab01490fc2ea7007494195dd5e6dc2c6931a4ca96795a","src/client/mod.rs":"8f588d4a035cf79d96f2500f06d5651c1a7c566127c456ffa5429811ddce3fd6","src/client/request.rs":"8841524e37d8195867bdf6ba98c75f610cf47a4644adeebd6372cc6713f2260a","src/client/state.rs":"4e31193ef2471c1dfabf1c6a391bcb95e14ddb45855786a4194ff187d5c9347c","src/client/status.rs":"f445a8765dac9789444e23b5145148413407bb1d18a15ef56682243997f591bf","src/client/storage_client.rs":"8de72d4ba3ca4f68c8e1898466de83a2b543545a18679800cb4f7fbda2dc3183","src/client/sync.rs":"b29abb512ec9d163f7883b71f78c9202802dcb17cad1fc5dc08087fb0bb66704","src/client/sync_multiple.rs":"6e92571132f89744b553190c596be8aff9b2d031d8f79d82c94cdf78b1683f4a","src/client/token.rs":"b268759d31e0fe17e0e2a428694cd9a317fcfbdd52f023d5d8c7cc6f00f1a102","src/client/util.rs":"71cc70ee41f821f53078675e636e9fad9c6046fa1a989e37f5487e340a2277d6","src/client_types.rs":"3c3cac1540b92482f43660d9e43bdde8481c4cc1a98253a68c80e791231f5976","src/clients_engine/engine.rs":"9e11b47be81fc63214f31879af74075674aa50a8f8989afe20fefa7990fa99b9","src/clients_engine/mod.rs":"461729e6f89b66b2cbd89b041a03d4d6a8ba582284ed4f3015cb13e1a0c6da97","src/clients_engine/record.rs":"b0d84bf420743d7638a45e4836633a45e50257d5548fe7ecd04bff4d724439b8","src/clients_engine/ser.rs":"ef12daeb11faf618fe3cafe91f20a031fe5bb6751369b6ee5aee03f196efe88c","src/device_type.rs":"dc2d4296d25e31471c8e68488f1043ff239b902036cd6aea8a686cf79b4ed335","src/enc_payload.rs":"aa3eea7df49b24cd59831680a47c417b73a3e36e6b0f3f4baf14ca66bd68be6b","src/engine/bridged_engine.rs":"f70f1bfce6e0c04b0c72ec9cbfbb12c82d4009a23fb9768792107d41b2865a4f","src/engine/mod.rs":"90f1f9760f5f712a337aebb04e59c736e4b6fbd89d6a188d969210c7f3f321ae","src/engine/request.rs":"5923025fb9550178339f880a1bf8526d8e853e7a0b2bce6d9d687cc808ac0085","src/engine/sync_engine.rs":"531b35d72ce9e04c3e543c0468c1e450fba2c0dc3d33d68d9b1c0a5c1ad7dd34","src/error.rs":"a45cfe02e6301f473c34678b694943c1a04308b8c292c6e0448bf495194c3b5e","src/key_bundle.rs":"abd0781f3be8c8e7c691f18bb71f3433b633803c48da9794e15ac6301ed60d6c","src/lib.rs":"f59f8817978d943518dfa03ab31fc0f6b1fc72ee9943a97aef1537e2769649f5","src/record_types.rs":"02bb3d352fb808131d298f9b90d9c95b7e9e0138b97c5401f3b9fdacc5562f44","src/server_timestamp.rs":"6272299c92b05b9ec9dc2e18402ebe927b07ccf1dcab5082301a09e0ee56ce24","src/sync15.udl":"005b2b056b93c959a04670f6f489afecb8e17093d8e4be34765a3a4cc0faeb8c","src/telemetry.rs":"e3a7e13e85f5e336526ebf07db04c81b8f1ba89ae1db4159a3a570826cb8cfd2","uniffi.toml":"34488f947497a9b05007445dd816024ef02e6b1696f1056ee868f039722828ee"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"739abc68b38e8468c5d1eb3f7a66f01e638765f9d8714080e07817f0939a2b66","README.md":"6d4ff5b079ac5340d18fa127f583e7ad793c5a2328b8ecd12c3fc723939804f2","build.rs":"aa971160d67ce8626b26e15c04c34b730f594c45c817aae34cfc9f3ea14ae284","src/bso/content.rs":"92935258745bdf0c3915a555cb6884a7fa69faa1290ec2c1815f6e2f3c0f0562","src/bso/crypto.rs":"27602dcccb37d3a55620ee4e16b705da455d49af575de115c7c79c0178eb1d6d","src/bso/mod.rs":"09e723dc7e99295ecafdcadffaf604d66ea27cf2b7f1fd9ab3cac4f4698ff6a7","src/bso/test_utils.rs":"4ec5a2df5e1c0ec14dc770681e959bdcef6ef04f6fde435999197f46a8ae4831","src/client/coll_state.rs":"13e6ef55273baf5536acc369be522e34a803a32cabf19cce43e426aea9b6223e","src/client/coll_update.rs":"dac04a90c29dd969f8b4250414609c9b6d61daf2dfa4ae77d1c4a165ba970b05","src/client/collection_keys.rs":"c27b2277a3a52033b58ab01490fc2ea7007494195dd5e6dc2c6931a4ca96795a","src/client/mod.rs":"8f588d4a035cf79d96f2500f06d5651c1a7c566127c456ffa5429811ddce3fd6","src/client/request.rs":"8841524e37d8195867bdf6ba98c75f610cf47a4644adeebd6372cc6713f2260a","src/client/state.rs":"4e31193ef2471c1dfabf1c6a391bcb95e14ddb45855786a4194ff187d5c9347c","src/client/status.rs":"f445a8765dac9789444e23b5145148413407bb1d18a15ef56682243997f591bf","src/client/storage_client.rs":"8de72d4ba3ca4f68c8e1898466de83a2b543545a18679800cb4f7fbda2dc3183","src/client/sync.rs":"b29abb512ec9d163f7883b71f78c9202802dcb17cad1fc5dc08087fb0bb66704","src/client/sync_multiple.rs":"6e92571132f89744b553190c596be8aff9b2d031d8f79d82c94cdf78b1683f4a","src/client/token.rs":"b268759d31e0fe17e0e2a428694cd9a317fcfbdd52f023d5d8c7cc6f00f1a102","src/client/util.rs":"71cc70ee41f821f53078675e636e9fad9c6046fa1a989e37f5487e340a2277d6","src/client_types.rs":"3c3cac1540b92482f43660d9e43bdde8481c4cc1a98253a68c80e791231f5976","src/clients_engine/engine.rs":"9e11b47be81fc63214f31879af74075674aa50a8f8989afe20fefa7990fa99b9","src/clients_engine/mod.rs":"461729e6f89b66b2cbd89b041a03d4d6a8ba582284ed4f3015cb13e1a0c6da97","src/clients_engine/record.rs":"b0d84bf420743d7638a45e4836633a45e50257d5548fe7ecd04bff4d724439b8","src/clients_engine/ser.rs":"ef12daeb11faf618fe3cafe91f20a031fe5bb6751369b6ee5aee03f196efe88c","src/device_type.rs":"dc2d4296d25e31471c8e68488f1043ff239b902036cd6aea8a686cf79b4ed335","src/enc_payload.rs":"aa3eea7df49b24cd59831680a47c417b73a3e36e6b0f3f4baf14ca66bd68be6b","src/engine/bridged_engine.rs":"f70f1bfce6e0c04b0c72ec9cbfbb12c82d4009a23fb9768792107d41b2865a4f","src/engine/mod.rs":"90f1f9760f5f712a337aebb04e59c736e4b6fbd89d6a188d969210c7f3f321ae","src/engine/request.rs":"5923025fb9550178339f880a1bf8526d8e853e7a0b2bce6d9d687cc808ac0085","src/engine/sync_engine.rs":"531b35d72ce9e04c3e543c0468c1e450fba2c0dc3d33d68d9b1c0a5c1ad7dd34","src/error.rs":"a45cfe02e6301f473c34678b694943c1a04308b8c292c6e0448bf495194c3b5e","src/key_bundle.rs":"abd0781f3be8c8e7c691f18bb71f3433b633803c48da9794e15ac6301ed60d6c","src/lib.rs":"f59f8817978d943518dfa03ab31fc0f6b1fc72ee9943a97aef1537e2769649f5","src/record_types.rs":"02bb3d352fb808131d298f9b90d9c95b7e9e0138b97c5401f3b9fdacc5562f44","src/server_timestamp.rs":"6272299c92b05b9ec9dc2e18402ebe927b07ccf1dcab5082301a09e0ee56ce24","src/sync15.udl":"005b2b056b93c959a04670f6f489afecb8e17093d8e4be34765a3a4cc0faeb8c","src/telemetry.rs":"e3a7e13e85f5e336526ebf07db04c81b8f1ba89ae1db4159a3a570826cb8cfd2","uniffi.toml":"34488f947497a9b05007445dd816024ef02e6b1696f1056ee868f039722828ee"},"package":null} \ No newline at end of file diff --git a/third_party/rust/sync15/Cargo.toml b/third_party/rust/sync15/Cargo.toml index 132213590a61..86f7da91f85d 100644 --- a/third_party/rust/sync15/Cargo.toml +++ b/third_party/rust/sync15/Cargo.toml @@ -68,7 +68,7 @@ path = "../viaduct" optional = true [dev-dependencies.env_logger] -version = "0.7" +version = "0.10" default-features = false [build-dependencies.uniffi] diff --git a/third_party/rust/tabs/.cargo-checksum.json b/third_party/rust/tabs/.cargo-checksum.json index d87bcbc48db5..1f1fa20bfa4b 100644 --- a/third_party/rust/tabs/.cargo-checksum.json +++ b/third_party/rust/tabs/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"b1aebb4475665781d03e81608afa113aee8343c3e9707fd91591244f0c08c8c7","README.md":"c48b8f391ef822c4f3971b5f453a1e7b43bea232752d520460d2f04803aead1a","build.rs":"33e61b811b19ed2b58e319cc65d5988bed258d2c4fea2d706301184c59847a0f","src/error.rs":"2694657aeb12f99c4b2fe102ad2b08b79955d209201831b3e071129f0b7d7eda","src/lib.rs":"7208f78955e015ef8bab7916307e551cd3c1bd56d7fe14f8b53cd53bc4b38555","src/schema.rs":"2b7b51f3c2edc0ca603495c10b917603fd9ac791c4a366080e40d090b13b91f2","src/storage.rs":"18f449b6daf1641dc351be451311495b7c05e16c4e2d4eaf12c1fa02fa750b67","src/store.rs":"ab0b6214b30b0f0fa7c6a89098ff3db1a8f76264f6711c4481c0be460afe522b","src/sync/bridge.rs":"18d3a7913a030b598d4b6cbd5b7e2ab4cef4cc7ea964f5bc84d7fb2f28787529","src/sync/engine.rs":"2d14d899a38ac72b9141d505babd94ef7b6fbc5a95be70f324a40bf01935793d","src/sync/mod.rs":"09ba3c87f1174a243bf5aaa481effd18929d54359ceb9b23ccb2c32ee3482f34","src/sync/record.rs":"eef6751c209d039958afbe245ddb006cfdf6b8b6b47f925f69c552b832b87922","src/tabs.udl":"2cefc7f6a27b5619bc536d4a19608cf24153d745199fbeaf192e24b4381dedfb","uniffi.toml":"f9125e8d55b109e86076ee88bfd640372f06b142b7db557e41816c7227dd445c"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"c06aa10f3dfa7be35c4fb54cb629704826da5bd9d08eaf09211343bb2b62bf74","README.md":"c48b8f391ef822c4f3971b5f453a1e7b43bea232752d520460d2f04803aead1a","build.rs":"33e61b811b19ed2b58e319cc65d5988bed258d2c4fea2d706301184c59847a0f","src/error.rs":"2694657aeb12f99c4b2fe102ad2b08b79955d209201831b3e071129f0b7d7eda","src/lib.rs":"7208f78955e015ef8bab7916307e551cd3c1bd56d7fe14f8b53cd53bc4b38555","src/schema.rs":"2b7b51f3c2edc0ca603495c10b917603fd9ac791c4a366080e40d090b13b91f2","src/storage.rs":"18f449b6daf1641dc351be451311495b7c05e16c4e2d4eaf12c1fa02fa750b67","src/store.rs":"ab0b6214b30b0f0fa7c6a89098ff3db1a8f76264f6711c4481c0be460afe522b","src/sync/bridge.rs":"18d3a7913a030b598d4b6cbd5b7e2ab4cef4cc7ea964f5bc84d7fb2f28787529","src/sync/engine.rs":"2d14d899a38ac72b9141d505babd94ef7b6fbc5a95be70f324a40bf01935793d","src/sync/mod.rs":"09ba3c87f1174a243bf5aaa481effd18929d54359ceb9b23ccb2c32ee3482f34","src/sync/record.rs":"eef6751c209d039958afbe245ddb006cfdf6b8b6b47f925f69c552b832b87922","src/tabs.udl":"2cefc7f6a27b5619bc536d4a19608cf24153d745199fbeaf192e24b4381dedfb","uniffi.toml":"f9125e8d55b109e86076ee88bfd640372f06b142b7db557e41816c7227dd445c"},"package":null} \ No newline at end of file diff --git a/third_party/rust/tabs/Cargo.toml b/third_party/rust/tabs/Cargo.toml index ebd558ee1c3f..c06ad84ca98f 100644 --- a/third_party/rust/tabs/Cargo.toml +++ b/third_party/rust/tabs/Cargo.toml @@ -60,12 +60,8 @@ features = ["sync-engine"] tempfile = "3.1" [dev-dependencies.env_logger] -version = "0.8.0" -features = [ - "termcolor", - "atty", - "humantime", -] +version = "0.10.0" +features = ["humantime"] default-features = false [build-dependencies.uniffi] diff --git a/third_party/rust/webext-storage/.cargo-checksum.json b/third_party/rust/webext-storage/.cargo-checksum.json index 3b61bc9dab25..98fa4d6a227b 100644 --- a/third_party/rust/webext-storage/.cargo-checksum.json +++ b/third_party/rust/webext-storage/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"Cargo.toml":"261161e1eea9bfa070c3025ce4398b5e53d9db44ca1539313e05b849723d625b","README.md":"1fd617294339930ee1ad5172377648b268cce0216fc3971facbfe7c6839e9ab1","android/build.gradle":"7c8ef7424dd3cc110cd96a0dca6fabef1b8479668bfc403902813efbf5187a83","android/src/main/AndroidManifest.xml":"0abfabd45a3a6415861c33532d4efcd658e9f78c30342e3e3e3570296a3cc8c2","android/src/test/java/mozilla/appservices/webextstorage/WebExtStorageTest.kt":"aff94a9798c7e91f8efd8e4329ead05ea3afa1d0579a78d86ab93a5d40a715b1","build.rs":"92f7d380f3d8fab1e6d80276915af57192e276321d132a5f800ea4520e9cb469","sql/create_schema.sql":"a17311a407ec10e033886b7125da4c8b84bc6d761f6b28edc9594de430e1d964","sql/create_sync_temp_tables.sql":"860ede362c94feb47d85522553fa2852f9bdb9f9b025d6438dd5dee3d4acd527","sql/tests/create_schema_v1.sql":"77cf0c90eaac3e1aea626537147e1b8ec349b68d6076c92fa7ae402aac613050","src/api.rs":"6fe362e4f437def2ad2249de385cca8f0d1d5d67679240351e9f57523fefe5e7","src/db.rs":"b95024c1d8f36a76a6f3098acea5a82bc49de144a24cdc280ed38e9bcc8e772b","src/error.rs":"6437e9a0edefac2707af85eef13bdbfcd53a84d7aa7859599155d10451d42361","src/ffi.rs":"f66a81393bebe7a4b7e7960cb426df106ff1f02bfebcaa6e335b4b8b56c5c936","src/lib.rs":"ab25e7c6ea67fb905fe6dad866c0d2c462b1e93bcff283db947513aeabbb2d73","src/migration.rs":"8d92f82b2ba38e1039fd054c8c75078a6b896a0d3cdc1a52571456b25a32c9c3","src/schema.rs":"d8dd8f66cad71e3e369722734e0d5d16fd9423d5f6a5abba1854a27e1e814724","src/store.rs":"d208689c46fb97cd2c60a0c610ba1998a7132fb50fffa2eefa1d6b169b7c34f0","src/sync/bridge.rs":"996de05beb2904f84b3cbfc9ef85c4844078fdb4867d9068390d496156bee614","src/sync/incoming.rs":"dd77c64e2ade4f39cba258decab6d3db8ad0b5f513aa018efbd56b9869a021d9","src/sync/mod.rs":"bd1bc5c428dfda6aee7efe53b6e74b8015da5129a303638a21ca8d63516e4061","src/sync/outgoing.rs":"dacb77b956f2546fd60a89367927a199d9b662b17201d0781145f7405b61fdce","src/sync/sync_tests.rs":"f3846ca7e463315ba9788826613b987ddcff7b21672ff257a98769ee94f4191a","src/webext-storage.udl":"0341d431ba837cf64ea210ef6157010c6664a0b5a194e89acb0414938636b391","uniffi.toml":"beeec89c2f877eb89be0090dc304dbc7c74e787385e7459bad78c6165bb66791"},"package":null} \ No newline at end of file +{"files":{"Cargo.toml":"a11f7fbc29c375034289e7bdd11da4aadac9cb2d939a4f2e5dc61aaea35cf465","README.md":"1fd617294339930ee1ad5172377648b268cce0216fc3971facbfe7c6839e9ab1","build.rs":"92f7d380f3d8fab1e6d80276915af57192e276321d132a5f800ea4520e9cb469","sql/create_schema.sql":"a17311a407ec10e033886b7125da4c8b84bc6d761f6b28edc9594de430e1d964","sql/create_sync_temp_tables.sql":"860ede362c94feb47d85522553fa2852f9bdb9f9b025d6438dd5dee3d4acd527","sql/tests/create_schema_v1.sql":"77cf0c90eaac3e1aea626537147e1b8ec349b68d6076c92fa7ae402aac613050","src/api.rs":"6fe362e4f437def2ad2249de385cca8f0d1d5d67679240351e9f57523fefe5e7","src/db.rs":"b95024c1d8f36a76a6f3098acea5a82bc49de144a24cdc280ed38e9bcc8e772b","src/error.rs":"6437e9a0edefac2707af85eef13bdbfcd53a84d7aa7859599155d10451d42361","src/ffi.rs":"f66a81393bebe7a4b7e7960cb426df106ff1f02bfebcaa6e335b4b8b56c5c936","src/lib.rs":"ab25e7c6ea67fb905fe6dad866c0d2c462b1e93bcff283db947513aeabbb2d73","src/migration.rs":"8d92f82b2ba38e1039fd054c8c75078a6b896a0d3cdc1a52571456b25a32c9c3","src/schema.rs":"d8dd8f66cad71e3e369722734e0d5d16fd9423d5f6a5abba1854a27e1e814724","src/store.rs":"d208689c46fb97cd2c60a0c610ba1998a7132fb50fffa2eefa1d6b169b7c34f0","src/sync/bridge.rs":"996de05beb2904f84b3cbfc9ef85c4844078fdb4867d9068390d496156bee614","src/sync/incoming.rs":"dd77c64e2ade4f39cba258decab6d3db8ad0b5f513aa018efbd56b9869a021d9","src/sync/mod.rs":"bd1bc5c428dfda6aee7efe53b6e74b8015da5129a303638a21ca8d63516e4061","src/sync/outgoing.rs":"dacb77b956f2546fd60a89367927a199d9b662b17201d0781145f7405b61fdce","src/sync/sync_tests.rs":"f3846ca7e463315ba9788826613b987ddcff7b21672ff257a98769ee94f4191a","src/webext-storage.udl":"0341d431ba837cf64ea210ef6157010c6664a0b5a194e89acb0414938636b391","uniffi.toml":"beeec89c2f877eb89be0090dc304dbc7c74e787385e7459bad78c6165bb66791"},"package":null} \ No newline at end of file diff --git a/third_party/rust/webext-storage/Cargo.toml b/third_party/rust/webext-storage/Cargo.toml index a43a91b2c2c9..c3f9b170df4e 100644 --- a/third_party/rust/webext-storage/Cargo.toml +++ b/third_party/rust/webext-storage/Cargo.toml @@ -14,6 +14,7 @@ edition = "2021" name = "webext-storage" version = "0.1.0" authors = ["sync-team@mozilla.com"] +exclude = ["/android"] readme = "README.md" license = "MPL-2.0" @@ -67,7 +68,7 @@ libsqlite3-sys = "0.27.0" tempfile = "3" [dev-dependencies.env_logger] -version = "0.7" +version = "0.10" default-features = false [dev-dependencies.sql-support] diff --git a/third_party/rust/webext-storage/android/build.gradle b/third_party/rust/webext-storage/android/build.gradle deleted file mode 100644 index 1ca49e5503ff..000000000000 --- a/third_party/rust/webext-storage/android/build.gradle +++ /dev/null @@ -1,22 +0,0 @@ -// TODO: Uncomment this code when webext-storage component is integrated in android - -// apply from: "$rootDir/build-scripts/component-common.gradle" -// apply from: "$rootDir/publish.gradle" - -// dependencies { -// // Part of the public API. -// api project(':sync15') - -// implementation "org.mozilla.telemetry:glean:$glean_version" -// implementation "androidx.core:core-ktx:$androidx_core_version" - -// testImplementation "androidx.test:core-ktx:$androidx_test_version" -// testImplementation "androidx.test.ext:junit-ktx:$androidx_test_junit_version" -// testImplementation "androidx.work:work-testing:$androidx_work_testing_version" -// testImplementation "org.mozilla.telemetry:glean-native-forUnitTests:$glean_version" -// } - - -// ext.configureUniFFIBindgen("../src/webext-storage.udl") -// ext.dependsOnTheMegazord() -// ext.configurePublish() diff --git a/third_party/rust/webext-storage/android/src/main/AndroidManifest.xml b/third_party/rust/webext-storage/android/src/main/AndroidManifest.xml deleted file mode 100644 index eb647ad3435f..000000000000 --- a/third_party/rust/webext-storage/android/src/main/AndroidManifest.xml +++ /dev/null @@ -1,2 +0,0 @@ - diff --git a/third_party/rust/webext-storage/android/src/test/java/mozilla/appservices/webextstorage/WebExtStorageTest.kt b/third_party/rust/webext-storage/android/src/test/java/mozilla/appservices/webextstorage/WebExtStorageTest.kt deleted file mode 100644 index 79a81ee91dac..000000000000 --- a/third_party/rust/webext-storage/android/src/test/java/mozilla/appservices/webextstorage/WebExtStorageTest.kt +++ /dev/null @@ -1,76 +0,0 @@ -/* Any copyright is dedicated to the Public Domain. - http://creativecommons.org/publicdomain/zero/1.0/ */ - -package mozilla.appservices.webextstorage - -import mozilla.appservices.Megazord -import org.junit.Assert -import org.junit.Rule -import org.junit.Test -import org.junit.rules.TemporaryFolder -import org.junit.runner.RunWith -import org.robolectric.RobolectricTestRunner -import org.robolectric.annotation.Config - -@RunWith(RobolectricTestRunner::class) -@Config(manifest = Config.NONE) -class WebExtStorageTest { - @Rule - @JvmField - val dbFolder = TemporaryFolder() - - fun createTestStore(): WebExtStorageStore { - Megazord.init() - val dbPath = dbFolder.newFile() - return WebExtStorageStore(path = dbPath.absolutePath) - } - - @Test - fun testSet() { - val store = createTestStore() - val extId = "ab" - val jsonString = """{"a":"a"}""" - - store.set(extId, jsonString) - val data = store.get(extId, "null") - Assert.assertEquals(jsonString, data) - store.close() - } - - @Test - fun testRemove() { - val store = createTestStore() - val extId = "ab" - val jsonString = """{"a":"a","b":"b"}""" - - store.set(extId, jsonString) - val change = store.remove("ab", """["b"]""").changes[0] - - Assert.assertEquals(change.key, "b") - Assert.assertEquals(change.oldValue, """"b"""") - Assert.assertNull(change.newValue) - store.close() - } - - @Test - fun testClear() { - val store = createTestStore() - val extId = "ab" - val jsonString = """{"a":"a","b":"b"}""" - - store.set(extId, jsonString) - val result = store.clear(extId) - - val firstChange = result.changes[0] - Assert.assertEquals(firstChange.key, "a") - Assert.assertEquals(firstChange.oldValue, """"a"""") - Assert.assertNull(firstChange.newValue) - - val secondChange = result.changes[1] - Assert.assertEquals(secondChange.key, "b") - Assert.assertEquals(secondChange.oldValue, """"b"""") - Assert.assertNull(secondChange.newValue) - - store.close() - } -}