impl ExportableProvider for ForkByErrorProvider and add tutorial (#5503)

Author: sffc

Cargo.lock

@@ -29,3 +29,7 @@ serde = { workspace = true, features = ["derive", "alloc"], optional = true }
 icu_provider = { path = "../../provider/core", features = ["macros", "deserialize_json"] }
 icu_locale = { path = "../../components/locale" }
 writeable = { path = "../../utils/writeable" }
+
+[features]
+std = []
+export = ["icu_provider/export", "std"]

provider/adapters/Cargo.toml

@@ -5,6 +5,8 @@
 //! Helpers for switching between multiple providers.
 
 use alloc::collections::BTreeSet;
+#[cfg(feature = "export")]
+use icu_provider::export::ExportableProvider;
 use icu_provider::prelude::*;
 
 /// A provider that is one of two types determined at runtime.
@@ -122,3 +124,18 @@ impl<M: DataMarker, P0: IterableDataProvider<M>, P1: IterableDataProvider<M>>
         }
     }
 }
+
+#[cfg(feature = "export")]
+impl<P0, P1> ExportableProvider for EitherProvider<P0, P1>
+where
+    P0: ExportableProvider,
+    P1: ExportableProvider,
+{
+    fn supported_markers(&self) -> std::collections::HashSet<DataMarkerInfo> {
+        use EitherProvider::*;
+        match self {
+            A(p) => p.supported_markers(),
+            B(p) => p.supported_markers(),
+        }
+    }
+}

provider/adapters/src/either.rs

@@ -7,6 +7,8 @@
 //! Use [`EmptyDataProvider`] as a stand-in for a provider that always fails.
 
 use alloc::collections::BTreeSet;
+#[cfg(feature = "export")]
+use icu_provider::export::ExportableProvider;
 use icu_provider::prelude::*;
 
 /// A data provider that always returns an error.
@@ -107,3 +109,10 @@ where
         Ok(Default::default())
     }
 }
+
+#[cfg(feature = "export")]
+impl ExportableProvider for EmptyDataProvider {
+    fn supported_markers(&self) -> std::collections::HashSet<DataMarkerInfo> {
+        Default::default()
+    }
+}

provider/adapters/src/empty.rs

@@ -23,6 +23,8 @@
 mod impls;
 
 use alloc::collections::BTreeSet;
+#[cfg(feature = "export")]
+use icu_provider::export::ExportableProvider;
 use icu_provider::prelude::*;
 
 /// A data provider that selectively filters out data requests.
@@ -165,3 +167,15 @@ where
         })
     }
 }
+
+#[cfg(feature = "export")]
+impl<P0, F> ExportableProvider for FilterDataProvider<P0, F>
+where
+    P0: ExportableProvider,
+    F: Fn(DataIdentifierBorrowed) -> bool + Sync,
+{
+    fn supported_markers(&self) -> std::collections::HashSet<DataMarkerInfo> {
+        // The predicate only takes DataIdentifier, not DataMarker, so we are not impacted
+        self.inner.supported_markers()
+    }
+}

provider/adapters/src/filter/mod.rs

@@ -4,6 +4,8 @@
 
 use super::ForkByErrorPredicate;
 use alloc::{collections::BTreeSet, vec::Vec};
+#[cfg(feature = "export")]
+use icu_provider::export::ExportableProvider;
 use icu_provider::prelude::*;
 
 /// A provider that returns data from one of two child providers based on a predicate function.
@@ -159,6 +161,20 @@ where
     }
 }
 
+#[cfg(feature = "export")]
+impl<P0, P1, F> ExportableProvider for ForkByErrorProvider<P0, P1, F>
+where
+    P0: ExportableProvider,
+    P1: ExportableProvider,
+    F: ForkByErrorPredicate + Sync,
+{
+    fn supported_markers(&self) -> std::collections::HashSet<DataMarkerInfo> {
+        let mut markers = self.0.supported_markers();
+        markers.extend(self.1.supported_markers());
+        markers
+    }
+}
+
 /// A provider that returns data from the first child provider passing a predicate function.
 ///
 /// This is an abstract forking provider that must be provided with a type implementing the
@@ -334,3 +350,17 @@ where
         Err(last_error)
     }
 }
+
+#[cfg(feature = "export")]
+impl<P, F> ExportableProvider for MultiForkByErrorProvider<P, F>
+where
+    P: ExportableProvider,
+    F: ForkByErrorPredicate + Sync,
+{
+    fn supported_markers(&self) -> std::collections::HashSet<DataMarkerInfo> {
+        self.providers
+            .iter()
+            .flat_map(|p| p.supported_markers())
+            .collect()
+    }
+}

provider/adapters/src/fork/by_error.rs

@@ -10,7 +10,7 @@
 //! - Use the [`fallback`] module to automatically resolve arbitrary locales for data loading.
 
 // https://github.com/unicode-org/icu4x/blob/main/documents/process/boilerplate.md#library-annotations
-#![cfg_attr(not(test), no_std)]
+#![cfg_attr(not(any(test, feature = "std")), no_std)]
 #![cfg_attr(
     not(test),
     deny(

provider/adapters/src/lib.rs

@@ -10,9 +10,10 @@ edition = "2021"
 
 [dev-dependencies]
 icu = { workspace = true, features = ["compiled_data", "serde"] }
-icu_provider_export = { workspace = true }
+icu_provider_export = { workspace = true, features = ["blob_exporter"] }
+icu_provider_source = { workspace = true, features = ["networking"] }
 icu_provider = { workspace = true, features = ["deserialize_json"] }
-icu_provider_adapters = { workspace = true, features = ["serde"] }
+icu_provider_adapters = { workspace = true, features = ["serde", "export"] }
 icu_provider_blob = { workspace = true }
 icu_provider_fs = { workspace = true }
 

tools/md-tests/Cargo.toml

@@ -253,6 +253,83 @@ assert_eq!(formatter.format_to_string(&100007i64.into()), "100🐮007");
 
 Forking providers can be implemented using `DataPayload::dynamic_cast`. For an example, see that function's documentation.
 
+## Exporting Custom Data Markers
+
+To add custom data markers to your baked data or postcard file, create a forking exportable provider:
+
+```rust
+use icu::locale::locale;
+use icu::plurals::provider::CardinalV1Marker;
+use icu_provider::prelude::*;
+use icu_provider::DataMarker;
+use icu_provider_adapters::fork::ForkByMarkerProvider;
+use icu_provider_blob::BlobDataProvider;
+use icu_provider_export::blob_exporter::BlobExporter;
+use icu_provider_export::prelude::*;
+use icu_provider_source::SourceDataProvider;
+use std::borrow::Cow;
+use std::collections::BTreeSet;
+
+#[icu_provider::data_struct(marker(CustomMarker, "x/custom@1"))]
+#[derive(Debug, PartialEq, serde::Deserialize, serde::Serialize, databake::Bake)]
+#[databake(path = crate)]
+pub struct Custom<'data> {
+    message: Cow<'data, str>,
+};
+
+struct CustomProvider;
+impl DataProvider<CustomMarker> for CustomProvider {
+    fn load(&self, req: DataRequest) -> Result<DataResponse<CustomMarker>, DataError> {
+        Ok(DataResponse {
+            metadata: Default::default(),
+            payload: DataPayload::from_owned(Custom {
+                message: format!("Custom data for locale {}!", req.id.locale).into(),
+            }),
+        })
+    }
+}
+
+impl IterableDataProvider<CustomMarker> for CustomProvider {
+    fn iter_ids(&self) -> Result<BTreeSet<DataIdentifierCow>, DataError> {
+        Ok([locale!("es"), locale!("ja")]
+            .into_iter()
+            .map(DataLocale::from)
+            .map(DataIdentifierCow::from_locale)
+            .collect())
+    }
+}
+
+icu_provider::export::make_exportable_provider!(CustomProvider, [CustomMarker,]);
+
+let icu4x_source_provider = SourceDataProvider::new_latest_tested();
+let custom_source_provider = CustomProvider;
+
+let mut buffer = Vec::<u8>::new();
+
+ExportDriver::new(
+    [DataLocaleFamily::FULL],
+    DeduplicationStrategy::None.into(),
+    LocaleFallbacker::try_new_unstable(&icu4x_source_provider).unwrap(),
+)
+.with_markers([CardinalV1Marker::INFO, CustomMarker::INFO])
+.export(
+    &ForkByMarkerProvider::new(icu4x_source_provider, custom_source_provider),
+    BlobExporter::new_v2_with_sink(Box::new(&mut buffer)),
+)
+.unwrap();
+
+let blob_provider = BlobDataProvider::try_new_from_blob(buffer.into()).unwrap();
+
+let locale = DataLocale::from(&locale!("ja"));
+let req = DataRequest {
+    id: DataIdentifierBorrowed::for_locale(&locale),
+    metadata: Default::default(),
+};
+
+assert!(blob_provider.load_data(CardinalV1Marker::INFO, req).is_ok());
+assert!(blob_provider.load_data(CustomMarker::INFO, req).is_ok());
+```
+
 ## Accessing the Resolved Locale
 
 ICU4X objects do not store their "resolved locale" because that is not a well-defined concept. Components can load data from many sources, and fallbacks to parent locales or root does not necessarily mean that a locale is not supported.