Remove -x- encoding for marker attributes (#5038)

#3632 

* baked: use `(&str, &str)` if needed, otherwise `&str` as before
* blob: use U+001E ("record separator") as the separator
* fs: use additional directory level

Author: robertbastian

components/experimental/tests/transliterate/data/baked/macros/transliterator_rules_v1.rs.data

@@ -16,30 +16,47 @@ pub fn bake(
 ) -> TokenStream {
     let mut data = data
         .into_iter()
-        .map(|((l, a), i)| {
-            (
-                DataRequest {
-                    locale: &l,
-                    marker_attributes: &a,
-                    ..Default::default()
-                }
-                .legacy_encode(),
-                quote!(#i),
-            )
-        })
+        .map(|((l, a), i)| ((a.to_string(), l.to_string()), quote!(#i)))
         .collect::<Vec<_>>();
 
     data.sort_by(|(a, _), (b, _)| a.cmp(b));
 
     let n = data.len();
-    let data = data.iter().map(|(r, i)| quote!((#r, &#i)));
-
-    quote! {
-        static DATA: [(&str, & #struct_type); #n] = [#(#data,)*];
-        fn lookup(req: icu_provider::DataRequest) -> Option<&'static #struct_type> {
-            DATA.binary_search_by(|(k, _)| req.legacy_cmp(k.as_bytes()).reverse())
-                .map(|i| (*unsafe { DATA.get_unchecked(i) }).1)
-                .ok()
+
+    if data.iter().all(|((a, _), _)| a.is_empty()) {
+        // Only DataLocales
+        let data = data.iter().map(|((_, l), i)| quote!((#l, &#i)));
+
+        quote! {
+            static DATA: [(&str, & #struct_type); #n] = [#(#data,)*];
+            fn lookup(req: icu_provider::DataRequest) -> Option<&'static #struct_type> {
+                DATA.binary_search_by(|(l, _)| req.locale.strict_cmp(l.as_bytes()).reverse())
+                    .map(|i| (*unsafe { DATA.get_unchecked(i) }).1)
+                    .ok()
+            }
+        }
+    } else if data.iter().all(|((_, l), _)| *l == "und") {
+        // Only marker attributes
+        let data = data.iter().map(|((a, _), i)| quote!((#a, &#i)));
+
+        quote! {
+            static DATA: [(&str, & #struct_type); #n] = [#(#data,)*];
+            fn lookup(req: icu_provider::DataRequest) -> Option<&'static #struct_type> {
+                DATA.binary_search_by(|(a, _)| a.cmp(&&**req.marker_attributes))
+                    .map(|i| (*unsafe { DATA.get_unchecked(i) }).1)
+                    .ok()
+            }
+        }
+    } else {
+        let data = data.iter().map(|((a, l), i)| quote!(((#a, #l), &#i)));
+
+        quote! {
+            static DATA: [((&str, &str), & #struct_type); #n] = [#(#data,)*];
+            fn lookup(req: icu_provider::DataRequest) -> Option<&'static #struct_type> {
+                DATA.binary_search_by(|((a, l), _)| a.cmp(&&**req.marker_attributes).then_with(|| req.locale.strict_cmp(l.as_bytes()).reverse()))
+                    .map(|i| (*unsafe { DATA.get_unchecked(i) }).1)
+                    .ok()
+            }
         }
     }
 }

provider/adapters/tests/data/blob.postcard

@@ -495,21 +495,16 @@ impl DataExporter for BakedExporter {
                 let mut idents = reqs
                     .iter()
                     .map(|(locale, marker_attributes)| {
-                        DataRequest {
-                            locale,
-                            marker_attributes,
-                            ..Default::default()
-                        }
-                        .legacy_encode()
-                        .chars()
-                        .map(|ch| {
-                            if ch == '-' {
-                                '_'
-                            } else {
-                                ch.to_ascii_uppercase()
-                            }
-                        })
-                        .collect::<String>()
+                        format!("_{}_{}", marker_attributes as &str, locale)
+                            .chars()
+                            .map(|ch| {
+                                if ch == '-' {
+                                    '_'
+                                } else {
+                                    ch.to_ascii_uppercase()
+                                }
+                            })
+                            .collect::<String>()
                     })
                     .collect::<Vec<_>>();
                 idents.sort();
@@ -573,27 +568,41 @@ impl DataExporter for BakedExporter {
                 }
             };
 
-            let mut legacy_encoded = deduplicated_values
+            let reqs = deduplicated_values
                 .values()
                 .flat_map(|s| {
                     s.iter().map(|(locale, marker_attributes)| {
-                        DataRequest {
-                            locale,
-                            marker_attributes,
-                            ..Default::default()
-                        }
-                        .legacy_encode()
+                        (marker_attributes.to_string(), locale.to_string())
                     })
                 })
-                .collect::<Vec<_>>();
-            legacy_encoded.sort();
-
-            let iterable_body = quote!(Ok(
-                [#(#legacy_encoded),*]
-                .into_iter()
-                .filter_map(icu_provider::DataRequest::legacy_decode)
-                .collect()
-            ));
+                .collect::<BTreeSet<_>>();
+            let locales = reqs.iter().map(|(_, l)| l);
+            let attrs = reqs.iter().map(|(a, _)| a);
+
+            let iterable_body = if attrs.clone().all(|a| a.is_empty()) {
+                // Only DataLocales
+                quote!(Ok(
+                    [#(icu_provider::_internal::locale_core::locale!(#locales)),*]
+                    .into_iter()
+                    .map(|l| (l.into(), Default::default()))
+                    .collect()
+                ))
+            } else if locales.clone().all(|l| l == "und") {
+                // Only attributes
+                quote!(Ok(
+                    [#(#attrs),*]
+                    .into_iter()
+                    .map(|a| (Default::default(), a.parse().unwrap()))
+                    .collect()
+                ))
+            } else {
+                quote!(Ok(
+                    [#((#attrs, icu_provider::_internal::locale_core::locale!(#locales))),*]
+                    .into_iter()
+                    .map(|(a, l)| (l.into(), a.parse().unwrap()))
+                    .collect()
+                ))
+            };
 
             (load_body, iterable_body)
         };