Merge branch 'main' into fullbake

unicode-org · Jun 7, 2023 · 9004450 · 9004450
2 parents 60191cf + 4925a50
commit 9004450
Show file tree

Hide file tree

Showing 220 changed files with 11,378 additions and 12,873 deletions.
diff --git a/components/datetime/src/lib.rs b/components/datetime/src/lib.rs
@@ -207,12 +207,12 @@ mod tests {
 
     #[test]
     fn check_sizes() {
-        check_size_of!(5784 | 4576, DateFormatter);
-        check_size_of!(6784 | 5448, DateTimeFormatter);
-        check_size_of!(7896 | 6464, ZonedDateTimeFormatter);
+        check_size_of!(5800 | 4592, DateFormatter);
+        check_size_of!(6792 | 5456, DateTimeFormatter);
+        check_size_of!(7904 | 6480, ZonedDateTimeFormatter);
         check_size_of!(1496 | 1320, TimeFormatter);
-        check_size_of!(1112 | 1016, TimeZoneFormatter);
-        check_size_of!(5744 | 4536, TypedDateFormatter::<Gregorian>);
+        check_size_of!(1112 | 1024, TimeZoneFormatter);
+        check_size_of!(5752 | 4544, TypedDateFormatter::<Gregorian>);
         check_size_of!(6744 | 5408, TypedDateTimeFormatter::<Gregorian>);
 
         check_size_of!(80, DateTimeError);
@@ -224,19 +224,19 @@ mod tests {
         type DP<M> = DataPayload<M>;
         check_size_of!(208, DP::<PatternPluralsFromPatternsV1Marker>);
         check_size_of!(1032 | 904, DP::<TimeSymbolsV1Marker>);
-        check_size_of!(32, DP::<GenericPatternV1Marker>);
+        check_size_of!(40, DP::<GenericPatternV1Marker>);
         check_size_of!(208, DP::<PatternPluralsFromPatternsV1Marker>);
         check_size_of!(5064 | 3904, DP::<ErasedDateSymbolsV1Marker>);
         check_size_of!(16, DP::<WeekDataV1Marker>);
-        check_size_of!(288 | 224, DP::<TimeZoneFormatsV1Marker>);
+        check_size_of!(288 | 232, DP::<TimeZoneFormatsV1Marker>);
         check_size_of!(64 | 56, DP::<ExemplarCitiesV1Marker>);
-        check_size_of!(120 | 104, DP::<MetazoneGenericNamesLongV1Marker>);
-        check_size_of!(120 | 104, DP::<MetazoneGenericNamesShortV1Marker>);
-        check_size_of!(216 | 200, DP::<MetazoneSpecificNamesLongV1Marker>);
-        check_size_of!(216 | 200, DP::<MetazoneSpecificNamesShortV1Marker>);
+        check_size_of!(120 | 112, DP::<MetazoneGenericNamesLongV1Marker>);
+        check_size_of!(120 | 112, DP::<MetazoneGenericNamesShortV1Marker>);
+        check_size_of!(216 | 208, DP::<MetazoneSpecificNamesLongV1Marker>);
+        check_size_of!(216 | 208, DP::<MetazoneSpecificNamesShortV1Marker>);
         check_size_of!(168, PluralRules);
         check_size_of!(256 | 208, FixedDecimalFormatter);
-        check_size_of!(1024 | 928, TimeZoneDataPayloads);
+        check_size_of!(1024 | 936, TimeZoneDataPayloads);
         check_size_of!(3, TimeZoneFormatterUnit);
     }
 }
diff --git a/components/plurals/data/macros.rs b/components/plurals/data/macros.rs
@@ -10,10 +10,6 @@ mod macros {
 pub use __impl_plurals_cardinal_v1 as impl_plurals_cardinal_v1;
 #[doc(inline)]
 pub use __impl_plurals_ordinal_v1 as impl_plurals_ordinal_v1;
-#[doc(hidden)]
-pub use __lookup_plurals_cardinal_v1 as lookup_plurals_cardinal_v1;
-#[doc(hidden)]
-pub use __lookup_plurals_ordinal_v1 as lookup_plurals_ordinal_v1;
 /// Implement [`DataProvider<M>`](icu_provider::DataProvider) on the given struct using the data
 /// hardcoded in this file. This allows the struct to be used with
 /// `icu`'s `_unstable` constructors.
@@ -52,8 +48,8 @@ macro_rules! __impl_any_provider {
                 const PLURALS_CARDINAL_V1: icu_provider::DataKeyHash = <icu_plurals::provider::CardinalV1Marker as icu_provider::KeyedDataMarker>::KEY.hashed();
                 const PLURALS_ORDINAL_V1: icu_provider::DataKeyHash = <icu_plurals::provider::OrdinalV1Marker as icu_provider::KeyedDataMarker>::KEY.hashed();
                 match key.hashed() {
-                    PLURALS_CARDINAL_V1 => lookup_plurals_cardinal_v1!(req.locale).map(icu_provider::AnyPayload::from_static_ref),
-                    PLURALS_ORDINAL_V1 => lookup_plurals_ordinal_v1!(req.locale).map(icu_provider::AnyPayload::from_static_ref),
+                    PLURALS_CARDINAL_V1 => Self::lookup_plurals_cardinal_v1(&req.locale).map(icu_provider::AnyPayload::from_static_ref),
+                    PLURALS_ORDINAL_V1 => Self::lookup_plurals_ordinal_v1(&req.locale).map(icu_provider::AnyPayload::from_static_ref),
                     _ => Err(icu_provider::DataErrorKind::MissingDataKey),
                 }
                 .map(|payload| icu_provider::AnyResponse { payload: Some(payload), metadata: Default::default() })

diff --git a/components/plurals/data/macros/plurals_cardinal_v1.rs b/components/plurals/data/macros/plurals_cardinal_v1.rs
diff --git a/components/plurals/data/macros/plurals_ordinal_v1.rs b/components/plurals/data/macros/plurals_ordinal_v1.rs
diff --git a/components/properties/src/provider/names.rs b/components/properties/src/provider/names.rs
@@ -14,14 +14,14 @@
 
 use alloc::boxed::Box;
 use core::cmp::Ordering;
-use core::fmt;
 
 use core::str;
 
 use icu_provider::prelude::*;
 
 use tinystr::TinyStr4;
-use zerovec::{maps::ZeroMapKV, ule::VarULE, VarZeroSlice, VarZeroVec, ZeroMap, ZeroVec};
+use zerovec::ule::{UnvalidatedStr, VarULE};
+use zerovec::{maps::ZeroMapKV, VarZeroSlice, VarZeroVec, ZeroMap, ZeroVec};
 
 /// This is a property name that can be "loose matched" as according to
 /// [PropertyValueAliases.txt](https://www.unicode.org/Public/UCD/latest/ucd/PropertyValueAliases.txt)
@@ -39,48 +39,62 @@ use zerovec::{maps::ZeroMapKV, ule::VarULE, VarZeroSlice, VarZeroVec, ZeroMap, Z
 /// including in SemVer minor releases. While the serde representation of data structs is guaranteed
 /// to be stable, their Rust representation might not be. Use with caution.
 /// </div>
+///
+/// # Examples
+///
+/// Using a [`NormalizedPropertyNameStr`] as the key of a [`ZeroMap`]:
+///
+/// ```
+/// use icu_properties::provider::names::NormalizedPropertyNameStr;
+/// use zerovec::ZeroMap;
+///
+/// let map: ZeroMap<NormalizedPropertyNameStr, usize> = [
+///     (NormalizedPropertyNameStr::from_str("A_BC"), 11),
+///     (NormalizedPropertyNameStr::from_str("dEf"), 22),
+///     (NormalizedPropertyNameStr::from_str("G_H-I"), 33),
+/// ]
+/// .into_iter()
+/// .collect();
+///
+/// let key_approx = NormalizedPropertyNameStr::from_str("AB-C");
+/// let key_exact = NormalizedPropertyNameStr::from_str("A_BC");
+///
+/// // Strict lookup:
+/// assert_eq!(None, map.get_copied(key_approx));
+/// assert_eq!(Some(11), map.get_copied(key_exact));
+///
+/// // Loose lookup:
+/// assert_eq!(Some(11), map.get_copied_by(|u| u.cmp_loose(key_approx)));
+/// assert_eq!(Some(11), map.get_copied_by(|u| u.cmp_loose(key_exact)));
+/// ```
 #[derive(PartialEq, Eq)] // VarULE wants these to be byte equality
-#[derive(VarULE)]
+#[derive(Debug, VarULE)]
+#[cfg_attr(feature = "serde", derive(serde::Serialize))]
 #[repr(transparent)]
-pub struct NormalizedPropertyNameStr([u8]);
+pub struct NormalizedPropertyNameStr(UnvalidatedStr);
 
-#[cfg(feature = "datagen")]
-impl serde::Serialize for NormalizedPropertyNameStr {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+/// This impl requires enabling the optional `serde` Cargo feature of the `icu_properties` crate
+#[cfg(feature = "serde")]
+impl<'de> serde::Deserialize<'de> for Box<NormalizedPropertyNameStr> {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
-        S: serde::Serializer,
+        D: serde::Deserializer<'de>,
     {
-        use serde::ser::Error;
-        if serializer.is_human_readable() {
-            let s = str::from_utf8(&self.0)
-                .map_err(|_| S::Error::custom("Attempted to datagen invalid string property"))?;
-            serializer.serialize_str(s)
-        } else {
-            serializer.serialize_bytes(&self.0)
-        }
+        <Box<UnvalidatedStr>>::deserialize(deserializer).map(NormalizedPropertyNameStr::cast_box)
     }
 }
 
+/// This impl requires enabling the optional `serde` Cargo feature of the `icu_properties` crate
 #[cfg(feature = "serde")]
-impl<'de> serde::Deserialize<'de> for Box<NormalizedPropertyNameStr> {
+impl<'de, 'a> serde::Deserialize<'de> for &'a NormalizedPropertyNameStr
+where
+    'de: 'a,
+{
     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
     where
         D: serde::Deserializer<'de>,
     {
-        use alloc::borrow::Cow;
-        let s; // lifetime lengthening
-        let b;
-        // Can be improved with https://github.com/unicode-org/icu4x/issues/2310
-        // the allocations here are fine, in normal ICU4X code they'll only get hit
-        // during human-readable deserialization
-        let bytes = if deserializer.is_human_readable() {
-            s = <Cow<str>>::deserialize(deserializer)?;
-            s.as_bytes()
-        } else {
-            b = <Cow<[u8]>>::deserialize(deserializer)?;
-            &b
-        };
-        Ok(NormalizedPropertyNameStr::boxed_from_bytes(bytes))
+        <&UnvalidatedStr>::deserialize(deserializer).map(NormalizedPropertyNameStr::cast_ref)
     }
 }
 
@@ -134,30 +148,34 @@ impl Ord for NormalizedPropertyNameStr {
     }
 }
 
-impl fmt::Debug for NormalizedPropertyNameStr {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        if let Ok(s) = str::from_utf8(&self.0) {
-            f.write_str(s)
-        } else {
-            f.write_str("(invalid utf8)")
-        }
-    }
-}
-
 impl NormalizedPropertyNameStr {
-    pub(crate) fn cmp_loose(&self, other: &Self) -> Ordering {
+    /// Perform the loose comparison as defined in [`NormalizedPropertyNameStr`].
+    pub fn cmp_loose(&self, other: &Self) -> Ordering {
         let self_iter = self.0.iter().copied().filter_map(normalize_char);
         let other_iter = other.0.iter().copied().filter_map(normalize_char);
         self_iter.cmp(other_iter)
     }
-    #[cfg(feature = "serde")]
-    /// Get a `Box<NormalizedPropertyName>` from a byte slice
-    pub fn boxed_from_bytes(b: &[u8]) -> Box<Self> {
-        #[allow(clippy::expect_used)] // Self has no invariants
-        // can be cleaned up with https://github.com/unicode-org/icu4x/issues/2310
-        let this = Self::parse_byte_slice(b).expect("NormalizedPropertyName has no invariants");
 
-        zerovec::ule::encode_varule_to_box(&this)
+    /// Convert a string reference to a [`NormalizedPropertyNameStr`].
+    pub const fn from_str(s: &str) -> &Self {
+        Self::cast_ref(UnvalidatedStr::from_str(s))
+    }
+
+    /// Convert a [`UnvalidatedStr`] reference to a [`NormalizedPropertyNameStr`] reference.
+    pub const fn cast_ref(value: &UnvalidatedStr) -> &Self {
+        // Safety: repr(transparent)
+        unsafe { core::mem::transmute(value) }
+    }
+
+    /// Convert a [`UnvalidatedStr`] box to a [`NormalizedPropertyNameStr`] box.
+    pub const fn cast_box(value: Box<UnvalidatedStr>) -> Box<Self> {
+        // Safety: repr(transparent)
+        unsafe { core::mem::transmute(value) }
+    }
+
+    /// Get a [`NormalizedPropertyName`] box from a byte slice.
+    pub fn boxed_from_bytes(b: &[u8]) -> Box<Self> {
+        Self::cast_box(UnvalidatedStr::from_boxed_bytes(b.into()))
     }
 }
 

diff --git a/components/timezone/data/macros.rs b/components/timezone/data/macros.rs
@@ -6,10 +6,6 @@ mod macros {
 }
 #[doc(inline)]
 pub use __impl_time_zone_metazone_period_v1 as impl_time_zone_metazone_period_v1;
-#[doc(hidden)]
-pub use __lookup_time_zone_metazone_period_v1 as lookup_time_zone_metazone_period_v1;
-#[doc(inline)]
-pub use __singleton_time_zone_metazone_period_v1 as singleton_time_zone_metazone_period_v1;
 /// Implement [`DataProvider<M>`](icu_provider::DataProvider) on the given struct using the data
 /// hardcoded in this file. This allows the struct to be used with
 /// `icu`'s `_unstable` constructors.
@@ -46,7 +42,7 @@ macro_rules! __impl_any_provider {
             fn load_any(&self, key: icu_provider::DataKey, req: icu_provider::DataRequest) -> Result<icu_provider::AnyResponse, icu_provider::DataError> {
                 const TIME_ZONE_METAZONE_PERIOD_V1: icu_provider::DataKeyHash = <icu_timezone::provider::MetazonePeriodV1Marker as icu_provider::KeyedDataMarker>::KEY.hashed();
                 match key.hashed() {
-                    TIME_ZONE_METAZONE_PERIOD_V1 => lookup_time_zone_metazone_period_v1!(req.locale).map(icu_provider::AnyPayload::from_static_ref),
+                    TIME_ZONE_METAZONE_PERIOD_V1 => Self::lookup_time_zone_metazone_period_v1(&req.locale).map(icu_provider::AnyPayload::from_static_ref),
                     _ => Err(icu_provider::DataErrorKind::MissingDataKey),
                 }
                 .map(|payload| icu_provider::AnyResponse { payload: Some(payload), metadata: Default::default() })

diff --git a/components/timezone/data/macros/time_zone_metazone_period_v1.rs b/components/timezone/data/macros/time_zone_metazone_period_v1.rs
diff --git a/components/timezone/src/lib.rs b/components/timezone/src/lib.rs
@@ -142,4 +142,8 @@ pub mod data {
     include!(concat!(core::env!("ICU4X_DATA_DIR"), "/macros.rs"));
     #[cfg(not(icu4x_custom_data))]
     include!("../data/macros.rs");
+
+    use crate as icu_timezone;
+    pub(crate) struct Provider;
+    impl_time_zone_metazone_period_v1!(Provider);
 }
diff --git a/components/timezone/src/metazone.rs b/components/timezone/src/metazone.rs
@@ -35,10 +35,9 @@ impl MetazoneCalculator {
     #[cfg(feature = "data")]
     #[inline]
     pub const fn new() -> Self {
-        use crate as icu_timezone;
         MetazoneCalculator {
-            metazone_period: DataPayload::from_owned(
-                crate::data::singleton_time_zone_metazone_period_v1!(),
+            metazone_period: DataPayload::from_static_ref(
+                crate::data::Provider::SINGLETON_TIME_ZONE_METAZONE_PERIOD_V1,
             ),
         }
     }

diff --git a/docs/tutorials/crates/Cargo.lock b/docs/tutorials/crates/Cargo.lock
diff --git a/provider/core/src/datagen/payload.rs b/provider/core/src/datagen/payload.rs
@@ -8,18 +8,17 @@ use alloc::boxed::Box;
 use databake::{Bake, CrateEnv, TokenStream};
 use yoke::*;
 
-trait ExportableYoke {
+trait ExportableDataPayload {
     fn bake_yoke(&self, env: &CrateEnv) -> TokenStream;
     fn serialize_yoke(
         &self,
         serializer: &mut dyn erased_serde::Serializer,
     ) -> Result<(), DataError>;
 }
 
-impl<Y, C> ExportableYoke for Yoke<Y, C>
+impl<M: DataMarker> ExportableDataPayload for DataPayload<M>
 where
-    Y: for<'a> Yokeable<'a>,
-    for<'a> <Y as Yokeable<'a>>::Output: Bake + serde::Serialize,
+    for<'a> <M::Yokeable as Yokeable<'a>>::Output: Bake + serde::Serialize,
 {
     fn bake_yoke(&self, ctx: &CrateEnv) -> TokenStream {
         self.get().bake(ctx)
@@ -40,7 +39,7 @@ where
 #[doc(hidden)] // exposed for make_exportable_provider
 #[derive(yoke::Yokeable)]
 pub struct ExportBox {
-    payload: Box<dyn ExportableYoke + Sync>,
+    payload: Box<dyn ExportableDataPayload + Sync>,
 }
 
 impl core::fmt::Debug for ExportBox {
@@ -59,7 +58,7 @@ where
 {
     fn upcast(other: DataPayload<M>) -> DataPayload<ExportMarker> {
         DataPayload::from_owned(ExportBox {
-            payload: Box::new(other.yoke),
+            payload: Box::new(other),
         })
     }
 }