Add optional icu4x unicode data

.github/workflows/main.yml

@@ -32,6 +32,9 @@ jobs:
     - name: Run tests
       run: cargo test --verbose
 
+    - name: Run tests with icu4x
+      run: cargo test --verbose --features icu
+
     - name: Rustfmt
       run: cargo fmt --all -- --check
 

harfrust/Cargo.toml

@@ -17,12 +17,15 @@ bitflags = "2.9"
 bytemuck = { version = "1.22", features = ["extern_crate_alloc"] }
 core_maths = { version = "0.1", optional = true }
 smallvec = "1.14"
+icu_properties = { version = "2.2.0", optional = true }
+icu_normalizer = { version = "2.2.0", optional = true }
 
 [features]
 default = ["std"]
 std = ["read-fonts/std"]
 libm = ["dep:core_maths", "read-fonts/libm"]
 experimental_font_api = []
+icu = ["dep:icu_properties", "dep:icu_normalizer"]
 
 [dev-dependencies]
 harfrust = { path = "../harfrust", features = ["experimental_font_api"] }

harfrust/src/hb/algs.rs → harfrust/src/algs.rs

@@ -14,12 +14,6 @@ pub fn rb_flag_unsafe(x: u32) -> u32 {
     }
 }
 
-// FLAG_RANGE macro in harfbuzz.
-#[inline]
-pub fn rb_flag_range(x: u32, y: u32) -> u32 {
-    (x < y) as u32 + rb_flag(y + 1) - rb_flag(x)
-}
-
 // FLAG64 macro in harfbuzz.
 #[inline]
 pub const fn rb_flag64(x: u32) -> u64 {

harfrust/src/hb/aat/mod.rs

@@ -81,7 +81,7 @@ pub struct AatTables<'a> {
     pub feat: Option<Feat<'a>>,
 }
 
-use crate::hb::algs::HB_CODEPOINT_ENCODE3 as encode3;
+use crate::algs::HB_CODEPOINT_ENCODE3 as encode3;
 
 /// Blocklist specific broken morx tables identified by the combination of
 /// morx, GSUB, and GDEF table lengths.

harfrust/src/hb/buffer.rs

@@ -1,11 +1,10 @@
 use super::hb_mask_t;
-use super::unicode::CharExt;
 use crate::hb::face::BasicFontMetrics;
 use crate::hb::glyph_metrics::GlyphMetrics;
 use crate::hb::glyph_names::GlyphNames;
 use crate::hb::set_digest::hb_set_digest_t;
 use crate::hb::tables::TableRanges;
-use crate::hb::unicode::Codepoint;
+use crate::unicode::{CharExt, Codepoint};
 use crate::U32Set;
 use crate::{script, BufferClusterLevel, BufferFlags, Direction, Language, Script, SerializeFlags};
 use alloc::{string::String, vec::Vec};

harfrust/src/hb/mod.rs

@@ -20,7 +20,6 @@
 #![allow(clippy::match_like_matches_macro)]
 #![allow(clippy::manual_range_contains)]
 
-mod algs;
 #[macro_use]
 pub mod buffer;
 mod aat;
@@ -72,11 +71,6 @@ mod tag;
 #[allow(clippy::collapsible_match)]
 mod tag_table;
 mod text_parser;
-#[rustfmt::skip]
-mod unicode_emoji_table;
-#[rustfmt::skip]
-mod ucd_table;
-mod unicode;
 
 use read_fonts::types::Tag as hb_tag_t;
 

harfrust/src/hb/ot/mod.rs

@@ -793,7 +793,7 @@ impl ClassDefInfo {
     }
 }
 
-use super::algs::HB_CODEPOINT_ENCODE3 as encode3;
+use crate::algs::HB_CODEPOINT_ENCODE3 as encode3;
 
 /// Blocklist specific broken GDEF tables identified by the combination of
 /// GDEF, GSUB, and GPOS table lengths. Nuke the GDEF tables to avoid

harfrust/src/hb/ot_layout.rs

@@ -7,10 +7,9 @@ use super::font_funcs::FontFuncsDispatch;
 use super::ot::lookup::LookupInfo;
 use super::ot_layout_gsubgpos::OT;
 use super::ot_shape_plan::hb_ot_shape_plan_t;
-use super::unicode::hb_unicode_funcs_t;
 use super::{hb_font_t, GlyphInfo};
 use crate::hb::ot_layout_gsubgpos::OT::check_glyph_property;
-use crate::hb::unicode::GeneralCategory;
+use crate::unicode::{hb_unicode_funcs_t, GeneralCategory};
 
 impl GlyphInfo {
     declare_buffer_var!(u16, 1, 0, GLYPH_PROPS_VAR, glyph_props, set_glyph_props);

harfrust/src/hb/ot_layout_gsubgpos.rs

@@ -11,7 +11,7 @@ use super::ot_layout_common::*;
 use super::set_digest::hb_set_digest_t;
 use crate::hb::ot::{ClassDefInfo, CoverageInfo};
 use crate::hb::ot_layout_gsubgpos::OT::check_glyph_property;
-use crate::hb::unicode::GeneralCategory;
+use crate::unicode::GeneralCategory;
 use alloc::boxed::Box;
 use read_fonts::tables::layout::SequenceLookupRecord;
 use read_fonts::types::GlyphId;

harfrust/src/hb/ot_shape.rs

@@ -6,18 +6,11 @@ use super::ot_layout_gpos_table::GPOS;
 use super::ot_map::*;
 use super::ot_shape_plan::hb_ot_shape_plan_t;
 use super::ot_shaper::*;
-use super::unicode::CharExt;
 use super::*;
 use super::{hb_font_t, hb_tag_t};
 use crate::hb::aat;
-use crate::hb::algs::{rb_flag, rb_flag_unsafe};
 use crate::hb::buffer::GlyphFlags;
-use crate::hb::unicode::hb_gc::{
-    HB_UNICODE_GENERAL_CATEGORY_LOWERCASE_LETTER, HB_UNICODE_GENERAL_CATEGORY_OTHER_LETTER,
-    HB_UNICODE_GENERAL_CATEGORY_SPACE_SEPARATOR, HB_UNICODE_GENERAL_CATEGORY_TITLECASE_LETTER,
-    HB_UNICODE_GENERAL_CATEGORY_UPPERCASE_LETTER,
-};
-use crate::hb::unicode::GeneralCategory;
+use crate::unicode::{CharExt, GeneralCategory};
 use crate::BufferFlags;
 use crate::{Direction, Feature, Language, Script};
 use core::ptr;
@@ -664,12 +657,12 @@ impl OtShapeContext<'_, '_> {
 
             let gen_cat = info.general_category();
 
-            if (rb_flag_unsafe(gen_cat.to_u8() as u32)
-                & (rb_flag(HB_UNICODE_GENERAL_CATEGORY_LOWERCASE_LETTER)
-                    | rb_flag(HB_UNICODE_GENERAL_CATEGORY_UPPERCASE_LETTER)
-                    | rb_flag(HB_UNICODE_GENERAL_CATEGORY_TITLECASE_LETTER)
-                    | rb_flag(HB_UNICODE_GENERAL_CATEGORY_OTHER_LETTER)
-                    | rb_flag(HB_UNICODE_GENERAL_CATEGORY_SPACE_SEPARATOR)))
+            if gen_cat.flag_unsafe()
+                & (GeneralCategory::LOWERCASE_LETTER.flag()
+                    | GeneralCategory::UPPERCASE_LETTER.flag()
+                    | GeneralCategory::TITLECASE_LETTER.flag()
+                    | GeneralCategory::OTHER_LETTER.flag()
+                    | GeneralCategory::SPACE_SEPARATOR.flag())
                 != 0
             {
                 i += 1;
@@ -767,7 +760,7 @@ impl OtShapeContext<'_, '_> {
             let rtlm_mask = self.plan.rtlm_mask;
 
             for info in &mut self.buffer.info[..len] {
-                if let Some(c) = info.as_codepoint().mirrored() {
+                if let Some(c) = info.as_codepoint().mirroring() {
                     if self.font_funcs.nominal_glyph(c).is_some() {
                         info.glyph_id = c;
                         continue;

harfrust/src/hb/ot_shape_fallback.rs

@@ -4,8 +4,8 @@ use super::buffer::{hb_buffer_t, GlyphPosition};
 use super::face::GlyphExtents;
 use super::font_funcs::FontFuncsDispatch;
 use super::ot_shape_plan::hb_ot_shape_plan_t;
-use super::unicode::*;
 use super::{hb_font_t, Direction};
+use crate::unicode::*;
 
 struct FallbackShapeContext<'a, 'x, 'u> {
     plan: &'a hb_ot_shape_plan_t,
@@ -456,7 +456,7 @@ pub fn _hb_ot_shape_fallback_spaces<'a, 'x>(
     buffer: &'x mut hb_buffer_t,
     font_funcs: &'x mut FontFuncsDispatch<'a, '_>,
 ) {
-    use super::unicode::hb_unicode_funcs_t as t;
+    use crate::unicode::hb_unicode_funcs_t as t;
 
     let _ = plan;
     let len = buffer.len;

harfrust/src/hb/ot_shape_normalize.rs

@@ -1,11 +1,9 @@
-use crate::hb::unicode::Codepoint;
-
 use super::buffer::*;
 use super::font_funcs::FontFuncsDispatch;
 use super::hb_font_t;
 use super::ot_shape_plan::hb_ot_shape_plan_t;
 use super::ot_shaper::{ComposeFn, DecomposeFn, MAX_COMBINING_MARKS};
-use super::unicode::{hb_unicode_funcs_t, CharExt};
+use crate::unicode::{hb_unicode_funcs_t, CharExt, Codepoint};
 use read_fonts::types::GlyphId;
 
 impl GlyphInfo {
@@ -96,15 +94,15 @@ fn decompose_unicode(
     _: &hb_ot_shape_normalize_context_t,
     ab: Codepoint,
 ) -> Option<(Codepoint, Codepoint)> {
-    super::unicode::decompose(ab)
+    crate::unicode::decompose(ab)
 }
 
 fn compose_unicode(
     _: &hb_ot_shape_normalize_context_t,
     a: Codepoint,
     b: Codepoint,
 ) -> Option<Codepoint> {
-    super::unicode::compose(a, b)
+    crate::unicode::compose(a, b)
 }
 
 fn output_char(buffer: &mut hb_buffer_t, unichar: u32, glyph: u32) {

harfrust/src/hb/ot_shaper.rs

@@ -4,8 +4,8 @@ use super::font_funcs::FontFuncsDispatch;
 use super::ot_shape::*;
 use super::ot_shape_normalize::*;
 use super::ot_shape_plan::hb_ot_shape_plan_t;
-use super::unicode::Codepoint;
 use super::{hb_tag_t, script, Direction, Script};
+use crate::unicode::Codepoint;
 use alloc::boxed::Box;
 use core::any::Any;
 

harfrust/src/hb/ot_shaper_arabic.rs

@@ -1,13 +1,12 @@
-use super::algs::*;
 use super::buffer::*;
 use super::font_funcs::FontFuncsDispatch;
 use super::ot_map::*;
 use super::ot_shape::*;
 use super::ot_shape_normalize::HB_OT_SHAPE_NORMALIZATION_MODE_AUTO;
 use super::ot_shape_plan::hb_ot_shape_plan_t;
 use super::ot_shaper::*;
-use super::unicode::*;
 use super::{hb_mask_t, hb_tag_t, script, GlyphInfo, Script};
+use crate::unicode::*;
 use crate::Direction;
 use alloc::boxed::Box;
 
@@ -17,21 +16,21 @@ const HB_BUFFER_SCRATCH_FLAG_ARABIC_HAS_STCH: hb_buffer_scratch_flags_t =
 // See:
 // https://github.com/harfbuzz/harfbuzz/commit/6e6f82b6f3dde0fc6c3c7d991d9ec6cfff57823d#commitcomment-14248516
 fn is_word_category(gc: GeneralCategory) -> bool {
-    (rb_flag_unsafe(gc.to_u8() as u32)
-        & (rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_UNASSIGNED)
-            | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_PRIVATE_USE)
-            | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_MODIFIER_LETTER)
-            | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_OTHER_LETTER)
-            | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_SPACING_MARK)
-            | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_ENCLOSING_MARK)
-            | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_NON_SPACING_MARK)
-            | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_DECIMAL_NUMBER)
-            | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_LETTER_NUMBER)
-            | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_OTHER_NUMBER)
-            | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_CURRENCY_SYMBOL)
-            | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_MODIFIER_SYMBOL)
-            | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_MATH_SYMBOL)
-            | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_OTHER_SYMBOL)))
+    (gc.flag_unsafe()
+        & (GeneralCategory::UNASSIGNED.flag()
+            | GeneralCategory::PRIVATE_USE.flag()
+            | GeneralCategory::MODIFIER_LETTER.flag()
+            | GeneralCategory::OTHER_LETTER.flag()
+            | GeneralCategory::SPACING_MARK.flag()
+            | GeneralCategory::ENCLOSING_MARK.flag()
+            | GeneralCategory::NON_SPACING_MARK.flag()
+            | GeneralCategory::DECIMAL_NUMBER.flag()
+            | GeneralCategory::LETTER_NUMBER.flag()
+            | GeneralCategory::OTHER_NUMBER.flag()
+            | GeneralCategory::CURRENCY_SYMBOL.flag()
+            | GeneralCategory::MODIFIER_SYMBOL.flag()
+            | GeneralCategory::MATH_SYMBOL.flag()
+            | GeneralCategory::OTHER_SYMBOL.flag()))
         != 0
 }
 
@@ -54,10 +53,10 @@ fn get_joining_type(u: Codepoint, gc: GeneralCategory) -> hb_arabic_joining_type
         return j_type;
     }
 
-    let ok = rb_flag_unsafe(gc.to_u8() as u32)
-        & (rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_NON_SPACING_MARK)
-            | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_ENCLOSING_MARK)
-            | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_FORMAT));
+    let ok = gc.flag_unsafe()
+        & (GeneralCategory::NON_SPACING_MARK.flag()
+            | GeneralCategory::ENCLOSING_MARK.flag()
+            | GeneralCategory::FORMAT.flag());
 
     if ok != 0 {
         hb_arabic_joining_type_t::T

harfrust/src/hb/ot_shaper_arabic_table.rs

@@ -16,7 +16,7 @@
 #![allow(unused_parens)]
 #![allow(clippy::unnecessary_cast, clippy::unreadable_literal, clippy::double_parens)]
 
-use crate::hb::unicode::Codepoint;
+use crate::unicode::Codepoint;
 
 use super::ot_shaper_arabic::hb_arabic_joining_type_t::{self, D, GroupAlaph, GroupDalathRish, L, R, T, U, X};
 

harfrust/src/hb/ot_shaper_hangul.rs

@@ -8,7 +8,7 @@ use super::ot_shape_normalize::HB_OT_SHAPE_NORMALIZATION_MODE_NONE;
 use super::ot_shape_plan::hb_ot_shape_plan_t;
 use super::ot_shaper::*;
 use super::*;
-use crate::hb::unicode::Codepoint;
+use crate::unicode::Codepoint;
 use crate::BufferFlags;
 
 const LJMO: u8 = 1;

harfrust/src/hb/ot_shaper_hebrew.rs

@@ -1,10 +1,9 @@
+use super::hb_tag_t;
 use super::ot_shape_normalize::*;
 use super::ot_shaper::*;
-use super::{hb_tag_t, unicode};
 use crate::hb::buffer::hb_buffer_t;
 use crate::hb::ot_shape_plan::hb_ot_shape_plan_t;
-use crate::hb::unicode::Codepoint;
-use crate::hb::unicode::{combining_class, modified_combining_class};
+use crate::unicode::{self, combining_class, modified_combining_class, Codepoint};
 
 pub const HEBREW_SHAPER: hb_ot_shaper_t = hb_ot_shaper_t {
     collect_features: None,

harfrust/src/hb/ot_shaper_indic.rs

@@ -4,7 +4,6 @@ use core::ops::Range;
 
 use read_fonts::types::GlyphId;
 
-use super::algs::*;
 use super::buffer::*;
 use super::font_funcs::FontFuncsDispatch;
 use super::ot_layout::*;
@@ -15,9 +14,10 @@ use super::ot_shape_normalize::*;
 use super::ot_shape_plan::hb_ot_shape_plan_t;
 use super::ot_shaper::*;
 use super::ot_shaper_syllabic::*;
-use super::unicode::Codepoint;
-use super::unicode::{hb_gc, CharExt};
 use super::{hb_font_t, hb_mask_t, hb_tag_t, script, GlyphInfo, Script};
+use crate::algs::*;
+use crate::unicode::GeneralCategory;
+use crate::unicode::{CharExt, Codepoint};
 
 pub const INDIC_SHAPER: hb_ot_shaper_t = hb_ot_shaper_t {
     collect_features: Some(collect_features),
@@ -591,7 +591,7 @@ fn decompose(_: &hb_ot_shape_normalize_context_t, ab: Codepoint) -> Option<(Code
         _ => {}
     }
 
-    crate::hb::unicode::decompose(ab)
+    crate::unicode::decompose(ab)
 }
 
 fn compose(_: &hb_ot_shape_normalize_context_t, a: Codepoint, b: Codepoint) -> Option<Codepoint> {
@@ -605,7 +605,7 @@ fn compose(_: &hb_ot_shape_normalize_context_t, a: Codepoint, b: Codepoint) -> O
         return Some(0x09DF);
     }
 
-    crate::hb::unicode::compose(a, b)
+    crate::unicode::compose(a, b)
 }
 
 fn setup_masks(_: &hb_ot_shape_plan_t, _: &mut FontFuncsDispatch, buffer: &mut hb_buffer_t) {
@@ -1833,12 +1833,9 @@ fn final_reordering_impl(
     // Apply 'init' to the Left Matra if it's a word start.
     if buffer.info[start].indic_position() == ot_position_t::POS_PRE_M {
         if start == 0
-            || (rb_flag_unsafe(buffer.info[start - 1].general_category().to_u8() as u32)
-                & rb_flag_range(
-                    hb_gc::HB_UNICODE_GENERAL_CATEGORY_FORMAT,
-                    hb_gc::HB_UNICODE_GENERAL_CATEGORY_NON_SPACING_MARK,
-                ))
-                == 0
+            || !buffer.info[start - 1]
+                .general_category()
+                .in_range_inclusive(GeneralCategory::FORMAT, GeneralCategory::NON_SPACING_MARK)
         {
             buffer.info[start].mask |= indic_plan.mask_array[indic_feature::INIT];
         } else {