From b75b15536975c2d593e324ab708b90414bc28d38 Mon Sep 17 00:00:00 2001 From: Chad Brokaw Date: Wed, 8 Jul 2026 12:56:26 -0400 Subject: [PATCH 1/5] reorganize unicode into top level module --- harfrust/Cargo.toml | 2 + harfrust/src/{hb => }/algs.rs | 0 harfrust/src/hb/aat/mod.rs | 2 +- harfrust/src/hb/buffer.rs | 3 +- harfrust/src/hb/mod.rs | 6 --- harfrust/src/hb/ot/mod.rs | 2 +- harfrust/src/hb/ot_layout.rs | 3 +- harfrust/src/hb/ot_layout_gsubgpos.rs | 2 +- harfrust/src/hb/ot_shape.rs | 15 +++---- harfrust/src/hb/ot_shape_fallback.rs | 4 +- harfrust/src/hb/ot_shape_normalize.rs | 8 ++-- harfrust/src/hb/ot_shaper.rs | 2 +- harfrust/src/hb/ot_shaper_arabic.rs | 4 +- harfrust/src/hb/ot_shaper_arabic_table.rs | 2 +- harfrust/src/hb/ot_shaper_hangul.rs | 2 +- harfrust/src/hb/ot_shaper_hebrew.rs | 5 +-- harfrust/src/hb/ot_shaper_indic.rs | 9 ++--- harfrust/src/hb/ot_shaper_khmer.rs | 7 ++-- harfrust/src/hb/ot_shaper_thai.rs | 2 +- harfrust/src/hb/ot_shaper_use.rs | 6 +-- harfrust/src/lib.rs | 2 + .../emoji_table.rs} | 2 +- .../src/{hb/unicode.rs => unicode/mod.rs} | 39 +++++++++++-------- harfrust/src/{hb => unicode}/ucd_table.rs | 4 +- 24 files changed, 65 insertions(+), 68 deletions(-) rename harfrust/src/{hb => }/algs.rs (100%) rename harfrust/src/{hb/unicode_emoji_table.rs => unicode/emoji_table.rs} (99%) rename harfrust/src/{hb/unicode.rs => unicode/mod.rs} (99%) rename harfrust/src/{hb => unicode}/ucd_table.rs (99%) diff --git a/harfrust/Cargo.toml b/harfrust/Cargo.toml index 92e4418e..9766d19f 100644 --- a/harfrust/Cargo.toml +++ b/harfrust/Cargo.toml @@ -17,12 +17,14 @@ bitflags = "2.9" bytemuck = { version = "1.22", features = ["extern_crate_alloc"] } core_maths = { version = "0.1", optional = true } smallvec = "1.14" +icu_properties = { version = "2.2.0", optional = true } [features] default = ["std"] std = ["read-fonts/std"] libm = ["dep:core_maths", "read-fonts/libm"] experimental_font_api = [] +icu = ["dep:icu_properties"] [dev-dependencies] harfrust = { path = "../harfrust", features = ["experimental_font_api"] } diff --git a/harfrust/src/hb/algs.rs b/harfrust/src/algs.rs similarity index 100% rename from harfrust/src/hb/algs.rs rename to harfrust/src/algs.rs diff --git a/harfrust/src/hb/aat/mod.rs b/harfrust/src/hb/aat/mod.rs index bfd4ca3c..75efa155 100644 --- a/harfrust/src/hb/aat/mod.rs +++ b/harfrust/src/hb/aat/mod.rs @@ -81,7 +81,7 @@ pub struct AatTables<'a> { pub feat: Option>, } -use crate::hb::algs::HB_CODEPOINT_ENCODE3 as encode3; +use crate::algs::HB_CODEPOINT_ENCODE3 as encode3; /// Blocklist specific broken morx tables identified by the combination of /// morx, GSUB, and GDEF table lengths. diff --git a/harfrust/src/hb/buffer.rs b/harfrust/src/hb/buffer.rs index b889a5d5..f8af0264 100644 --- a/harfrust/src/hb/buffer.rs +++ b/harfrust/src/hb/buffer.rs @@ -1,11 +1,10 @@ use super::hb_mask_t; -use super::unicode::CharExt; use crate::hb::face::BasicFontMetrics; use crate::hb::glyph_metrics::GlyphMetrics; use crate::hb::glyph_names::GlyphNames; use crate::hb::set_digest::hb_set_digest_t; use crate::hb::tables::TableRanges; -use crate::hb::unicode::Codepoint; +use crate::unicode::{CharExt, Codepoint}; use crate::U32Set; use crate::{script, BufferClusterLevel, BufferFlags, Direction, Language, Script, SerializeFlags}; use alloc::{string::String, vec::Vec}; diff --git a/harfrust/src/hb/mod.rs b/harfrust/src/hb/mod.rs index c6dba1de..0ee74c93 100644 --- a/harfrust/src/hb/mod.rs +++ b/harfrust/src/hb/mod.rs @@ -20,7 +20,6 @@ #![allow(clippy::match_like_matches_macro)] #![allow(clippy::manual_range_contains)] -mod algs; #[macro_use] pub mod buffer; mod aat; @@ -72,11 +71,6 @@ mod tag; #[allow(clippy::collapsible_match)] mod tag_table; mod text_parser; -#[rustfmt::skip] -mod unicode_emoji_table; -#[rustfmt::skip] -mod ucd_table; -mod unicode; use read_fonts::types::Tag as hb_tag_t; diff --git a/harfrust/src/hb/ot/mod.rs b/harfrust/src/hb/ot/mod.rs index 4b5783ba..d2568622 100644 --- a/harfrust/src/hb/ot/mod.rs +++ b/harfrust/src/hb/ot/mod.rs @@ -793,7 +793,7 @@ impl ClassDefInfo { } } -use super::algs::HB_CODEPOINT_ENCODE3 as encode3; +use crate::algs::HB_CODEPOINT_ENCODE3 as encode3; /// Blocklist specific broken GDEF tables identified by the combination of /// GDEF, GSUB, and GPOS table lengths. Nuke the GDEF tables to avoid diff --git a/harfrust/src/hb/ot_layout.rs b/harfrust/src/hb/ot_layout.rs index 2065d6d4..f91ff5f9 100644 --- a/harfrust/src/hb/ot_layout.rs +++ b/harfrust/src/hb/ot_layout.rs @@ -7,10 +7,9 @@ use super::font_funcs::FontFuncsDispatch; use super::ot::lookup::LookupInfo; use super::ot_layout_gsubgpos::OT; use super::ot_shape_plan::hb_ot_shape_plan_t; -use super::unicode::hb_unicode_funcs_t; use super::{hb_font_t, GlyphInfo}; use crate::hb::ot_layout_gsubgpos::OT::check_glyph_property; -use crate::hb::unicode::GeneralCategory; +use crate::unicode::{hb_unicode_funcs_t, GeneralCategory}; impl GlyphInfo { declare_buffer_var!(u16, 1, 0, GLYPH_PROPS_VAR, glyph_props, set_glyph_props); diff --git a/harfrust/src/hb/ot_layout_gsubgpos.rs b/harfrust/src/hb/ot_layout_gsubgpos.rs index a8771281..cc363eef 100644 --- a/harfrust/src/hb/ot_layout_gsubgpos.rs +++ b/harfrust/src/hb/ot_layout_gsubgpos.rs @@ -11,7 +11,7 @@ use super::ot_layout_common::*; use super::set_digest::hb_set_digest_t; use crate::hb::ot::{ClassDefInfo, CoverageInfo}; use crate::hb::ot_layout_gsubgpos::OT::check_glyph_property; -use crate::hb::unicode::GeneralCategory; +use crate::unicode::GeneralCategory; use alloc::boxed::Box; use read_fonts::tables::layout::SequenceLookupRecord; use read_fonts::types::GlyphId; diff --git a/harfrust/src/hb/ot_shape.rs b/harfrust/src/hb/ot_shape.rs index 1a9c9e98..1c742ae5 100644 --- a/harfrust/src/hb/ot_shape.rs +++ b/harfrust/src/hb/ot_shape.rs @@ -6,18 +6,19 @@ use super::ot_layout_gpos_table::GPOS; use super::ot_map::*; use super::ot_shape_plan::hb_ot_shape_plan_t; use super::ot_shaper::*; -use super::unicode::CharExt; use super::*; use super::{hb_font_t, hb_tag_t}; +use crate::algs::{rb_flag, rb_flag_unsafe}; use crate::hb::aat; -use crate::hb::algs::{rb_flag, rb_flag_unsafe}; use crate::hb::buffer::GlyphFlags; -use crate::hb::unicode::hb_gc::{ - HB_UNICODE_GENERAL_CATEGORY_LOWERCASE_LETTER, HB_UNICODE_GENERAL_CATEGORY_OTHER_LETTER, - HB_UNICODE_GENERAL_CATEGORY_SPACE_SEPARATOR, HB_UNICODE_GENERAL_CATEGORY_TITLECASE_LETTER, - HB_UNICODE_GENERAL_CATEGORY_UPPERCASE_LETTER, +use crate::unicode::{ + hb_gc::{ + HB_UNICODE_GENERAL_CATEGORY_LOWERCASE_LETTER, HB_UNICODE_GENERAL_CATEGORY_OTHER_LETTER, + HB_UNICODE_GENERAL_CATEGORY_SPACE_SEPARATOR, HB_UNICODE_GENERAL_CATEGORY_TITLECASE_LETTER, + HB_UNICODE_GENERAL_CATEGORY_UPPERCASE_LETTER, + }, + CharExt, GeneralCategory, }; -use crate::hb::unicode::GeneralCategory; use crate::BufferFlags; use crate::{Direction, Feature, Language, Script}; use core::ptr; diff --git a/harfrust/src/hb/ot_shape_fallback.rs b/harfrust/src/hb/ot_shape_fallback.rs index 89e8a450..4ff54560 100644 --- a/harfrust/src/hb/ot_shape_fallback.rs +++ b/harfrust/src/hb/ot_shape_fallback.rs @@ -4,8 +4,8 @@ use super::buffer::{hb_buffer_t, GlyphPosition}; use super::face::GlyphExtents; use super::font_funcs::FontFuncsDispatch; use super::ot_shape_plan::hb_ot_shape_plan_t; -use super::unicode::*; use super::{hb_font_t, Direction}; +use crate::unicode::*; struct FallbackShapeContext<'a, 'x, 'u> { plan: &'a hb_ot_shape_plan_t, @@ -456,7 +456,7 @@ pub fn _hb_ot_shape_fallback_spaces<'a, 'x>( buffer: &'x mut hb_buffer_t, font_funcs: &'x mut FontFuncsDispatch<'a, '_>, ) { - use super::unicode::hb_unicode_funcs_t as t; + use crate::unicode::hb_unicode_funcs_t as t; let _ = plan; let len = buffer.len; diff --git a/harfrust/src/hb/ot_shape_normalize.rs b/harfrust/src/hb/ot_shape_normalize.rs index 9bf0a08d..7384e866 100644 --- a/harfrust/src/hb/ot_shape_normalize.rs +++ b/harfrust/src/hb/ot_shape_normalize.rs @@ -1,11 +1,9 @@ -use crate::hb::unicode::Codepoint; - use super::buffer::*; use super::font_funcs::FontFuncsDispatch; use super::hb_font_t; use super::ot_shape_plan::hb_ot_shape_plan_t; use super::ot_shaper::{ComposeFn, DecomposeFn, MAX_COMBINING_MARKS}; -use super::unicode::{hb_unicode_funcs_t, CharExt}; +use crate::unicode::{hb_unicode_funcs_t, CharExt, Codepoint}; use read_fonts::types::GlyphId; impl GlyphInfo { @@ -96,7 +94,7 @@ fn decompose_unicode( _: &hb_ot_shape_normalize_context_t, ab: Codepoint, ) -> Option<(Codepoint, Codepoint)> { - super::unicode::decompose(ab) + crate::unicode::decompose(ab) } fn compose_unicode( @@ -104,7 +102,7 @@ fn compose_unicode( a: Codepoint, b: Codepoint, ) -> Option { - super::unicode::compose(a, b) + crate::unicode::compose(a, b) } fn output_char(buffer: &mut hb_buffer_t, unichar: u32, glyph: u32) { diff --git a/harfrust/src/hb/ot_shaper.rs b/harfrust/src/hb/ot_shaper.rs index 755e7605..70f2a07a 100644 --- a/harfrust/src/hb/ot_shaper.rs +++ b/harfrust/src/hb/ot_shaper.rs @@ -4,8 +4,8 @@ use super::font_funcs::FontFuncsDispatch; use super::ot_shape::*; use super::ot_shape_normalize::*; use super::ot_shape_plan::hb_ot_shape_plan_t; -use super::unicode::Codepoint; use super::{hb_tag_t, script, Direction, Script}; +use crate::unicode::Codepoint; use alloc::boxed::Box; use core::any::Any; diff --git a/harfrust/src/hb/ot_shaper_arabic.rs b/harfrust/src/hb/ot_shaper_arabic.rs index 8a395347..b1a3330f 100644 --- a/harfrust/src/hb/ot_shaper_arabic.rs +++ b/harfrust/src/hb/ot_shaper_arabic.rs @@ -1,4 +1,3 @@ -use super::algs::*; use super::buffer::*; use super::font_funcs::FontFuncsDispatch; use super::ot_map::*; @@ -6,8 +5,9 @@ use super::ot_shape::*; use super::ot_shape_normalize::HB_OT_SHAPE_NORMALIZATION_MODE_AUTO; use super::ot_shape_plan::hb_ot_shape_plan_t; use super::ot_shaper::*; -use super::unicode::*; use super::{hb_mask_t, hb_tag_t, script, GlyphInfo, Script}; +use crate::algs::*; +use crate::unicode::*; use crate::Direction; use alloc::boxed::Box; diff --git a/harfrust/src/hb/ot_shaper_arabic_table.rs b/harfrust/src/hb/ot_shaper_arabic_table.rs index 8bdb8316..2f764925 100644 --- a/harfrust/src/hb/ot_shaper_arabic_table.rs +++ b/harfrust/src/hb/ot_shaper_arabic_table.rs @@ -16,7 +16,7 @@ #![allow(unused_parens)] #![allow(clippy::unnecessary_cast, clippy::unreadable_literal, clippy::double_parens)] -use crate::hb::unicode::Codepoint; +use crate::unicode::Codepoint; use super::ot_shaper_arabic::hb_arabic_joining_type_t::{self, D, GroupAlaph, GroupDalathRish, L, R, T, U, X}; diff --git a/harfrust/src/hb/ot_shaper_hangul.rs b/harfrust/src/hb/ot_shaper_hangul.rs index b747d061..cf0d8d14 100644 --- a/harfrust/src/hb/ot_shaper_hangul.rs +++ b/harfrust/src/hb/ot_shaper_hangul.rs @@ -8,7 +8,7 @@ use super::ot_shape_normalize::HB_OT_SHAPE_NORMALIZATION_MODE_NONE; use super::ot_shape_plan::hb_ot_shape_plan_t; use super::ot_shaper::*; use super::*; -use crate::hb::unicode::Codepoint; +use crate::unicode::Codepoint; use crate::BufferFlags; const LJMO: u8 = 1; diff --git a/harfrust/src/hb/ot_shaper_hebrew.rs b/harfrust/src/hb/ot_shaper_hebrew.rs index a4b3a2fc..61136fa0 100644 --- a/harfrust/src/hb/ot_shaper_hebrew.rs +++ b/harfrust/src/hb/ot_shaper_hebrew.rs @@ -1,10 +1,9 @@ +use super::hb_tag_t; use super::ot_shape_normalize::*; use super::ot_shaper::*; -use super::{hb_tag_t, unicode}; use crate::hb::buffer::hb_buffer_t; use crate::hb::ot_shape_plan::hb_ot_shape_plan_t; -use crate::hb::unicode::Codepoint; -use crate::hb::unicode::{combining_class, modified_combining_class}; +use crate::unicode::{self, combining_class, modified_combining_class, Codepoint}; pub const HEBREW_SHAPER: hb_ot_shaper_t = hb_ot_shaper_t { collect_features: None, diff --git a/harfrust/src/hb/ot_shaper_indic.rs b/harfrust/src/hb/ot_shaper_indic.rs index 3a759077..259b1081 100644 --- a/harfrust/src/hb/ot_shaper_indic.rs +++ b/harfrust/src/hb/ot_shaper_indic.rs @@ -4,7 +4,6 @@ use core::ops::Range; use read_fonts::types::GlyphId; -use super::algs::*; use super::buffer::*; use super::font_funcs::FontFuncsDispatch; use super::ot_layout::*; @@ -15,9 +14,9 @@ use super::ot_shape_normalize::*; use super::ot_shape_plan::hb_ot_shape_plan_t; use super::ot_shaper::*; use super::ot_shaper_syllabic::*; -use super::unicode::Codepoint; -use super::unicode::{hb_gc, CharExt}; use super::{hb_font_t, hb_mask_t, hb_tag_t, script, GlyphInfo, Script}; +use crate::algs::*; +use crate::unicode::{hb_gc, CharExt, Codepoint}; pub const INDIC_SHAPER: hb_ot_shaper_t = hb_ot_shaper_t { collect_features: Some(collect_features), @@ -591,7 +590,7 @@ fn decompose(_: &hb_ot_shape_normalize_context_t, ab: Codepoint) -> Option<(Code _ => {} } - crate::hb::unicode::decompose(ab) + crate::unicode::decompose(ab) } fn compose(_: &hb_ot_shape_normalize_context_t, a: Codepoint, b: Codepoint) -> Option { @@ -605,7 +604,7 @@ fn compose(_: &hb_ot_shape_normalize_context_t, a: Codepoint, b: Codepoint) -> O return Some(0x09DF); } - crate::hb::unicode::compose(a, b) + crate::unicode::compose(a, b) } fn setup_masks(_: &hb_ot_shape_plan_t, _: &mut FontFuncsDispatch, buffer: &mut hb_buffer_t) { diff --git a/harfrust/src/hb/ot_shaper_khmer.rs b/harfrust/src/hb/ot_shaper_khmer.rs index 3679f530..bcf2b9ab 100644 --- a/harfrust/src/hb/ot_shaper_khmer.rs +++ b/harfrust/src/hb/ot_shaper_khmer.rs @@ -7,9 +7,8 @@ use super::ot_shape_plan::hb_ot_shape_plan_t; use super::ot_shaper::*; use super::ot_shaper_indic::ot_category_t; use super::ot_shaper_syllabic::*; -use super::unicode::CharExt; -use super::unicode::Codepoint; use super::{hb_mask_t, hb_tag_t, GlyphInfo}; +use crate::unicode::{CharExt, Codepoint}; use alloc::boxed::Box; pub const KHMER_SHAPER: hb_ot_shaper_t = hb_ot_shaper_t { @@ -295,7 +294,7 @@ fn decompose(_: &hb_ot_shape_normalize_context_t, ab: Codepoint) -> Option<(Code // Decompose split matras that don't have Unicode decompositions. match ab { 0x17BE | 0x17BF | 0x17C0 | 0x17C4 | 0x17C5 => Some((0x17C1, ab)), - _ => crate::hb::unicode::decompose(ab), + _ => crate::unicode::decompose(ab), } } @@ -305,7 +304,7 @@ fn compose(_: &hb_ot_shape_normalize_context_t, a: Codepoint, b: Codepoint) -> O return None; } - crate::hb::unicode::compose(a, b) + crate::unicode::compose(a, b) } fn setup_masks(_: &hb_ot_shape_plan_t, _: &mut FontFuncsDispatch, buffer: &mut hb_buffer_t) { diff --git a/harfrust/src/hb/ot_shaper_thai.rs b/harfrust/src/hb/ot_shaper_thai.rs index 39a9f3b9..7502ed51 100644 --- a/harfrust/src/hb/ot_shaper_thai.rs +++ b/harfrust/src/hb/ot_shaper_thai.rs @@ -5,7 +5,7 @@ use super::ot_shape_normalize::HB_OT_SHAPE_NORMALIZATION_MODE_AUTO; use super::ot_shape_plan::hb_ot_shape_plan_t; use super::ot_shaper::*; use super::script; -use super::unicode::GeneralCategory; +use crate::unicode::GeneralCategory; pub const THAI_SHAPER: hb_ot_shaper_t = hb_ot_shaper_t { collect_features: None, diff --git a/harfrust/src/hb/ot_shaper_use.rs b/harfrust/src/hb/ot_shaper_use.rs index b340bf1d..0c654a83 100644 --- a/harfrust/src/hb/ot_shaper_use.rs +++ b/harfrust/src/hb/ot_shaper_use.rs @@ -1,4 +1,3 @@ -use super::algs::*; use super::buffer::*; use super::font_funcs::FontFuncsDispatch; use super::ot_layout::*; @@ -9,8 +8,9 @@ use super::ot_shape_plan::hb_ot_shape_plan_t; use super::ot_shaper::*; use super::ot_shaper_arabic::arabic_shape_plan_t; use super::ot_shaper_syllabic::*; -use super::unicode::{CharExt, Codepoint}; use super::{hb_mask_t, hb_tag_t, script, GlyphInfo, Script}; +use crate::algs::*; +use crate::unicode::{CharExt, Codepoint}; use alloc::boxed::Box; pub const UNIVERSAL_SHAPER: hb_ot_shaper_t = hb_ot_shaper_t { @@ -567,7 +567,7 @@ fn compose(_: &hb_ot_shape_normalize_context_t, a: Codepoint, b: Codepoint) -> O return None; } - crate::hb::unicode::compose(a, b) + crate::unicode::compose(a, b) } fn setup_masks(plan: &hb_ot_shape_plan_t, _: &mut FontFuncsDispatch, buffer: &mut hb_buffer_t) { diff --git a/harfrust/src/lib.rs b/harfrust/src/lib.rs index ed748f38..2b5f8ddf 100644 --- a/harfrust/src/lib.rs +++ b/harfrust/src/lib.rs @@ -10,7 +10,9 @@ A complete [harfbuzz](https://github.com/harfbuzz/harfbuzz) shaping algorithm po extern crate alloc; +mod algs; mod hb; +mod unicode; #[cfg(feature = "std")] pub(crate) type U32Set = read_fonts::collections::int_set::U32Set; diff --git a/harfrust/src/hb/unicode_emoji_table.rs b/harfrust/src/unicode/emoji_table.rs similarity index 99% rename from harfrust/src/hb/unicode_emoji_table.rs rename to harfrust/src/unicode/emoji_table.rs index 4e194bb1..2f25c3e9 100644 --- a/harfrust/src/hb/unicode_emoji_table.rs +++ b/harfrust/src/unicode/emoji_table.rs @@ -21,7 +21,7 @@ #![allow(unused_parens)] #![allow(clippy::unnecessary_cast, clippy::unreadable_literal, clippy::double_parens)] -use crate::hb::unicode::Codepoint; +use crate::unicode::Codepoint; #[allow(dead_code, non_upper_case_globals, clippy::allow_attributes_without_reason, missing_docs)] pub(crate) static _hb_emoji_u8: [u8; 624]= diff --git a/harfrust/src/hb/unicode.rs b/harfrust/src/unicode/mod.rs similarity index 99% rename from harfrust/src/hb/unicode.rs rename to harfrust/src/unicode/mod.rs index 9ca0928c..34a447d8 100644 --- a/harfrust/src/hb/unicode.rs +++ b/harfrust/src/unicode/mod.rs @@ -1,6 +1,13 @@ -use super::ucd_table::ucd::*; -use crate::hb::algs::*; +#![expect(non_camel_case_types, non_snake_case, non_upper_case_globals)] + +#[rustfmt::skip] +mod emoji_table; +#[rustfmt::skip] +mod ucd_table; + +use crate::algs::*; use crate::Script; +use ucd_table::ucd::*; pub type Codepoint = u32; @@ -388,6 +395,19 @@ impl CharExt for Codepoint { GeneralCategory(_hb_ucd_gc(self as usize)) } + fn combining_class(self) -> u8 { + _hb_ucd_ccc(self as usize) + } + + fn mirrored(self) -> Option { + let delta = _hb_ucd_bmg(self as usize); + if delta == 0 { + None + } else { + Some(((self as i32).wrapping_add(delta as i32)) as u32) + } + } + fn space_fallback(self) -> hb_unicode_funcs_t::space_t { use hb_unicode_funcs_t::*; @@ -413,10 +433,6 @@ impl CharExt for Codepoint { } } - fn combining_class(self) -> u8 { - _hb_ucd_ccc(self as usize) - } - fn modified_combining_class(self) -> u8 { let u = self; @@ -440,17 +456,8 @@ impl CharExt for Codepoint { MODIFIED_COMBINING_CLASS[k as usize] } - fn mirrored(self) -> Option { - let delta = _hb_ucd_bmg(self as usize); - if delta == 0 { - None - } else { - Some(((self as i32).wrapping_add(delta as i32)) as u32) - } - } - fn is_emoji_extended_pictographic(self) -> bool { - super::unicode_emoji_table::is_Extended_Pictographic(self) + emoji_table::is_Extended_Pictographic(self) } /// Default_Ignorable codepoints: diff --git a/harfrust/src/hb/ucd_table.rs b/harfrust/src/unicode/ucd_table.rs similarity index 99% rename from harfrust/src/hb/ucd_table.rs rename to harfrust/src/unicode/ucd_table.rs index 980b32a2..c3d2e66d 100644 --- a/harfrust/src/hb/ucd_table.rs +++ b/harfrust/src/unicode/ucd_table.rs @@ -12,9 +12,7 @@ pub(crate) mod ucd { #![allow(unused_parens)] #![allow(clippy::unnecessary_cast, clippy::unreadable_literal, clippy::double_parens)] -use crate::hb::algs::{HB_CODEPOINT_ENCODE3, HB_CODEPOINT_ENCODE3_11_7_14}; -use crate::hb::common::script; -use crate::hb::common::Script as hb_script_t; +use crate::{algs::{HB_CODEPOINT_ENCODE3, HB_CODEPOINT_ENCODE3_11_7_14}, script, Script as hb_script_t}; #[allow(dead_code, non_upper_case_globals, clippy::allow_attributes_without_reason, missing_docs)] pub(crate) static _hb_ucd_sc_map: [hb_script_t; 176]= From f6ce19e73e32f544628f4c560c5f7ca395ac034b Mon Sep 17 00:00:00 2001 From: Chad Brokaw Date: Wed, 8 Jul 2026 15:49:31 -0400 Subject: [PATCH 2/5] add icu4x ucd backend --- .github/workflows/main.yml | 3 + harfrust/Cargo.toml | 3 +- harfrust/src/algs.rs | 6 - harfrust/src/hb/ot_shape.rs | 24 +- harfrust/src/hb/ot_shaper_arabic.rs | 39 +- harfrust/src/hb/ot_shaper_indic.rs | 12 +- harfrust/src/unicode/mod.rs | 815 +++++++++++++++++++++------- 7 files changed, 649 insertions(+), 253 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b0e7659b..fb2af07d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -32,6 +32,9 @@ jobs: - name: Run tests run: cargo test --verbose + - name: Run tests with icu4x + run: cargo test --verbose --features icu + - name: Rustfmt run: cargo fmt --all -- --check diff --git a/harfrust/Cargo.toml b/harfrust/Cargo.toml index 9766d19f..d791b355 100644 --- a/harfrust/Cargo.toml +++ b/harfrust/Cargo.toml @@ -18,13 +18,14 @@ bytemuck = { version = "1.22", features = ["extern_crate_alloc"] } core_maths = { version = "0.1", optional = true } smallvec = "1.14" icu_properties = { version = "2.2.0", optional = true } +icu_normalizer = { version = "2.2.0", optional = true } [features] default = ["std"] std = ["read-fonts/std"] libm = ["dep:core_maths", "read-fonts/libm"] experimental_font_api = [] -icu = ["dep:icu_properties"] +icu = ["dep:icu_properties", "dep:icu_normalizer"] [dev-dependencies] harfrust = { path = "../harfrust", features = ["experimental_font_api"] } diff --git a/harfrust/src/algs.rs b/harfrust/src/algs.rs index 4a4b78fb..b7e1a421 100644 --- a/harfrust/src/algs.rs +++ b/harfrust/src/algs.rs @@ -14,12 +14,6 @@ pub fn rb_flag_unsafe(x: u32) -> u32 { } } -// FLAG_RANGE macro in harfbuzz. -#[inline] -pub fn rb_flag_range(x: u32, y: u32) -> u32 { - (x < y) as u32 + rb_flag(y + 1) - rb_flag(x) -} - // FLAG64 macro in harfbuzz. #[inline] pub const fn rb_flag64(x: u32) -> u64 { diff --git a/harfrust/src/hb/ot_shape.rs b/harfrust/src/hb/ot_shape.rs index 1c742ae5..8d87d209 100644 --- a/harfrust/src/hb/ot_shape.rs +++ b/harfrust/src/hb/ot_shape.rs @@ -8,17 +8,9 @@ use super::ot_shape_plan::hb_ot_shape_plan_t; use super::ot_shaper::*; use super::*; use super::{hb_font_t, hb_tag_t}; -use crate::algs::{rb_flag, rb_flag_unsafe}; use crate::hb::aat; use crate::hb::buffer::GlyphFlags; -use crate::unicode::{ - hb_gc::{ - HB_UNICODE_GENERAL_CATEGORY_LOWERCASE_LETTER, HB_UNICODE_GENERAL_CATEGORY_OTHER_LETTER, - HB_UNICODE_GENERAL_CATEGORY_SPACE_SEPARATOR, HB_UNICODE_GENERAL_CATEGORY_TITLECASE_LETTER, - HB_UNICODE_GENERAL_CATEGORY_UPPERCASE_LETTER, - }, - CharExt, GeneralCategory, -}; +use crate::unicode::{CharExt, GeneralCategory}; use crate::BufferFlags; use crate::{Direction, Feature, Language, Script}; use core::ptr; @@ -665,12 +657,12 @@ impl OtShapeContext<'_, '_> { let gen_cat = info.general_category(); - if (rb_flag_unsafe(gen_cat.to_u8() as u32) - & (rb_flag(HB_UNICODE_GENERAL_CATEGORY_LOWERCASE_LETTER) - | rb_flag(HB_UNICODE_GENERAL_CATEGORY_UPPERCASE_LETTER) - | rb_flag(HB_UNICODE_GENERAL_CATEGORY_TITLECASE_LETTER) - | rb_flag(HB_UNICODE_GENERAL_CATEGORY_OTHER_LETTER) - | rb_flag(HB_UNICODE_GENERAL_CATEGORY_SPACE_SEPARATOR))) + if gen_cat.flag_unsafe() + & (GeneralCategory::LOWERCASE_LETTER.flag() + | GeneralCategory::UPPERCASE_LETTER.flag() + | GeneralCategory::TITLECASE_LETTER.flag() + | GeneralCategory::OTHER_LETTER.flag() + | GeneralCategory::SPACE_SEPARATOR.flag()) != 0 { i += 1; @@ -768,7 +760,7 @@ impl OtShapeContext<'_, '_> { let rtlm_mask = self.plan.rtlm_mask; for info in &mut self.buffer.info[..len] { - if let Some(c) = info.as_codepoint().mirrored() { + if let Some(c) = info.as_codepoint().mirroring() { if self.font_funcs.nominal_glyph(c).is_some() { info.glyph_id = c; continue; diff --git a/harfrust/src/hb/ot_shaper_arabic.rs b/harfrust/src/hb/ot_shaper_arabic.rs index b1a3330f..634de33c 100644 --- a/harfrust/src/hb/ot_shaper_arabic.rs +++ b/harfrust/src/hb/ot_shaper_arabic.rs @@ -6,7 +6,6 @@ use super::ot_shape_normalize::HB_OT_SHAPE_NORMALIZATION_MODE_AUTO; use super::ot_shape_plan::hb_ot_shape_plan_t; use super::ot_shaper::*; use super::{hb_mask_t, hb_tag_t, script, GlyphInfo, Script}; -use crate::algs::*; use crate::unicode::*; use crate::Direction; use alloc::boxed::Box; @@ -17,21 +16,21 @@ const HB_BUFFER_SCRATCH_FLAG_ARABIC_HAS_STCH: hb_buffer_scratch_flags_t = // See: // https://github.com/harfbuzz/harfbuzz/commit/6e6f82b6f3dde0fc6c3c7d991d9ec6cfff57823d#commitcomment-14248516 fn is_word_category(gc: GeneralCategory) -> bool { - (rb_flag_unsafe(gc.to_u8() as u32) - & (rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_UNASSIGNED) - | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_PRIVATE_USE) - | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_MODIFIER_LETTER) - | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_OTHER_LETTER) - | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_SPACING_MARK) - | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_ENCLOSING_MARK) - | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_NON_SPACING_MARK) - | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_DECIMAL_NUMBER) - | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_LETTER_NUMBER) - | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_OTHER_NUMBER) - | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_CURRENCY_SYMBOL) - | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_MODIFIER_SYMBOL) - | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_MATH_SYMBOL) - | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_OTHER_SYMBOL))) + (gc.flag_unsafe() + & (GeneralCategory::UNASSIGNED.flag() + | GeneralCategory::PRIVATE_USE.flag() + | GeneralCategory::MODIFIER_LETTER.flag() + | GeneralCategory::OTHER_LETTER.flag() + | GeneralCategory::SPACING_MARK.flag() + | GeneralCategory::ENCLOSING_MARK.flag() + | GeneralCategory::NON_SPACING_MARK.flag() + | GeneralCategory::DECIMAL_NUMBER.flag() + | GeneralCategory::LETTER_NUMBER.flag() + | GeneralCategory::OTHER_NUMBER.flag() + | GeneralCategory::CURRENCY_SYMBOL.flag() + | GeneralCategory::MODIFIER_SYMBOL.flag() + | GeneralCategory::MATH_SYMBOL.flag() + | GeneralCategory::OTHER_SYMBOL.flag())) != 0 } @@ -54,10 +53,10 @@ fn get_joining_type(u: Codepoint, gc: GeneralCategory) -> hb_arabic_joining_type return j_type; } - let ok = rb_flag_unsafe(gc.to_u8() as u32) - & (rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_NON_SPACING_MARK) - | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_ENCLOSING_MARK) - | rb_flag(hb_gc::HB_UNICODE_GENERAL_CATEGORY_FORMAT)); + let ok = gc.flag_unsafe() + & (GeneralCategory::NON_SPACING_MARK.flag() + | GeneralCategory::ENCLOSING_MARK.flag() + | GeneralCategory::FORMAT.flag()); if ok != 0 { hb_arabic_joining_type_t::T diff --git a/harfrust/src/hb/ot_shaper_indic.rs b/harfrust/src/hb/ot_shaper_indic.rs index 259b1081..0738befb 100644 --- a/harfrust/src/hb/ot_shaper_indic.rs +++ b/harfrust/src/hb/ot_shaper_indic.rs @@ -16,7 +16,8 @@ use super::ot_shaper::*; use super::ot_shaper_syllabic::*; use super::{hb_font_t, hb_mask_t, hb_tag_t, script, GlyphInfo, Script}; use crate::algs::*; -use crate::unicode::{hb_gc, CharExt, Codepoint}; +use crate::unicode::GeneralCategory; +use crate::unicode::{CharExt, Codepoint}; pub const INDIC_SHAPER: hb_ot_shaper_t = hb_ot_shaper_t { collect_features: Some(collect_features), @@ -1832,12 +1833,9 @@ fn final_reordering_impl( // Apply 'init' to the Left Matra if it's a word start. if buffer.info[start].indic_position() == ot_position_t::POS_PRE_M { if start == 0 - || (rb_flag_unsafe(buffer.info[start - 1].general_category().to_u8() as u32) - & rb_flag_range( - hb_gc::HB_UNICODE_GENERAL_CATEGORY_FORMAT, - hb_gc::HB_UNICODE_GENERAL_CATEGORY_NON_SPACING_MARK, - )) - == 0 + || !buffer.info[start - 1] + .general_category() + .in_range_inclusive(GeneralCategory::FORMAT, GeneralCategory::NON_SPACING_MARK) { buffer.info[start].mask |= indic_plan.mask_array[indic_feature::INIT]; } else { diff --git a/harfrust/src/unicode/mod.rs b/harfrust/src/unicode/mod.rs index 34a447d8..b454bd4d 100644 --- a/harfrust/src/unicode/mod.rs +++ b/harfrust/src/unicode/mod.rs @@ -2,12 +2,11 @@ #[rustfmt::skip] mod emoji_table; +#[cfg(not(feature = "icu"))] #[rustfmt::skip] mod ucd_table; -use crate::algs::*; use crate::Script; -use ucd_table::ucd::*; pub type Codepoint = u32; @@ -31,60 +30,76 @@ pub mod hb_unicode_funcs_t { pub const SPACE_NARROW: u8 = 21; } +/// Data type for the "General_Category" (gc) property from the Unicode +/// Character Database. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub struct GeneralCategory(pub u8); #[allow(unused)] impl GeneralCategory { - pub const CONTROL: Self = Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_CONTROL as _); - pub const FORMAT: Self = Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_FORMAT as _); - pub const UNASSIGNED: Self = Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_UNASSIGNED as _); - pub const PRIVATE_USE: Self = Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_PRIVATE_USE as _); - pub const SURROGATE: Self = Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_SURROGATE as _); - pub const LOWERCASE_LETTER: Self = - Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_LOWERCASE_LETTER as _); - pub const MODIFIER_LETTER: Self = Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_MODIFIER_LETTER as _); - pub const OTHER_LETTER: Self = Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_OTHER_LETTER as _); - pub const TITLECASE_LETTER: Self = - Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_TITLECASE_LETTER as _); - pub const UPPERCASE_LETTER: Self = - Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_UPPERCASE_LETTER as _); - pub const SPACING_MARK: Self = Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_SPACING_MARK as _); - pub const ENCLOSING_MARK: Self = Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_ENCLOSING_MARK as _); - pub const NON_SPACING_MARK: Self = - Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_NON_SPACING_MARK as _); - pub const DECIMAL_NUMBER: Self = Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_DECIMAL_NUMBER as _); - pub const LETTER_NUMBER: Self = Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_LETTER_NUMBER as _); - pub const OTHER_NUMBER: Self = Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_OTHER_NUMBER as _); - pub const CONNECT_PUNCTUATION: Self = - Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_CONNECT_PUNCTUATION as _); - pub const DASH_PUNCTUATION: Self = - Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_DASH_PUNCTUATION as _); - pub const CLOSE_PUNCTUATION: Self = - Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_CLOSE_PUNCTUATION as _); - pub const FINAL_PUNCTUATION: Self = - Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_FINAL_PUNCTUATION as _); - pub const INITIAL_PUNCTUATION: Self = - Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_INITIAL_PUNCTUATION as _); - pub const OTHER_PUNCTUATION: Self = - Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_OTHER_PUNCTUATION as _); - pub const OPEN_PUNCTUATION: Self = - Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_OPEN_PUNCTUATION as _); - pub const CURRENCY_SYMBOL: Self = Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_CURRENCY_SYMBOL as _); - pub const MODIFIER_SYMBOL: Self = Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_MODIFIER_SYMBOL as _); - pub const MATH_SYMBOL: Self = Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_MATH_SYMBOL as _); - pub const OTHER_SYMBOL: Self = Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_OTHER_SYMBOL as _); - pub const LINE_SEPARATOR: Self = Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_LINE_SEPARATOR as _); - pub const PARAGRAPH_SEPARATOR: Self = - Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_PARAGRAPH_SEPARATOR as _); - pub const SPACE_SEPARATOR: Self = Self(hb_gc::HB_UNICODE_GENERAL_CATEGORY_SPACE_SEPARATOR as _); + /// Control (`Cc`). + pub const CONTROL: Self = Self(0); + /// Format (`Cf`). + pub const FORMAT: Self = Self(1); + /// Unassigned (`Cn`). + pub const UNASSIGNED: Self = Self(2); + /// Private Use (`Co`). + pub const PRIVATE_USE: Self = Self(3); + /// Surrogate (`Cs`). + pub const SURROGATE: Self = Self(4); + /// Lowercase Letter (`Ll`). + pub const LOWERCASE_LETTER: Self = Self(5); + /// Modifier Letter (`Lm`). + pub const MODIFIER_LETTER: Self = Self(6); + /// Other Letter (`Lo`). + pub const OTHER_LETTER: Self = Self(7); + /// Titlecase Letter (`Lt`). + pub const TITLECASE_LETTER: Self = Self(8); + /// Uppercase Letter (`Lu`). + pub const UPPERCASE_LETTER: Self = Self(9); + /// Spacing Mark (`Mc`). + pub const SPACING_MARK: Self = Self(10); + /// Enclosing Mark (`Me`). + pub const ENCLOSING_MARK: Self = Self(11); + /// Nonspacing Mark (`Mn`). + pub const NON_SPACING_MARK: Self = Self(12); + /// Decimal Number (`Nd`). + pub const DECIMAL_NUMBER: Self = Self(13); + /// Letter Number (`Nl`). + pub const LETTER_NUMBER: Self = Self(14); + /// Other Number (`No`). + pub const OTHER_NUMBER: Self = Self(15); + /// Connector Punctuation (`Pc`). + pub const CONNECT_PUNCTUATION: Self = Self(16); + /// Dash Punctuation (`Pd`). + pub const DASH_PUNCTUATION: Self = Self(17); + /// Close Punctuation (`Pe`). + pub const CLOSE_PUNCTUATION: Self = Self(18); + /// Final Punctuation (`Pf`). + pub const FINAL_PUNCTUATION: Self = Self(19); + /// Initial Punctuation (`Pi`). + pub const INITIAL_PUNCTUATION: Self = Self(20); + /// Other Punctuation (`Po`). + pub const OTHER_PUNCTUATION: Self = Self(21); + /// Open Punctuation (`Ps`). + pub const OPEN_PUNCTUATION: Self = Self(22); + /// Currency Symbol (`Sc`). + pub const CURRENCY_SYMBOL: Self = Self(23); + /// Modifier Symbol (`Sk`). + pub const MODIFIER_SYMBOL: Self = Self(24); + /// Math Symbol (`Sm`). + pub const MATH_SYMBOL: Self = Self(25); + /// Other Symbol (`So`). + pub const OTHER_SYMBOL: Self = Self(26); + /// Line Separator (`Zl`). + pub const LINE_SEPARATOR: Self = Self(27); + /// Paragraph Separator (`Zp`). + pub const PARAGRAPH_SEPARATOR: Self = Self(28); + /// Space Separator (`Zs`). + pub const SPACE_SEPARATOR: Self = Self(29); } impl GeneralCategory { - pub fn to_u8(self) -> u8 { - self.0 - } - pub fn is_mark(&self) -> bool { matches!( *self, @@ -102,6 +117,27 @@ impl GeneralCategory { | Self::UPPERCASE_LETTER ) } + + #[inline(always)] + pub(crate) const fn flag(self) -> u32 { + 1 << self.0 + } + + #[inline(always)] + pub(crate) const fn flag_unsafe(self) -> u32 { + if self.0 < 32 { + 1 << self.0 + } else { + 0 + } + } + + #[inline(always)] + pub(crate) const fn in_range_inclusive(self, start: Self, end: Self) -> bool { + debug_assert!(start.0 < end.0); + let range_mask = (1 << (end.0 + 1)) - (1 << start.0); + self.flag_unsafe() & range_mask != 0 + } } #[allow(dead_code)] @@ -379,7 +415,7 @@ pub trait CharExt { fn space_fallback(self) -> hb_unicode_funcs_t::space_t; fn combining_class(self) -> u8; fn modified_combining_class(self) -> u8; - fn mirrored(self) -> Option; + fn mirroring(self) -> Option; fn is_emoji_extended_pictographic(self) -> bool; fn is_default_ignorable(self) -> bool; fn is_variation_selector(self) -> bool; @@ -388,24 +424,19 @@ pub trait CharExt { impl CharExt for Codepoint { fn script(self) -> Script { - _hb_ucd_sc_map[_hb_ucd_sc(self as usize) as usize] + script_for(self) } fn general_category(self) -> GeneralCategory { - GeneralCategory(_hb_ucd_gc(self as usize)) + general_category_for(self) } fn combining_class(self) -> u8 { - _hb_ucd_ccc(self as usize) + combining_class_for(self) } - fn mirrored(self) -> Option { - let delta = _hb_ucd_bmg(self as usize); - if delta == 0 { - None - } else { - Some(((self as i32).wrapping_add(delta as i32)) as u32) - } + fn mirroring(self) -> Option { + mirroring_for(self) } fn space_fallback(self) -> hb_unicode_funcs_t::space_t { @@ -587,172 +618,550 @@ impl CharExt for Codepoint { } } -const S_BASE: u32 = 0xAC00; -const L_BASE: u32 = 0x1100; -const V_BASE: u32 = 0x1161; -const T_BASE: u32 = 0x11A7; -const L_COUNT: u32 = 19; -const V_COUNT: u32 = 21; -const T_COUNT: u32 = 28; -const N_COUNT: u32 = V_COUNT * T_COUNT; -const S_COUNT: u32 = L_COUNT * N_COUNT; - -pub fn compose(a: Codepoint, b: Codepoint) -> Option { - // Hangul is handled algorithmically. - if let Some(ab) = compose_hangul(a, b) { - return Some(ab); - } - - let u: u32; - - if (a & 0xFFFF_F800) == 0x0000 && (b & 0xFFFF_FF80) == 0x0300 { - /* If "a" is small enough and "b" is in the U+0300 range, - * the composition data is encoded in a 32bit array sorted - * by "a,b" pair. */ - let k = HB_CODEPOINT_ENCODE3_11_7_14(a, b, 0); - let v = _hb_ucd_dm2_u32_map - .binary_search_by(|probe| { - let key = probe & HB_CODEPOINT_ENCODE3_11_7_14(0x001F_FFFF, 0x001F_FFFF, 0); - key.cmp(&k) - }) - .ok() - .map(|index| _hb_ucd_dm2_u32_map[index]); - - if let Some(value) = v { - u = HB_CODEPOINT_DECODE3_11_7_14_3(value); - } else { - return None; - } - } else { - /* Otherwise it is stored in a 64bit array sorted by - * "a,b" pair. */ - let k = HB_CODEPOINT_ENCODE3(a, b, 0); - let v = _hb_ucd_dm2_u64_map - .binary_search_by(|probe| { - let key = probe & HB_CODEPOINT_ENCODE3(0x001F_FFFF, 0x001F_FFFF, 0); - key.cmp(&k) - }) - .ok() - .map(|index| _hb_ucd_dm2_u64_map[index]); - - if let Some(value) = v { - u = HB_CODEPOINT_DECODE3_3(value); - } else { - return None; - } +#[cfg(feature = "icu")] +pub(crate) use icu::*; + +#[cfg(not(feature = "icu"))] +pub(crate) use builtin::*; + +#[cfg(feature = "icu")] +mod icu { + use super::{Codepoint, GeneralCategory, Script}; + use icu_normalizer::properties::{ + CanonicalCompositionBorrowed, CanonicalDecompositionBorrowed, Decomposed, + }; + use icu_properties::{ + props::{ + BidiMirroringGlyph, CanonicalCombiningClass as IcuCanonicalCombiningClass, + GeneralCategory as IcuGeneralCategory, Script as IcuScript, + }, + CodePointMapData, + }; + + pub(crate) fn script_for(c: u32) -> Script { + let icu_script = CodePointMapData::::new().get32(c); + SCRIPT_MAP + .get(icu_script.to_icu4c_value() as usize) + .copied() + .unwrap_or(crate::script::UNKNOWN) } - if u == 0 { - None - } else { - Some(u) + pub(crate) fn general_category_for(c: u32) -> GeneralCategory { + GeneralCategory::from_icu(CodePointMapData::::new().get32(c)) + } + + pub(crate) fn combining_class_for(c: u32) -> u8 { + CodePointMapData::::new() + .get32(c) + .to_icu4c_value() + } + + pub(crate) fn mirroring_for(c: u32) -> Option { + CodePointMapData::::new() + .get32(c) + .mirroring_glyph + .map(Codepoint::from) + } + + pub(crate) fn compose(a: Codepoint, b: Codepoint) -> Option { + let a = char::from_u32(a)?; + let b = char::from_u32(b)?; + CanonicalCompositionBorrowed::new() + .compose(a, b) + .map(Codepoint::from) } -} -fn compose_hangul(a: Codepoint, b: Codepoint) -> Option { - let l = a; - let v = b; - if L_BASE <= l && l < (L_BASE + L_COUNT) && V_BASE <= v && v < (V_BASE + V_COUNT) { - let r = S_BASE + (l - L_BASE) * N_COUNT + (v - V_BASE) * T_COUNT; - Some(r) - } else if S_BASE <= l - && l <= (S_BASE + S_COUNT - T_COUNT) - && T_BASE <= v - && v < (T_BASE + T_COUNT) - && (l - S_BASE) % T_COUNT == 0 - { - let r = l + (v - T_BASE); - Some(r) - } else { - None + pub(crate) fn decompose(ab: Codepoint) -> Option<(Codepoint, Codepoint)> { + let ch = char::from_u32(ab)?; + match CanonicalDecompositionBorrowed::new().decompose(ch) { + Decomposed::Default => None, + Decomposed::Singleton(a) => Some((Codepoint::from(a), 0)), + Decomposed::Expansion(a, b) => Some((Codepoint::from(a), Codepoint::from(b))), + } + } + + impl GeneralCategory { + fn from_icu(category: IcuGeneralCategory) -> Self { + use IcuGeneralCategory::*; + match category { + Control => Self::CONTROL, + Format => Self::FORMAT, + Unassigned => Self::UNASSIGNED, + PrivateUse => Self::PRIVATE_USE, + Surrogate => Self::SURROGATE, + LowercaseLetter => Self::LOWERCASE_LETTER, + ModifierLetter => Self::MODIFIER_LETTER, + OtherLetter => Self::OTHER_LETTER, + TitlecaseLetter => Self::TITLECASE_LETTER, + UppercaseLetter => Self::UPPERCASE_LETTER, + SpacingMark => Self::SPACING_MARK, + EnclosingMark => Self::ENCLOSING_MARK, + NonspacingMark => Self::NON_SPACING_MARK, + DecimalNumber => Self::DECIMAL_NUMBER, + LetterNumber => Self::LETTER_NUMBER, + OtherNumber => Self::OTHER_NUMBER, + ConnectorPunctuation => Self::CONNECT_PUNCTUATION, + DashPunctuation => Self::DASH_PUNCTUATION, + ClosePunctuation => Self::CLOSE_PUNCTUATION, + FinalPunctuation => Self::FINAL_PUNCTUATION, + InitialPunctuation => Self::INITIAL_PUNCTUATION, + OtherPunctuation => Self::OTHER_PUNCTUATION, + OpenPunctuation => Self::OPEN_PUNCTUATION, + CurrencySymbol => Self::CURRENCY_SYMBOL, + ModifierSymbol => Self::MODIFIER_SYMBOL, + MathSymbol => Self::MATH_SYMBOL, + OtherSymbol => Self::OTHER_SYMBOL, + LineSeparator => Self::LINE_SEPARATOR, + ParagraphSeparator => Self::PARAGRAPH_SEPARATOR, + SpaceSeparator => Self::SPACE_SEPARATOR, + } + } } + + pub(crate) static SCRIPT_MAP: [Script; 212] = [ + crate::script::COMMON, // 0 + crate::script::INHERITED, // 1 + crate::script::ARABIC, // 2 + crate::script::ARMENIAN, // 3 + crate::script::BENGALI, // 4 + crate::script::BOPOMOFO, // 5 + crate::script::CHEROKEE, // 6 + crate::script::COPTIC, // 7 + crate::script::CYRILLIC, // 8 + crate::script::DESERET, // 9 + crate::script::DEVANAGARI, // 10 + crate::script::ETHIOPIC, // 11 + crate::script::GEORGIAN, // 12 + crate::script::GOTHIC, // 13 + crate::script::GREEK, // 14 + crate::script::GUJARATI, // 15 + crate::script::GURMUKHI, // 16 + crate::script::HAN, // 17 + crate::script::HANGUL, // 18 + crate::script::HEBREW, // 19 + crate::script::HIRAGANA, // 20 + crate::script::KANNADA, // 21 + crate::script::KATAKANA, // 22 + crate::script::KHMER, // 23 + crate::script::LAO, // 24 + crate::script::LATIN, // 25 + crate::script::MALAYALAM, // 26 + crate::script::MONGOLIAN, // 27 + crate::script::MYANMAR, // 28 + crate::script::OGHAM, // 29 + crate::script::OLD_ITALIC, // 30 + crate::script::ORIYA, // 31 + crate::script::RUNIC, // 32 + crate::script::SINHALA, // 33 + crate::script::SYRIAC, // 34 + crate::script::TAMIL, // 35 + crate::script::TELUGU, // 36 + crate::script::THAANA, // 37 + crate::script::THAI, // 38 + crate::script::TIBETAN, // 39 + crate::script::CANADIAN_SYLLABICS, // 40 + crate::script::YI, // 41 + crate::script::TAGALOG, // 42 + crate::script::HANUNOO, // 43 + crate::script::BUHID, // 44 + crate::script::TAGBANWA, // 45 + crate::script::BRAILLE, // 46 + crate::script::CYPRIOT, // 47 + crate::script::LIMBU, // 48 + crate::script::LINEAR_B, // 49 + crate::script::OSMANYA, // 50 + crate::script::SHAVIAN, // 51 + crate::script::TAI_LE, // 52 + crate::script::UGARITIC, // 53 + crate::script::UNKNOWN, // 54 + crate::script::BUGINESE, // 55 + crate::script::GLAGOLITIC, // 56 + crate::script::KHAROSHTHI, // 57 + crate::script::SYLOTI_NAGRI, // 58 + crate::script::NEW_TAI_LUE, // 59 + crate::script::TIFINAGH, // 60 + crate::script::OLD_PERSIAN, // 61 + crate::script::BALINESE, // 62 + crate::script::BATAK, // 63 + crate::script::UNKNOWN, // 64 + crate::script::BRAHMI, // 65 + crate::script::CHAM, // 66 + crate::script::UNKNOWN, // 67 + crate::script::CYRILLIC, // 68 (Cyrs) + crate::script::UNKNOWN, // 69 + crate::script::UNKNOWN, // 70 + crate::script::EGYPTIAN_HIEROGLYPHS, // 71 + crate::script::GEORGIAN, // 72 (Geok) + crate::script::HAN, // 73 (Hans) + crate::script::HAN, // 74 (Hant) + crate::script::PAHAWH_HMONG, // 75 + crate::script::OLD_HUNGARIAN, // 76 + crate::script::UNKNOWN, // 77 + crate::script::JAVANESE, // 78 + crate::script::KAYAH_LI, // 79 + crate::script::LATIN, // 80 (Latf) + crate::script::LATIN, // 81 (Latg) + crate::script::LEPCHA, // 82 + crate::script::LINEAR_A, // 83 + crate::script::MANDAIC, // 84 + crate::script::UNKNOWN, // 85 + crate::script::MEROITIC_HIEROGLYPHS, // 86 + crate::script::NKO, // 87 + crate::script::OLD_TURKIC, // 88 + crate::script::OLD_PERMIC, // 89 + crate::script::PHAGS_PA, // 90 + crate::script::PHOENICIAN, // 91 + crate::script::MIAO, // 92 + crate::script::UNKNOWN, // 93 + crate::script::UNKNOWN, // 94 + crate::script::SYRIAC, // 95 (Syre) + crate::script::SYRIAC, // 96 (Syrj) + crate::script::SYRIAC, // 97 (Syrn) + crate::script::UNKNOWN, // 98 + crate::script::VAI, // 99 + crate::script::UNKNOWN, // 100 + crate::script::CUNEIFORM, // 101 + crate::script::UNKNOWN, // 102 + crate::script::UNKNOWN, // 103 + crate::script::CARIAN, // 104 + crate::script::UNKNOWN, // 105 + crate::script::TAI_THAM, // 106 + crate::script::LYCIAN, // 107 + crate::script::LYDIAN, // 108 + crate::script::OL_CHIKI, // 109 + crate::script::REJANG, // 110 + crate::script::SAURASHTRA, // 111 + crate::script::SIGNWRITING, // 112 + crate::script::SUNDANESE, // 113 + crate::script::UNKNOWN, // 114 + crate::script::MEETEI_MAYEK, // 115 + crate::script::IMPERIAL_ARAMAIC, // 116 + crate::script::AVESTAN, // 117 + crate::script::CHAKMA, // 118 + crate::script::UNKNOWN, // 119 + crate::script::KAITHI, // 120 + crate::script::MANICHAEAN, // 121 + crate::script::INSCRIPTIONAL_PAHLAVI, // 122 + crate::script::PSALTER_PAHLAVI, // 123 + crate::script::UNKNOWN, // 124 + crate::script::INSCRIPTIONAL_PARTHIAN, // 125 + crate::script::SAMARITAN, // 126 + crate::script::TAI_VIET, // 127 + crate::script::UNKNOWN, // 128 + crate::script::UNKNOWN, // 129 + crate::script::BAMUM, // 130 + crate::script::LISU, // 131 + crate::script::UNKNOWN, // 132 + crate::script::OLD_SOUTH_ARABIAN, // 133 + crate::script::BASSA_VAH, // 134 + crate::script::DUPLOYAN, // 135 + crate::script::ELBASAN, // 136 + crate::script::GRANTHA, // 137 + crate::script::UNKNOWN, // 138 + crate::script::UNKNOWN, // 139 + crate::script::MENDE_KIKAKUI, // 140 + crate::script::MEROITIC_CURSIVE, // 141 + crate::script::OLD_NORTH_ARABIAN, // 142 + crate::script::NABATAEAN, // 143 + crate::script::PALMYRENE, // 144 + crate::script::KHUDAWADI, // 145 + crate::script::WARANG_CITI, // 146 + crate::script::UNKNOWN, // 147 + crate::script::UNKNOWN, // 148 + crate::script::MRO, // 149 + crate::script::NUSHU, // 150 + crate::script::SHARADA, // 151 + crate::script::SORA_SOMPENG, // 152 + crate::script::TAKRI, // 153 + crate::script::TANGUT, // 154 + crate::script::UNKNOWN, // 155 + crate::script::ANATOLIAN_HIEROGLYPHS, // 156 + crate::script::KHOJKI, // 157 + crate::script::TIRHUTA, // 158 + crate::script::CAUCASIAN_ALBANIAN, // 159 + crate::script::MAHAJANI, // 160 + crate::script::AHOM, // 161 + crate::script::HATRAN, // 162 + crate::script::MODI, // 163 + crate::script::MULTANI, // 164 + crate::script::PAU_CIN_HAU, // 165 + crate::script::SIDDHAM, // 166 + crate::script::ADLAM, // 167 + crate::script::BHAIKSUKI, // 168 + crate::script::MARCHEN, // 169 + crate::script::NEWA, // 170 + crate::script::OSAGE, // 171 + crate::script::UNKNOWN, // 172 + crate::script::HANGUL, // 173 (Jamo) + crate::script::UNKNOWN, // 174 + crate::script::MASARAM_GONDI, // 175 + crate::script::SOYOMBO, // 176 + crate::script::ZANABAZAR_SQUARE, // 177 + crate::script::DOGRA, // 178 + crate::script::GUNJALA_GONDI, // 179 + crate::script::MAKASAR, // 180 + crate::script::MEDEFAIDRIN, // 181 + crate::script::HANIFI_ROHINGYA, // 182 + crate::script::SOGDIAN, // 183 + crate::script::OLD_SOGDIAN, // 184 + crate::script::ELYMAIC, // 185 + crate::script::NYIAKENG_PUACHUE_HMONG, // 186 + crate::script::NANDINAGARI, // 187 + crate::script::WANCHO, // 188 + crate::script::CHORASMIAN, // 189 + crate::script::DIVES_AKURU, // 190 + crate::script::KHITAN_SMALL_SCRIPT, // 191 + crate::script::YEZIDI, // 192 + crate::script::CYPRO_MINOAN, // 193 + crate::script::OLD_UYGHUR, // 194 + crate::script::TANGSA, // 195 + crate::script::TOTO, // 196 + crate::script::VITHKUQI, // 197 + crate::script::KAWI, // 198 + crate::script::NAG_MUNDARI, // 199 + crate::script::ARABIC, // 200 + crate::script::GARAY, // 201 + crate::script::GURUNG_KHEMA, // 202 + crate::script::KIRAT_RAI, // 203 + crate::script::OL_ONAL, // 204 + crate::script::SUNUWAR, // 205 + crate::script::TODHRI, // 206 + crate::script::TULU_TIGALARI, // 207 + crate::script::BERIA_ERFE, // 208 + crate::script::SIDETIC, // 209 + crate::script::TAI_YO, // 210 + crate::script::TOLONG_SIKI, // 211 + ]; } -pub fn decompose(ab: Codepoint) -> Option<(Codepoint, Codepoint)> { - if let Some((a, b)) = decompose_hangul(ab) { - return Some((a, b)); +#[cfg(not(feature = "icu"))] +mod builtin { + use super::{super::algs::*, ucd_table::ucd::*, Codepoint, GeneralCategory, Script}; + + pub(crate) fn script_for(c: u32) -> Script { + _hb_ucd_sc_map[_hb_ucd_sc(c as usize) as usize] } - let mut i = _hb_ucd_dm(ab as usize) as usize; + pub(crate) fn general_category_for(c: u32) -> GeneralCategory { + GeneralCategory(_hb_ucd_gc(c as usize)) + } - // If no data, there's no decomposition. - if i == 0 { - return None; + pub(crate) fn combining_class_for(c: u32) -> u8 { + _hb_ucd_ccc(c as usize) } - i -= 1; - if i < _hb_ucd_dm1_p0_map.len() + _hb_ucd_dm1_p2_map.len() { - let a = if i < _hb_ucd_dm1_p0_map.len() { - _hb_ucd_dm1_p0_map[i] as u32 + pub(crate) fn mirroring_for(c: u32) -> Option { + let delta = _hb_ucd_bmg(c as usize); + if delta == 0 { + None } else { - let j = i - _hb_ucd_dm1_p0_map.len(); - 0x20000 | _hb_ucd_dm1_p2_map[j] as u32 - }; - return Some((a, 0)); + Some(((c as i32).wrapping_add(delta as i32)) as u32) + } } - i -= _hb_ucd_dm1_p0_map.len() + _hb_ucd_dm1_p2_map.len(); + pub(crate) fn compose(a: Codepoint, b: Codepoint) -> Option { + // Hangul is handled algorithmically. + if let Some(ab) = compose_hangul(a, b) { + return Some(ab); + } + + let u: u32; + + if (a & 0xFFFF_F800) == 0x0000 && (b & 0xFFFF_FF80) == 0x0300 { + /* If "a" is small enough and "b" is in the U+0300 range, + * the composition data is encoded in a 32bit array sorted + * by "a,b" pair. */ + let k = HB_CODEPOINT_ENCODE3_11_7_14(a, b, 0); + let v = _hb_ucd_dm2_u32_map + .binary_search_by(|probe| { + let key = probe & HB_CODEPOINT_ENCODE3_11_7_14(0x001F_FFFF, 0x001F_FFFF, 0); + key.cmp(&k) + }) + .ok() + .map(|index| _hb_ucd_dm2_u32_map[index]); + + if let Some(value) = v { + u = HB_CODEPOINT_DECODE3_11_7_14_3(value); + } else { + return None; + } + } else { + /* Otherwise it is stored in a 64bit array sorted by + * "a,b" pair. */ + let k = HB_CODEPOINT_ENCODE3(a, b, 0); + let v = _hb_ucd_dm2_u64_map + .binary_search_by(|probe| { + let key = probe & HB_CODEPOINT_ENCODE3(0x001F_FFFF, 0x001F_FFFF, 0); + key.cmp(&k) + }) + .ok() + .map(|index| _hb_ucd_dm2_u64_map[index]); + + if let Some(value) = v { + u = HB_CODEPOINT_DECODE3_3(value); + } else { + return None; + } + } - if i < _hb_ucd_dm2_u32_map.len() { - let v = _hb_ucd_dm2_u32_map[i]; - let a = HB_CODEPOINT_DECODE3_11_7_14_1(v); - let b = HB_CODEPOINT_DECODE3_11_7_14_2(v); - return Some((a, b)); + if u == 0 { + None + } else { + Some(u) + } } - i -= _hb_ucd_dm2_u32_map.len(); + pub(crate) fn decompose(ab: Codepoint) -> Option<(Codepoint, Codepoint)> { + if let Some((a, b)) = decompose_hangul(ab) { + return Some((a, b)); + } - let v = _hb_ucd_dm2_u64_map[i]; - let a = HB_CODEPOINT_DECODE3_1(v); - let b = HB_CODEPOINT_DECODE3_2(v); - Some((a, b)) -} + let mut i = _hb_ucd_dm(ab as usize) as usize; + + // If no data, there's no decomposition. + if i == 0 { + return None; + } + i -= 1; + + if i < _hb_ucd_dm1_p0_map.len() + _hb_ucd_dm1_p2_map.len() { + let a = if i < _hb_ucd_dm1_p0_map.len() { + _hb_ucd_dm1_p0_map[i] as u32 + } else { + let j = i - _hb_ucd_dm1_p0_map.len(); + 0x20000 | _hb_ucd_dm1_p2_map[j] as u32 + }; + return Some((a, 0)); + } + + i -= _hb_ucd_dm1_p0_map.len() + _hb_ucd_dm1_p2_map.len(); + + if i < _hb_ucd_dm2_u32_map.len() { + let v = _hb_ucd_dm2_u32_map[i]; + let a = HB_CODEPOINT_DECODE3_11_7_14_1(v); + let b = HB_CODEPOINT_DECODE3_11_7_14_2(v); + return Some((a, b)); + } + + i -= _hb_ucd_dm2_u32_map.len(); -pub fn decompose_hangul(ab: Codepoint) -> Option<(Codepoint, Codepoint)> { - let si = ab.wrapping_sub(S_BASE); - if si >= S_COUNT { - return None; + let v = _hb_ucd_dm2_u64_map[i]; + let a = HB_CODEPOINT_DECODE3_1(v); + let b = HB_CODEPOINT_DECODE3_2(v); + Some((a, b)) } - let (a, b) = if si % T_COUNT != 0 { - // LV,T - (S_BASE + (si / T_COUNT) * T_COUNT, T_BASE + (si % T_COUNT)) - } else { - // L,V - (L_BASE + (si / N_COUNT), V_BASE + (si % N_COUNT) / T_COUNT) - }; - Some((a, b)) + const S_BASE: u32 = 0xAC00; + const L_BASE: u32 = 0x1100; + const V_BASE: u32 = 0x1161; + const T_BASE: u32 = 0x11A7; + const L_COUNT: u32 = 19; + const V_COUNT: u32 = 21; + const T_COUNT: u32 = 28; + const N_COUNT: u32 = V_COUNT * T_COUNT; + const S_COUNT: u32 = L_COUNT * N_COUNT; + + fn compose_hangul(a: Codepoint, b: Codepoint) -> Option { + let l = a; + let v = b; + if L_BASE <= l && l < (L_BASE + L_COUNT) && V_BASE <= v && v < (V_BASE + V_COUNT) { + let r = S_BASE + (l - L_BASE) * N_COUNT + (v - V_BASE) * T_COUNT; + Some(r) + } else if S_BASE <= l + && l <= (S_BASE + S_COUNT - T_COUNT) + && T_BASE <= v + && v < (T_BASE + T_COUNT) + && (l - S_BASE) % T_COUNT == 0 + { + let r = l + (v - T_BASE); + Some(r) + } else { + None + } + } + + fn decompose_hangul(ab: Codepoint) -> Option<(Codepoint, Codepoint)> { + let si = ab.wrapping_sub(S_BASE); + if si >= S_COUNT { + return None; + } + let (a, b) = if si % T_COUNT != 0 { + // LV,T + (S_BASE + (si / T_COUNT) * T_COUNT, T_BASE + (si % T_COUNT)) + } else { + // L,V + (L_BASE + (si / N_COUNT), V_BASE + (si % N_COUNT) / T_COUNT) + }; + Some((a, b)) + } } -pub mod hb_gc { - pub const HB_UNICODE_GENERAL_CATEGORY_CONTROL: u32 = 0; - pub const HB_UNICODE_GENERAL_CATEGORY_FORMAT: u32 = 1; - pub const HB_UNICODE_GENERAL_CATEGORY_UNASSIGNED: u32 = 2; - pub const HB_UNICODE_GENERAL_CATEGORY_PRIVATE_USE: u32 = 3; - pub const HB_UNICODE_GENERAL_CATEGORY_SURROGATE: u32 = 4; - pub const HB_UNICODE_GENERAL_CATEGORY_LOWERCASE_LETTER: u32 = 5; - pub const HB_UNICODE_GENERAL_CATEGORY_MODIFIER_LETTER: u32 = 6; - pub const HB_UNICODE_GENERAL_CATEGORY_OTHER_LETTER: u32 = 7; - pub const HB_UNICODE_GENERAL_CATEGORY_TITLECASE_LETTER: u32 = 8; - pub const HB_UNICODE_GENERAL_CATEGORY_UPPERCASE_LETTER: u32 = 9; - pub const HB_UNICODE_GENERAL_CATEGORY_SPACING_MARK: u32 = 10; - pub const HB_UNICODE_GENERAL_CATEGORY_ENCLOSING_MARK: u32 = 11; - pub const HB_UNICODE_GENERAL_CATEGORY_NON_SPACING_MARK: u32 = 12; - pub const HB_UNICODE_GENERAL_CATEGORY_DECIMAL_NUMBER: u32 = 13; - pub const HB_UNICODE_GENERAL_CATEGORY_LETTER_NUMBER: u32 = 14; - pub const HB_UNICODE_GENERAL_CATEGORY_OTHER_NUMBER: u32 = 15; - pub const HB_UNICODE_GENERAL_CATEGORY_CONNECT_PUNCTUATION: u32 = 16; - pub const HB_UNICODE_GENERAL_CATEGORY_DASH_PUNCTUATION: u32 = 17; - pub const HB_UNICODE_GENERAL_CATEGORY_CLOSE_PUNCTUATION: u32 = 18; - pub const HB_UNICODE_GENERAL_CATEGORY_FINAL_PUNCTUATION: u32 = 19; - pub const HB_UNICODE_GENERAL_CATEGORY_INITIAL_PUNCTUATION: u32 = 20; - pub const HB_UNICODE_GENERAL_CATEGORY_OTHER_PUNCTUATION: u32 = 21; - pub const HB_UNICODE_GENERAL_CATEGORY_OPEN_PUNCTUATION: u32 = 22; - pub const HB_UNICODE_GENERAL_CATEGORY_CURRENCY_SYMBOL: u32 = 23; - pub const HB_UNICODE_GENERAL_CATEGORY_MODIFIER_SYMBOL: u32 = 24; - pub const HB_UNICODE_GENERAL_CATEGORY_MATH_SYMBOL: u32 = 25; - pub const HB_UNICODE_GENERAL_CATEGORY_OTHER_SYMBOL: u32 = 26; - pub const HB_UNICODE_GENERAL_CATEGORY_LINE_SEPARATOR: u32 = 27; - pub const HB_UNICODE_GENERAL_CATEGORY_PARAGRAPH_SEPARATOR: u32 = 28; - pub const HB_UNICODE_GENERAL_CATEGORY_SPACE_SEPARATOR: u32 = 29; +#[cfg(test)] +mod tests { + #[test] + fn general_category_ranges() { + use super::GeneralCategory as Gc; + assert!(!Gc::CONTROL.in_range_inclusive(Gc::FORMAT, Gc::NON_SPACING_MARK)); + assert!(Gc::LOWERCASE_LETTER.in_range_inclusive(Gc::LOWERCASE_LETTER, Gc::UPPERCASE_LETTER)); + assert!(Gc::UPPERCASE_LETTER.in_range_inclusive(Gc::LOWERCASE_LETTER, Gc::UPPERCASE_LETTER)); + assert!(!Gc::SPACING_MARK.in_range_inclusive(Gc::LOWERCASE_LETTER, Gc::UPPERCASE_LETTER)); + } + + #[cfg(feature = "icu")] + #[test] + fn script_mapping() { + use icu_properties::props::Script as IcuScript; + use icu_properties::PropertyNamesShort; + // These ICU script codes are aliases, placeholders, or meta scripts. + // No Unicode code points map directly to them via the Script property, + // so our direct-script table intentionally keeps them as UNKNOWN. + const ALLOWED_UNKNOWN_SCRIPT_VALUES: &[usize] = &[ + 54, 64, 67, 69, 70, 77, 85, 93, 94, 98, 100, 102, 105, 114, 119, 124, 128, 129, 132, + 138, 139, 147, 148, 155, 172, 174, + ]; + let short_names = PropertyNamesShort::::new(); + let mut mismatches = Vec::new(); + for (icu4c_value, script) in super::icu::SCRIPT_MAP.iter().enumerate() { + let icu_script = IcuScript::from_icu4c_value(icu4c_value as u16); + match short_names.get(icu_script) { + Some(short_name) => { + let expected = if short_name.len() == 4 { + let bytes = short_name.as_bytes(); + let tag = + crate::Tag::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]); + crate::Script::from_iso15924_tag(tag).unwrap_or(crate::script::UNKNOWN) + } else { + crate::script::UNKNOWN + }; + if *script == crate::script::UNKNOWN + && ALLOWED_UNKNOWN_SCRIPT_VALUES.contains(&icu4c_value) + { + continue; + } + if expected != *script { + let expected_tag = expected.tag().to_be_bytes(); + let actual_tag = script.tag().to_be_bytes(); + mismatches.push(format!( + "value {icu4c_value}: Script::from_iso15924_tag({short_name}) -> {}, mapped tag {}", + core::str::from_utf8(&expected_tag).unwrap_or("????"), + core::str::from_utf8(&actual_tag).unwrap_or("????") + )); + } + } + None => { + if *script != crate::script::UNKNOWN { + mismatches.push(format!( + "value {icu4c_value}: no ICU short name but mapped tag {}", + core::str::from_utf8(&script.tag().to_be_bytes()).unwrap_or("????") + )); + } + } + } + } + assert!( + mismatches.is_empty(), + "script mapping mismatches:\n{}", + mismatches.join("\n") + ); + } } From 69d086ab4c979efbf5e8ed9bbfba96ab8de65cb6 Mon Sep 17 00:00:00 2001 From: Chad Brokaw Date: Wed, 8 Jul 2026 16:28:08 -0400 Subject: [PATCH 3/5] silence clippy --- harfrust/src/unicode/mod.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/harfrust/src/unicode/mod.rs b/harfrust/src/unicode/mod.rs index b454bd4d..58e25696 100644 --- a/harfrust/src/unicode/mod.rs +++ b/harfrust/src/unicode/mod.rs @@ -1,5 +1,13 @@ -#![expect(non_camel_case_types, non_snake_case, non_upper_case_globals)] - +#![expect( + non_camel_case_types, + non_snake_case, + non_upper_case_globals, + clippy::manual_range_contains, + clippy::inline_always, + clippy::wrong_self_convention +)] + +#[expect(clippy::identity_op)] #[rustfmt::skip] mod emoji_table; #[cfg(not(feature = "icu"))] From fb0d18805dabcb1f813cd16af34e72acf675251d Mon Sep 17 00:00:00 2001 From: Chad Brokaw Date: Wed, 8 Jul 2026 16:35:54 -0400 Subject: [PATCH 4/5] clippy again --- harfrust/src/unicode/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/harfrust/src/unicode/mod.rs b/harfrust/src/unicode/mod.rs index 58e25696..3f826dfe 100644 --- a/harfrust/src/unicode/mod.rs +++ b/harfrust/src/unicode/mod.rs @@ -2,7 +2,6 @@ non_camel_case_types, non_snake_case, non_upper_case_globals, - clippy::manual_range_contains, clippy::inline_always, clippy::wrong_self_convention )] @@ -690,6 +689,7 @@ mod icu { impl GeneralCategory { fn from_icu(category: IcuGeneralCategory) -> Self { + #[expect(clippy::enum_glob_use)] use IcuGeneralCategory::*; match category { Control => Self::CONTROL, @@ -943,6 +943,7 @@ mod icu { } #[cfg(not(feature = "icu"))] +#[expect(clippy::manual_range_contains)] mod builtin { use super::{super::algs::*, ucd_table::ucd::*, Codepoint, GeneralCategory, Script}; From ca71283bf1bf4c695c4e5212cb7234a1ac504320 Mon Sep 17 00:00:00 2001 From: Chad Brokaw Date: Wed, 8 Jul 2026 17:02:07 -0400 Subject: [PATCH 5/5] again --- harfrust/src/unicode/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/harfrust/src/unicode/mod.rs b/harfrust/src/unicode/mod.rs index 3f826dfe..da818e96 100644 --- a/harfrust/src/unicode/mod.rs +++ b/harfrust/src/unicode/mod.rs @@ -1,9 +1,10 @@ -#![expect( +#![allow( non_camel_case_types, non_snake_case, non_upper_case_globals, clippy::inline_always, - clippy::wrong_self_convention + clippy::wrong_self_convention, + clippy::enum_glob_use )] #[expect(clippy::identity_op)] @@ -689,7 +690,6 @@ mod icu { impl GeneralCategory { fn from_icu(category: IcuGeneralCategory) -> Self { - #[expect(clippy::enum_glob_use)] use IcuGeneralCategory::*; match category { Control => Self::CONTROL,