From 48b6954dce7ce62a1caa541eb073f0df5f625aba Mon Sep 17 00:00:00 2001 From: Fey Naomi Schrewe Date: Fri, 11 Sep 2026 08:04:38 +0200 Subject: [PATCH] feat: implement hyphenation --- Cargo.lock | 33 +++++++- Cargo.toml | 2 +- examples/linebreak.rs | 15 ++-- src/lib.rs | 193 ++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 225 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 34fc137..1ea5890 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -36,12 +36,24 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "clone_dyn_types" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5226d8fe69efe07200ea831d71b9d8fbfb1a894945fc4a44f41df17575de099f" + [[package]] name = "crunchy" version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "either" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" + [[package]] name = "find-msvc-tools" version = "0.1.12" @@ -77,6 +89,25 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef68590049bab63a464eee1a1158ac04c6f6613a546d8d90f78636b8b94f171" +[[package]] +name = "iter_tools" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32403e96b23e11832956d1e989f5f9c0f2aa43bd8324c24f586dc0c8ccb1010e" +dependencies = [ + "clone_dyn_types", + "itertools", +] + +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "libc" version = "0.2.189" @@ -88,7 +119,7 @@ name = "lua-hyphenate" version = "0.1.0" dependencies = [ "hypher", - "libc", + "iter_tools", "mlua-sys", "text_layout", "unicode-linebreak", diff --git a/Cargo.toml b/Cargo.toml index e015f26..b7cd4c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ crate-type = [ "cdylib", "lib" ] [dependencies] hypher = "0.1.7" -libc = "0.2.189" +iter_tools = "0.50.0" mlua-sys = { version = "0.12.0", features = ["luajit"] } text_layout = "0.3.0" unicode-linebreak = "0.1.5" diff --git a/examples/linebreak.rs b/examples/linebreak.rs index 68bbab0..9c2ee61 100644 --- a/examples/linebreak.rs +++ b/examples/linebreak.rs @@ -1,11 +1,12 @@ +use hypher::Lang; use lua_hyphenate::layout; pub fn main() { - println!( - "{}", - layout( - "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Etiam tincidunt, lectus a fringilla fringilla, velit purus accumsan nulla, sed vehicula nunc eros eget nibh. Donec leo erat, efficitur vel faucibus sit amet, tristique nec ante. Quisque vestibulum lacinia nisl in iaculis. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Nulla ultricies blandit massa ut interdum. Nam non posuere velit. Curabitur magna lorem, eleifend eget tellus eu, accumsan pharetra arcu. Morbi consequat in sapien ut tincidunt. Interdum et malesuada fames ac ante ipsum primis in faucibus. Nam metus elit, cursus eu ornare quis, bibendum sollicitudin magna. Donec at ornare tellus. Proin sollicitudin dictum blandit. Duis at ornare felis. Cras lacus sem, congue vel nunc nec, condimentum convallis ex. Sed sed tincidunt justo.", - 30 - ) - ); + println!("{}", layout( + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Etiam tincidunt, lectus a fringilla fringilla, velit purus accumsan nulla, sed vehicula nunc eros eget nibh. Donec leo erat, efficitur vel faucibus sit amet, tristique nec ante. Quisque vestibulum lacinia nisl in iaculis. Pellentesque habitant morbi tristique senectus et netus et malesuada fames ac turpis egestas. Nulla ultricies blandit massa ut interdum. Nam non posuere velit. + Curabitur magna lorem, eleifend eget tellus eu, accumsan pharetra arcu. Morbi consequat in sapien ut tincidunt. Interdum et malesuada fames ac ante ipsum primis in faucibus. Nam metus elit, cursus eu ornare quis, bibendum sollicitudin magna. Donec at ornare tellus. Proin sollicitudin dictum blandit. Duis at ornare felis. Cras lacus sem, congue vel nunc nec, condimentum convallis ex. Sed sed tincidunt justo.", + 45, + Lang::Latin + ).join("\n")); + println!("{}", layout("short line", 45, Lang::English).join("\n")); } diff --git a/src/lib.rs b/src/lib.rs index 1cdd642..dc0cfb4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,15 +1,190 @@ -use hypher::{Lang, hyphenate}; -use mlua_sys::lua51::lua_State; -use text_layout::{Item, KnuthPlass}; -use unicode_linebreak::linebreaks; +use std::ffi::{CStr, CString}; -pub fn layout(s: &str, width: usize) -> String { - let layout = KnuthPlass::::new(); - s.to_string() +use hypher::{Lang, hyphenate}; +use iter_tools::Itertools; +use mlua_sys::{ + lua_isnumber, lua_pushnumber, lua_settable, lua_tonumber, + lua51::{ + lua_State, lua_createtable, lua_gettop, lua_isinteger, lua_isstring, lua_pushcfunction, + lua_pushlstring, lua_pushnil, lua_pushstring, lua_setfield, lua_tointeger, lua_tostring, + lua_typename, + }, +}; +use text_layout::{Item, KnuthPlass, ParagraphLayout}; +use unicode_linebreak::{BreakOpportunity, linebreaks}; +use unicode_segmentation::UnicodeSegmentation; +use unicode_width::UnicodeWidthStr; + +#[derive(Clone)] +enum Hyphen<'a> { + Hyphen, + Syllable(&'a str), +} +fn box_value(value: &str) -> Item<&str, &str, &str> { + Item::Box { + width: UnicodeWidthStr::width(value) as f32, + data: value, + } +} +pub fn layout(text: impl AsRef, width: usize, lang: Lang) -> Vec { + let s = text.as_ref(); + let layout = KnuthPlass::new().with_threshold(f32::INFINITY); + + let mut items: Vec> = vec![]; + let mut last_break_opportunity = 0; + for (idx, break_opportunity) in linebreaks(s) { + let trimmed = if idx == s.len() { + &s[last_break_opportunity..idx] + } else { + &s[last_break_opportunity..idx - 1] + }; + let mut last_word_end = 0; + for (start, word) in trimmed.unicode_word_indices() { + if last_word_end < start { + items.push(box_value(&trimmed[last_word_end..start])) + } + for syl in (hyphenate(&word, lang)) + .map(Hyphen::Syllable) + .intersperse(Hyphen::Hyphen) + { + items.push(match syl { + Hyphen::Syllable(x) => box_value(x), + Hyphen::Hyphen => Item::Penalty { + width: 1f32, + cost: 1f32, + flagged: true, + data: "-", + }, + }) + } + last_word_end = start + word.len(); + } + if last_word_end < trimmed.len() { + items.push(box_value(&trimmed[last_word_end..])); + } + + if break_opportunity == BreakOpportunity::Mandatory { + println!("{} {}", idx, s.len()); + items.push(Item::Glue { + width: 0.0, + stretch: 100000.0, + shrink: 0.0, + data: "", + }); + items.push(Item::Penalty { + width: 0.0, + cost: f32::NEG_INFINITY, + flagged: true, + data: "", + }); + } else { + (s[idx - 1..].chars().next()); + items.push(Item::Glue { + width: 1f32, + stretch: 1f32, + shrink: 0f32, + data: " ", + }); + } + + last_break_opportunity = idx; + } + + let lines = layout.layout_paragraph(&items, width as f32); + + let mut last_idx = 0; + let mut ret = Vec::with_capacity(lines.len()); + for line in lines { + let mut s = String::new(); + for i in last_idx..line.break_at { + let data = match items[i] { + Item::Box { data, width: _ } => data, + Item::Glue { data, .. } => data, + Item::Penalty { .. } => "", + }; + s.push_str(data); + last_idx = line.break_at + 1; + } + if let Item::Penalty { data, .. } = items[line.break_at] { + s.push_str(data); + } else if let Item::Box { data, .. } = items[line.break_at] { + s.push_str(data); + } + ret.push(s); + } + ret } -pub unsafe extern "C" fn break_line(state: *mut lua_State) -> isize { - 0 +fn err(state: *mut lua_State, s: impl AsRef) -> i32 { + unsafe { + lua_pushnil(state); + lua_pushstring(state, CString::new(s.as_ref()).unwrap().as_ptr()); + 2 + } +} +unsafe extern "C-unwind" fn break_line(state: *mut lua_State) -> i32 { + unsafe { + if lua_gettop(state) < 3 { + return err( + state, + format!("Expected 3 arguments, got {}", lua_gettop(state)), + ); + } + if lua_isstring(state, 1) == 0 { + return err( + state, + format!( + "Expected a string as first arg, got {}", + CStr::from_ptr(lua_typename(state, 1)).to_string_lossy() + ), + ); + } + if lua_isnumber(state, 2) == 0 { + return err( + state, + format!( + "Expected a number as second arg, got {}", + CStr::from_ptr(lua_typename(state, 2)).to_string_lossy() + ), + ); + } + if lua_isstring(state, 3) == 0 { + return err( + state, + format!( + "Expected a string as third arg, got {}", + CStr::from_ptr(lua_typename(state, 3)).to_string_lossy() + ), + ); + } + let text = CStr::from_ptr(lua_tostring(state, 1)); + let width: usize = lua_tonumber(state, 2).to_int_unchecked(); + let lang_code = CStr::from_ptr(lua_tostring(state, 3)).to_bytes(); + let lang = if lang_code.len() == 2 { + Lang::from_iso([lang_code[0], lang_code[1]]).unwrap_or(Lang::English) + } else { + Lang::English + }; + println!("(hyphenate \"{text:?}\" {width} :{lang_code:?})"); + let hyphenated = layout(text.to_string_lossy(), width as usize, lang); + lua_createtable(state, hyphenated.len() as i32, 0); + for (idx, line) in hyphenated.iter().enumerate() { + lua_pushnumber(state, idx as f64 + 1f64); + lua_pushlstring(state, line.as_ptr() as *const i8, line.len()); + lua_settable(state, -3); + } + 1 + } +} + +#[unsafe(no_mangle)] +pub unsafe extern "C-unwind" fn luaopen_hyphenate(state: *mut lua_State) -> i32 { + unsafe { + lua_createtable(state, 0, 1); + lua_pushcfunction(state, break_line); + lua_setfield(state, -2, c"hyphenate".as_ptr()); + 1 + } } #[cfg(test)]