// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie // SPDX-FileCopyrightText: © 2011,2012 Google, Inc. // SPDX-License-Identifier: MIT AND MIT-Modern-Variant //! Normalization as a shaper wants it, which is not quite Unicode's: a //! character is decomposed only when the font lacks a glyph for it and has //! glyphs for its parts, marks are put in canonical order, and composed //! back where the font has a glyph for the composite. The aim is the //! sequence of characters the font can actually draw. HarfBuzz's //! `hb-ot-shape-normalize.cc`. const std = @import("std"); const Buffer = @import("Buffer.zig"); const Face = @import("Face.zig"); const unicode = @import("unicode.zig"); const props = @import("props.zig"); const Error = std.mem.Allocator.Error; const GlyphInfo = Buffer.GlyphInfo; pub const Mode = enum { none, decomposed, /// Never composes base to base. composed_diacritics, /// Always fully decomposes, and then composes back. composed_diacritics_no_short_circuit, auto, }; /// The most marks in a row that are reordered; a longer run is left alone, /// since sorting it is quadratic. const max_combining_marks = 32; pub const Hooks = struct { decompose: *const fn (ctx: *const anyopaque, ab: u21) ?unicode.Decomposition = defaultDecomposeFn, compose: *const fn (ctx: *const anyopaque, a: u21, b: u21) ?u21 = defaultComposeFn, reorder_marks: ?*const fn (ctx: *const anyopaque, buffer: *Buffer, start: usize, end: usize) void = null, context: *const anyopaque = undefined, pub fn defaultDecomposeFn(_: *const anyopaque, ab: u21) ?unicode.Decomposition { return unicode.decompose(ab); } pub fn defaultComposeFn(_: *const anyopaque, a: u21, b: u21) ?u21 { return unicode.compose(a, b); } }; const Context = struct { buffer: *Buffer, face: *const Face, hooks: Hooks, fn decompose(c: *const Context, ab: u21) ?unicode.Decomposition { return c.hooks.decompose(c.hooks.context, ab); } fn compose(c: *const Context, a: u21, b: u21) ?u21 { return c.hooks.compose(c.hooks.context, a, b); } }; fn nominal(face: *const Face, cp: u32) ?u32 { return face.nominalGlyph(cp); } fn setGlyph(info: *GlyphInfo, face: *const Face) void { info.glyph_index = nominal(face, info.codepoint) orelse 0; } /// Writes `unichar`, with `glyph` noted for it, as a copy of the character /// at the cursor, and gives it its own Unicode properties. fn outputChar(buffer: *Buffer, unichar: u21, glyph: u32) Error!void { // This is very confusing indeed. buffer.cur(0).glyph_index = glyph; try buffer.outputGlyph(unichar); props.setUnicodeProps(buffer.prev(), buffer); } fn nextChar(buffer: *Buffer, glyph: u32) Error!void { buffer.cur(0).glyph_index = glyph; try buffer.nextGlyph(); } /// Returns 0 if `ab` did not decompose, or how many characters it became. fn decompose(c: *const Context, shortest: bool, ab: u21) Error!usize { const buffer = c.buffer; const face = c.face; const d = c.decompose(ab) orelse return 0; var b_glyph: u32 = 0; if (d.b != 0) b_glyph = nominal(face, d.b) orelse return 0; const a_glyph = nominal(face, d.a); if (shortest and a_glyph != null) { try outputChar(buffer, d.a, a_glyph.?); if (d.b != 0) { try outputChar(buffer, d.b, b_glyph); return 2; } return 1; } const ret = try decompose(c, shortest, d.a); if (ret != 0) { if (d.b != 0) { try outputChar(buffer, d.b, b_glyph); return ret + 1; } return ret; } if (a_glyph) |ag| { try outputChar(buffer, d.a, ag); if (d.b != 0) { try outputChar(buffer, d.b, b_glyph); return 2; } return 1; } return 0; } fn decomposeCurrentCharacter(c: *const Context, shortest: bool) Error!void { const buffer = c.buffer; const u: u21 = @intCast(buffer.cur(0).codepoint); var glyph: u32 = 0; if (shortest) { if (nominal(c.face, u)) |g| { try nextChar(buffer, g); return; } glyph = buffer.not_found; } if (try decompose(c, shortest, u) != 0) { buffer.skipGlyph(); return; } if (!shortest) { if (nominal(c.face, u)) |g| { try nextChar(buffer, g); return; } glyph = buffer.not_found; } if (buffer.cur(0).isUnicodeSpace()) { const space_type = unicode.spaceFallbackType(u); if (space_type != .not_space) { const space_glyph = nominal(c.face, 0x0020) orelse buffer.invisible; if (space_glyph != 0) { buffer.cur(0).setSpaceFallbackType(space_type); try nextChar(buffer, space_glyph); buffer.scratch_flags |= Buffer.ScratchFlags.has_space_fallback; return; } } } if (u == 0x2011) { // U+2011 is the only sensible character that is a no-break version of // another character and not a space. The spaces are handled above. if (nominal(c.face, 0x2010)) |other| { try nextChar(buffer, other); return; } } try nextChar(buffer, glyph); } fn handleVariationSelectorCluster(c: *const Context, end: usize) Error!void { // With a variation selector, normalization gives up; it's just too hard. const buffer = c.buffer; const face = c.face; while (buffer.idx + 1 < end and buffer.successful) { if (unicode.isVariationSelector(@intCast(buffer.cur(1).codepoint))) { if (face.variationGlyph(buffer.cur(0).codepoint, buffer.cur(1).codepoint)) |g| { buffer.cur(0).glyph_index = g; const u = buffer.cur(0).codepoint; try buffer.replaceGlyphs(2, &.{u}); } else { // Pass the two characters on separately; GSUB may do // something with them. setGlyph(buffer.cur(0), face); try buffer.nextGlyph(); buffer.scratch_flags |= Buffer.ScratchFlags.has_variation_selector_fallback; buffer.cur(0).setVariationSelector(true); if (buffer.not_found_variation_selector != null) buffer.cur(0).clearDefaultIgnorable(); setGlyph(buffer.cur(0), face); try buffer.nextGlyph(); } // Skip any further variation selectors. while (buffer.idx < end and buffer.successful and unicode.isVariationSelector(@intCast(buffer.cur(0).codepoint))) { setGlyph(buffer.cur(0), face); try buffer.nextGlyph(); } } else { setGlyph(buffer.cur(0), face); try buffer.nextGlyph(); } } if (buffer.idx < end) { setGlyph(buffer.cur(0), face); try buffer.nextGlyph(); } } fn decomposeMultiCharCluster(c: *const Context, end: usize, short_circuit: bool) Error!void { const buffer = c.buffer; var i = buffer.idx; while (i < end and buffer.successful) : (i += 1) { if (unicode.isVariationSelector(@intCast(buffer.info.items[i].codepoint))) { return handleVariationSelectorCluster(c, end); } } while (buffer.idx < end and buffer.successful) try decomposeCurrentCharacter(c, short_circuit); } fn compareCombiningClass(a: *const GlyphInfo, b: *const GlyphInfo) i32 { const ca = a.modifiedCombiningClass(); const cb = b.modifiedCombiningClass(); return if (ca < cb) -1 else if (ca == cb) 0 else 1; } /// `_hb_ot_shape_normalize`. pub fn normalize(buffer: *Buffer, face: *const Face, mode_in: Mode, hooks: Hooks) Error!void { if (buffer.len() == 0) return; const mode: Mode = if (mode_in == .auto) .composed_diacritics else mode_in; const c: Context = .{ .buffer = buffer, .face = face, .hooks = hooks }; const always_short_circuit = mode == .none; const might_short_circuit = always_short_circuit or (mode != .decomposed and mode != .composed_diacritics_no_short_circuit); // First round, decompose. var all_simple = true; { buffer.clearOutput(); const count = buffer.len(); buffer.idx = 0; while (true) { var end = buffer.idx + 1; while (end < count) : (end += 1) { if (buffer.info.items[end].isUnicodeMark()) break; } // Leave one base for the marks to cluster with. if (end < count) end -= 1; // From idx to end are simple clusters. if (might_short_circuit) { var done: usize = 0; while (buffer.idx + done < end) : (done += 1) { const info = &buffer.info.items[buffer.idx + done]; info.glyph_index = nominal(face, info.codepoint) orelse break; } try buffer.nextGlyphs(done); } while (buffer.idx < end and buffer.successful) try decomposeCurrentCharacter(&c, might_short_circuit); if (buffer.idx == count or !buffer.successful) break; all_simple = false; // Find all the marks now. end = buffer.idx + 1; while (end < count) : (end += 1) { if (!buffer.info.items[end].isUnicodeMark()) break; } // idx to end is one non-simple cluster. try decomposeMultiCharCluster(&c, end, always_short_circuit); if (!(buffer.idx < count and buffer.successful)) break; } _ = buffer.sync(); } // Second round, reorder, in place. if (!all_simple) { const count = buffer.len(); const info = buffer.info.items; var i: usize = 0; while (i < count) : (i += 1) { if (info[i].modifiedCombiningClass() == 0) continue; var end = i + 1; while (end < count) : (end += 1) { if (info[end].modifiedCombiningClass() == 0) break; } // This is quadratic; only do it for short sequences. if (end - i > max_combining_marks) { i = end; continue; } buffer.sort(i, end, compareCombiningClass); if (hooks.reorder_marks) |f| f(hooks.context, buffer, i, end); i = end; } } if (buffer.scratch_flags & Buffer.ScratchFlags.has_cgj != 0) { // A CGJ that did not prevent any reordering can be skipped. const count = buffer.len(); const info = buffer.info.items; var i: usize = 1; while (i + 1 < count) : (i += 1) { if (info[i].codepoint == 0x034F and (info[i + 1].modifiedCombiningClass() == 0 or info[i - 1].modifiedCombiningClass() <= info[i + 1].modifiedCombiningClass())) { info[i].unhide(); } } } // Third round, recompose. if (!all_simple and buffer.successful and (mode == .composed_diacritics or mode == .composed_diacritics_no_short_circuit)) { // Characters of combining class 0 are not composed with the starter // before them. buffer.clearOutput(); const count = buffer.len(); var starter: usize = 0; try buffer.nextGlyph(); while (buffer.idx < count) { // A non-mark is not composed with the starter before it: an // optimization, and what Hangul fonts want, since they are not // designed to mix precomposed syllables with jamo. if (buffer.cur(0).isUnicodeMark()) { const out = buffer.out_info.items; // Anything between the starter and this character has to // have a lower combining class. if (starter == out.len - 1 or buffer.prev().modifiedCombiningClass() < buffer.cur(0).modifiedCombiningClass()) { if (c.compose(@intCast(out[starter].codepoint), @intCast(buffer.cur(0).codepoint))) |composed| { if (nominal(face, composed)) |glyph| { // Composes. try buffer.nextGlyph(); buffer.mergeOutClusters(starter, buffer.out_info.items.len); // Remove the second composable. buffer.out_info.items.len -= 1; // Modify the starter and carry on. const s = &buffer.out_info.items[starter]; s.codepoint = composed; s.glyph_index = glyph; props.setUnicodeProps(s, buffer); continue; } } } } // Blocked, or does not compose. try buffer.nextGlyph(); if (buffer.prev().modifiedCombiningClass() == 0) starter = buffer.out_info.items.len - 1; } _ = buffer.sync(); } }