//! Streaming UTF-8 decoder for use by `TextDecoderStream` and similar APIs. //! //! Wraps the WHATWG-compliant byte-by-byte UTF-8 state machine so callers can //! feed bytes incrementally. A partial multi-byte sequence at the end of one //! chunk is held in internal state and combined with the next chunk's leading //! bytes — no replacement character is emitted at chunk boundaries (only at //! end-of-stream when fatal mode is off). use crate::error::EncodingError; /// Streaming UTF-8 decoder. /// /// `fatal` controls whether malformed sequences raise an error (`true`) or /// are replaced with U+FFFD (`false`). /// /// `ignore_bom` (when `false`) causes a leading UTF-8 BOM (EF BB BF) seen at /// the very start of the stream to be skipped. When `true`, the BOM is /// preserved as the code point U+FEFF. pub struct StreamingUtf8Decoder { code_point: u32, bytes_seen: u8, bytes_needed: u8, lower_boundary: u8, upper_boundary: u8, /// `true` once at least one byte has been observed. saw_input: bool, /// Possible pending BOM bytes at the start of the stream. bom_buf: Vec, bom_done: bool, fatal: bool, ignore_bom: bool, } impl StreamingUtf8Decoder { pub fn new(fatal: bool, ignore_bom: bool) -> Self { Self { code_point: 0, bytes_seen: 0, bytes_needed: 0, lower_boundary: 0x80, upper_boundary: 0xBF, saw_input: false, bom_buf: Vec::with_capacity(3), bom_done: false, fatal, ignore_bom, } } /// Decode bytes into a string. /// /// In fatal mode, returns an error on the first invalid sequence. pub fn decode(&mut self, bytes: &[u8]) -> Result { if bytes.is_empty() { return Ok(String::new()); } self.saw_input = true; let mut out = String::with_capacity(bytes.len()); let mut idx = 0; // Handle the leading BOM. if !self.bom_done { while idx < bytes.len() && self.bom_buf.len() < 3 { self.bom_buf.push(bytes[idx]); idx += 1; if self.bom_check_failed() { // No BOM — flush bom_buf back through the decoder. let buf = std::mem::take(&mut self.bom_buf); self.bom_done = true; self.decode_inner(&buf, &mut out)?; break; } if self.bom_buf.len() == 3 { // Full BOM observed. if !self.ignore_bom { // Per WHATWG, when ignore_bom is false, the BOM is // *consumed* (not emitted); when true, the BOM bytes // are decoded as the U+FEFF code point. // Wait — the spec is the opposite: by default the BOM // is consumed; if `ignoreBOM` is true the bytes are // surfaced as U+FEFF. // We follow the spec: ignore_bom=false → strip; // ignore_bom=true → emit U+FEFF. } if self.ignore_bom { out.push('\u{FEFF}'); } self.bom_done = true; self.bom_buf.clear(); break; } } if idx >= bytes.len() && !self.bom_done { return Ok(out); } } if idx < bytes.len() { self.decode_inner(&bytes[idx..], &mut out)?; } Ok(out) } /// Returns true once we've seen the first three bytes and they aren't BOM. fn bom_check_failed(&self) -> bool { !matches!( self.bom_buf.as_slice(), [0xEF] | [0xEF, 0xBB] | [0xEF, 0xBB, 0xBF] ) } fn decode_inner(&mut self, bytes: &[u8], out: &mut String) -> Result<(), EncodingError> { let mut i = 0; while i < bytes.len() { match self.process_byte(bytes[i]) { ProcessResult::CodePoint(ch) => { out.push(ch); i += 1; } ProcessResult::Error => { if self.fatal { return Err(EncodingError::InvalidSequence { encoding: "UTF-8", position: i, }); } out.push('\u{FFFD}'); i += 1; } ProcessResult::ErrorPrepend => { if self.fatal { return Err(EncodingError::InvalidSequence { encoding: "UTF-8", position: i, }); } out.push('\u{FFFD}'); // do not advance i; re-process this byte } ProcessResult::Continue => { i += 1; } } } Ok(()) } /// Flush any state at end-of-stream. /// /// Returns `\u{FFFD}` if a partial multi-byte sequence is pending and /// `fatal` is false; returns an error if `fatal` is true; otherwise /// returns an empty string. pub fn flush(&mut self) -> Result { if !self.bom_done && !self.bom_buf.is_empty() { // Bytes pending in BOM detector that aren't BOM bytes. let buf = std::mem::take(&mut self.bom_buf); self.bom_done = true; let mut out = String::new(); self.decode_inner(&buf, &mut out)?; if self.bytes_needed > 0 { if self.fatal { return Err(EncodingError::InvalidSequence { encoding: "UTF-8", position: 0, }); } out.push('\u{FFFD}'); self.reset(); } return Ok(out); } if self.bytes_needed > 0 { if self.fatal { return Err(EncodingError::InvalidSequence { encoding: "UTF-8", position: 0, }); } self.reset(); return Ok(String::from('\u{FFFD}')); } Ok(String::new()) } fn process_byte(&mut self, byte: u8) -> ProcessResult { if self.bytes_needed == 0 { match byte { 0x00..=0x7F => ProcessResult::CodePoint(byte as char), 0xC2..=0xDF => { self.bytes_needed = 1; self.code_point = (byte & 0x1F) as u32; ProcessResult::Continue } 0xE0 => { self.bytes_needed = 2; self.lower_boundary = 0xA0; self.code_point = (byte & 0x0F) as u32; ProcessResult::Continue } 0xE1..=0xEC | 0xEE..=0xEF => { self.bytes_needed = 2; self.code_point = (byte & 0x0F) as u32; ProcessResult::Continue } 0xED => { self.bytes_needed = 2; self.upper_boundary = 0x9F; self.code_point = (byte & 0x0F) as u32; ProcessResult::Continue } 0xF0 => { self.bytes_needed = 3; self.lower_boundary = 0x90; self.code_point = (byte & 0x07) as u32; ProcessResult::Continue } 0xF1..=0xF3 => { self.bytes_needed = 3; self.code_point = (byte & 0x07) as u32; ProcessResult::Continue } 0xF4 => { self.bytes_needed = 3; self.upper_boundary = 0x8F; self.code_point = (byte & 0x07) as u32; ProcessResult::Continue } _ => ProcessResult::Error, } } else if byte < self.lower_boundary || byte > self.upper_boundary { self.reset(); ProcessResult::ErrorPrepend } else { self.lower_boundary = 0x80; self.upper_boundary = 0xBF; self.code_point = (self.code_point << 6) | (byte & 0x3F) as u32; self.bytes_seen += 1; if self.bytes_seen == self.bytes_needed { let cp = self.code_point; self.reset(); let ch = char::from_u32(cp).unwrap_or('\u{FFFD}'); ProcessResult::CodePoint(ch) } else { ProcessResult::Continue } } } fn reset(&mut self) { self.code_point = 0; self.bytes_seen = 0; self.bytes_needed = 0; self.lower_boundary = 0x80; self.upper_boundary = 0xBF; } } enum ProcessResult { CodePoint(char), Error, ErrorPrepend, Continue, } #[cfg(test)] mod tests { use super::*; #[test] fn decode_whole_string() { let mut d = StreamingUtf8Decoder::new(false, false); let out = d.decode("hello".as_bytes()).unwrap(); assert_eq!(out, "hello"); assert!(d.flush().unwrap().is_empty()); } #[test] fn decode_split_two_byte() { // U+00E9 = 0xC3 0xA9 let mut d = StreamingUtf8Decoder::new(false, false); let a = d.decode(&[0xC3]).unwrap(); assert_eq!(a, ""); let b = d.decode(&[0xA9]).unwrap(); assert_eq!(b, "\u{00E9}"); assert!(d.flush().unwrap().is_empty()); } #[test] fn decode_split_four_byte() { // U+1F600 = 0xF0 0x9F 0x98 0x80 let mut d = StreamingUtf8Decoder::new(false, false); assert_eq!(d.decode(&[0xF0]).unwrap(), ""); assert_eq!(d.decode(&[0x9F, 0x98]).unwrap(), ""); assert_eq!(d.decode(&[0x80]).unwrap(), "\u{1F600}"); } #[test] fn decode_split_after_bom() { let mut d = StreamingUtf8Decoder::new(false, false); // BOM split across two chunks. assert_eq!(d.decode(&[0xEF, 0xBB]).unwrap(), ""); assert_eq!(d.decode(&[0xBF, b'A']).unwrap(), "A"); } #[test] fn decode_ignore_bom_emits_feff() { let mut d = StreamingUtf8Decoder::new(false, true); assert_eq!(d.decode(&[0xEF, 0xBB, 0xBF, b'A']).unwrap(), "\u{FEFF}A"); } #[test] fn flush_partial_replaces_with_fffd() { let mut d = StreamingUtf8Decoder::new(false, false); assert_eq!(d.decode(&[0xC3]).unwrap(), ""); assert_eq!(d.flush().unwrap(), "\u{FFFD}"); } #[test] fn fatal_mode_errors_on_truncated() { let mut d = StreamingUtf8Decoder::new(true, false); d.decode(&[0xC3]).unwrap(); assert!(d.flush().is_err()); } #[test] fn fatal_mode_errors_on_invalid() { let mut d = StreamingUtf8Decoder::new(true, false); assert!(d.decode(&[0xFF]).is_err()); } #[test] fn flush_with_pending_non_bom_bytes() { let mut d = StreamingUtf8Decoder::new(false, false); // Bytes that look like the start of a BOM but aren't. assert_eq!(d.decode(&[0xEF, 0xBB]).unwrap(), ""); let tail = d.flush().unwrap(); // 0xEF 0xBB looks like the lead+continuation of a 3-byte sequence // expecting a third byte; flush emits a single replacement char. assert_eq!(tail, "\u{FFFD}"); } }