Something went wrong. Try again.
we (web engine): Experimental web browser project to understand the limits of Claude
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677//! HTML5 tokenizer and tree builder.//!//! Implements the WHATWG HTML5 tokenizer state machine (§13.2.5)//! and a simplified tree builder for constructing DOM trees from tokens.
mod entities;pub mod speculative;mod tokenizer;mod tree_builder;
pub use tokenizer::{ParseError, Tokenizer};pub use tree_builder::{parse_html, TreeBuilder};
/// A token emitted by the HTML tokenizer.#[derive(Debug, Clone, PartialEq)]pub enum Token { /// `<!DOCTYPE name public_id system_id>` Doctype { name: Option<String>, public_id: Option<String>, system_id: Option<String>, force_quirks: bool, }, /// `<tag attr="val">` StartTag { name: String, attributes: Vec<(String, String)>, self_closing: bool, }, /// `</tag>` EndTag { name: String }, /// Character data (may be coalesced). Character(String), /// `<!-- comment -->` Comment(String), /// End of file. Eof,}
/// Tokenize an HTML input string into a sequence of tokens.////// Runs the HTML5 tokenizer state machine and returns all emitted tokens/// (excluding Eof). Adjacent Character tokens are coalesced.pub fn tokenize(input: &str) -> Vec<Token> { let mut tok = Tokenizer::new(input); let mut tokens = Vec::new(); loop { let token = tok.next_token(); if let Token::StartTag { name, .. } = &token { // Mirror tree-construction state switching so script/style/title // content tokenizes correctly when the caller drives the // tokenizer through this convenience wrapper. match name.as_str() { "script" => tok.switch_to_script_data(), "style" | "iframe" | "xmp" | "noembed" | "noframes" | "noscript" => { tok.switch_to_rawtext(); } "title" | "textarea" => tok.switch_to_rcdata(), _ => {} } } match token { Token::Eof => break, Token::Character(ref s) => { // Coalesce adjacent character tokens. if let Some(Token::Character(ref mut prev)) = tokens.last_mut() { prev.push_str(s); } else { tokens.push(token); } } _ => tokens.push(token), } } tokens}