Something went wrong. Try again.
source dump of claude code forked from oppi.li/claude-code
Something went wrong. Try again.
9.3 kB · 319 lines
TypeScript
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320/** * Input Tokenizer - Escape sequence boundary detection * * Splits terminal input into tokens: text chunks and raw escape sequences. * Unlike the Parser which interprets sequences semantically, this just * identifies boundaries for use by keyboard input parsing. */
import { C0, ESC_TYPE, isEscFinal } from './ansi.js'import { isCSIFinal, isCSIIntermediate, isCSIParam } from './csi.js'
export type Token = | { type: 'text'; value: string } | { type: 'sequence'; value: string }
type State = | 'ground' | 'escape' | 'escapeIntermediate' | 'csi' | 'ss3' | 'osc' | 'dcs' | 'apc'
export type Tokenizer = { /** Feed input and get resulting tokens */ feed(input: string): Token[] /** Flush any buffered incomplete sequences */ flush(): Token[] /** Reset tokenizer state */ reset(): void /** Get any buffered incomplete sequence */ buffer(): string}
type TokenizerOptions = { /** * Treat `CSI M` as an X10 mouse event prefix and consume 3 payload bytes. * Only enable for stdin input — `\x1b[M` is also CSI DL (Delete Lines) in * output streams, and enabling this there swallows display text. Default false. */ x10Mouse?: boolean}
/** * Create a streaming tokenizer for terminal input. * * Usage: * ```typescript * const tokenizer = createTokenizer() * const tokens1 = tokenizer.feed('hello\x1b[') * const tokens2 = tokenizer.feed('A') // completes the escape sequence * const remaining = tokenizer.flush() // force output incomplete sequences * ``` */export function createTokenizer(options?: TokenizerOptions): Tokenizer { let currentState: State = 'ground' let currentBuffer = '' const x10Mouse = options?.x10Mouse ?? false
return { feed(input: string): Token[] { const result = tokenize( input, currentState, currentBuffer, false, x10Mouse, ) currentState = result.state.state currentBuffer = result.state.buffer return result.tokens },
flush(): Token[] { const result = tokenize('', currentState, currentBuffer, true, x10Mouse) currentState = result.state.state currentBuffer = result.state.buffer return result.tokens },
reset(): void { currentState = 'ground' currentBuffer = '' },
buffer(): string { return currentBuffer }, }}
type InternalState = { state: State buffer: string}
function tokenize( input: string, initialState: State, initialBuffer: string, flush: boolean, x10Mouse: boolean,): { tokens: Token[]; state: InternalState } { const tokens: Token[] = [] const result: InternalState = { state: initialState, buffer: '', }
const data = initialBuffer + input let i = 0 let textStart = 0 let seqStart = 0
const flushText = (): void => { if (i > textStart) { const text = data.slice(textStart, i) if (text) { tokens.push({ type: 'text', value: text }) } } textStart = i }
const emitSequence = (seq: string): void => { if (seq) { tokens.push({ type: 'sequence', value: seq }) } result.state = 'ground' textStart = i }
while (i < data.length) { const code = data.charCodeAt(i)
switch (result.state) { case 'ground': if (code === C0.ESC) { flushText() seqStart = i result.state = 'escape' i++ } else { i++ } break
case 'escape': if (code === ESC_TYPE.CSI) { result.state = 'csi' i++ } else if (code === ESC_TYPE.OSC) { result.state = 'osc' i++ } else if (code === ESC_TYPE.DCS) { result.state = 'dcs' i++ } else if (code === ESC_TYPE.APC) { result.state = 'apc' i++ } else if (code === 0x4f) { // 'O' - SS3 result.state = 'ss3' i++ } else if (isCSIIntermediate(code)) { // Intermediate byte (e.g., ESC ( for charset) - continue buffering result.state = 'escapeIntermediate' i++ } else if (isEscFinal(code)) { // Two-character escape sequence i++ emitSequence(data.slice(seqStart, i)) } else if (code === C0.ESC) { // Double escape - emit first, start new emitSequence(data.slice(seqStart, i)) seqStart = i result.state = 'escape' i++ } else { // Invalid - treat ESC as text result.state = 'ground' textStart = seqStart } break
case 'escapeIntermediate': // After intermediate byte(s), wait for final byte if (isCSIIntermediate(code)) { // More intermediate bytes i++ } else if (isEscFinal(code)) { // Final byte - complete the sequence i++ emitSequence(data.slice(seqStart, i)) } else { // Invalid - treat as text result.state = 'ground' textStart = seqStart } break
case 'csi': // X10 mouse: CSI M + 3 raw payload bytes (Cb+32, Cx+32, Cy+32). // M immediately after [ (offset 2) means no params — SGR mouse // (CSI < … M) has a `<` param byte first and reaches M at offset > 2. // Terminals that ignore DECSET 1006 but honor 1000/1002 emit this // legacy encoding; without this branch the 3 payload bytes leak // through as text (`` `rK `` / `arK` garbage in the prompt). // // Gated on x10Mouse — `\x1b[M` is also CSI DL (Delete Lines) and // blindly consuming 3 chars corrupts output rendering (Parser/Ansi) // and fragments bracketed-paste PASTE_END. Only stdin enables this. // The ≥0x20 check on each payload slot is belt-and-suspenders: X10 // guarantees Cb≥32, Cx≥33, Cy≥33, so a control byte (ESC=0x1B) in // any slot means this is CSI DL adjacent to another sequence, not a // mouse event. Checking all three slots prevents PASTE_END's ESC // from being consumed when paste content ends in `\x1b[M`+0-2 chars. // // Known limitation: this counts JS string chars, but X10 is byte- // oriented and stdin uses utf8 encoding (App.tsx). At col 162-191 × // row 96-159 the two coord bytes (0xC2-0xDF, 0x80-0xBF) form a valid // UTF-8 2-byte sequence and collapse to one char — the length check // fails and the event buffers until the next keypress absorbs it. // Fixing this requires latin1 stdin; X10's 223-coord cap is exactly // why SGR was invented, and no-SGR terminals at 162+ cols are rare. if ( x10Mouse && code === 0x4d /* M */ && i - seqStart === 2 && (i + 1 >= data.length || data.charCodeAt(i + 1) >= 0x20) && (i + 2 >= data.length || data.charCodeAt(i + 2) >= 0x20) && (i + 3 >= data.length || data.charCodeAt(i + 3) >= 0x20) ) { if (i + 4 <= data.length) { i += 4 emitSequence(data.slice(seqStart, i)) } else { // Incomplete — exit loop; end-of-input buffers from seqStart. // Re-entry re-tokenizes from ground via the invalid-CSI fallthrough. i = data.length } break } if (isCSIFinal(code)) { i++ emitSequence(data.slice(seqStart, i)) } else if (isCSIParam(code) || isCSIIntermediate(code)) { i++ } else { // Invalid CSI - abort, treat as text result.state = 'ground' textStart = seqStart } break
case 'ss3': // SS3 sequences: ESC O followed by a single final byte if (code >= 0x40 && code <= 0x7e) { i++ emitSequence(data.slice(seqStart, i)) } else { // Invalid - treat as text result.state = 'ground' textStart = seqStart } break
case 'osc': if (code === C0.BEL) { i++ emitSequence(data.slice(seqStart, i)) } else if ( code === C0.ESC && i + 1 < data.length && data.charCodeAt(i + 1) === ESC_TYPE.ST ) { i += 2 emitSequence(data.slice(seqStart, i)) } else { i++ } break
case 'dcs': case 'apc': if (code === C0.BEL) { i++ emitSequence(data.slice(seqStart, i)) } else if ( code === C0.ESC && i + 1 < data.length && data.charCodeAt(i + 1) === ESC_TYPE.ST ) { i += 2 emitSequence(data.slice(seqStart, i)) } else { i++ } break } }
// Handle end of input if (result.state === 'ground') { flushText() } else if (flush) { // Force output incomplete sequence const remaining = data.slice(seqStart) if (remaining) tokens.push({ type: 'sequence', value: remaining }) result.state = 'ground' } else { // Buffer incomplete sequence for next call result.buffer = data.slice(seqStart) }
return { tokens, state: result }}