Something went wrong. Try again.
we (web engine): Experimental web browser project to understand the limits of Claude
Something went wrong. Try again.
Rust
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711//! Fetch API: `fetch()`, Response, Headers.//!//! Network I/O runs on background threads; the event loop polls for completed//! requests and resolves/rejects the returned promises.
use std::cell::RefCell;use std::sync::{Arc, Mutex};
use crate::builtins::{ create_promise_object_pub, make_native, reject_promise_internal, resolve_promise_internal, set_builtin_prop,};use crate::gc::{Gc, GcRef};use crate::vm::*;
// ── Pending fetch state ─────────────────────────────────────────
/// Result of a completed HTTP fetch.pub struct FetchResult { pub status: u16, pub status_text: String, pub headers: Vec<(String, String)>, pub body: Vec<u8>, pub url: String,}
/// A fetch that is in-flight or just completed.struct PendingFetch { /// The promise returned to JS. promise: GcRef, /// Shared slot: the background thread writes the result here. result: Arc<Mutex<Option<Result<FetchResult, String>>>>,}
thread_local! { static FETCH_STATE: RefCell<Vec<PendingFetch>> = const { RefCell::new(Vec::new()) };}
/// Reset fetch state (useful for tests).pub fn reset_fetch_state() { FETCH_STATE.with(|s| s.borrow_mut().clear());}
/// Returns true if there are any in-flight fetches.pub fn has_pending_fetches() -> bool { FETCH_STATE.with(|s| !s.borrow().is_empty())}
/// Collect GC roots for pending fetch promises.pub fn fetch_gc_roots() -> Vec<GcRef> { FETCH_STATE.with(|s| s.borrow().iter().map(|f| f.promise).collect())}
/// A completed fetch ready to be resolved.pub struct CompletedFetch { pub promise: GcRef, pub result: Result<FetchResult, String>,}
/// Take all completed fetches (leaving in-flight ones).pub fn take_completed_fetches() -> Vec<CompletedFetch> { FETCH_STATE.with(|s| { let mut state = s.borrow_mut(); let mut completed = Vec::new(); let mut still_pending = Vec::new();
for pending in state.drain(..) { let taken = pending.result.lock().unwrap().take(); if let Some(result) = taken { completed.push(CompletedFetch { promise: pending.promise, result, }); } else { still_pending.push(pending); } }
*state = still_pending; completed })}
// ── fetch() native function ─────────────────────────────────────
/// `fetch(url, options?)` — initiate an HTTP request, return Promise<Response>.pub fn fetch_native(args: &[Value], ctx: &mut NativeContext) -> Result<Value, RuntimeError> { // Parse URL argument. let url_str = match args.first() { Some(v) => v.to_js_string(ctx.gc), None => { return Err(RuntimeError::type_error( "fetch requires at least one argument", )) } };
// Parse options (method, headers, body). let mut method = "GET".to_string(); let mut req_headers: Vec<(String, String)> = Vec::new(); let mut body: Option<Vec<u8>> = None;
if let Some(Value::Object(opts_ref)) = args.get(1) { if let Some(HeapObject::Object(data)) = ctx.gc.get(*opts_ref) { // method if let Some(prop) = data.properties.get("method") { if let Value::String(m) = &prop.value { method = m.to_uppercase(); } } // body if let Some(prop) = data.properties.get("body") { if let Value::String(b) = &prop.value { body = Some(b.as_bytes().to_vec()); } } // headers — read from nested object if let Some(prop) = data.properties.get("headers") { if let Some(hdrs_ref) = prop.value.gc_ref() { if let Some(HeapObject::Object(hdr_data)) = ctx.gc.get(hdrs_ref) { for (key, hdr_prop) in &hdr_data.properties { if hdr_prop.enumerable { if let Value::String(v) = &hdr_prop.value { req_headers.push((key.clone(), v.clone())); } } } } } } } }
// Create a pending promise. let promise = create_promise_object_pub(ctx.gc);
// Shared slot for the result. let result_slot: Arc<Mutex<Option<Result<FetchResult, String>>>> = Arc::new(Mutex::new(None)); let slot_clone = result_slot.clone();
// Register the pending fetch. FETCH_STATE.with(|s| { s.borrow_mut().push(PendingFetch { promise, result: result_slot, }); });
// Spawn a background thread for the network I/O. std::thread::spawn(move || { let result = do_fetch(&url_str, &method, &req_headers, body.as_deref()); let mut lock = slot_clone.lock().unwrap(); *lock = Some(result); });
Ok(Value::Object(promise))}
/// Perform the actual HTTP fetch (runs on a background thread).fn do_fetch( url_str: &str, method: &str, headers: &[(String, String)], body: Option<&[u8]>,) -> Result<FetchResult, String> { let url = we_url::Url::parse(url_str).map_err(|e| format!("Invalid URL: {e}"))?;
let mut req_headers = we_net::http::Headers::new(); for (key, val) in headers { req_headers.add(key, val); }
let http_method = match method { "GET" => we_net::http::Method::Get, "POST" => we_net::http::Method::Post, "PUT" => we_net::http::Method::Put, "DELETE" => we_net::http::Method::Delete, "HEAD" => we_net::http::Method::Head, "OPTIONS" => we_net::http::Method::Options, "PATCH" => we_net::http::Method::Patch, other => return Err(format!("Unsupported HTTP method: {other}")), };
let mut client = we_net::client::HttpClient::new(); let response = client .request(http_method, &url, &req_headers, body) .map_err(|e| format!("Network error: {e}"))?;
let resp_headers: Vec<(String, String)> = response .headers .iter() .map(|(k, v)| (k.to_string(), v.to_string())) .collect();
Ok(FetchResult { status: response.status_code, status_text: response.reason, headers: resp_headers, body: response.body, url: url.serialize(), })}
// ── Response object creation ────────────────────────────────────
/// Create a JS Response object from a completed fetch result.pub fn create_response_object(gc: &mut Gc<HeapObject>, result: &FetchResult) -> GcRef { let mut data = ObjectData::new();
data.properties.insert( "status".to_string(), Property::data(Value::Number(result.status as f64)), ); data.properties.insert( "statusText".to_string(), Property::data(Value::String(result.status_text.clone())), ); data.properties.insert( "ok".to_string(), Property::data(Value::Boolean((200..300).contains(&result.status))), ); data.properties.insert( "url".to_string(), Property::data(Value::String(result.url.clone())), );
// Store body as hidden property for text()/json(). let body_str = String::from_utf8_lossy(&result.body).to_string(); data.properties.insert( "__body__".to_string(), Property::builtin(Value::String(body_str)), );
let resp_ref = gc.alloc(HeapObject::Object(data));
// Create Headers object. let headers_ref = create_headers_object(gc, &result.headers); set_builtin_prop(gc, resp_ref, "headers", Value::Object(headers_ref));
// Register methods. let text_fn = make_native(gc, "text", response_text); set_builtin_prop(gc, resp_ref, "text", Value::Function(text_fn));
let json_fn = make_native(gc, "json", response_json); set_builtin_prop(gc, resp_ref, "json", Value::Function(json_fn));
resp_ref}
/// `response.text()` — returns Promise<String> with the response body.fn response_text(_args: &[Value], ctx: &mut NativeContext) -> Result<Value, RuntimeError> { let body_str = get_hidden_body(ctx.gc, &ctx.this); let promise = create_promise_object_pub(ctx.gc); resolve_promise_internal(ctx.gc, promise, Value::String(body_str)); Ok(Value::Object(promise))}
/// `response.json()` — returns Promise<Object> with parsed JSON body.fn response_json(_args: &[Value], ctx: &mut NativeContext) -> Result<Value, RuntimeError> { let body_str = get_hidden_body(ctx.gc, &ctx.this); let promise = create_promise_object_pub(ctx.gc); let json_args = [Value::String(body_str)]; match crate::builtins::json_parse_pub(&json_args, ctx) { Ok(parsed) => resolve_promise_internal(ctx.gc, promise, parsed), Err(e) => { let err_val = Value::String(e.to_string()); reject_promise_internal(ctx.gc, promise, err_val); } } Ok(Value::Object(promise))}
/// Read the hidden `__body__` string from a Response object.fn get_hidden_body(gc: &Gc<HeapObject>, this: &Value) -> String { if let Some(obj_ref) = this.gc_ref() { if let Some(HeapObject::Object(data)) = gc.get(obj_ref) { if let Some(prop) = data.properties.get("__body__") { if let Value::String(s) = &prop.value { return s.clone(); } } } } String::new()}
// ── Headers object ──────────────────────────────────────────────
/// Create a JS Headers object from response headers.////// Headers are stored as hidden indexed properties for case-insensitive/// lookup. Duplicate header names are combined with ", " per HTTP spec.fn create_headers_object(gc: &mut Gc<HeapObject>, headers: &[(String, String)]) -> GcRef { // Combine headers with same name (case-insensitive). let mut combined: Vec<(String, String)> = Vec::new(); for (name, value) in headers { let lower = name.to_lowercase(); if let Some(existing) = combined.iter_mut().find(|(n, _)| *n == lower) { existing.1 = format!("{}, {value}", existing.1); } else { combined.push((lower, value.clone())); } }
let mut data = ObjectData::new(); data.properties.insert( "__hdr_count__".to_string(), Property::builtin(Value::Number(combined.len() as f64)), ); for (i, (name, value)) in combined.iter().enumerate() { data.properties.insert( format!("__hdr_{i}_n__"), Property::builtin(Value::String(name.clone())), ); data.properties.insert( format!("__hdr_{i}_v__"), Property::builtin(Value::String(value.clone())), ); }
let obj_ref = gc.alloc(HeapObject::Object(data));
let get_fn = make_native(gc, "get", headers_get); set_builtin_prop(gc, obj_ref, "get", Value::Function(get_fn));
let has_fn = make_native(gc, "has", headers_has); set_builtin_prop(gc, obj_ref, "has", Value::Function(has_fn));
let set_fn = make_native(gc, "set", headers_set); set_builtin_prop(gc, obj_ref, "set", Value::Function(set_fn));
let delete_fn = make_native(gc, "delete", headers_delete); set_builtin_prop(gc, obj_ref, "delete", Value::Function(delete_fn));
obj_ref}
/// Read header entries from a Headers object: returns (count, Vec<(name, value)>).fn read_header_entries(gc: &Gc<HeapObject>, obj_ref: GcRef) -> Vec<(String, String)> { let mut entries = Vec::new(); if let Some(HeapObject::Object(data)) = gc.get(obj_ref) { let count = data .properties .get("__hdr_count__") .and_then(|p| { if let Value::Number(n) = &p.value { Some(*n as usize) } else { None } }) .unwrap_or(0);
for i in 0..count { let name = data .properties .get(&format!("__hdr_{i}_n__")) .and_then(|p| { if let Value::String(s) = &p.value { Some(s.clone()) } else { None } }) .unwrap_or_default(); let value = data .properties .get(&format!("__hdr_{i}_v__")) .and_then(|p| { if let Value::String(s) = &p.value { Some(s.clone()) } else { None } }) .unwrap_or_default(); entries.push((name, value)); } } entries}
/// `headers.get(name)` — case-insensitive lookup, returns string or null.fn headers_get(args: &[Value], ctx: &mut NativeContext) -> Result<Value, RuntimeError> { let name = match args.first() { Some(Value::String(s)) => s.to_lowercase(), Some(v) => v.to_js_string(ctx.gc).to_lowercase(), None => return Ok(Value::Null), }; let obj_ref = match ctx.this.gc_ref() { Some(r) => r, None => return Ok(Value::Null), }; let entries = read_header_entries(ctx.gc, obj_ref); for (n, v) in entries { if n == name { return Ok(Value::String(v)); } } Ok(Value::Null)}
/// `headers.has(name)` — case-insensitive check.fn headers_has(args: &[Value], ctx: &mut NativeContext) -> Result<Value, RuntimeError> { let name = match args.first() { Some(Value::String(s)) => s.to_lowercase(), Some(v) => v.to_js_string(ctx.gc).to_lowercase(), None => return Ok(Value::Boolean(false)), }; let obj_ref = match ctx.this.gc_ref() { Some(r) => r, None => return Ok(Value::Boolean(false)), }; let entries = read_header_entries(ctx.gc, obj_ref); Ok(Value::Boolean(entries.iter().any(|(n, _)| n == &name)))}
/// `headers.set(name, value)` — set or replace a header.fn headers_set(args: &[Value], ctx: &mut NativeContext) -> Result<Value, RuntimeError> { let name = match args.first() { Some(Value::String(s)) => s.to_lowercase(), Some(v) => v.to_js_string(ctx.gc).to_lowercase(), None => return Ok(Value::Undefined), }; let value = match args.get(1) { Some(Value::String(s)) => s.clone(), Some(v) => v.to_js_string(ctx.gc), None => String::new(), }; let obj_ref = match ctx.this.gc_ref() { Some(r) => r, None => return Ok(Value::Undefined), }; let mut entries = read_header_entries(ctx.gc, obj_ref); let mut found = false; for entry in &mut entries { if entry.0 == name { entry.1 = value.clone(); found = true; break; } } if !found { entries.push((name, value)); } write_header_entries(ctx.gc, obj_ref, &entries); Ok(Value::Undefined)}
/// `headers.delete(name)` — remove a header.fn headers_delete(args: &[Value], ctx: &mut NativeContext) -> Result<Value, RuntimeError> { let name = match args.first() { Some(Value::String(s)) => s.to_lowercase(), Some(v) => v.to_js_string(ctx.gc).to_lowercase(), None => return Ok(Value::Undefined), }; let obj_ref = match ctx.this.gc_ref() { Some(r) => r, None => return Ok(Value::Undefined), }; let mut entries = read_header_entries(ctx.gc, obj_ref); entries.retain(|(n, _)| n != &name); write_header_entries(ctx.gc, obj_ref, &entries); Ok(Value::Undefined)}
/// Rewrite header entries into a Headers object.fn write_header_entries(gc: &mut Gc<HeapObject>, obj_ref: GcRef, entries: &[(String, String)]) { if let Some(HeapObject::Object(data)) = gc.get_mut(obj_ref) { // Remove old header entries. data.properties.retain(|k, _| !k.starts_with("__hdr_"));
// Write new entries. data.properties.insert( "__hdr_count__".to_string(), Property::builtin(Value::Number(entries.len() as f64)), ); for (i, (name, value)) in entries.iter().enumerate() { data.properties.insert( format!("__hdr_{i}_n__"), Property::builtin(Value::String(name.clone())), ); data.properties.insert( format!("__hdr_{i}_v__"), Property::builtin(Value::String(value.clone())), ); } }}
// ── Registration ────────────────────────────────────────────────
/// Register the `fetch` global function in the VM.pub fn init_fetch_api(vm: &mut Vm) { let fetch_fn = make_native(&mut vm.gc, "fetch", fetch_native); vm.set_global("fetch", Value::Function(fetch_fn));}
// ── Tests ───────────────────────────────────────────────────────
#[cfg(test)]mod tests { use super::*; use crate::builtins; use crate::compiler; use crate::parser::Parser; use std::cell::RefCell; use std::rc::Rc;
struct CapturedConsole { log_messages: RefCell<Vec<String>>, }
impl CapturedConsole { fn new() -> Self { Self { log_messages: RefCell::new(Vec::new()), } } }
impl ConsoleOutput for CapturedConsole { fn log(&self, message: &str) { self.log_messages.borrow_mut().push(message.to_string()); } fn error(&self, _message: &str) {} fn warn(&self, _message: &str) {} }
struct RcConsole(Rc<CapturedConsole>);
impl ConsoleOutput for RcConsole { fn log(&self, message: &str) { self.0.log(message); } fn error(&self, message: &str) { self.0.error(message); } fn warn(&self, message: &str) { self.0.warn(message); } }
#[test] fn test_fetch_returns_promise() { reset_fetch_state(); crate::timers::reset_timers();
let source = r#"typeof fetch"#; let program = Parser::parse(source).expect("parse failed"); let func = compiler::compile(&program).expect("compile failed"); let mut vm = Vm::new(); init_fetch_api(&mut vm); let result = vm.execute(&func).expect("execute failed"); assert_eq!(result.to_js_string(&vm.gc), "function"); }
#[test] fn test_fetch_returns_promise_object() { reset_fetch_state(); crate::timers::reset_timers();
// fetch() should return something (a promise) even if the URL is bogus. // We can't easily test network fetches in unit tests, so we test the // synchronous parts: argument parsing, promise creation. let source = r#" var p = fetch("http://0.0.0.0:1/nonexistent"); typeof p "#; let program = Parser::parse(source).expect("parse failed"); let func = compiler::compile(&program).expect("compile failed"); let mut vm = Vm::new(); init_fetch_api(&mut vm); let result = vm.execute(&func).expect("execute failed"); assert_eq!(result.to_js_string(&vm.gc), "object"); }
#[test] fn test_fetch_no_args_error() { reset_fetch_state(); crate::timers::reset_timers();
let source = r#" try { fetch(); "no error"; } catch(e) { "caught: " + e.message; } "#; let program = Parser::parse(source).expect("parse failed"); let func = compiler::compile(&program).expect("compile failed"); let mut vm = Vm::new(); init_fetch_api(&mut vm); let result = vm.execute(&func).expect("execute failed"); let s = result.to_js_string(&vm.gc); assert!( s.contains("fetch requires"), "expected error about missing args, got: {s}" ); }
/// Execute JS with fetch API, pump event loop, return console logs. fn eval_with_fetch(source: &str, max_iterations: usize) -> Vec<String> { reset_fetch_state(); crate::timers::reset_timers(); let console = Rc::new(CapturedConsole::new()); let program = Parser::parse(source).expect("parse failed"); let func = compiler::compile(&program).expect("compile failed"); let mut vm = Vm::new(); init_fetch_api(&mut vm); vm.set_console_output(Box::new(RcConsole(console.clone()))); vm.execute(&func).expect("execute failed"); vm.run_event_loop(max_iterations) .expect("event loop failed"); let result = console.log_messages.borrow().clone(); result }
#[test] fn test_fetch_rejects_on_network_error() { // Connect to a port that almost certainly won't be listening. let logs = eval_with_fetch( r#" fetch("http://127.0.0.1:1/fail").then( function(resp) { console.log("resolved: " + resp.status); }, function(err) { console.log("rejected"); } ); "#, 200, ); assert_eq!(logs, vec!["rejected"]); }
#[test] fn test_response_object_properties() { // Test that create_response_object creates the right structure. let mut gc = crate::gc::Gc::new(); builtins::init_promise_proto_for_test(&mut gc);
let result = FetchResult { status: 200, status_text: "OK".to_string(), headers: vec![ ("Content-Type".to_string(), "text/plain".to_string()), ("X-Custom".to_string(), "hello".to_string()), ], body: b"hello world".to_vec(), url: "http://example.com/".to_string(), };
let resp_ref = create_response_object(&mut gc, &result); let resp_obj = gc.get(resp_ref).unwrap(); if let HeapObject::Object(data) = resp_obj { // Check status assert!(matches!( data.properties.get("status").map(|p| &p.value), Some(Value::Number(n)) if *n == 200.0 )); // Check ok assert!(matches!( data.properties.get("ok").map(|p| &p.value), Some(Value::Boolean(true)) )); // Check statusText assert!(matches!( data.properties.get("statusText").map(|p| &p.value), Some(Value::String(s)) if s == "OK" )); // Check url assert!(matches!( data.properties.get("url").map(|p| &p.value), Some(Value::String(s)) if s == "http://example.com/" )); } else { panic!("expected Object"); } }
#[test] fn test_headers_get_case_insensitive() { let mut gc = crate::gc::Gc::new(); let headers = vec![ ("Content-Type".to_string(), "text/html".to_string()), ("X-Foo".to_string(), "bar".to_string()), ]; let hdr_ref = create_headers_object(&mut gc, &headers); let entries = read_header_entries(&gc, hdr_ref); assert_eq!(entries.len(), 2); // All stored lowercased assert!(entries .iter() .any(|(n, v)| n == "content-type" && v == "text/html")); assert!(entries.iter().any(|(n, v)| n == "x-foo" && v == "bar")); }
#[test] fn test_headers_duplicate_combining() { let mut gc = crate::gc::Gc::new(); let headers = vec![ ("Set-Cookie".to_string(), "a=1".to_string()), ("Set-Cookie".to_string(), "b=2".to_string()), ]; let hdr_ref = create_headers_object(&mut gc, &headers); let entries = read_header_entries(&gc, hdr_ref); assert_eq!(entries.len(), 1); assert_eq!(entries[0].0, "set-cookie"); assert_eq!(entries[0].1, "a=1, b=2"); }}