Something went wrong. Try again.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221use crate::cache::Validators;use mlua::Lua;use reqwest::StatusCode;use reqwest::header::{ETAG, IF_MODIFIED_SINCE, IF_NONE_MATCH, LAST_MODIFIED};use std::sync::LazyLock;
static CLIENT: LazyLock<reqwest::blocking::Client> = LazyLock::new(|| { reqwest::blocking::Client::builder() .user_agent(concat!("luafeed/", env!("CARGO_PKG_VERSION"))) .build() .expect("building the HTTP client should not fail")});
pub enum FetchResult { /// The server sent a fresh body, along with whatever validators it /// attached for next time (either may be absent, or unchanged from what /// was sent). Modified { body: String, validators: Validators, }, /// The server confirmed (via 304) that `validators` still describes the /// page: the caller's cached copy is still current and there's nothing /// new to parse. NotModified,}
/// Fetches `url`, sending `validators` as `If-None-Match`/`If-Modified-Since`/// so a server that supports conditional requests can answer with a cheap/// 304 instead of resending a page that hasn't changed.pub fn fetch(url: &str, validators: &Validators) -> Result<FetchResult, reqwest::Error> { let mut request = CLIENT.get(url); if let Some(etag) = &validators.etag { request = request.header(IF_NONE_MATCH, etag); } if let Some(last_modified) = &validators.last_modified { request = request.header(IF_MODIFIED_SINCE, last_modified); }
let response = request.send()?; if response.status() == StatusCode::NOT_MODIFIED { return Ok(FetchResult::NotModified); }
let response = response.error_for_status()?; let header_str = |name| { response .headers() .get(name) .and_then(|v| v.to_str().ok()) .map(str::to_string) }; let validators = Validators { etag: header_str(ETAG), last_modified: header_str(LAST_MODIFIED), }; let body = response.text()?; Ok(FetchResult::Modified { body, validators })}
/// Fetches without treating any outcome as fatal: a non-2xx status, or a/// connection failure that produces no response at all (reported as status/// 0), are both handed back rather than erroring, since the caller already/// has to decide what to do with an unsuccessful status regardless.fn fetch_status(url: &str) -> (u16, String) { match CLIENT.get(url).send() { Ok(response) => { let status = response.status().as_u16(); let body = response.text().unwrap_or_default(); (status, body) } Err(_) => (0, String::new()), }}
/// Registers `fetch(url) -> {status=, body=}` as a Lua global, letting a/// method script fetch additional pages itself -- an article's own page for/// its full body, say -- beyond the one page luafeed already fetched for it.pub fn register(lua: &Lua) -> mlua::Result<()> { let fetch_fn = lua.create_function(|lua, url: String| { let (status, body) = fetch_status(&url); let table = lua.create_table()?; table.set("status", status)?; table.set("body", body)?; Ok(table) })?; lua.globals().set("fetch", fetch_fn)}
#[cfg(test)]mod tests { use super::*; use std::io::{Read, Write}; use std::net::TcpListener;
fn serve_once(response: &'static str) -> String { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let port = listener.local_addr().unwrap().port(); std::thread::spawn(move || { let (mut stream, _) = listener.accept().unwrap(); let mut buf = [0u8; 1024]; let _ = stream.read(&mut buf); stream.write_all(response.as_bytes()).unwrap(); }); format!("http://127.0.0.1:{port}/") }
/// Like `serve_once`, but also hands back everything the client sent, /// lowercased, so a test can assert on which headers were present. fn serve_once_capturing(response: &'static str) -> (String, std::sync::mpsc::Receiver<String>) { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let port = listener.local_addr().unwrap().port(); let (tx, rx) = std::sync::mpsc::channel(); std::thread::spawn(move || { let (mut stream, _) = listener.accept().unwrap(); let mut buf = [0u8; 4096]; let n = stream.read(&mut buf).unwrap_or(0); let _ = tx.send(String::from_utf8_lossy(&buf[..n]).to_lowercase()); stream.write_all(response.as_bytes()).unwrap(); }); (format!("http://127.0.0.1:{port}/"), rx) }
#[test] fn fetch_sends_validators_as_conditional_headers() { let (url, requests) = serve_once_capturing("HTTP/1.1 304 Not Modified\r\nConnection: close\r\n\r\n"); let validators = Validators { etag: Some("\"abc\"".to_string()), last_modified: Some("Sun, 06 Sep 2026 00:00:00 GMT".to_string()), };
let result = fetch(&url, &validators).unwrap();
assert!(matches!(result, FetchResult::NotModified)); let request = requests.recv().unwrap(); assert!(request.contains("if-none-match: \"abc\"")); assert!(request.contains("if-modified-since: sun, 06 sep 2026 00:00:00 gmt")); }
#[test] fn fetch_omits_conditional_headers_when_no_validators_are_cached() { let (url, requests) = serve_once_capturing( "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok", );
fetch(&url, &Validators::default()).unwrap();
let request = requests.recv().unwrap(); assert!(!request.contains("if-none-match")); assert!(!request.contains("if-modified-since")); }
#[test] fn fetch_returns_modified_body_and_new_validators_on_200() { let url = serve_once(concat!( "HTTP/1.1 200 OK\r\n", "ETag: \"xyz\"\r\n", "Last-Modified: Mon, 07 Sep 2026 00:00:00 GMT\r\n", "Content-Length: 5\r\n", "Connection: close\r\n\r\n", "hello" ));
let result = fetch(&url, &Validators::default()).unwrap();
match result { FetchResult::Modified { body, validators } => { assert_eq!(body, "hello"); assert_eq!(validators.etag.as_deref(), Some("\"xyz\"")); assert_eq!( validators.last_modified.as_deref(), Some("Mon, 07 Sep 2026 00:00:00 GMT") ); } FetchResult::NotModified => panic!("expected a Modified result"), } }
#[test] fn fetch_status_returns_status_and_body() { let url = serve_once("HTTP/1.1 200 OK\r\nContent-Length: 5\r\nConnection: close\r\n\r\nhello"); assert_eq!(fetch_status(&url), (200, "hello".to_string())); }
#[test] fn fetch_status_reports_non_2xx_without_erroring() { let url = serve_once("HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"); assert_eq!(fetch_status(&url), (404, String::new())); }
#[test] fn fetch_status_reports_zero_on_connection_failure() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let port = listener.local_addr().unwrap().port(); drop(listener); // nothing is listening on this port now
assert_eq!( fetch_status(&format!("http://127.0.0.1:{port}/")), (0, String::new()) ); }
#[test] fn lua_fetch_exposes_status_and_body_table() { let url = serve_once("HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok"); let lua = Lua::new(); register(&lua).unwrap(); lua.globals().set("target", url).unwrap();
let (status, body): (u16, String) = lua .load("local r = fetch(target); return r.status, r.body") .eval() .unwrap(); assert_eq!(status, 200); assert_eq!(body, "ok"); }}