diff --git a/uplc/src/hosts.rs b/uplc/src/hosts.rs index aa6ab00..def7f17 100644 --- a/uplc/src/hosts.rs +++ b/uplc/src/hosts.rs @@ -54,6 +54,8 @@ pub async fn run_spam_classifier(db: DB) -> Result<()> { } } +/// Check whether a host is clearly not going to be a valid pds purely based +/// on analysing the url pub fn is_clear_spam(url: &Url) -> bool { // Non http(s) urls are spam if !matches!(url.scheme(), "http" | "https") { @@ -67,6 +69,8 @@ pub fn is_clear_spam(url: &Url) -> bool { || matches!(host, "localhost" | "127.0.0.1" | "::1" | "0.0.0.0") } +/// Check whether the host is a valid pds by trying to make a +/// com.atproto.server.describeServer xrpc request async fn is_real_pds(host: &str, client: &reqwest::Client) -> bool { let mut url = match Url::parse(&format!("https://{}", host)) { Ok(url) => url, @@ -162,10 +166,12 @@ impl DB { Ok(()) } - pub fn upsert_service_hosts( + /// Insert new pds/service urls to be automatically classified as spam or not + /// Takes in the current spam hosts from the database to make the insertion faster + pub fn upsert_pds_hosts( &self, services: &Vec<&String>, - spam_hosts: &HashSet, + current_spam_hosts: &HashSet, ) -> Result<()> { let db = self.handle()?; let mut stmt = @@ -179,7 +185,7 @@ impl DB { .not() .then_some(url.host_str().map(|host| host.to_string())?) }) - .filter(|host| !spam_hosts.contains(host)) + .filter(|host| !current_spam_hosts.contains(host)) .collect::>(); info!("upserting {} hosts", hosts.len()); diff --git a/uplc/src/plc.rs b/uplc/src/plc.rs index 9eacbb4..24beb33 100644 --- a/uplc/src/plc.rs +++ b/uplc/src/plc.rs @@ -136,7 +136,7 @@ pub async fn run_plc_sync(db: DB, stats: Arc>) -> Result<()> { .filter_map(|op| op.service.as_ref()) .collect::>(); - db.upsert_service_hosts(&hosts, &stats.blocking_read().spam_hosts)?; + db.upsert_pds_hosts(&hosts, &stats.blocking_read().spam_hosts)?; db.upsert_operations(ops) } }) diff --git a/uplc/src/stats.rs b/uplc/src/stats.rs index f8027cf..0926439 100644 --- a/uplc/src/stats.rs +++ b/uplc/src/stats.rs @@ -113,11 +113,11 @@ impl Stats { .query_map([], |row| { Ok(( row.get::<_, i64>(0)?, // total_ops - row.get::<_, Option>(1)?, // last_op + row.get::<_, Option>(1)?, // last_op row.get::<_, i64>(2)?, // dids - row.get::<_, Option>(3)?, // deactivated - row.get::<_, Option>(4)?, // pds - row.get::<_, Option>(5)?, // other + row.get::<_, Option>(3)?, // deactivated + row.get::<_, Option>(4)?, // pds + row.get::<_, Option>(5)?, // other row.get::<_, Option>(6)?, // service row.get::<_, Option>(7)?, // count )) @@ -138,9 +138,10 @@ impl Stats { } info!("getting hosts"); - // this doesn't do a lot because it's just the spam hosts - // although I guess since a good proportion of the hosts are spam - // it still helps later + // this doesn't do a lot on startup because it's just the spam hosts + // which means that it still tries to insert thousands of hosts unnecessarily + // but it's still worth reading this here for api efficiency, and there's a lot + // of spam so it does save some time/resources self.spam_hosts = db.get_spam_hosts()?; info!("found {} spam hosts, upserting", self.spam_hosts.len()); @@ -149,7 +150,7 @@ impl Stats { .filter_map(|(_, _, _, _, _, _, service, _)| service.as_ref()) .collect::>(); - db.upsert_service_hosts(&hosts, &self.spam_hosts)?; + db.upsert_pds_hosts(&hosts, &self.spam_hosts)?; info!("finishing stats mapping");