diff --git a/vorebot/src/webparse/mod.rs b/vorebot/src/webparse/mod.rs index d6d6580..ba84f74 100644 --- a/vorebot/src/webparse/mod.rs +++ b/vorebot/src/webparse/mod.rs @@ -67,7 +67,7 @@ pub async fn web_parse( .create_key_value(kv::Config { bucket: "hosts".to_string(), description: "prevent the same host from being scraped too quickly".to_string(), - max_age: Duration::from_secs(60 * 5), + max_age: Duration::from_secs(60 * 10), ..Default::default() }) .await; @@ -93,7 +93,7 @@ pub async fn web_parse( if let Ok(Some(host)) = hosts_bucket.get(hash.to_string()).await { let count = *host.first().unwrap_or(&0); - if count > 100 { + if count > 10 { warn!("scraping {} too quickly, avoiding for one minute", robots_url); return Err(()); }