bring over latest vorebot tweaks #4

Merged
husky merged 2 commits from feature/nikocs/vorebot-tweaks into develop 2025-03-14 18:03:29 -07:00
Showing only changes of commit c754338bf4 - Show all commits

View file

@ -67,7 +67,7 @@ pub async fn web_parse(
.create_key_value(kv::Config { .create_key_value(kv::Config {
bucket: "hosts".to_string(), bucket: "hosts".to_string(),
description: "prevent the same host from being scraped too quickly".to_string(), description: "prevent the same host from being scraped too quickly".to_string(),
max_age: Duration::from_secs(60 * 5), max_age: Duration::from_secs(60 * 10),
..Default::default() ..Default::default()
}) })
.await; .await;
@ -93,7 +93,7 @@ pub async fn web_parse(
if let Ok(Some(host)) = hosts_bucket.get(hash.to_string()).await { if let Ok(Some(host)) = hosts_bucket.get(hash.to_string()).await {
let count = *host.first().unwrap_or(&0); let count = *host.first().unwrap_or(&0);
if count > 100 { if count > 10 {
warn!("scraping {} too quickly, avoiding for one minute", robots_url); warn!("scraping {} too quickly, avoiding for one minute", robots_url);
return Err(()); return Err(());
} }