bring over latest vorebot tweaks #4
1 changed files with 2 additions and 2 deletions
|
@ -67,7 +67,7 @@ pub async fn web_parse(
|
||||||
.create_key_value(kv::Config {
|
.create_key_value(kv::Config {
|
||||||
bucket: "hosts".to_string(),
|
bucket: "hosts".to_string(),
|
||||||
description: "prevent the same host from being scraped too quickly".to_string(),
|
description: "prevent the same host from being scraped too quickly".to_string(),
|
||||||
max_age: Duration::from_secs(60 * 5),
|
max_age: Duration::from_secs(60 * 10),
|
||||||
..Default::default()
|
..Default::default()
|
||||||
})
|
})
|
||||||
.await;
|
.await;
|
||||||
|
@ -93,7 +93,7 @@ pub async fn web_parse(
|
||||||
|
|
||||||
if let Ok(Some(host)) = hosts_bucket.get(hash.to_string()).await {
|
if let Ok(Some(host)) = hosts_bucket.get(hash.to_string()).await {
|
||||||
let count = *host.first().unwrap_or(&0);
|
let count = *host.first().unwrap_or(&0);
|
||||||
if count > 100 {
|
if count > 10 {
|
||||||
warn!("scraping {} too quickly, avoiding for one minute", robots_url);
|
warn!("scraping {} too quickly, avoiding for one minute", robots_url);
|
||||||
return Err(());
|
return Err(());
|
||||||
}
|
}
|
||||||
|
|
Loading…
Add table
Reference in a new issue