87 lines
No EOL
2.5 KiB
Rust
87 lines
No EOL
2.5 KiB
Rust
/*
|
|
* asklyphe-common nats/injest.rs
|
|
* - nats structs for crawler injest service
|
|
*
|
|
* Copyright (C) 2025 Real Microsoft, LLC
|
|
*
|
|
* This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, version 3.
|
|
*
|
|
* This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU Affero General Public License along with this program. If not, see <https://www.gnu.org/licenses/>.
|
|
*/
|
|
|
|
use log::error;
|
|
use serde::{Deserialize, Serialize};
|
|
pub const INJEST_SERVICE: &str = "injest";
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub enum InjestRequest {
|
|
WebsiteCreateRequest(WebsiteCreateRequest),
|
|
WebsiteLastParseFinish(WebsiteLastParseFinish),
|
|
WebsiteRobotsFlag(WebsiteRobotsFlag),
|
|
SetMetawords(SetMetawords),
|
|
SetWords(SetWords),
|
|
SetLinks(SetLinks),
|
|
SetInfo(SetInfo),
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct WebsiteCreateRequest {
|
|
pub url: String,
|
|
pub last_parse_begin: i64,
|
|
pub damping: f64,
|
|
}
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct SetInfo {
|
|
pub url: String,
|
|
pub title: String,
|
|
pub description: String,
|
|
pub favicon_hash: i64,
|
|
}
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct WebsiteLastParseFinish {
|
|
pub url: String,
|
|
pub last_parse_finish: i64,
|
|
pub last_parse_finish_true: i64,
|
|
}
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct WebsiteRobotsFlag {
|
|
pub url: String,
|
|
pub robots: bool,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct Metaword {
|
|
pub word: String,
|
|
pub property: String,
|
|
pub encoded: Vec<u8>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct SetMetawords {
|
|
pub url: String,
|
|
pub metawords: Vec<Metaword>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct Word {
|
|
pub word: String,
|
|
pub encoded: Vec<u8>,
|
|
}
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct SetWords {
|
|
pub url: String,
|
|
pub words: Vec<Word>,
|
|
}
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct Link {
|
|
pub from: String,
|
|
pub to: String,
|
|
pub flags: u8,
|
|
pub words: Vec<(String, Vec<u8>)>
|
|
}
|
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
|
pub struct SetLinks {
|
|
pub links: Vec<Link>,
|
|
} |