From 109e20c7b47a9b4ed181f526ba28225db2037821 Mon Sep 17 00:00:00 2001 From: Book-reader Date: Thu, 8 May 2025 15:03:00 +1200 Subject: [PATCH 01/15] initial mostly working thing --- asklyphe-frontend/src/bangs.rs | 106 +++++++++++++++++++++++++ asklyphe-frontend/src/main.rs | 1 + asklyphe-frontend/src/routes/search.rs | 6 ++ 3 files changed, 113 insertions(+) create mode 100644 asklyphe-frontend/src/bangs.rs diff --git a/asklyphe-frontend/src/bangs.rs b/asklyphe-frontend/src/bangs.rs new file mode 100644 index 0000000..5055223 --- /dev/null +++ b/asklyphe-frontend/src/bangs.rs @@ -0,0 +1,106 @@ +use tracing::{debug, error}; +use once_cell::sync::Lazy; +use std::collections::BTreeMap; + +pub static BANG_PREFIX: &str = "!"; + +pub static BUILTIN_BANGS: Lazy> = Lazy::new(|| { + let mut bangs = BTreeMap::new(); + bangs.insert("Google", ("https://google.com/search?q={}", &["g", "google"] as &[&str])); + + bangs.insert("DuckDuckGo", ("https://duckduckgo.com/?q={}", &["d", "ddg", "duckduckgo"] as &[&str])); + + bangs.insert("Wikipedia", ("https://wikipedia.org/w/index.php?search={}", &["w", "wiki", "wikipedia"] as &[&str])); + bangs +}); + +enum Bangs { + Google, + DuckDuckGo, + Wikipedia, +} + +impl Bangs { + pub fn get_suffix(self) -> &'static [&'static str] { + match self { + Bangs::Google => &["g", "google"], + Bangs::DuckDuckGo => &["d", "ddg", "duckduckgo"], + Bangs::Wikipedia => &["w", "wiki", "wikipedia"], + } + } + + pub fn get_url_format(self) -> &'static str { + match self { + Bangs::Google => "https://google.com/search?q={}", + Bangs::DuckDuckGo => "https://duckduckgo.com/?q={}", + Bangs::Wikipedia => "https://wikipedia.org/w/index.php?search={}", + } + } + +} + +pub fn do_bangs_or_whatever(query: &String) -> Option { + #[derive(Debug, Clone)] + struct Bang<'a> { + pub name: &'a str, + pub start_idx: usize, + pub end_idx: usize, + } + impl<'a> Bang<'_> { + fn init(name: &'a str, start_idx: usize, end_idx: usize) -> Bang { + Bang {name, start_idx, end_idx} + } + } + error!("Hello, World!, you searched \"{}\"", query); + return match query.match_indices(BANG_PREFIX).map(|idx| { + let bang_start_idx = idx.0; + match bang_start_idx == 0 || query.chars().nth(bang_start_idx - 1).unwrap().is_ascii_whitespace() { + true => match query.get(bang_start_idx + 1 .. query.len()) { + Some(rest) => + { + debug!("rest is {}", rest); + match BUILTIN_BANGS.iter().map( + |bang_full| + match bang_full.1.1.iter() + .filter(|short| rest.starts_with(**short)) + .max_by(|a, b| a.len().cmp(&b.len())) { + Some(bang_text) => { + let bang_end_idx = bang_start_idx + 1 + bang_text.len(); + match query.chars().nth(bang_end_idx).unwrap_or(' ').is_ascii_whitespace() { + true => Some(Bang::init(bang_full.1.0, bang_start_idx, bang_end_idx)), + false => None + } + } + None => None, + } + ).filter(|i| i.is_some()).map(|bang| bang.unwrap()) + .collect::>().last() + { + Some(bang) => { + debug!("got:'{:?}'", bang); + Some(bang.clone()) + }, + None => None + } + }, + None => None + } + false => None + } + }).filter(|bang| bang.is_some()).map(|bang| bang.unwrap()).collect::>().first() { + Some(bang) => { + let query_trimmed = format!( + "{}{}", + query.get(0..bang.start_idx).unwrap(), + if bang.end_idx == query.len() { + "" + } else { + query.get(bang.end_idx..query.len()).unwrap() + }); + debug!("Old query: \"{}\"", query); + debug!("New query: \"{}\"", query_trimmed); + None + }, + None => None + } +} diff --git a/asklyphe-frontend/src/main.rs b/asklyphe-frontend/src/main.rs index 2b87121..e99fe51 100644 --- a/asklyphe-frontend/src/main.rs +++ b/asklyphe-frontend/src/main.rs @@ -14,6 +14,7 @@ pub mod searchbot; pub mod wikipedia; pub mod unit_converter; +pub mod bangs; pub mod routes; use std::{env, process}; diff --git a/asklyphe-frontend/src/routes/search.rs b/asklyphe-frontend/src/routes/search.rs index f1d43b7..78172c5 100644 --- a/asklyphe-frontend/src/routes/search.rs +++ b/asklyphe-frontend/src/routes/search.rs @@ -18,6 +18,7 @@ use crate::unit_converter; use crate::unit_converter::UnitConversion; use crate::wikipedia::WikipediaSummary; use crate::{wikipedia, Opts, ALPHA, BUILT_ON, GIT_COMMIT, VERSION, YEAR}; +use crate::bangs; use askama::Template; use asklyphe_common::nats; use asklyphe_common::nats::bingservice::{ @@ -170,6 +171,10 @@ pub async fn search_js( unit_query = unit_query.replace("metre", "meter"); let unit_comp = unit_converter::convert_unit(&unit_query); complications.unit_converter = unit_comp; + /*if let Some(bang, result_query) = bangs::do_bangs_or_whatever(&query) { + error!("Would redirect to '{}', but it's not implemented", format!(bang, result_query)); + }*/ + let _ = bangs::do_bangs_or_whatever(&query); } else { complications.disabled = true; query = query.replace("-complications", ""); @@ -282,6 +287,7 @@ pub async fn search_nojs( unit_query = unit_query.replace("metre", "meter"); let unit_comp = unit_converter::convert_unit(&unit_query); complications.unit_converter = unit_comp; + let bang = bangs::do_bangs_or_whatever(&query); } else { complications.disabled = true; query = query.replace("-complications", ""); From 245744a3175b222e78c1d9d5fa716c80778635a8 Mon Sep 17 00:00:00 2001 From: Book-reader Date: Thu, 8 May 2025 15:46:18 +1200 Subject: [PATCH 02/15] properly format bang urls --- asklyphe-frontend/src/bangs.rs | 53 +++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/asklyphe-frontend/src/bangs.rs b/asklyphe-frontend/src/bangs.rs index 5055223..8307961 100644 --- a/asklyphe-frontend/src/bangs.rs +++ b/asklyphe-frontend/src/bangs.rs @@ -39,18 +39,20 @@ impl Bangs { } +#[derive(Debug, Clone)] +struct BangLoc<'a> { + pub url: &'a str, + pub start_idx: usize, + pub end_idx: usize, +} +impl<'a> BangLoc<'_> { + fn init(url: &'a str, start_idx: usize, end_idx: usize) -> BangLoc { + BangLoc {url, start_idx, end_idx} + } +} + pub fn do_bangs_or_whatever(query: &String) -> Option { - #[derive(Debug, Clone)] - struct Bang<'a> { - pub name: &'a str, - pub start_idx: usize, - pub end_idx: usize, - } - impl<'a> Bang<'_> { - fn init(name: &'a str, start_idx: usize, end_idx: usize) -> Bang { - Bang {name, start_idx, end_idx} - } - } + // TODO: make this a little more procedural and not as confusing error!("Hello, World!, you searched \"{}\"", query); return match query.match_indices(BANG_PREFIX).map(|idx| { let bang_start_idx = idx.0; @@ -67,7 +69,7 @@ pub fn do_bangs_or_whatever(query: &String) -> Option { Some(bang_text) => { let bang_end_idx = bang_start_idx + 1 + bang_text.len(); match query.chars().nth(bang_end_idx).unwrap_or(' ').is_ascii_whitespace() { - true => Some(Bang::init(bang_full.1.0, bang_start_idx, bang_end_idx)), + true => Some(BangLoc::init(bang_full.1.0, bang_start_idx, bang_end_idx + 1)), false => None } } @@ -89,17 +91,22 @@ pub fn do_bangs_or_whatever(query: &String) -> Option { } }).filter(|bang| bang.is_some()).map(|bang| bang.unwrap()).collect::>().first() { Some(bang) => { - let query_trimmed = format!( - "{}{}", - query.get(0..bang.start_idx).unwrap(), - if bang.end_idx == query.len() { - "" - } else { - query.get(bang.end_idx..query.len()).unwrap() - }); - debug!("Old query: \"{}\"", query); - debug!("New query: \"{}\"", query_trimmed); - None + debug!("Initial query: \"{}\"", query); + let query_split = query.split_once(query.get(bang.start_idx..bang.end_idx).unwrap()).unwrap(); + debug!("Split query: {:?}", query_split); + let query_trimmed = format!("{}{}", query_split.0, query_split.1); + debug!("Trimmed query: \"{}\"", query_trimmed); + + let bang_url_split = bang.url.split_once("{}").unwrap(); + debug!("Split bang URL: {:?}", bang_url_split); + let bang_url = format!( + "{}{}{}", + bang_url_split.0, + query_trimmed, + bang_url_split.1 + ); + debug!("Final URL: \"{}\"", bang_url); + Some(bang_url) }, None => None } From bac21898c9636695ad27b6ce786d454814239591 Mon Sep 17 00:00:00 2001 From: Book-reader Date: Thu, 8 May 2025 18:40:38 +1200 Subject: [PATCH 03/15] just make the bangs better --- asklyphe-frontend/src/bangs.rs | 169 +++++++++++-------------- asklyphe-frontend/src/routes/search.rs | 5 +- 2 files changed, 81 insertions(+), 93 deletions(-) diff --git a/asklyphe-frontend/src/bangs.rs b/asklyphe-frontend/src/bangs.rs index 8307961..83eb1e8 100644 --- a/asklyphe-frontend/src/bangs.rs +++ b/asklyphe-frontend/src/bangs.rs @@ -1,113 +1,98 @@ use tracing::{debug, error}; use once_cell::sync::Lazy; use std::collections::BTreeMap; +use url_encoded_data; pub static BANG_PREFIX: &str = "!"; -pub static BUILTIN_BANGS: Lazy> = Lazy::new(|| { +#[derive(Debug)] +struct Bang<'a> { + pub url: &'a str, + pub keys: &'a [&'a str] +} + +impl<'a> Bang<'_> { + fn new(url: &'a str, keys: &'a [&'a str]) -> Bang<'a> { + Bang {url, keys} + } +} + +static BUILTIN_BANGS: Lazy> = Lazy::new(|| { let mut bangs = BTreeMap::new(); - bangs.insert("Google", ("https://google.com/search?q={}", &["g", "google"] as &[&str])); + bangs.insert("Google", Bang::new("https://google.com/search?q={}", &["g", "google"] as &[&str])); - bangs.insert("DuckDuckGo", ("https://duckduckgo.com/?q={}", &["d", "ddg", "duckduckgo"] as &[&str])); + bangs.insert("DuckDuckGo", Bang::new("https://duckduckgo.com/?q={}", &["d", "ddg", "duckduckgo"] as &[&str])); - bangs.insert("Wikipedia", ("https://wikipedia.org/w/index.php?search={}", &["w", "wiki", "wikipedia"] as &[&str])); + bangs.insert("Wikipedia", Bang::new("https://wikipedia.org/w/index.php?search={}", &["w", "wiki", "wikipedia"] as &[&str])); bangs }); -enum Bangs { - Google, - DuckDuckGo, - Wikipedia, -} - -impl Bangs { - pub fn get_suffix(self) -> &'static [&'static str] { - match self { - Bangs::Google => &["g", "google"], - Bangs::DuckDuckGo => &["d", "ddg", "duckduckgo"], - Bangs::Wikipedia => &["w", "wiki", "wikipedia"], - } - } - - pub fn get_url_format(self) -> &'static str { - match self { - Bangs::Google => "https://google.com/search?q={}", - Bangs::DuckDuckGo => "https://duckduckgo.com/?q={}", - Bangs::Wikipedia => "https://wikipedia.org/w/index.php?search={}", - } - } - -} - #[derive(Debug, Clone)] -struct BangLoc<'a> { - pub url: &'a str, +struct BangLoc<'b> { + pub url: &'b str, pub start_idx: usize, - pub end_idx: usize, + pub len: usize } -impl<'a> BangLoc<'_> { - fn init(url: &'a str, start_idx: usize, end_idx: usize) -> BangLoc { - BangLoc {url, start_idx, end_idx} + +impl<'b> BangLoc<'_> { + fn new(url: &'b str, start_idx: usize, len: usize) -> BangLoc<'b> { + BangLoc {url, start_idx, len} } } -pub fn do_bangs_or_whatever(query: &String) -> Option { - // TODO: make this a little more procedural and not as confusing - error!("Hello, World!, you searched \"{}\"", query); - return match query.match_indices(BANG_PREFIX).map(|idx| { - let bang_start_idx = idx.0; - match bang_start_idx == 0 || query.chars().nth(bang_start_idx - 1).unwrap().is_ascii_whitespace() { - true => match query.get(bang_start_idx + 1 .. query.len()) { - Some(rest) => - { - debug!("rest is {}", rest); - match BUILTIN_BANGS.iter().map( - |bang_full| - match bang_full.1.1.iter() - .filter(|short| rest.starts_with(**short)) - .max_by(|a, b| a.len().cmp(&b.len())) { - Some(bang_text) => { - let bang_end_idx = bang_start_idx + 1 + bang_text.len(); - match query.chars().nth(bang_end_idx).unwrap_or(' ').is_ascii_whitespace() { - true => Some(BangLoc::init(bang_full.1.0, bang_start_idx, bang_end_idx + 1)), - false => None - } - } - None => None, - } - ).filter(|i| i.is_some()).map(|bang| bang.unwrap()) - .collect::>().last() - { - Some(bang) => { - debug!("got:'{:?}'", bang); - Some(bang.clone()) - }, - None => None - } - }, - None => None - } - false => None +#[derive(Debug)] +pub struct BangRedirect { + pub url: String +} + +pub fn do_bangs_or_whatever(query: &String) -> Option { + error!("Hello, World!"); + if !query.contains(BANG_PREFIX) { + return None; + } + let bangs = query.match_indices(BANG_PREFIX).filter(|(bang_start_idx, _)| { + if *bang_start_idx == 0 || query.chars().nth(*bang_start_idx - 1).unwrap().is_whitespace() { + true + } else { + false } - }).filter(|bang| bang.is_some()).map(|bang| bang.unwrap()).collect::>().first() { - Some(bang) => { - debug!("Initial query: \"{}\"", query); - let query_split = query.split_once(query.get(bang.start_idx..bang.end_idx).unwrap()).unwrap(); - debug!("Split query: {:?}", query_split); - let query_trimmed = format!("{}{}", query_split.0, query_split.1); - debug!("Trimmed query: \"{}\"", query_trimmed); + }).map(|(bang_start_idx, _)| { + let rest = query.get(bang_start_idx + 1..query.len()).unwrap(); + BUILTIN_BANGS.iter().map(|(_, bang)| { + let key = bang.keys.iter() + .filter(|key| rest.starts_with(**key)) + .filter( + |key| rest.chars() + .nth(key.len()) + .unwrap_or(' ') + .is_whitespace()) + .max_by(|a, b| a.len().cmp(&b.len()))?; + Some(BangLoc::new(bang.url, bang_start_idx, key.len())) + }).filter(|bang| bang.is_some()).map(|bang| bang.unwrap()).next() + }).filter(|bang| bang.is_some()) + .map(|bang| bang.unwrap()) + .collect::>(); + debug!("Result: {:?}", bangs); - let bang_url_split = bang.url.split_once("{}").unwrap(); - debug!("Split bang URL: {:?}", bang_url_split); - let bang_url = format!( - "{}{}{}", - bang_url_split.0, - query_trimmed, - bang_url_split.1 - ); - debug!("Final URL: \"{}\"", bang_url); - Some(bang_url) - }, - None => None - } + let bang = bangs.first()?; + let end_idx = bang.start_idx + 1 + bang.len; + debug!("Initial query: \"{}\"", query); + let query_split = query.split_once(query.get(bang.start_idx..end_idx).unwrap()).unwrap(); + debug!("Split query: {:?}", query_split); + let query_trimmed = format!("{}{}", query_split.0, query_split.1); + debug!("Trimmed query: \"{}\"", query_trimmed); + let query_encoded = url_encoded_data::stringify(&[("", query_trimmed.as_str())]); + let query_encoded = query_encoded.get(1..query_encoded.len()).unwrap().to_owned(); + debug!("Encoded query: {:?}", query_encoded); + + let bang_url_split = bang.url.split_once("{}").unwrap(); + debug!("Split bang URL: {:?}", bang_url_split); + let bang_url = format!( + "{}{}{}", + bang_url_split.0, + query_encoded, + bang_url_split.1 + ); + debug!("Final URL: \"{}\"", bang_url); + Some(BangRedirect{url: bang_url}) } diff --git a/asklyphe-frontend/src/routes/search.rs b/asklyphe-frontend/src/routes/search.rs index 78172c5..af33081 100644 --- a/asklyphe-frontend/src/routes/search.rs +++ b/asklyphe-frontend/src/routes/search.rs @@ -19,6 +19,7 @@ use crate::unit_converter::UnitConversion; use crate::wikipedia::WikipediaSummary; use crate::{wikipedia, Opts, ALPHA, BUILT_ON, GIT_COMMIT, VERSION, YEAR}; use crate::bangs; +use crate::bangs::BangRedirect; use askama::Template; use asklyphe_common::nats; use asklyphe_common::nats::bingservice::{ @@ -69,6 +70,7 @@ pub struct Complications { disabled: bool, wikipedia: Option, unit_converter: Option, + bang: Option } pub async fn search( @@ -174,7 +176,8 @@ pub async fn search_js( /*if let Some(bang, result_query) = bangs::do_bangs_or_whatever(&query) { error!("Would redirect to '{}', but it's not implemented", format!(bang, result_query)); }*/ - let _ = bangs::do_bangs_or_whatever(&query); + let bang_redirect = bangs::do_bangs_or_whatever(&query); + complications.bang = bang_redirect; } else { complications.disabled = true; query = query.replace("-complications", ""); From 64a771f8cc2b135f65cbe7f8d4ba1126ce4caa3f Mon Sep 17 00:00:00 2001 From: Book-reader Date: Thu, 8 May 2025 19:21:13 +1200 Subject: [PATCH 04/15] make it work fully --- asklyphe-frontend/src/bangs.rs | 15 +++++++-------- asklyphe-frontend/src/routes/search.rs | 15 +++++++++------ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/asklyphe-frontend/src/bangs.rs b/asklyphe-frontend/src/bangs.rs index 83eb1e8..682bd32 100644 --- a/asklyphe-frontend/src/bangs.rs +++ b/asklyphe-frontend/src/bangs.rs @@ -46,7 +46,6 @@ pub struct BangRedirect { } pub fn do_bangs_or_whatever(query: &String) -> Option { - error!("Hello, World!"); if !query.contains(BANG_PREFIX) { return None; } @@ -72,27 +71,27 @@ pub fn do_bangs_or_whatever(query: &String) -> Option { }).filter(|bang| bang.is_some()) .map(|bang| bang.unwrap()) .collect::>(); - debug!("Result: {:?}", bangs); + let bang = bangs.first()?; let end_idx = bang.start_idx + 1 + bang.len; - debug!("Initial query: \"{}\"", query); + let query_split = query.split_once(query.get(bang.start_idx..end_idx).unwrap()).unwrap(); - debug!("Split query: {:?}", query_split); + let query_trimmed = format!("{}{}", query_split.0, query_split.1); - debug!("Trimmed query: \"{}\"", query_trimmed); + let query_encoded = url_encoded_data::stringify(&[("", query_trimmed.as_str())]); let query_encoded = query_encoded.get(1..query_encoded.len()).unwrap().to_owned(); - debug!("Encoded query: {:?}", query_encoded); + let bang_url_split = bang.url.split_once("{}").unwrap(); - debug!("Split bang URL: {:?}", bang_url_split); + let bang_url = format!( "{}{}{}", bang_url_split.0, query_encoded, bang_url_split.1 ); - debug!("Final URL: \"{}\"", bang_url); + Some(BangRedirect{url: bang_url}) } diff --git a/asklyphe-frontend/src/routes/search.rs b/asklyphe-frontend/src/routes/search.rs index af33081..4509740 100644 --- a/asklyphe-frontend/src/routes/search.rs +++ b/asklyphe-frontend/src/routes/search.rs @@ -70,7 +70,6 @@ pub struct Complications { disabled: bool, wikipedia: Option, unit_converter: Option, - bang: Option } pub async fn search( @@ -173,11 +172,11 @@ pub async fn search_js( unit_query = unit_query.replace("metre", "meter"); let unit_comp = unit_converter::convert_unit(&unit_query); complications.unit_converter = unit_comp; - /*if let Some(bang, result_query) = bangs::do_bangs_or_whatever(&query) { - error!("Would redirect to '{}', but it's not implemented", format!(bang, result_query)); - }*/ + let bang_redirect = bangs::do_bangs_or_whatever(&query); - complications.bang = bang_redirect; + if let Some(redirect) = bang_redirect { + return Redirect::to(&redirect.url).into_response(); + } } else { complications.disabled = true; query = query.replace("-complications", ""); @@ -290,7 +289,11 @@ pub async fn search_nojs( unit_query = unit_query.replace("metre", "meter"); let unit_comp = unit_converter::convert_unit(&unit_query); complications.unit_converter = unit_comp; - let bang = bangs::do_bangs_or_whatever(&query); + + let bang_redirect = bangs::do_bangs_or_whatever(&query); + if let Some(redirect) = bang_redirect { + return Redirect::to(&redirect.url).into_response(); + } } else { complications.disabled = true; query = query.replace("-complications", ""); From 87458f30b6ca60c6d928e99bd18512f1b1e7829a Mon Sep 17 00:00:00 2001 From: Book-reader Date: Thu, 8 May 2025 19:40:35 +1200 Subject: [PATCH 05/15] bangs: fix redict including extra space sometimes --- asklyphe-frontend/src/bangs.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/asklyphe-frontend/src/bangs.rs b/asklyphe-frontend/src/bangs.rs index 682bd32..8ce78a8 100644 --- a/asklyphe-frontend/src/bangs.rs +++ b/asklyphe-frontend/src/bangs.rs @@ -74,9 +74,22 @@ pub fn do_bangs_or_whatever(query: &String) -> Option { let bang = bangs.first()?; - let end_idx = bang.start_idx + 1 + bang.len; + let end_idx = { + let mut end_idx = bang.start_idx + 1 + bang.len; + if end_idx < query.len() { + end_idx += 1; + } + end_idx + }; - let query_split = query.split_once(query.get(bang.start_idx..end_idx).unwrap()).unwrap(); + let start_idx = if end_idx == query.len() && bang.start_idx > 0 { + bang.start_idx - 1 + } else { + bang.start_idx + }; + + + let query_split = query.split_once(query.get(start_idx..end_idx).unwrap()).unwrap(); let query_trimmed = format!("{}{}", query_split.0, query_split.1); From 43aae463e869ba192b59436d6ad8834bba28511f Mon Sep 17 00:00:00 2001 From: Book-reader Date: Thu, 8 May 2025 19:57:38 +1200 Subject: [PATCH 06/15] use a more sensible name --- asklyphe-frontend/src/bangs.rs | 2 +- asklyphe-frontend/src/routes/search.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/asklyphe-frontend/src/bangs.rs b/asklyphe-frontend/src/bangs.rs index 8ce78a8..91b4828 100644 --- a/asklyphe-frontend/src/bangs.rs +++ b/asklyphe-frontend/src/bangs.rs @@ -45,7 +45,7 @@ pub struct BangRedirect { pub url: String } -pub fn do_bangs_or_whatever(query: &String) -> Option { +pub fn redirect_bang(query: &String) -> Option { if !query.contains(BANG_PREFIX) { return None; } diff --git a/asklyphe-frontend/src/routes/search.rs b/asklyphe-frontend/src/routes/search.rs index 4509740..f1814f5 100644 --- a/asklyphe-frontend/src/routes/search.rs +++ b/asklyphe-frontend/src/routes/search.rs @@ -173,7 +173,7 @@ pub async fn search_js( let unit_comp = unit_converter::convert_unit(&unit_query); complications.unit_converter = unit_comp; - let bang_redirect = bangs::do_bangs_or_whatever(&query); + let bang_redirect = bangs::redirect_bang(&query); if let Some(redirect) = bang_redirect { return Redirect::to(&redirect.url).into_response(); } @@ -290,7 +290,7 @@ pub async fn search_nojs( let unit_comp = unit_converter::convert_unit(&unit_query); complications.unit_converter = unit_comp; - let bang_redirect = bangs::do_bangs_or_whatever(&query); + let bang_redirect = bangs::redirect_bang(&query); if let Some(redirect) = bang_redirect { return Redirect::to(&redirect.url).into_response(); } From f06b84bf661b01b52c8d9830afb983567d1de530 Mon Sep 17 00:00:00 2001 From: Book-reader Date: Thu, 8 May 2025 20:11:15 +1200 Subject: [PATCH 07/15] a little cleanup --- asklyphe-frontend/src/bangs.rs | 10 +++------- asklyphe-frontend/src/routes/search.rs | 5 ++--- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/asklyphe-frontend/src/bangs.rs b/asklyphe-frontend/src/bangs.rs index 91b4828..16a444f 100644 --- a/asklyphe-frontend/src/bangs.rs +++ b/asklyphe-frontend/src/bangs.rs @@ -40,12 +40,7 @@ impl<'b> BangLoc<'_> { } } -#[derive(Debug)] -pub struct BangRedirect { - pub url: String -} - -pub fn redirect_bang(query: &String) -> Option { +pub fn redirect_bang(query: &String) -> Option { if !query.contains(BANG_PREFIX) { return None; } @@ -93,6 +88,7 @@ pub fn redirect_bang(query: &String) -> Option { let query_trimmed = format!("{}{}", query_split.0, query_split.1); + // A hack to get URL escaping without using a proper URL layout, hopefully has no other issues apart from prepending '=' to the string let query_encoded = url_encoded_data::stringify(&[("", query_trimmed.as_str())]); let query_encoded = query_encoded.get(1..query_encoded.len()).unwrap().to_owned(); @@ -106,5 +102,5 @@ pub fn redirect_bang(query: &String) -> Option { bang_url_split.1 ); - Some(BangRedirect{url: bang_url}) + Some(bang_url) } diff --git a/asklyphe-frontend/src/routes/search.rs b/asklyphe-frontend/src/routes/search.rs index f1814f5..7304718 100644 --- a/asklyphe-frontend/src/routes/search.rs +++ b/asklyphe-frontend/src/routes/search.rs @@ -19,7 +19,6 @@ use crate::unit_converter::UnitConversion; use crate::wikipedia::WikipediaSummary; use crate::{wikipedia, Opts, ALPHA, BUILT_ON, GIT_COMMIT, VERSION, YEAR}; use crate::bangs; -use crate::bangs::BangRedirect; use askama::Template; use asklyphe_common::nats; use asklyphe_common::nats::bingservice::{ @@ -175,7 +174,7 @@ pub async fn search_js( let bang_redirect = bangs::redirect_bang(&query); if let Some(redirect) = bang_redirect { - return Redirect::to(&redirect.url).into_response(); + return Redirect::to(&redirect).into_response(); } } else { complications.disabled = true; @@ -292,7 +291,7 @@ pub async fn search_nojs( let bang_redirect = bangs::redirect_bang(&query); if let Some(redirect) = bang_redirect { - return Redirect::to(&redirect.url).into_response(); + return Redirect::to(&redirect).into_response(); } } else { complications.disabled = true; From 3696d4cb6dd485f59c4824af361bb97a85e21dc6 Mon Sep 17 00:00:00 2001 From: Book-reader Date: Thu, 8 May 2025 20:12:03 +1200 Subject: [PATCH 08/15] spaces to follow at least a little code style :( --- asklyphe-frontend/src/bangs.rs | 134 ++++++++++++++++----------------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/asklyphe-frontend/src/bangs.rs b/asklyphe-frontend/src/bangs.rs index 16a444f..eed4aad 100644 --- a/asklyphe-frontend/src/bangs.rs +++ b/asklyphe-frontend/src/bangs.rs @@ -7,100 +7,100 @@ pub static BANG_PREFIX: &str = "!"; #[derive(Debug)] struct Bang<'a> { - pub url: &'a str, - pub keys: &'a [&'a str] + pub url: &'a str, + pub keys: &'a [&'a str] } impl<'a> Bang<'_> { - fn new(url: &'a str, keys: &'a [&'a str]) -> Bang<'a> { - Bang {url, keys} - } + fn new(url: &'a str, keys: &'a [&'a str]) -> Bang<'a> { + Bang {url, keys} + } } static BUILTIN_BANGS: Lazy> = Lazy::new(|| { - let mut bangs = BTreeMap::new(); - bangs.insert("Google", Bang::new("https://google.com/search?q={}", &["g", "google"] as &[&str])); + let mut bangs = BTreeMap::new(); + bangs.insert("Google", Bang::new("https://google.com/search?q={}", &["g", "google"] as &[&str])); - bangs.insert("DuckDuckGo", Bang::new("https://duckduckgo.com/?q={}", &["d", "ddg", "duckduckgo"] as &[&str])); + bangs.insert("DuckDuckGo", Bang::new("https://duckduckgo.com/?q={}", &["d", "ddg", "duckduckgo"] as &[&str])); - bangs.insert("Wikipedia", Bang::new("https://wikipedia.org/w/index.php?search={}", &["w", "wiki", "wikipedia"] as &[&str])); - bangs + bangs.insert("Wikipedia", Bang::new("https://wikipedia.org/w/index.php?search={}", &["w", "wiki", "wikipedia"] as &[&str])); + bangs }); #[derive(Debug, Clone)] struct BangLoc<'b> { - pub url: &'b str, - pub start_idx: usize, - pub len: usize + pub url: &'b str, + pub start_idx: usize, + pub len: usize } impl<'b> BangLoc<'_> { - fn new(url: &'b str, start_idx: usize, len: usize) -> BangLoc<'b> { - BangLoc {url, start_idx, len} - } + fn new(url: &'b str, start_idx: usize, len: usize) -> BangLoc<'b> { + BangLoc {url, start_idx, len} + } } pub fn redirect_bang(query: &String) -> Option { - if !query.contains(BANG_PREFIX) { - return None; - } - let bangs = query.match_indices(BANG_PREFIX).filter(|(bang_start_idx, _)| { - if *bang_start_idx == 0 || query.chars().nth(*bang_start_idx - 1).unwrap().is_whitespace() { - true - } else { - false - } - }).map(|(bang_start_idx, _)| { - let rest = query.get(bang_start_idx + 1..query.len()).unwrap(); - BUILTIN_BANGS.iter().map(|(_, bang)| { - let key = bang.keys.iter() - .filter(|key| rest.starts_with(**key)) - .filter( - |key| rest.chars() - .nth(key.len()) - .unwrap_or(' ') - .is_whitespace()) - .max_by(|a, b| a.len().cmp(&b.len()))?; - Some(BangLoc::new(bang.url, bang_start_idx, key.len())) - }).filter(|bang| bang.is_some()).map(|bang| bang.unwrap()).next() - }).filter(|bang| bang.is_some()) - .map(|bang| bang.unwrap()) - .collect::>(); + if !query.contains(BANG_PREFIX) { + return None; + } + let bangs = query.match_indices(BANG_PREFIX).filter(|(bang_start_idx, _)| { + if *bang_start_idx == 0 || query.chars().nth(*bang_start_idx - 1).unwrap().is_whitespace() { + true + } else { + false + } + }).map(|(bang_start_idx, _)| { + let rest = query.get(bang_start_idx + 1..query.len()).unwrap(); + BUILTIN_BANGS.iter().map(|(_, bang)| { + let key = bang.keys.iter() + .filter(|key| rest.starts_with(**key)) + .filter( + |key| rest.chars() + .nth(key.len()) + .unwrap_or(' ') + .is_whitespace()) + .max_by(|a, b| a.len().cmp(&b.len()))?; + Some(BangLoc::new(bang.url, bang_start_idx, key.len())) + }).filter(|bang| bang.is_some()).map(|bang| bang.unwrap()).next() + }).filter(|bang| bang.is_some()) + .map(|bang| bang.unwrap()) + .collect::>(); - let bang = bangs.first()?; - let end_idx = { - let mut end_idx = bang.start_idx + 1 + bang.len; - if end_idx < query.len() { - end_idx += 1; - } - end_idx - }; + let bang = bangs.first()?; + let end_idx = { + let mut end_idx = bang.start_idx + 1 + bang.len; + if end_idx < query.len() { + end_idx += 1; + } + end_idx + }; - let start_idx = if end_idx == query.len() && bang.start_idx > 0 { - bang.start_idx - 1 - } else { - bang.start_idx - }; + let start_idx = if end_idx == query.len() && bang.start_idx > 0 { + bang.start_idx - 1 + } else { + bang.start_idx + }; - let query_split = query.split_once(query.get(start_idx..end_idx).unwrap()).unwrap(); + let query_split = query.split_once(query.get(start_idx..end_idx).unwrap()).unwrap(); - let query_trimmed = format!("{}{}", query_split.0, query_split.1); + let query_trimmed = format!("{}{}", query_split.0, query_split.1); - // A hack to get URL escaping without using a proper URL layout, hopefully has no other issues apart from prepending '=' to the string - let query_encoded = url_encoded_data::stringify(&[("", query_trimmed.as_str())]); - let query_encoded = query_encoded.get(1..query_encoded.len()).unwrap().to_owned(); + // A hack to get URL escaping without using a proper URL layout, hopefully has no other issues apart from prepending '=' to the string + let query_encoded = url_encoded_data::stringify(&[("", query_trimmed.as_str())]); + let query_encoded = query_encoded.get(1..query_encoded.len()).unwrap().to_owned(); - let bang_url_split = bang.url.split_once("{}").unwrap(); + let bang_url_split = bang.url.split_once("{}").unwrap(); - let bang_url = format!( - "{}{}{}", - bang_url_split.0, - query_encoded, - bang_url_split.1 - ); + let bang_url = format!( + "{}{}{}", + bang_url_split.0, + query_encoded, + bang_url_split.1 + ); - Some(bang_url) + Some(bang_url) } From 24067eca99ad8e286057b853c24d622aa9a17c6d Mon Sep 17 00:00:00 2001 From: Book-reader Date: Thu, 8 May 2025 20:18:00 +1200 Subject: [PATCH 09/15] I was just writing the PR and realised that aliases is a better name than keys --- asklyphe-frontend/src/bangs.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/asklyphe-frontend/src/bangs.rs b/asklyphe-frontend/src/bangs.rs index eed4aad..ddd383b 100644 --- a/asklyphe-frontend/src/bangs.rs +++ b/asklyphe-frontend/src/bangs.rs @@ -8,12 +8,12 @@ pub static BANG_PREFIX: &str = "!"; #[derive(Debug)] struct Bang<'a> { pub url: &'a str, - pub keys: &'a [&'a str] + pub aliases: &'a [&'a str] } impl<'a> Bang<'_> { - fn new(url: &'a str, keys: &'a [&'a str]) -> Bang<'a> { - Bang {url, keys} + fn new(url: &'a str, aliases: &'a [&'a str]) -> Bang<'a> { + Bang {url, aliases} } } @@ -53,15 +53,15 @@ pub fn redirect_bang(query: &String) -> Option { }).map(|(bang_start_idx, _)| { let rest = query.get(bang_start_idx + 1..query.len()).unwrap(); BUILTIN_BANGS.iter().map(|(_, bang)| { - let key = bang.keys.iter() - .filter(|key| rest.starts_with(**key)) + let alias = bang.aliases.iter() + .filter(|alias| rest.starts_with(**alias)) .filter( - |key| rest.chars() - .nth(key.len()) + |alias| rest.chars() + .nth(alias.len()) .unwrap_or(' ') .is_whitespace()) .max_by(|a, b| a.len().cmp(&b.len()))?; - Some(BangLoc::new(bang.url, bang_start_idx, key.len())) + Some(BangLoc::new(bang.url, bang_start_idx, alias.len())) }).filter(|bang| bang.is_some()).map(|bang| bang.unwrap()).next() }).filter(|bang| bang.is_some()) .map(|bang| bang.unwrap()) From 1370bc8a43c86bd63ac938e0d5e136c9c68b4d98 Mon Sep 17 00:00:00 2001 From: Book-reader Date: Thu, 4 Sep 2025 15:53:53 +1200 Subject: [PATCH 10/15] Docker compose setup --- .gitignore | 6 +- Dockerfile.auth-frontend | 16 ++++ Dockerfile.authservice | 16 ++++ Dockerfile.bingservice | 14 ++++ Dockerfile.frontend | 17 +++++ Dockerfile.googleservice | 14 ++++ Dockerfile.vorebot | 14 ++++ asklyphe-auth-frontend/src/main.rs | 4 +- asklyphe-frontend/src/main.rs | 4 +- docker-compose.yml | 113 +++++++++++++++++++++++++++++ nginx/default.conf | 27 +++++++ 11 files changed, 240 insertions(+), 5 deletions(-) create mode 100644 Dockerfile.auth-frontend create mode 100644 Dockerfile.authservice create mode 100644 Dockerfile.bingservice create mode 100644 Dockerfile.frontend create mode 100644 Dockerfile.googleservice create mode 100644 Dockerfile.vorebot create mode 100644 docker-compose.yml create mode 100644 nginx/default.conf diff --git a/.gitignore b/.gitignore index 4d2be69..344be0a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ .idea -/target \ No newline at end of file +/target +# /nginx +database +database.bak +.env diff --git a/Dockerfile.auth-frontend b/Dockerfile.auth-frontend new file mode 100644 index 0000000..5247ad5 --- /dev/null +++ b/Dockerfile.auth-frontend @@ -0,0 +1,16 @@ +FROM rust:1.89.0 AS builder + +WORKDIR /usr/src/asklyphe/ +COPY asklyphe-auth-frontend asklyphe-auth-frontend +COPY asklyphe-common asklyphe-common +COPY lyphedb lyphedb + +RUN cargo install --debug --path asklyphe-auth-frontend/ + +FROM debian:trixie-slim +RUN apt-get update && apt-get upgrade -y && apt-get install -y libssl3 && rm -rf /var/lib/apt-get/lists/* +COPY --from=builder /usr/local/cargo/bin/asklyphe-auth-frontend /usr/local/bin/ +COPY --from=builder /usr/src/asklyphe/asklyphe-auth-frontend/static /data/static +VOLUME /data + +CMD ["asklyphe-auth-frontend"] diff --git a/Dockerfile.authservice b/Dockerfile.authservice new file mode 100644 index 0000000..80f8a7a --- /dev/null +++ b/Dockerfile.authservice @@ -0,0 +1,16 @@ +FROM rust:1.89.0 AS builder + +WORKDIR /usr/src/asklyphe/ +COPY authservice authservice +COPY asklyphe-common asklyphe-common +COPY lyphedb lyphedb + +RUN cargo install --debug --path authservice/ +RUN cargo install --debug --path authservice/migration/ + +FROM debian:trixie-slim +RUN apt-get update && apt-get upgrade -y && apt-get install -y libssl3 && rm -rf /var/lib/apt-get/lists/* +COPY --from=builder /usr/local/cargo/bin/authservice /usr/local/bin/ +COPY --from=builder /usr/local/cargo/bin/migration /usr/local/bin/ + +CMD ["authservice"] diff --git a/Dockerfile.bingservice b/Dockerfile.bingservice new file mode 100644 index 0000000..81e0118 --- /dev/null +++ b/Dockerfile.bingservice @@ -0,0 +1,14 @@ +FROM rust:1.89.0 AS builder + +WORKDIR /usr/src/asklyphe/ +COPY bingservice bingservice +COPY asklyphe-common asklyphe-common +COPY lyphedb lyphedb + +RUN cargo install --debug --path bingservice/ + +FROM debian:trixie-slim +RUN apt-get update && apt-get upgrade -y && apt-get install -y libssl3 && rm -rf /var/lib/apt-get/lists/* +COPY --from=builder /usr/local/cargo/bin/bingservice /usr/local/bin/ + +CMD ["bingservice"] diff --git a/Dockerfile.frontend b/Dockerfile.frontend new file mode 100644 index 0000000..61503a7 --- /dev/null +++ b/Dockerfile.frontend @@ -0,0 +1,17 @@ +FROM rust:1.89.0 AS builder + +WORKDIR /usr/src/asklyphe/ +COPY asklyphe-frontend asklyphe-frontend +COPY asklyphe-common asklyphe-common +COPY lyphedb lyphedb +COPY unit_converter unit_converter + +RUN cargo install --debug --path asklyphe-frontend/ + +FROM debian:trixie-slim +RUN apt-get update && apt-get upgrade -y && apt-get install -y libssl3 && rm -rf /var/lib/apt-get/lists/* +COPY --from=builder /usr/local/cargo/bin/asklyphe-frontend /usr/local/bin/ +COPY --from=builder /usr/src/asklyphe/asklyphe-frontend/static /data/static +VOLUME /data + +CMD ["asklyphe-frontend"] diff --git a/Dockerfile.googleservice b/Dockerfile.googleservice new file mode 100644 index 0000000..ed1ed1d --- /dev/null +++ b/Dockerfile.googleservice @@ -0,0 +1,14 @@ +FROM rust:1.89.0 AS builder + +WORKDIR /usr/src/asklyphe/ +COPY googleservice googleservice +COPY asklyphe-common asklyphe-common +COPY lyphedb lyphedb + +RUN cargo install --debug --path googleservice/ + +FROM debian:trixie-slim +RUN apt-get update && apt-get upgrade -y && apt-get install -y libssl3 && rm -rf /var/lib/apt-get/lists/* +COPY --from=builder /usr/local/cargo/bin/googleservice /usr/local/bin/ + +CMD ["googleservice"] diff --git a/Dockerfile.vorebot b/Dockerfile.vorebot new file mode 100644 index 0000000..c9490c3 --- /dev/null +++ b/Dockerfile.vorebot @@ -0,0 +1,14 @@ +FROM rust:1.89.0 AS builder + +WORKDIR /usr/src/asklyphe/ +COPY vorebot vorebot +COPY asklyphe-common asklyphe-common +COPY lyphedb lyphedb + +RUN cargo install --debug --path vorebot/ + +FROM debian:trixie-slim +RUN apt-get update && apt-get upgrade -y && apt-get install -y libssl3 && rm -rf /var/lib/apt-get/lists/* +COPY --from=builder /usr/local/cargo/bin/vorebot /usr/local/bin/ + +CMD ["vorebot"] diff --git a/asklyphe-auth-frontend/src/main.rs b/asklyphe-auth-frontend/src/main.rs index cd16d94..d8324f0 100644 --- a/asklyphe-auth-frontend/src/main.rs +++ b/asklyphe-auth-frontend/src/main.rs @@ -17,7 +17,7 @@ mod login; use std::{env, process}; use std::collections::HashMap; -use std::net::SocketAddr; +use std::net::{SocketAddr, ToSocketAddrs}; use std::ops::Deref; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; @@ -66,7 +66,7 @@ async fn main() { env_logger::init(); let opts = Opts { bind_addr: env::var("BIND_ADDR").unwrap_or("0.0.0.0:5843".to_string()).parse().expect("Badly formed BIND_ADDR (Needs to be SocketAddr)"), - nats_addr: env::var("NATS_ADDR").unwrap_or("127.0.0.1:4222".to_string()).parse().expect("Badly formed NATS_ADDR (Needs to be SocketAddr)"), + nats_addr: env::var("NATS_ADDR").unwrap_or("127.0.0.1:4222".to_string()).to_socket_addrs().expect("Badly formed NATS_ADDR (Needs to be SocketAddr)").nth(0).unwrap(), nats_cert: env::var("NATS_CERT").expect("NATS_CERT needs to be set"), nats_key: env::var("NATS_KEY").expect("NATS_KEY needs to be set"), asklyphe_url: env::var("ASKLYPHE_URL").unwrap_or("https://asklyphe.com".to_string()), diff --git a/asklyphe-frontend/src/main.rs b/asklyphe-frontend/src/main.rs index e99fe51..1dc7fd0 100644 --- a/asklyphe-frontend/src/main.rs +++ b/asklyphe-frontend/src/main.rs @@ -19,7 +19,7 @@ pub mod routes; use std::{env, process}; use std::collections::HashMap; -use std::net::SocketAddr; +use std::net::{SocketAddr, ToSocketAddrs}; use std::ops::Deref; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; @@ -85,7 +85,7 @@ async fn main() { env_logger::init(); let opts = Opts { bind_addr: env::var("BIND_ADDR").unwrap_or("0.0.0.0:5842".to_string()).parse().expect("Badly formed BIND_ADDR (Needs to be SocketAddr)"), - nats_addr: env::var("NATS_ADDR").unwrap_or("127.0.0.1:4222".to_string()).parse().expect("Badly formed NATS_ADDR (Needs to be SocketAddr)"), + nats_addr: env::var("NATS_ADDR").unwrap_or("127.0.0.1:4222".to_string()).to_socket_addrs().expect("Badly formed NATS_ADDR (Needs to be SocketAddr)").nth(0).unwrap(), nats_cert: env::var("NATS_CERT").expect("NATS_CERT needs to be set"), nats_key: env::var("NATS_KEY").expect("NATS_KEY needs to be set"), auth_url: env::var("AUTH_URL").unwrap_or("https://auth.asklyphe.com".to_string()), diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..7d8bca1 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,113 @@ +services: + auth-frontend: + restart: unless-stopped + networks: + - lyphenet + env_file: ".env" + depends_on: + - nats + - authservice + build: + dockerfile: Dockerfile.auth-frontend + volumes: + - auth_frontend_data:/data + image: asklyphe/auth-frontend + authservice: + restart: unless-stopped + networks: + - lyphenet + env_file: ".env" + depends_on: + - nats + build: + dockerfile: Dockerfile.authservice + image: asklyphe/authservice + frontend: + restart: unless-stopped + networks: + - lyphenet + env_file: ".env" + depends_on: + - nats + build: + dockerfile: Dockerfile.frontend + volumes: + - frontend_data:/data + image: asklyphe/frontend +# vorebot: +# restart: unless-stopped +# networks: +# - lyphenet +# - outer +# env_file: ".env" +# depends_on: +# - nats +# build: +# dockerfile: Dockerfile.vorebot +# image: asklyphe/vorebot + +# bingservice: +# restart: unless-stopped +# networks: +# - lyphenet +# - outer +# env_file: ".env" +# depends_on: +# - nats +# build: +# dockerfile: Dockerfile.bingservice +# image: asklyphe/bingservice +# googleservice: +# restart: unless-stopped +# networks: +# - lyphenet +# env_file: ".env" +# depends_on: +# - nats +# - bingservice +# build: +# dockerfile: Dockerfile.googleservice +# image: asklyphe/googleservice + + nats: + restart: unless-stopped + networks: + - lyphenet + depends_on: + - db + env_file: ".env" + image: nats:2.11.8 + cmd: "nats-server -js" + db: + image: postgres:17 + networks: + - lyphenet + env_file: ".env" + healthcheck: + test: ['CMD', 'pg_isready', '-U', 'postgres'] + volumes: + - ./database:/var/lib/postgresql/data + proxy: + image: nginx:latest + networks: + - lyphenet + - outer + depends_on: + - frontend + - auth-frontend + volumes: + - ./nginx:/etc/nginx/conf.d + - frontend_data:/data/frontend + - auth_frontend_data:/data/auth-frontend + ports: + - "1234:80" + - "1235:81" + +networks: + lyphenet: + internal: true + outer: + +volumes: + frontend_data: + auth_frontend_data: diff --git a/nginx/default.conf b/nginx/default.conf new file mode 100644 index 0000000..aea95c8 --- /dev/null +++ b/nginx/default.conf @@ -0,0 +1,27 @@ +server { + listen 81; + server_name 0.0.0.0; + access_log off; + + location /static/ { + root /data/auth-frontend; + } + + location / { + proxy_pass http://auth-frontend:5843; + } +} + +server { + listen 80; + server_name 0.0.0.0; + access_log off; + + location /static/ { + root /data/frontend; + } + + location / { + proxy_pass http://frontend:5842; + } +} From 3ed54bf7e8683869175867383942a9a2ecf5d791 Mon Sep 17 00:00:00 2001 From: Book-reader Date: Thu, 4 Sep 2025 19:15:57 +1200 Subject: [PATCH 11/15] Fix docker compose & the authservice `admin_count` function --- authservice/src/db/user.rs | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/authservice/src/db/user.rs b/authservice/src/db/user.rs index 989b58d..106060b 100644 --- a/authservice/src/db/user.rs +++ b/authservice/src/db/user.rs @@ -458,7 +458,7 @@ pub async fn user_count(db: &DatabaseConnection) -> Result Result { // dont fucking touch this, i don't know why it works but it does, it's actually evil // note: doesn't work - Ok(user::Entity::find().filter(user::Column::Flags.into_expr().binary(BinOper::LShift, Expr::value(63 - 2)).lt(1 << (63 - 2))) + Ok(user::Entity::find().filter(user::Column::Flags.into_expr().binary(BinOper::BitAnd, UserFlag::Administrator as i64).binary(BinOper::NotEqual, 0)) .count(db).await.map_err(|e| { error!("DATABASE ERROR WHILE ADMINCOUNT: {e}"); FetchUserError::DatabaseError diff --git a/docker-compose.yml b/docker-compose.yml index 7d8bca1..73cc7b1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,7 +77,7 @@ services: - db env_file: ".env" image: nats:2.11.8 - cmd: "nats-server -js" + command: "-js" db: image: postgres:17 networks: From 7fbe876c54c238daf81050d893f80115e149ef85 Mon Sep 17 00:00:00 2001 From: Book-reader Date: Thu, 4 Sep 2025 20:49:25 +1200 Subject: [PATCH 12/15] Finally, build caching --- Dockerfile.auth-frontend | 5 ++++- Dockerfile.authservice | 12 ++++++++++-- Dockerfile.bingservice | 5 ++++- Dockerfile.frontend | 5 ++++- Dockerfile.googleservice | 5 ++++- Dockerfile.vorebot | 5 ++++- 6 files changed, 30 insertions(+), 7 deletions(-) diff --git a/Dockerfile.auth-frontend b/Dockerfile.auth-frontend index 5247ad5..8bbe115 100644 --- a/Dockerfile.auth-frontend +++ b/Dockerfile.auth-frontend @@ -5,7 +5,10 @@ COPY asklyphe-auth-frontend asklyphe-auth-frontend COPY asklyphe-common asklyphe-common COPY lyphedb lyphedb -RUN cargo install --debug --path asklyphe-auth-frontend/ +RUN --mount=type=cache,target=$CARGO_HOME/registry \ + --mount=type=cache,target=/usr/src/asklyphe/asklyphe-auth-frontend/target \ + --mount=type=cache,target=$CARGO_HOME/git/db \ + cargo install --debug --path asklyphe-auth-frontend/ FROM debian:trixie-slim RUN apt-get update && apt-get upgrade -y && apt-get install -y libssl3 && rm -rf /var/lib/apt-get/lists/* diff --git a/Dockerfile.authservice b/Dockerfile.authservice index 80f8a7a..c26116e 100644 --- a/Dockerfile.authservice +++ b/Dockerfile.authservice @@ -5,8 +5,16 @@ COPY authservice authservice COPY asklyphe-common asklyphe-common COPY lyphedb lyphedb -RUN cargo install --debug --path authservice/ -RUN cargo install --debug --path authservice/migration/ + +RUN --mount=type=cache,target=$CARGO_HOME/registry \ + --mount=type=cache,target=/usr/src/asklyphe/authservice/target \ + --mount=type=cache,target=$CARGO_HOME/git/db \ + cargo install --debug --path authservice/ + +RUN --mount=type=cache,target=$CARGO_HOME/registry \ + --mount=type=cache,target=/usr/src/asklyphe/authservice/migration/target \ + --mount=type=cache,target=$CARGO_HOME/git/db \ + cargo install --debug --path authservice/migration/ FROM debian:trixie-slim RUN apt-get update && apt-get upgrade -y && apt-get install -y libssl3 && rm -rf /var/lib/apt-get/lists/* diff --git a/Dockerfile.bingservice b/Dockerfile.bingservice index 81e0118..9534d14 100644 --- a/Dockerfile.bingservice +++ b/Dockerfile.bingservice @@ -5,7 +5,10 @@ COPY bingservice bingservice COPY asklyphe-common asklyphe-common COPY lyphedb lyphedb -RUN cargo install --debug --path bingservice/ +RUN --mount=type=cache,target=$CARGO_HOME/registry \ + --mount=type=cache,target=/usr/src/asklyphe/bingservice/target \ + --mount=type=cache,target=$CARGO_HOME/git/db \ + cargo install --debug --path bingservice/ FROM debian:trixie-slim RUN apt-get update && apt-get upgrade -y && apt-get install -y libssl3 && rm -rf /var/lib/apt-get/lists/* diff --git a/Dockerfile.frontend b/Dockerfile.frontend index 61503a7..ed80b37 100644 --- a/Dockerfile.frontend +++ b/Dockerfile.frontend @@ -6,7 +6,10 @@ COPY asklyphe-common asklyphe-common COPY lyphedb lyphedb COPY unit_converter unit_converter -RUN cargo install --debug --path asklyphe-frontend/ +RUN --mount=type=cache,target=$CARGO_HOME/registry \ + --mount=type=cache,target=/usr/src/asklyphe/asklyphe-frontend/target \ + --mount=type=cache,target=$CARGO_HOME/git/db \ + cargo install --debug --path asklyphe-frontend/ FROM debian:trixie-slim RUN apt-get update && apt-get upgrade -y && apt-get install -y libssl3 && rm -rf /var/lib/apt-get/lists/* diff --git a/Dockerfile.googleservice b/Dockerfile.googleservice index ed1ed1d..b462f1d 100644 --- a/Dockerfile.googleservice +++ b/Dockerfile.googleservice @@ -5,7 +5,10 @@ COPY googleservice googleservice COPY asklyphe-common asklyphe-common COPY lyphedb lyphedb -RUN cargo install --debug --path googleservice/ +RUN --mount=type=cache,target=$CARGO_HOME/registry \ + --mount=type=cache,target=/usr/src/asklyphe/googleservice/target \ + --mount=type=cache,target=$CARGO_HOME/git/db \ + cargo install --debug --path googleservice/ FROM debian:trixie-slim RUN apt-get update && apt-get upgrade -y && apt-get install -y libssl3 && rm -rf /var/lib/apt-get/lists/* diff --git a/Dockerfile.vorebot b/Dockerfile.vorebot index c9490c3..02b29a3 100644 --- a/Dockerfile.vorebot +++ b/Dockerfile.vorebot @@ -5,7 +5,10 @@ COPY vorebot vorebot COPY asklyphe-common asklyphe-common COPY lyphedb lyphedb -RUN cargo install --debug --path vorebot/ +RUN --mount=type=cache,target=$CARGO_HOME/registry \ + --mount=type=cache,target=/usr/src/asklyphe/vorebot/target \ + --mount=type=cache,target=$CARGO_HOME/git/db \ + cargo install --debug --path vorebot/ FROM debian:trixie-slim RUN apt-get update && apt-get upgrade -y && apt-get install -y libssl3 && rm -rf /var/lib/apt-get/lists/* From 430e0fce8568484a7331d05fac65fd5f0efd7b2e Mon Sep 17 00:00:00 2001 From: Book-reader Date: Thu, 4 Sep 2025 21:35:20 +1200 Subject: [PATCH 13/15] reenable bing & google services in dockerfile, fix bingservice crash when proxies file is empty --- .gitignore | 1 + bingservice/src/main.rs | 2 +- docker-compose.yml | 49 ++++++++++++++++++++++------------------- 3 files changed, 28 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index 344be0a..33daff8 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ database database.bak .env +proxies.txt diff --git a/bingservice/src/main.rs b/bingservice/src/main.rs index e8223fd..5fce86a 100644 --- a/bingservice/src/main.rs +++ b/bingservice/src/main.rs @@ -34,7 +34,7 @@ pub static PROXIES: Lazy> = Lazy::new(|| { let contents = std::fs::read_to_string(proxy_file); let mut proxies = vec![]; - for line in contents.expect("FAILED TO READ FILE").lines() { + for line in contents.expect("FAILED TO READ FILE").lines().filter(|l| l.len() > 0) { proxies.push(Proxy::from_str(line).expect("INVALID PROXY")); } diff --git a/docker-compose.yml b/docker-compose.yml index 73cc7b1..d8d370d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,28 +46,31 @@ services: # dockerfile: Dockerfile.vorebot # image: asklyphe/vorebot -# bingservice: -# restart: unless-stopped -# networks: -# - lyphenet -# - outer -# env_file: ".env" -# depends_on: -# - nats -# build: -# dockerfile: Dockerfile.bingservice -# image: asklyphe/bingservice -# googleservice: -# restart: unless-stopped -# networks: -# - lyphenet -# env_file: ".env" -# depends_on: -# - nats -# - bingservice -# build: -# dockerfile: Dockerfile.googleservice -# image: asklyphe/googleservice + bingservice: + restart: unless-stopped + networks: + - lyphenet + - outer + env_file: ".env" + depends_on: + - nats + build: + dockerfile: Dockerfile.bingservice + volumes: + - ./proxies.txt:/data/proxies.txt + image: asklyphe/bingservice + googleservice: + restart: unless-stopped + networks: + - lyphenet + - outer + env_file: ".env" + depends_on: + - nats + - bingservice + build: + dockerfile: Dockerfile.googleservice + image: asklyphe/googleservice nats: restart: unless-stopped @@ -104,9 +107,9 @@ services: - "1235:81" networks: + outer: lyphenet: internal: true - outer: volumes: frontend_data: From c39a425d48b75249b346117fdedaa31356e256cb Mon Sep 17 00:00:00 2001 From: Book-reader Date: Fri, 5 Sep 2025 14:20:42 +1200 Subject: [PATCH 14/15] Lock dockerfile cargo cache --- Dockerfile.auth-frontend | 4 ++-- Dockerfile.authservice | 8 ++++---- Dockerfile.bingservice | 4 ++-- Dockerfile.frontend | 4 ++-- Dockerfile.googleservice | 4 ++-- Dockerfile.vorebot | 4 ++-- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Dockerfile.auth-frontend b/Dockerfile.auth-frontend index 8bbe115..053cc99 100644 --- a/Dockerfile.auth-frontend +++ b/Dockerfile.auth-frontend @@ -5,9 +5,9 @@ COPY asklyphe-auth-frontend asklyphe-auth-frontend COPY asklyphe-common asklyphe-common COPY lyphedb lyphedb -RUN --mount=type=cache,target=$CARGO_HOME/registry \ +RUN --mount=type=cache,target=$CARGO_HOME/registry,sharing=locked \ --mount=type=cache,target=/usr/src/asklyphe/asklyphe-auth-frontend/target \ - --mount=type=cache,target=$CARGO_HOME/git/db \ + --mount=type=cache,target=$CARGO_HOME/git/db,sharing=locked \ cargo install --debug --path asklyphe-auth-frontend/ FROM debian:trixie-slim diff --git a/Dockerfile.authservice b/Dockerfile.authservice index c26116e..ab340c3 100644 --- a/Dockerfile.authservice +++ b/Dockerfile.authservice @@ -6,14 +6,14 @@ COPY asklyphe-common asklyphe-common COPY lyphedb lyphedb -RUN --mount=type=cache,target=$CARGO_HOME/registry \ +RUN --mount=type=cache,target=$CARGO_HOME/registry,sharing=locked \ --mount=type=cache,target=/usr/src/asklyphe/authservice/target \ - --mount=type=cache,target=$CARGO_HOME/git/db \ + --mount=type=cache,target=$CARGO_HOME/git/db,sharing=locked \ cargo install --debug --path authservice/ -RUN --mount=type=cache,target=$CARGO_HOME/registry \ +RUN --mount=type=cache,target=$CARGO_HOME/registry,sharing=locked \ --mount=type=cache,target=/usr/src/asklyphe/authservice/migration/target \ - --mount=type=cache,target=$CARGO_HOME/git/db \ + --mount=type=cache,target=$CARGO_HOME/git/db,sharing=locked \ cargo install --debug --path authservice/migration/ FROM debian:trixie-slim diff --git a/Dockerfile.bingservice b/Dockerfile.bingservice index 9534d14..94cf1d0 100644 --- a/Dockerfile.bingservice +++ b/Dockerfile.bingservice @@ -5,9 +5,9 @@ COPY bingservice bingservice COPY asklyphe-common asklyphe-common COPY lyphedb lyphedb -RUN --mount=type=cache,target=$CARGO_HOME/registry \ +RUN --mount=type=cache,target=$CARGO_HOME/registry,sharing=locked \ --mount=type=cache,target=/usr/src/asklyphe/bingservice/target \ - --mount=type=cache,target=$CARGO_HOME/git/db \ + --mount=type=cache,target=$CARGO_HOME/git/db,sharing=locked \ cargo install --debug --path bingservice/ FROM debian:trixie-slim diff --git a/Dockerfile.frontend b/Dockerfile.frontend index ed80b37..6aaa3e1 100644 --- a/Dockerfile.frontend +++ b/Dockerfile.frontend @@ -6,9 +6,9 @@ COPY asklyphe-common asklyphe-common COPY lyphedb lyphedb COPY unit_converter unit_converter -RUN --mount=type=cache,target=$CARGO_HOME/registry \ +RUN --mount=type=cache,target=$CARGO_HOME/registry,sharing=locked \ --mount=type=cache,target=/usr/src/asklyphe/asklyphe-frontend/target \ - --mount=type=cache,target=$CARGO_HOME/git/db \ + --mount=type=cache,target=$CARGO_HOME/git/db,sharing=locked \ cargo install --debug --path asklyphe-frontend/ FROM debian:trixie-slim diff --git a/Dockerfile.googleservice b/Dockerfile.googleservice index b462f1d..a4fd902 100644 --- a/Dockerfile.googleservice +++ b/Dockerfile.googleservice @@ -5,9 +5,9 @@ COPY googleservice googleservice COPY asklyphe-common asklyphe-common COPY lyphedb lyphedb -RUN --mount=type=cache,target=$CARGO_HOME/registry \ +RUN --mount=type=cache,target=$CARGO_HOME/registry,sharing=locked \ --mount=type=cache,target=/usr/src/asklyphe/googleservice/target \ - --mount=type=cache,target=$CARGO_HOME/git/db \ + --mount=type=cache,target=$CARGO_HOME/git/db,sharing=locked \ cargo install --debug --path googleservice/ FROM debian:trixie-slim diff --git a/Dockerfile.vorebot b/Dockerfile.vorebot index 02b29a3..b0d54b7 100644 --- a/Dockerfile.vorebot +++ b/Dockerfile.vorebot @@ -5,9 +5,9 @@ COPY vorebot vorebot COPY asklyphe-common asklyphe-common COPY lyphedb lyphedb -RUN --mount=type=cache,target=$CARGO_HOME/registry \ +RUN --mount=type=cache,target=$CARGO_HOME/registry,sharing=locked \ --mount=type=cache,target=/usr/src/asklyphe/vorebot/target \ - --mount=type=cache,target=$CARGO_HOME/git/db \ + --mount=type=cache,target=$CARGO_HOME/git/db,sharing=locked \ cargo install --debug --path vorebot/ FROM debian:trixie-slim From f11aa990381cb2e7d0d47928b5aa09cca425194a Mon Sep 17 00:00:00 2001 From: Book-reader Date: Fri, 5 Sep 2025 15:44:29 +1200 Subject: [PATCH 15/15] Add protocol to proxy file format (may remove), misc changes --- Caddyfile | 8 ++++---- bingservice/src/proxy.rs | 5 +++-- docker-compose.yml | 13 +++++++++++++ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/Caddyfile b/Caddyfile index 0a55afb..10f884b 100644 --- a/Caddyfile +++ b/Caddyfile @@ -1,4 +1,4 @@ -http://127.0.0.1:8001 { +http://127.0.0.1:1235 { route /static/* { uri strip_prefix /static file_server { @@ -6,10 +6,10 @@ http://127.0.0.1:8001 { } } - reverse_proxy 127.0.0.1:5843 + reverse_proxy http://auth-frontend:5843 } -http://127.0.0.1:8002 { +http://127.0.0.1:1234 { route /static/* { uri strip_prefix /static file_server { @@ -17,5 +17,5 @@ http://127.0.0.1:8002 { } } - reverse_proxy 127.0.0.1:5842 + reverse_proxy http://frontend:5842 } diff --git a/bingservice/src/proxy.rs b/bingservice/src/proxy.rs index 4559a95..ce0c568 100644 --- a/bingservice/src/proxy.rs +++ b/bingservice/src/proxy.rs @@ -30,6 +30,7 @@ impl FromStr for Proxy { fn from_str(s: &str) -> Result { let mut parts = s.split(':'); + let protocol = parts.next().ok_or(ProxyError::InvalidProxyFormat)?; let host = parts.next().ok_or(ProxyError::InvalidProxyFormat)?; let port = parts.next().ok_or(ProxyError::InvalidProxyFormat)?; let auth = if let Some(user) = parts.next() { @@ -39,8 +40,8 @@ impl FromStr for Proxy { None }; Ok(Proxy { - address: format!("{}:{}", host, port), + address: format!("{}://{}:{}", protocol, host, port), credentials: auth, }) } -} \ No newline at end of file +} diff --git a/docker-compose.yml b/docker-compose.yml index d8d370d..089694d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -105,6 +105,19 @@ services: ports: - "1234:80" - "1235:81" +# caddy: +# image: caddy:latest +# networks: +# - lyphenet +# - outer +# depends_on: +# - frontend +# - auth-frontend +# volumes: +# - ./Caddyfile:/etc/caddy/Caddyfile +# ports: +# - 1234:1234 +# - 1235:1235 networks: outer: