asklyphe-public/asklyphe
asklyphe-public/asklyphe
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

bring over new crawler and lyphedb

Browse source
This commit is contained in:
husky 2025-03-12 12:52:24 -07:00
parent b1202cc889
commit 906c2ed8df
25 changed files with 2919 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

594
Cargo.lock generated
View file

@ -133,6 +133,12 @@ version = "1.0.97"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f"
[[package]]
name = "arc-swap"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457"
[[package]] [[package]]
name = "argon2" name = "argon2"
version = "0.5.3" version = "0.5.3"
@ -151,6 +157,12 @@ version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "ascii"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16"
[[package]] [[package]]
name = "askama" name = "askama"
version = "0.12.1" version = "0.12.1"
@ -236,6 +248,8 @@ dependencies = [
"foundationdb", "foundationdb",
"futures", "futures",
"log", "log",
"lyphedb",
"percent-encoding",
"rand 0.8.5", "rand 0.8.5",
"rmp-serde", "rmp-serde",
"serde", "serde",
@ -406,7 +420,7 @@ version = "0.38.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76433c4de73442daedb3a59e991d94e85c14ebfc33db53dfcd347a21cd6ef4f8" checksum = "76433c4de73442daedb3a59e991d94e85c14ebfc33db53dfcd347a21cd6ef4f8"
dependencies = [ dependencies = [
"base64", "base64 0.22.1",
"bytes", "bytes",
"futures", "futures",
"memchr", "memchr",
@ -651,6 +665,12 @@ dependencies = [
"windows-targets 0.52.6", "windows-targets 0.52.6",
] ]
[[package]]
name = "base64"
version = "0.21.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
[[package]] [[package]]
name = "base64" name = "base64"
version = "0.22.1" version = "0.22.1"
@ -703,7 +723,7 @@ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"regex", "regex",
"rustc-hash", "rustc-hash 1.1.0",
"shlex", "shlex",
"syn 2.0.99", "syn 2.0.99",
"which", "which",
@ -726,6 +746,12 @@ dependencies = [
"url_encoded_data", "url_encoded_data",
] ]
[[package]]
name = "bit_field"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61"
[[package]] [[package]]
name = "bitflags" name = "bitflags"
version = "1.3.2" version = "1.3.2"
@ -807,6 +833,17 @@ dependencies = [
"syn 2.0.99", "syn 2.0.99",
] ]
[[package]]
name = "bstr"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"
dependencies = [
"lazy_static",
"memchr",
"regex-automata 0.1.10",
]
[[package]] [[package]]
name = "bumpalo" name = "bumpalo"
version = "3.17.0" version = "3.17.0"
@ -835,6 +872,12 @@ dependencies = [
"syn 1.0.109", "syn 1.0.109",
] ]
[[package]]
name = "bytemuck"
version = "1.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540"
[[package]] [[package]]
name = "byteorder" name = "byteorder"
version = "1.5.0" version = "1.5.0"
@ -911,6 +954,12 @@ dependencies = [
"stacker", "stacker",
] ]
[[package]]
name = "chunked_transfer"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901"
[[package]] [[package]]
name = "clang-sys" name = "clang-sys"
version = "1.8.1" version = "1.8.1"
@ -962,6 +1011,12 @@ version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"
[[package]]
name = "color_quant"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
[[package]] [[package]]
name = "colorchoice" name = "colorchoice"
version = "1.0.3" version = "1.0.3"
@ -983,6 +1038,26 @@ version = "0.9.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"
[[package]]
name = "const_format"
version = "0.2.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "126f97965c8ad46d6d9163268ff28432e8f6a1196a55578867832e3049df63dd"
dependencies = [
"const_format_proc_macros",
]
[[package]]
name = "const_format_proc_macros"
version = "0.2.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744"
dependencies = [
"proc-macro2",
"quote",
"unicode-xid",
]
[[package]] [[package]]
name = "cookie" name = "cookie"
version = "0.18.1" version = "0.18.1"
@ -1044,6 +1119,34 @@ version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
[[package]]
name = "crc32fast"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
dependencies = [
"crossbeam-utils",
]
[[package]] [[package]]
name = "crossbeam-queue" name = "crossbeam-queue"
version = "0.3.12" version = "0.3.12"
@ -1059,6 +1162,12 @@ version = "0.8.21"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
[[package]]
name = "crunchy"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929"
[[package]] [[package]]
name = "crypto-common" name = "crypto-common"
version = "0.1.6" version = "0.1.6"
@ -1308,7 +1417,7 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea3d894bbbab314476b265f9b2d46bf24b123a36dd0e96b06a1b49545b9d9dcc" checksum = "ea3d894bbbab314476b265f9b2d46bf24b123a36dd0e96b06a1b49545b9d9dcc"
dependencies = [ dependencies = [
"base64", "base64 0.22.1",
"memchr", "memchr",
] ]
@ -1424,6 +1533,21 @@ dependencies = [
"pin-project-lite", "pin-project-lite",
] ]
[[package]]
name = "exr"
version = "1.73.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f83197f59927b46c04a183a619b7c29df34e63e63c7869320862268c0ef687e0"
dependencies = [
"bit_field",
"half",
"lebe",
"miniz_oxide",
"rayon-core",
"smallvec",
"zune-inflate",
]
[[package]] [[package]]
name = "fastrand" name = "fastrand"
version = "1.9.0" version = "1.9.0"
@ -1439,12 +1563,31 @@ version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "fdeflate"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c"
dependencies = [
"simd-adler32",
]
[[package]] [[package]]
name = "fiat-crypto" name = "fiat-crypto"
version = "0.2.9" version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d"
[[package]]
name = "flate2"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc"
dependencies = [
"crc32fast",
"miniz_oxide",
]
[[package]] [[package]]
name = "flume" name = "flume"
version = "0.11.1" version = "0.11.1"
@ -1722,8 +1865,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"js-sys",
"libc", "libc",
"wasi 0.11.0+wasi-snapshot-preview1", "wasi 0.11.0+wasi-snapshot-preview1",
"wasm-bindgen",
] ]
[[package]] [[package]]
@ -1738,6 +1883,16 @@ dependencies = [
"windows-targets 0.52.6", "windows-targets 0.52.6",
] ]
[[package]]
name = "gif"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fb2d69b19215e18bb912fa30f7ce15846e301408695e44e0ef719f1da9e19f2"
dependencies = [
"color_quant",
"weezl",
]
[[package]] [[package]]
name = "gimli" name = "gimli"
version = "0.31.1" version = "0.31.1"
@ -1779,6 +1934,16 @@ dependencies = [
"url_encoded_data", "url_encoded_data",
] ]
[[package]]
name = "half"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888"
dependencies = [
"cfg-if",
"crunchy",
]
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.12.3" version = "0.12.3"
@ -2001,6 +2166,24 @@ dependencies = [
"want", "want",
] ]
[[package]]
name = "hyper-rustls"
version = "0.27.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2"
dependencies = [
"futures-util",
"http 1.2.0",
"hyper",
"hyper-util",
"rustls",
"rustls-pki-types",
"tokio",
"tokio-rustls",
"tower-service",
"webpki-roots",
]
[[package]] [[package]]
name = "hyper-tls" name = "hyper-tls"
version = "0.6.0" version = "0.6.0"
@ -2204,6 +2387,24 @@ dependencies = [
"icu_properties", "icu_properties",
] ]
[[package]]
name = "image"
version = "0.24.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5690139d2f55868e080017335e4b94cb7414274c74f1669c84fb5feba2c9f69d"
dependencies = [
"bytemuck",
"byteorder",
"color_quant",
"exr",
"gif",
"jpeg-decoder",
"num-traits",
"png",
"qoi",
"tiff",
]
[[package]] [[package]]
name = "indexmap" name = "indexmap"
version = "2.7.1" version = "2.7.1"
@ -2310,6 +2511,15 @@ version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
[[package]]
name = "jpeg-decoder"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0"
dependencies = [
"rayon",
]
[[package]] [[package]]
name = "js-sys" name = "js-sys"
version = "0.3.77" version = "0.3.77"
@ -2344,13 +2554,33 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]]
name = "ldbtesttool"
version = "0.1.0"
dependencies = [
"async-nats",
"futures",
"lyphedb",
"rand 0.9.0",
"rmp-serde",
"serde",
"tokio",
"toml",
]
[[package]]
name = "lebe"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8"
[[package]] [[package]]
name = "lettre" name = "lettre"
version = "0.11.14" version = "0.11.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d476fe7a4a798f392ce34947aa7d53d981127e37523c5251da3c927f7fa901f" checksum = "5d476fe7a4a798f392ce34947aa7d53d981127e37523c5251da3c927f7fa901f"
dependencies = [ dependencies = [
"base64", "base64 0.22.1",
"chumsky", "chumsky",
"email-encoding", "email-encoding",
"email_address", "email_address",
@ -2470,6 +2700,24 @@ dependencies = [
"prost-types", "prost-types",
] ]
[[package]]
name = "lyphedb"
version = "0.1.0"
dependencies = [
"async-nats",
"chrono",
"env_logger 0.11.6",
"futures",
"log",
"once_cell",
"percent-encoding",
"rand 0.9.0",
"rmp-serde",
"serde",
"tokio",
"toml",
]
[[package]] [[package]]
name = "mac" name = "mac"
version = "0.1.1" version = "0.1.1"
@ -2558,6 +2806,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5"
dependencies = [ dependencies = [
"adler2", "adler2",
"simd-adler32",
] ]
[[package]] [[package]]
@ -2588,6 +2837,16 @@ dependencies = [
"version_check", "version_check",
] ]
[[package]]
name = "mutex-timeouts"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2e9a1502e1bb4bbb96a448f684d77f17023e5a14ff49fc87c3161df6aaac2d9"
dependencies = [
"once_cell",
"tokio",
]
[[package]] [[package]]
name = "native-tls" name = "native-tls"
version = "0.2.14" version = "0.2.14"
@ -3061,6 +3320,19 @@ version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
[[package]]
name = "png"
version = "0.17.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526"
dependencies = [
"bitflags 1.3.2",
"crc32fast",
"fdeflate",
"flate2",
"miniz_oxide",
]
[[package]] [[package]]
name = "polling" name = "polling"
version = "2.8.0" version = "2.8.0"
@ -3182,6 +3454,34 @@ dependencies = [
"yansi", "yansi",
] ]
[[package]]
name = "prometheus"
version = "0.13.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1"
dependencies = [
"cfg-if",
"fnv",
"lazy_static",
"memchr",
"parking_lot",
"thiserror 1.0.69",
]
[[package]]
name = "prometheus_exporter"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "caf17cbebe0bfdf4f279ef84eeefe0d50468b0b7116f078acf41d456e48fe81a"
dependencies = [
"ascii",
"lazy_static",
"log",
"prometheus",
"thiserror 1.0.69",
"tiny_http",
]
[[package]] [[package]]
name = "prost" name = "prost"
version = "0.13.5" version = "0.13.5"
@ -3243,6 +3543,67 @@ dependencies = [
"syn 1.0.109", "syn 1.0.109",
] ]
[[package]]
name = "qoi"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001"
dependencies = [
"bytemuck",
]
[[package]]
name = "quinn"
version = "0.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef"
dependencies = [
"bytes",
"pin-project-lite",
"quinn-proto",
"quinn-udp",
"rustc-hash 2.1.1",
"rustls",
"socket2",
"thiserror 2.0.12",
"tokio",
"tracing",
]
[[package]]
name = "quinn-proto"
version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d"
dependencies = [
"bytes",
"getrandom 0.2.15",
"rand 0.8.5",
"ring",
"rustc-hash 2.1.1",
"rustls",
"rustls-pki-types",
"slab",
"thiserror 2.0.12",
"tinyvec",
"tracing",
"web-time",
]
[[package]]
name = "quinn-udp"
version = "0.5.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e46f3055866785f6b92bc6164b76be02ca8f2eb4b002c0354b28cf4c119e5944"
dependencies = [
"cfg_aliases",
"libc",
"once_cell",
"socket2",
"tracing",
"windows-sys 0.59.0",
]
[[package]] [[package]]
name = "quote" name = "quote"
version = "1.0.39" version = "1.0.39"
@ -3324,6 +3685,26 @@ dependencies = [
"getrandom 0.3.1", "getrandom 0.3.1",
] ]
[[package]]
name = "rayon"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
dependencies = [
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
dependencies = [
"crossbeam-deque",
"crossbeam-utils",
]
[[package]] [[package]]
name = "redox_syscall" name = "redox_syscall"
version = "0.5.10" version = "0.5.10"
@ -3392,7 +3773,7 @@ version = "0.12.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da" checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da"
dependencies = [ dependencies = [
"base64", "base64 0.22.1",
"bytes", "bytes",
"futures-core", "futures-core",
"futures-util", "futures-util",
@ -3400,6 +3781,7 @@ dependencies = [
"http-body", "http-body",
"http-body-util", "http-body-util",
"hyper", "hyper",
"hyper-rustls",
"hyper-tls", "hyper-tls",
"hyper-util", "hyper-util",
"ipnet", "ipnet",
@ -3410,19 +3792,24 @@ dependencies = [
"once_cell", "once_cell",
"percent-encoding", "percent-encoding",
"pin-project-lite", "pin-project-lite",
"quinn",
"rustls",
"rustls-pemfile", "rustls-pemfile",
"rustls-pki-types",
"serde", "serde",
"serde_json", "serde_json",
"serde_urlencoded", "serde_urlencoded",
"sync_wrapper", "sync_wrapper",
"tokio", "tokio",
"tokio-native-tls", "tokio-native-tls",
"tokio-rustls",
"tower", "tower",
"tower-service", "tower-service",
"url", "url",
"wasm-bindgen", "wasm-bindgen",
"wasm-bindgen-futures", "wasm-bindgen-futures",
"web-sys", "web-sys",
"webpki-roots",
"windows-registry", "windows-registry",
] ]
@ -3539,6 +3926,12 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
[[package]]
name = "rustc-hash"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
[[package]] [[package]]
name = "rustc_version" name = "rustc_version"
version = "0.4.1" version = "0.4.1"
@ -3627,6 +4020,9 @@ name = "rustls-pki-types"
version = "1.11.0" version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c"
dependencies = [
"web-time",
]
[[package]] [[package]]
name = "rustls-webpki" name = "rustls-webpki"
@ -3938,9 +4334,9 @@ dependencies = [
[[package]] [[package]]
name = "serde_bytes" name = "serde_bytes"
version = "0.11.16" version = "0.11.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "364fec0df39c49a083c9a8a18a23a6bcfd9af130fe9fe321d18520a0d113e09e" checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96"
dependencies = [ dependencies = [
"serde", "serde",
] ]
@ -3962,6 +4358,7 @@ version = "1.0.140"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373"
dependencies = [ dependencies = [
"indexmap",
"itoa", "itoa",
"memchr", "memchr",
"ryu", "ryu",
@ -3998,6 +4395,15 @@ dependencies = [
"syn 2.0.99", "syn 2.0.99",
] ]
[[package]]
name = "serde_spanned"
version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1"
dependencies = [
"serde",
]
[[package]] [[package]]
name = "serde_urlencoded" name = "serde_urlencoded"
version = "0.7.1" version = "0.7.1"
@ -4087,6 +4493,12 @@ dependencies = [
"rand_core 0.6.4", "rand_core 0.6.4",
] ]
[[package]]
name = "simd-adler32"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
[[package]] [[package]]
name = "simdutf8" name = "simdutf8"
version = "0.1.5" version = "0.1.5"
@ -4270,7 +4682,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4560278f0e00ce64938540546f59f590d60beee33fffbd3b9cd47851e5fff233" checksum = "4560278f0e00ce64938540546f59f590d60beee33fffbd3b9cd47851e5fff233"
dependencies = [ dependencies = [
"atoi", "atoi",
"base64", "base64 0.22.1",
"bigdecimal", "bigdecimal",
"bitflags 2.9.0", "bitflags 2.9.0",
"byteorder", "byteorder",
@ -4317,7 +4729,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5b98a57f363ed6764d5b3a12bfedf62f07aa16e1856a7ddc2a0bb190a959613" checksum = "c5b98a57f363ed6764d5b3a12bfedf62f07aa16e1856a7ddc2a0bb190a959613"
dependencies = [ dependencies = [
"atoi", "atoi",
"base64", "base64 0.22.1",
"bigdecimal", "bigdecimal",
"bitflags 2.9.0", "bitflags 2.9.0",
"byteorder", "byteorder",
@ -4404,6 +4816,16 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "stopwords"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e4508a6e132e6ea159112d42ed1f29927460dd45a118eed298d7666c81b713e"
dependencies = [
"lazy_static",
"thiserror 1.0.69",
]
[[package]] [[package]]
name = "string_cache" name = "string_cache"
version = "0.8.8" version = "0.8.8"
@ -4429,6 +4851,15 @@ dependencies = [
"quote", "quote",
] ]
[[package]]
name = "stringmatch"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6aadc0801d92f0cdc26127c67c4b8766284f52a5ba22894f285e3101fa57d05d"
dependencies = [
"regex",
]
[[package]] [[package]]
name = "stringprep" name = "stringprep"
version = "0.1.5" version = "0.1.5"
@ -4540,6 +4971,61 @@ dependencies = [
"winapi-util", "winapi-util",
] ]
[[package]]
name = "texting_robots"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b82a718a28dda2e67ad6e0464597b58eae39e2e4d0451e03d1028d71e81bb4a"
dependencies = [
"anyhow",
"bstr",
"lazy_static",
"nom 7.1.3",
"percent-encoding",
"regex",
"thiserror 1.0.69",
"url",
]
[[package]]
name = "thirtyfour"
version = "0.35.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44d1dd5271076f9cf5cb6a673497d1476cda2633e6cf2523edd00ac16218d651"
dependencies = [
"arc-swap",
"async-trait",
"base64 0.22.1",
"bytes",
"cfg-if",
"const_format",
"futures-util",
"http 1.2.0",
"indexmap",
"paste",
"reqwest",
"serde",
"serde_json",
"serde_repr",
"stringmatch",
"thirtyfour-macros",
"thiserror 1.0.69",
"tokio",
"tracing",
"url",
]
[[package]]
name = "thirtyfour-macros"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5cf0ffc3ba4368e99597bd6afd83f4ff6febad66d9ae541ab46e697d32285fc0"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.99",
]
[[package]] [[package]]
name = "thiserror" name = "thiserror"
version = "1.0.69" version = "1.0.69"
@ -4590,6 +5076,17 @@ dependencies = [
"once_cell", "once_cell",
] ]
[[package]]
name = "tiff"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba1310fcea54c6a9a4fd1aad794ecc02c31682f6bfbecdf460bf19533eed1e3e"
dependencies = [
"flate2",
"jpeg-decoder",
"weezl",
]
[[package]] [[package]]
name = "time" name = "time"
version = "0.3.39" version = "0.3.39"
@ -4621,6 +5118,19 @@ dependencies = [
"time-core", "time-core",
] ]
[[package]]
name = "tiny_http"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5f8734c6d6943ad6df6b588d228a87b4af184998bcffa268ceddf05c2055a8c"
dependencies = [
"ascii",
"chunked_transfer",
"log",
"time",
"url",
]
[[package]] [[package]]
name = "tinystr" name = "tinystr"
version = "0.7.6" version = "0.7.6"
@ -4725,7 +5235,7 @@ version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f591660438b3038dd04d16c938271c79e7e06260ad2ea2885a4861bfb238605d" checksum = "f591660438b3038dd04d16c938271c79e7e06260ad2ea2885a4861bfb238605d"
dependencies = [ dependencies = [
"base64", "base64 0.22.1",
"bytes", "bytes",
"futures-core", "futures-core",
"futures-sink", "futures-sink",
@ -4740,11 +5250,26 @@ dependencies = [
"tokio-util", "tokio-util",
] ]
[[package]]
name = "toml"
version = "0.8.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148"
dependencies = [
"serde",
"serde_spanned",
"toml_datetime",
"toml_edit",
]
[[package]] [[package]]
name = "toml_datetime" name = "toml_datetime"
version = "0.6.8" version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
dependencies = [
"serde",
]
[[package]] [[package]]
name = "toml_edit" name = "toml_edit"
@ -4753,6 +5278,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474"
dependencies = [ dependencies = [
"indexmap", "indexmap",
"serde",
"serde_spanned",
"toml_datetime", "toml_datetime",
"winnow", "winnow",
] ]
@ -4982,6 +5509,12 @@ version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
[[package]]
name = "unicode-xid"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
[[package]] [[package]]
name = "unit_converter" name = "unit_converter"
version = "0.1.0" version = "0.1.0"
@ -5080,6 +5613,32 @@ version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "vorebot"
version = "0.2.0"
dependencies = [
"asklyphe-common",
"async-nats",
"base64 0.21.7",
"chrono",
"env_logger 0.10.2",
"futures",
"image",
"isahc",
"log",
"mutex-timeouts",
"once_cell",
"prometheus_exporter",
"rand 0.8.5",
"rmp-serde",
"serde",
"stopwords",
"texting_robots",
"thirtyfour",
"tokio",
"ulid",
]
[[package]] [[package]]
name = "waker-fn" name = "waker-fn"
version = "1.2.0" version = "1.2.0"
@ -5216,6 +5775,12 @@ dependencies = [
"rustls-pki-types", "rustls-pki-types",
] ]
[[package]]
name = "weezl"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082"
[[package]] [[package]]
name = "which" name = "which"
version = "4.4.2" version = "4.4.2"
@ -5636,3 +6201,12 @@ dependencies = [
"quote", "quote",
"syn 2.0.99", "syn 2.0.99",
] ]
[[package]]
name = "zune-inflate"
version = "0.2.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02"
dependencies = [
"simd-adler32",
]

2
Cargo.toml
View file

@ -1,2 +1,2 @@
[workspace] [workspace]
members = ["asklyphe-common", "asklyphe-frontend", "searchservice", "asklyphe-auth-frontend", "unit_converter", "authservice", "authservice/migration", "authservice/entity", "bingservice", "googleservice"] members = ["asklyphe-common", "asklyphe-frontend", "searchservice", "asklyphe-auth-frontend", "unit_converter", "authservice", "authservice/migration", "authservice/entity", "bingservice", "googleservice", "vorebot", "lyphedb", "lyphedb/ldbtesttool"]

4
asklyphe-common/Cargo.toml
View file

@ -11,10 +11,12 @@ license-file = "LICENSE"
tokio = { version = "1.0", features = ["full"] } tokio = { version = "1.0", features = ["full"] }
chrono = "0.4.31" chrono = "0.4.31"
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
lyphedb = { path = "../lyphedb" }
foundationdb = { version = "0.8.0", features = ["embedded-fdb-include"] } foundationdb = { version = "0.8.0", features = ["embedded-fdb-include"] }
log = "0.4.20" log = "0.4.20"
rmp-serde = "1.1.2" rmp-serde = "1.1.2"
futures = "0.3.30" futures = "0.3.30"
async-nats = "0.38.0" async-nats = "0.38.0"
ulid = "1.1.0" ulid = "1.1.0"
rand = "0.8.5" rand = "0.8.5"
percent-encoding = "2.3.1"

63
asklyphe-common/src/ldb/linkstore.rs Normal file
View file

@ -0,0 +1,63 @@
use log::{error, warn};
use lyphedb::{KVList, KeyDirectory, LDBNatsMessage, LypheDBCommand, PropagationStrategy};
use crate::ldb::{construct_path, hash, DBConn};
pub const LINKSTORE: &str = "linkstore";
pub async fn add_url_to_linkwords(db: &DBConn, linkwords: &[&str], url: &str) -> Result<(), ()> {
let mut key_sets = Vec::new();
for linkword in linkwords {
let path = construct_path(&[LINKSTORE, linkword, &hash(url)]).as_bytes().to_vec();
key_sets.push((path, url.as_bytes().to_vec()));
}
let cmd = LDBNatsMessage::Command(LypheDBCommand::SetKeys(KVList {
kvs: key_sets,
}, PropagationStrategy::OnRead));
match db.query(cmd).await {
LDBNatsMessage::Success => {
Ok(())
}
LDBNatsMessage::BadRequest => {
error!("bad request for add_url_to_linkwords");
Err(())
}
LDBNatsMessage::NotFound => {
error!("not found for add_url_to_linkwords");
Err(())
}
_ => {
warn!("lyphedb sent weird message as response, treating as error");
Err(())
}
}
}
pub async fn get_urls_from_linkword(db: &DBConn, keyword: &str) -> Result<Vec<String>, ()> {
let path = construct_path(&[LINKSTORE, keyword]).as_bytes().to_vec();
let cmd = LDBNatsMessage::Command(LypheDBCommand::GetKeyDirectory(KeyDirectory {
key: path,
}));
match db.query(cmd).await {
LDBNatsMessage::Entries(kvs) => {
Ok(kvs.kvs.into_iter().map(|v| String::from_utf8_lossy(&v.1).to_string()).collect())
}
LDBNatsMessage::Success => {
warn!("lyphedb responded with \"success\" to get_urls_from_linkwords, treating as error");
Err(())
}
LDBNatsMessage::BadRequest => {
warn!("bad request for get_urls_from_linkwords");
Err(())
}
LDBNatsMessage::NotFound => {
Ok(vec![])
}
_ => {
warn!("lyphedb sent weird message as response, treating as error");
Err(())
}
}
}

63
asklyphe-common/src/ldb/metastore.rs Normal file
View file

@ -0,0 +1,63 @@
use log::{error, warn};
use lyphedb::{KVList, KeyDirectory, LDBNatsMessage, LypheDBCommand, PropagationStrategy};
use crate::ldb::{construct_path, hash, DBConn};
pub const METASTORE: &str = "metastore";
pub async fn add_url_to_metawords(db: &DBConn, metawords: &[&str], url: &str) -> Result<(), ()> {
let mut key_sets = Vec::new();
for metaword in metawords {
let path = construct_path(&[METASTORE, metaword, &hash(url)]).as_bytes().to_vec();
key_sets.push((path, url.as_bytes().to_vec()));
}
let cmd = LDBNatsMessage::Command(LypheDBCommand::SetKeys(KVList {
kvs: key_sets,
}, PropagationStrategy::OnRead));
match db.query(cmd).await {
LDBNatsMessage::Success => {
Ok(())
}
LDBNatsMessage::BadRequest => {
error!("bad request for add_url_to_metawords");
Err(())
}
LDBNatsMessage::NotFound => {
error!("not found for add_url_to_metawords");
Err(())
}
_ => {
warn!("lyphedb sent weird message as response, treating as error");
Err(())
}
}
}
pub async fn get_urls_from_metaword(db: &DBConn, metaword: &str) -> Result<Vec<String>, ()> {
let path = construct_path(&[METASTORE, metaword]).as_bytes().to_vec();
let cmd = LDBNatsMessage::Command(LypheDBCommand::GetKeyDirectory(KeyDirectory {
key: path,
}));
match db.query(cmd).await {
LDBNatsMessage::Entries(kvs) => {
Ok(kvs.kvs.into_iter().map(|v| String::from_utf8_lossy(&v.1).to_string()).collect())
}
LDBNatsMessage::Success => {
warn!("lyphedb responded with \"success\" to get_urls_from_metawords, treating as error");
Err(())
}
LDBNatsMessage::BadRequest => {
warn!("bad request for get_urls_from_metawords");
Err(())
}
LDBNatsMessage::NotFound => {
Ok(vec![])
}
_ => {
warn!("lyphedb sent weird message as response, treating as error");
Err(())
}
}
}

62
asklyphe-common/src/ldb/mod.rs Normal file
View file

@ -0,0 +1,62 @@
pub mod wordstore;
pub mod sitestore;
pub mod linkstore;
pub mod metastore;
pub mod titlestore;
use std::hash::{DefaultHasher, Hasher};
use std::sync::Arc;
use std::sync::atomic::AtomicU64;
use futures::StreamExt;
use log::warn;
use lyphedb::LDBNatsMessage;
use percent_encoding::{percent_encode, AsciiSet, CONTROLS, NON_ALPHANUMERIC};
static NEXT_REPLY_ID: AtomicU64 = AtomicU64::new(0);
pub const NOT_ALLOWED_ASCII: AsciiSet = CONTROLS.add(b' ').add(b'/').add(b'.').add(b'\\');
pub fn hash(str: &str) -> String {
let mut hasher = DefaultHasher::new();
hasher.write(str.as_bytes());
hasher.finish().to_string()
}
pub fn construct_path(path_elements: &[&str]) -> String {
let mut buf = String::new();
buf.push_str("ASKLYPHE/");
for el in path_elements {
buf.push_str(&percent_encode(el.as_bytes(), &NOT_ALLOWED_ASCII).to_string());
buf.push('/');
}
buf.pop();
buf
}
#[derive(Clone)]
pub struct DBConn {
nats: async_nats::Client,
name: String,
}
impl DBConn {
pub fn new(nats: async_nats::Client, name: impl ToString) -> DBConn {
DBConn { nats, name: name.to_string() }
}
pub async fn query(&self, message: LDBNatsMessage) -> LDBNatsMessage {
let data = rmp_serde::to_vec(&message).unwrap();
let replyto = format!("ldb-reply-{}", NEXT_REPLY_ID.fetch_add(1, std::sync::atomic::Ordering::SeqCst));
let mut subscriber = self.nats.subscribe(replyto.clone()).await.expect("NATS ERROR");
self.nats.publish_with_reply(self.name.clone(), replyto, data.into()).await.expect("NATS ERROR");
if let Some(reply) = subscriber.next().await {
let reply = rmp_serde::from_slice::<LDBNatsMessage>(&reply.payload);
if reply.is_err() {
warn!("DECODED BAD MESSAGE FROM LYPHEDB: {}", reply.err().unwrap());
return LDBNatsMessage::NotFound;
}
return reply.unwrap();
}
LDBNatsMessage::NotFound
}
}

178
asklyphe-common/src/ldb/sitestore.rs Normal file
View file

@ -0,0 +1,178 @@
use crate::ldb::{construct_path, hash, DBConn};
use log::{error, warn};
use lyphedb::{KVList, KeyDirectory, KeyList, LDBNatsMessage, LypheDBCommand, PropagationStrategy};
pub const SITESTORE: &str = "sitestore";
pub const TITLE: &str = "title";
pub const DESCRIPTION: &str = "desc";
pub const KEYWORDS: &str = "keywords";
pub const PAGE_TEXT_RANKING: &str = "ptranks";
pub const PAGE_TEXT_RAW: &str = "textraw";
pub const DAMPING: &str = "damping";
pub async fn add_website(
db: &DBConn,
url: &str,
title: Option<String>,
description: Option<String>,
keywords: Option<Vec<String>>,
page_text_ranking: &[(String, f64)],
page_text_raw: String,
damping: f64,
) -> Result<(), ()> {
let keyurl = hash(url);
let mut kvs = vec![
(
construct_path(&[SITESTORE, &keyurl]).as_bytes().to_vec(),
url.as_bytes().to_vec(),
),
(
construct_path(&[SITESTORE, &keyurl, PAGE_TEXT_RANKING])
.as_bytes()
.to_vec(),
rmp_serde::to_vec(page_text_ranking).unwrap(),
),
(
construct_path(&[SITESTORE, &keyurl, PAGE_TEXT_RAW])
.as_bytes()
.to_vec(),
page_text_raw.as_bytes().to_vec(),
),
(
construct_path(&[SITESTORE, &keyurl, DAMPING])
.as_bytes()
.to_vec(),
damping.to_be_bytes().to_vec(),
),
];
if let Some(title) = title {
kvs.push((
construct_path(&[SITESTORE, &keyurl, TITLE]).as_bytes().to_vec(),
title.as_bytes().to_vec(),
))
}
if let Some(description) = description {
kvs.push((
construct_path(&[SITESTORE, &keyurl, DESCRIPTION])
.as_bytes()
.to_vec(),
description.as_bytes().to_vec(),
))
}
if let Some(keywords) = keywords {
kvs.push((
construct_path(&[SITESTORE, &keyurl, KEYWORDS])
.as_bytes()
.to_vec(),
rmp_serde::to_vec(&keywords).unwrap(),
))
}
let cmd = LDBNatsMessage::Command(LypheDBCommand::SetKeys(
KVList { kvs },
PropagationStrategy::OnRead,
));
match db.query(cmd).await {
LDBNatsMessage::Success => Ok(()),
LDBNatsMessage::BadRequest => {
error!("bad request for add_website");
Err(())
}
LDBNatsMessage::NotFound => {
error!("not found for add_website");
Err(())
}
_ => {
warn!("lyphedb sent weird message as response, treating as error");
Err(())
}
}
}
#[derive(Default, Debug, Clone)]
pub struct WebsiteData {
pub title: Option<String>,
pub description: Option<String>,
pub keywords: Option<Vec<String>>,
pub page_text_ranking: Vec<(String, f64)>,
pub page_text_raw: String,
pub damping: f64,
}
pub async fn get_website(db: &DBConn, url: &str) -> Result<WebsiteData, ()> {
let keyurl = hash(url);
let keys = [
construct_path(&[SITESTORE, &keyurl, TITLE]).as_bytes().to_vec(),
construct_path(&[SITESTORE, &keyurl, DESCRIPTION]).as_bytes().to_vec(),
construct_path(&[SITESTORE, &keyurl, KEYWORDS]).as_bytes().to_vec(),
construct_path(&[SITESTORE, &keyurl, PAGE_TEXT_RANKING]).as_bytes().to_vec(),
construct_path(&[SITESTORE, &keyurl, PAGE_TEXT_RAW]).as_bytes().to_vec(),
construct_path(&[SITESTORE, &keyurl, DAMPING]).as_bytes().to_vec(),
].to_vec();
let cmd = LDBNatsMessage::Command(LypheDBCommand::GetKeys(KeyList { keys }));
match db.query(cmd).await {
LDBNatsMessage::Entries(kvlist) => {
let mut data = WebsiteData::default();
for (key, value) in kvlist.kvs {
let key = String::from_utf8_lossy(&key).to_string();
match key.as_str() {
_ if key.ends_with(TITLE) => {
data.title = Some(String::from_utf8_lossy(&value).to_string());
}
_ if key.ends_with(DESCRIPTION) => {
data.description = Some(String::from_utf8_lossy(&value).to_string());
}
_ if key.ends_with(KEYWORDS) => {
let deser = rmp_serde::from_slice::<Vec<String>>(&value);
if let Err(e) = deser {
error!("bad keywords entry for {}, deser error: {:?}", key, e);
} else {
data.keywords = Some(deser.unwrap());
}
}
_ if key.ends_with(PAGE_TEXT_RANKING) => {
let deser = rmp_serde::from_slice::<Vec<(String, f64)>>(&value);
if let Err(e) = deser {
error!("bad page_text_ranking entry for {}, deser error: {:?}", key, e);
} else {
data.page_text_ranking = deser.unwrap();
}
}
_ if key.ends_with(PAGE_TEXT_RAW) => {
data.page_text_raw = String::from_utf8_lossy(&value).to_string();
}
_ if key.ends_with(DAMPING) => {
data.damping = f64::from_be_bytes(value.try_into().unwrap_or(0.85f64.to_be_bytes()));
}
_ => {
warn!("encountered weird returned key for get_website");
}
}
}
Ok(data)
}
LDBNatsMessage::Success => {
warn!("lyphedb responded with \"success\" to get_website, treating as error");
Err(())
}
LDBNatsMessage::BadRequest => {
error!("bad request for get_website");
Err(())
}
LDBNatsMessage::NotFound => {
warn!("not found for get_website");
Err(())
}
_ => {
warn!("lyphedb sent weird message as response, treating as error");
Err(())
}
}
}

63
asklyphe-common/src/ldb/titlestore.rs Normal file
View file

@ -0,0 +1,63 @@
use log::{error, warn};
use lyphedb::{KVList, KeyDirectory, LDBNatsMessage, LypheDBCommand, PropagationStrategy};
use crate::ldb::{construct_path, hash, DBConn};
pub const TITLESTORE: &str = "titlestore";
pub async fn add_url_to_titlewords(db: &DBConn, titlewords: &[&str], url: &str) -> Result<(), ()> {
let mut key_sets = Vec::new();
for titleword in titlewords {
let path = construct_path(&[TITLESTORE, titleword, &hash(url)]).as_bytes().to_vec();
key_sets.push((path, url.as_bytes().to_vec()));
}
let cmd = LDBNatsMessage::Command(LypheDBCommand::SetKeys(KVList {
kvs: key_sets,
}, PropagationStrategy::OnRead));
match db.query(cmd).await {
LDBNatsMessage::Success => {
Ok(())
}
LDBNatsMessage::BadRequest => {
error!("bad request for add_url_to_titlewords");
Err(())
}
LDBNatsMessage::NotFound => {
error!("not found for add_url_to_titlewords");
Err(())
}
_ => {
warn!("lyphedb sent weird message as response, treating as error");
Err(())
}
}
}
pub async fn get_urls_from_titleword(db: &DBConn, titleword: &str) -> Result<Vec<String>, ()> {
let path = construct_path(&[TITLESTORE, titleword]).as_bytes().to_vec();
let cmd = LDBNatsMessage::Command(LypheDBCommand::GetKeyDirectory(KeyDirectory {
key: path,
}));
match db.query(cmd).await {
LDBNatsMessage::Entries(kvs) => {
Ok(kvs.kvs.into_iter().map(|v| String::from_utf8_lossy(&v.1).to_string()).collect())
}
LDBNatsMessage::Success => {
warn!("lyphedb responded with \"success\" to get_urls_from_titlewords, treating as error");
Err(())
}
LDBNatsMessage::BadRequest => {
warn!("bad request for get_urls_from_titlewords");
Err(())
}
LDBNatsMessage::NotFound => {
Ok(vec![])
}
_ => {
warn!("lyphedb sent weird message as response, treating as error");
Err(())
}
}
}

64
asklyphe-common/src/ldb/wordstore.rs Normal file
View file

@ -0,0 +1,64 @@
use log::{error, warn};
use lyphedb::{KVList, KeyDirectory, LDBNatsMessage, LypheDBCommand, PropagationStrategy};
use crate::ldb::{construct_path, hash, DBConn};
pub const WORDSTORE: &str = "wordstore";
pub async fn add_url_to_keywords(db: &DBConn, keywords: &[&str], url: &str) -> Result<(), ()> {
let mut key_sets = Vec::new();
for keyword in keywords {
let path = construct_path(&[WORDSTORE, keyword, &hash(url)]).as_bytes().to_vec();
let data = url.as_bytes().to_vec();
key_sets.push((path, data));
}
let cmd = LDBNatsMessage::Command(LypheDBCommand::SetKeys(KVList {
kvs: key_sets,
}, PropagationStrategy::OnRead));
match db.query(cmd).await {
LDBNatsMessage::Success => {
Ok(())
}
LDBNatsMessage::BadRequest => {
error!("bad request for add_url_to_keywords");
Err(())
}
LDBNatsMessage::NotFound => {
error!("not found for add_url_to_keywords");
Err(())
}
_ => {
warn!("lyphedb sent weird message as response, treating as error");
Err(())
}
}
}
pub async fn get_urls_from_keyword(db: &DBConn, keyword: &str) -> Result<Vec<String>, ()> {
let path = construct_path(&[WORDSTORE, keyword]).as_bytes().to_vec();
let cmd = LDBNatsMessage::Command(LypheDBCommand::GetKeyDirectory(KeyDirectory {
key: path,
}));
match db.query(cmd).await {
LDBNatsMessage::Entries(kvs) => {
Ok(kvs.kvs.into_iter().map(|v| String::from_utf8_lossy(&v.1).to_string()).collect())
}
LDBNatsMessage::Success => {
warn!("lyphedb responded with \"success\" to get_urls_from_keywords, treating as error");
Err(())
}
LDBNatsMessage::BadRequest => {
warn!("bad request for get_urls_from_keywords");
Err(())
}
LDBNatsMessage::NotFound => {
Ok(vec![])
}
_ => {
warn!("lyphedb sent weird message as response, treating as error");
Err(())
}
}
}

2
asklyphe-common/src/lib.rs
View file

@ -13,7 +13,9 @@
pub mod nats; pub mod nats;
pub mod db; pub mod db;
pub mod ldb;
pub use lyphedb;
pub use foundationdb; pub use foundationdb;
pub fn add(left: usize, right: usize) -> usize { pub fn add(left: usize, right: usize) -> usize {

6
asklyphe-common/src/nats/vorebot.rs
View file

@ -18,7 +18,7 @@ pub const VOREBOT_NEWHOSTNAME_SERVICE: &str = "websiteparse_highpriority";
pub const VOREBOT_SUGGESTED_SERVICE: &str = "websiteparse_highestpriority"; pub const VOREBOT_SUGGESTED_SERVICE: &str = "websiteparse_highestpriority";
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WebParseRequest { pub struct CrawlRequest {
pub url: String, pub url: String,
pub damping_factor: f32, pub damping: f64,
} }

30
lyphedb/.gitignore vendored Normal file
View file

@ -0,0 +1,30 @@
# ---> Rust
# Generated by Cargo
# will have compiled files and executables
debug/
target/
# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
Cargo.lock
# These are backup files generated by rustfmt
**/*.rs.bk
# MSVC Windows builds of rustc generate these, which store debugging information
*.pdb
# RustRover
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
# Added by cargo
/target
# don't commit employee test configs (:
husky_config.toml

18
lyphedb/Cargo.toml Normal file
View file

@ -0,0 +1,18 @@
[package]
name = "lyphedb"
version = "0.1.0"
edition = "2024"
[dependencies]
log = "0.4.20"
rmp-serde = "1.1.2"
futures = "0.3.30"
async-nats = "0.38.0"
tokio = { version = "1.0", features = ["full"] }
chrono = "0.4.31"
serde = { version = "1.0", features = ["derive"] }
rand = "0.9.0"
toml = "0.8.20"
env_logger = "0.11.6"
once_cell = "1.20.3"
percent-encoding = "2.3.1"

10
lyphedb/config.toml Normal file
View file

@ -0,0 +1,10 @@
name = "lyphedb-test"
write = true
master = "/lyphedb_master"
ram_limit = "1mb" # supported suffixes: b,kb,mb,gb
gc_limit = "512kb"
avg_entry_size = "1kb"
log = "debug"
nats_cert = "nats/nats.cert"
nats_key = "nats/nats.pem"
nats_url = "127.0.0.1:4222"

14
lyphedb/ldbtesttool/Cargo.toml Normal file
View file

@ -0,0 +1,14 @@
[package]
name = "ldbtesttool"
version = "0.1.0"
edition = "2024"
[dependencies]
rmp-serde = "1.1.2"
futures = "0.3.30"
async-nats = "0.38.0"
tokio = { version = "1.0", features = ["full"] }
serde = { version = "1.0", features = ["derive"] }
lyphedb = { path = "../" }
toml = "0.8.20"
rand = "0.9.0"

40
lyphedb/ldbtesttool/src/config.rs Normal file
View file

@ -0,0 +1,40 @@
use serde::Deserialize;
#[derive(Deserialize)]
struct ConfigFile {
pub name: String,
pub nats_cert: String,
pub nats_key: String,
pub nats_url: String,
}
#[derive(Clone)]
pub struct LypheDBConfig {
pub name: String,
pub nats_cert: String,
pub nats_key: String,
pub nats_url: String,
}
pub fn load_config(path: &str) -> LypheDBConfig {
let s = std::fs::read_to_string(path);
if let Err(e) = s {
panic!("failed to read config file: {}", e);
}
let s = s.unwrap();
let cnf = toml::from_str::<ConfigFile>(&s);
if let Err(e) = cnf {
panic!("failed to parse config file: {}", e);
}
let cnf = cnf.unwrap();
// quick checks and conversions
let mut config = LypheDBConfig {
name: cnf.name,
nats_cert: cnf.nats_cert,
nats_key: cnf.nats_key,
nats_url: cnf.nats_url,
};
config
}

114
lyphedb/ldbtesttool/src/main.rs Normal file
View file

@ -0,0 +1,114 @@
use crate::config::load_config;
use lyphedb::{KVList, KeyList, LDBNatsMessage, LypheDBCommand, PropagationStrategy};
use std::collections::BTreeMap;
use async_nats::Message;
use futures::StreamExt;
use tokio::sync::mpsc;
mod config;
#[tokio::main]
async fn main() {
let config = load_config(&std::env::args().nth(1).expect("please specify config file"));
let nats = async_nats::ConnectOptions::new()
.add_client_certificate(
config.nats_cert.as_str().into(),
config.nats_key.as_str().into(),
)
.connect(config.nats_url.as_str())
.await;
if let Err(e) = nats {
eprintln!("FATAL ERROR, COULDN'T CONNECT TO NATS: {}", e);
return;
}
let nats = nats.unwrap();
// test 1: create 10000 keys, send in chunks of 10
println!("test 1: create 10_000 keys, send in chunks of 100");
let (key_tx, mut key_rx) = mpsc::unbounded_channel();
let start = std::time::Instant::now();
let mut tasks = vec![];
for _ in 0..(10_000 / 100) {
let name = config.name.clone();
let nats = nats.clone();
let key_tx = key_tx.clone();
tasks.push(tokio::spawn(async move {
let mut keys = BTreeMap::new();
for _ in 0..100 {
let key = rand::random::<[u8; 16]>();
let data = rand::random::<[u8; 16]>();
key_tx.send((key, data)).unwrap();
keys.insert(key, data);
}
let data = rmp_serde::to_vec(&LDBNatsMessage::Command(LypheDBCommand::SetKeys(
KVList {
kvs: keys
.into_iter()
.map(|(k, v)| (k.to_vec(), v.to_vec()))
.collect(),
},
PropagationStrategy::Immediate,
)))
.unwrap();
let replyto_sub = "lyphedb_test_1".to_string();
let mut subscriber = nats.subscribe(replyto_sub.clone()).await.unwrap();
nats.publish_with_reply(name.clone(), replyto_sub, data.into()).await.unwrap();
if let Some(reply) = subscriber.next().await {
let reply = rmp_serde::from_slice::<LDBNatsMessage>(&reply.payload).unwrap();
if let LDBNatsMessage::Success = reply {} else {
eprintln!("failure");
}
}
}));
}
for task in tasks {
task.await.unwrap();
}
let end = std::time::Instant::now();
println!("test 1: {}ms", end.duration_since(start).as_millis());
let mut our_copy = BTreeMap::new();
let mut buffer = vec![];
key_rx.recv_many(&mut buffer, 10_000).await;
for (k, v) in buffer {
our_copy.insert(k, v);
}
// test 2: read back all keys and check for accuracy
println!("test 2: read back all keys and check for accuracy");
let kv: Vec<_> = our_copy.into_iter().map(|(k, v)| (k.to_vec(), v.to_vec())).collect();
let start = std::time::Instant::now();
let mut tasks = vec![];
for i in 0..100 {
let batch = kv[i * 100..((i + 1) * 100).min(kv.len())].to_vec();
let name = config.name.clone();
let nats = nats.clone();
tasks.push(tokio::spawn(async move {
let data = rmp_serde::to_vec(&LDBNatsMessage::Command(LypheDBCommand::GetKeys(
KeyList {
keys: batch.iter().map(|(k, _)| k.to_vec()).collect()
}
))).unwrap();
let replyto_sub = format!("lyphedb_test_2_{}", i);
let mut subscriber = nats.subscribe(replyto_sub.clone()).await.unwrap();
nats.publish_with_reply(name.clone(), replyto_sub, data.into()).await.unwrap();
if let Some(reply) = subscriber.next().await {
let reply = rmp_serde::from_slice::<LDBNatsMessage>(&reply.payload).unwrap();
if let LDBNatsMessage::Entries(kvlist) = reply {
// check all keys
for (k1, v1) in batch {
let v2 = kvlist.kvs.iter().find(|(k, v)| k == &k1).expect("key not found");
assert_eq!(v1, v2.1);
}
} else {
eprintln!("failure");
}
}
}));
}
for task in tasks {
task.await.unwrap();
}
let end = std::time::Instant::now();
println!("test 2: {}ms", end.duration_since(start).as_millis());
}

129
lyphedb/src/config.rs Normal file
View file

@ -0,0 +1,129 @@
use serde::Deserialize;
#[derive(Deserialize)]
struct ConfigFile {
pub name: String,
pub write: bool,
pub master: String,
pub ram_limit: String,
pub gc_limit: String,
pub avg_entry_size: String,
pub log: String,
pub nats_cert: String,
pub nats_key: String,
pub nats_url: String,
}
#[derive(Clone)]
pub struct LypheDBConfig {
pub name: String,
pub write: bool,
pub master: String,
pub ram_limit: usize,
pub gc_limit: usize,
pub avg_entry_size: usize,
pub log: String,
pub nats_cert: String,
pub nats_key: String,
pub nats_url: String,
}
pub fn load_config(path: &str) -> LypheDBConfig {
let s = std::fs::read_to_string(path);
if let Err(e) = s {
panic!("failed to read config file: {}", e);
}
let s = s.unwrap();
let cnf = toml::from_str::<ConfigFile>(&s);
if let Err(e) = cnf {
panic!("failed to parse config file: {}", e);
}
let cnf = cnf.unwrap();
// quick checks and conversions
let mut config = LypheDBConfig {
name: cnf.name,
write: cnf.write,
master: cnf.master,
ram_limit: 0,
gc_limit: 0,
avg_entry_size: 0,
log: cnf.log,
nats_cert: cnf.nats_cert,
nats_key: cnf.nats_key,
nats_url: cnf.nats_url,
};
if !(cnf.ram_limit.ends_with("b") &&
(
cnf.ram_limit.trim_end_matches("b").ends_with("k") ||
cnf.ram_limit.trim_end_matches("b").ends_with("m") ||
cnf.ram_limit.trim_end_matches("b").ends_with("g")
)
) {
panic!("invalid ram limit");
}
config.ram_limit = if cnf.ram_limit.ends_with("gb") {
cnf.ram_limit.trim_end_matches("gb").parse::<usize>().unwrap() * 1024 * 1024 * 1024
} else if cnf.ram_limit.ends_with("mb") {
cnf.ram_limit.trim_end_matches("mb").parse::<usize>().unwrap() * 1024 * 1024
} else if cnf.ram_limit.ends_with("kb") {
cnf.ram_limit.trim_end_matches("kb").parse::<usize>().unwrap() * 1024
} else {
cnf.ram_limit.trim_end_matches("b").parse::<usize>().unwrap()
};
if !(cnf.gc_limit.ends_with("b") &&
(
cnf.gc_limit.trim_end_matches("b").ends_with("k") ||
cnf.gc_limit.trim_end_matches("b").ends_with("m") ||
cnf.gc_limit.trim_end_matches("b").ends_with("g")
)
) {
panic!("invalid ram limit");
}
config.gc_limit = if cnf.gc_limit.ends_with("gb") {
cnf.gc_limit.trim_end_matches("gb").parse::<usize>().unwrap() * 1024 * 1024 * 1024
} else if cnf.gc_limit.ends_with("mb") {
cnf.gc_limit.trim_end_matches("mb").parse::<usize>().unwrap() * 1024 * 1024
} else if cnf.gc_limit.ends_with("kb") {
cnf.gc_limit.trim_end_matches("kb").parse::<usize>().unwrap() * 1024
} else {
cnf.gc_limit.trim_end_matches("b").parse::<usize>().unwrap()
};
if !(cnf.avg_entry_size.ends_with("b") &&
(
cnf.avg_entry_size.trim_end_matches("b").ends_with("k") ||
cnf.avg_entry_size.trim_end_matches("b").ends_with("m") ||
cnf.avg_entry_size.trim_end_matches("b").ends_with("g")
)
) {
panic!("invalid ram limit");
}
config.avg_entry_size = if cnf.avg_entry_size.ends_with("gb") {
cnf.avg_entry_size.trim_end_matches("gb").parse::<usize>().unwrap() * 1024 * 1024 * 1024
} else if cnf.avg_entry_size.ends_with("mb") {
cnf.avg_entry_size.trim_end_matches("mb").parse::<usize>().unwrap() * 1024 * 1024
} else if cnf.avg_entry_size.ends_with("kb") {
cnf.avg_entry_size.trim_end_matches("kb").parse::<usize>().unwrap() * 1024
} else {
cnf.avg_entry_size.trim_end_matches("b").parse::<usize>().unwrap()
};
if config.avg_entry_size > config.ram_limit {
panic!("avg entry size is larger than ram limit");
}
if config.gc_limit > config.ram_limit {
panic!("gc limit is larger than ram limit");
}
if config.log.is_empty() {
config.log = "info".to_string();
}
if config.log != "debug" && config.log != "info" && config.log != "warn" && config.log != "error" {
panic!("invalid log level");
}
config
}

321
lyphedb/src/dbimpl/mod.rs Normal file
View file

@ -0,0 +1,321 @@
use crate::config::LypheDBConfig;
use log::*;
use lyphedb::PropagationStrategy;
use once_cell::sync::Lazy;
use std::borrow::Cow;
use std::collections::BTreeMap;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use percent_encoding::{percent_decode_str, percent_encode, AsciiSet, CONTROLS};
use tokio::sync::RwLock;
pub const NOT_ALLOWED_ASCII: AsciiSet = CONTROLS.add(b' ').add(b'/').add(b'.').add(b'\\');
pub type LDBCache = BTreeMap<Arc<Vec<u8>>, Arc<Vec<u8>>>;
/// Reads will read from this cache
pub static PRIMARY_CACHE: Lazy<RwLock<LDBCache>> =
Lazy::new(|| RwLock::new(BTreeMap::new()));
/// Writes will be written to this cache, and then depending on the propagation method, the
/// Primary cache will be set to a copy
pub static SECONDARY_CACHE: Lazy<RwLock<LDBCache>> =
Lazy::new(|| RwLock::new(BTreeMap::new()));
/// how often are keys accessed? this will influence the garbage collector
pub static KEY_ACCESS_COUNTER: Lazy<RwLock<BTreeMap<Arc<Vec<u8>>, AtomicU64>>> =
Lazy::new(|| RwLock::new(BTreeMap::new()));
pub fn key_to_path(config: &LypheDBConfig, key: &[u8]) -> PathBuf {
let mut path = PathBuf::new();
path.push(&config.master);
let mut fnbuf = Vec::new();
for (i, byte) in key.iter().enumerate() {
if *byte == b'/' {
let encoded = percent_encode(&fnbuf, &NOT_ALLOWED_ASCII).to_string();
path.push(encoded);
fnbuf.clear();
} else {
fnbuf.push(*byte);
}
}
if !fnbuf.is_empty() {
let encoded = percent_encode(&fnbuf, &NOT_ALLOWED_ASCII).to_string();
path.push(encoded);
fnbuf.clear();
}
path.push(".self");
path
}
#[derive(Debug)]
pub enum OperationError {
KeyCannotBeEmpty,
FilesystemPermissionError,
BadFilesystemEntry,
}
pub async fn set_key_disk(
config: &LypheDBConfig,
key: &[u8],
value: &[u8],
) -> Result<(), OperationError> {
if key.is_empty() {
return Err(OperationError::KeyCannotBeEmpty);
}
let path = key_to_path(config, key);
let directory = path.parent().ok_or(OperationError::KeyCannotBeEmpty)?;
if let Ok(directory_exists) = directory.try_exists() {
if !directory_exists {
std::fs::create_dir_all(directory).map_err(|e| {
warn!("couldn't create directory: {:?}", e);
OperationError::FilesystemPermissionError
})?;
}
} else {
return Err(OperationError::FilesystemPermissionError);
}
std::fs::write(path, value).map_err(|e| {
warn!("couldn't write file: {:?}", e);
OperationError::FilesystemPermissionError
})?;
Ok(())
}
pub async fn delete_key_disk(
config: &LypheDBConfig,
key: &[u8],
) -> Result<(), OperationError> {
if key.is_empty() {
return Err(OperationError::KeyCannotBeEmpty);
}
let path = key_to_path(config, key);
if let Ok(exists) = path.try_exists() {
if !exists {
return Ok(());
}
}
std::fs::remove_file(path).map_err(|e| {
warn!("couldn't remove file: {:?}", e);
OperationError::FilesystemPermissionError
})
}
pub async fn get_key_disk(
config: &LypheDBConfig,
key: &[u8],
) -> Result<Option<Vec<u8>>, OperationError> {
if key.is_empty() {
return Err(OperationError::KeyCannotBeEmpty);
}
let path = key_to_path(config, key);
if let Ok(exists) = path.try_exists() {
if !exists {
return Ok(None);
}
}
let data = std::fs::read(path).map_err(|e| {
warn!("couldn't read file: {:?}", e);
OperationError::FilesystemPermissionError
})?;
Ok(Some(data))
}
/// this function allows empty keys, so you can get all keys under root by doing
/// get_keys_under_keydir_disk(..., b"")
pub async fn get_keys_under_keydir_disk(
config: &LypheDBConfig,
keydir: &[u8],
) -> Result<Vec<Vec<u8>>, OperationError> {
let path = key_to_path(config, keydir);
let path = path.parent().expect("bad master");
let mut keys = Vec::new();
for entry in std::fs::read_dir(path).map_err(|e| {
warn!("couldn't read directory: {:?}", e);
OperationError::FilesystemPermissionError
})? {
let entry = entry.map_err(|e| {
warn!("couldn't read directory entry: {:?}", e);
OperationError::FilesystemPermissionError
})?;
let path = entry.path();
let filename = path
.to_str()
.ok_or(OperationError::FilesystemPermissionError)?;
if filename.ends_with(".self") {
// this is a value file, ignore
continue;
}
let filename = filename.trim_start_matches(&config.master);
let key = percent_decode_str(filename).collect();
keys.push(key);
}
Ok(keys)
}
pub async fn set_key(
config: LypheDBConfig,
key: &[u8],
value: &[u8],
strat: &PropagationStrategy,
) -> Result<(), OperationError> {
let k1 = Arc::new(key.to_vec());
let v1 = Arc::new(value.to_vec());
let disk_task = {
let k1 = k1.clone();
let v1 = v1.clone();
tokio::spawn(async move { set_key_disk(&config, &k1, &v1).await })
};
let prop_task = match strat {
PropagationStrategy::Immediate => {
let k1 = k1.clone();
let v1 = v1.clone();
tokio::spawn(async move {
let mut secondary_cache = SECONDARY_CACHE.write().await;
secondary_cache.insert(k1, v1);
let mut primary_cache = PRIMARY_CACHE.write().await;
*primary_cache = secondary_cache.clone();
})
}
PropagationStrategy::Timeout => {
let k1 = k1.clone();
let v1 = v1.clone();
tokio::spawn(async move {
tokio::spawn(async move {
{
let mut secondary_cache = SECONDARY_CACHE.write().await;
secondary_cache.insert(k1, v1);
}
tokio::spawn(async move {
let start = std::time::Instant::now();
loop {
if start.elapsed().as_secs() > 60 {
break;
}
let pc = PRIMARY_CACHE.try_write();
if pc.is_err() {
tokio::time::sleep(std::time::Duration::from_secs(10)).await;
continue;
} else {
break;
}
}
let mut primary_cache = PRIMARY_CACHE.write().await;
let secondary_cache = SECONDARY_CACHE.read().await;
*primary_cache = secondary_cache.clone();
});
});
})
}
PropagationStrategy::OnRead => {
let k1 = k1.clone();
tokio::spawn(async move {
{
let mut secondary_cache = SECONDARY_CACHE.write().await;
secondary_cache.remove(&k1);
}
{
let mut primary_cache = PRIMARY_CACHE.write().await;
primary_cache.remove(&k1);
}
})
}
};
if let Ok(Err(e)) = disk_task.await {
error!("couldn't set key on disk: {:?} ({:?})", e, key);
// undo propagation
prop_task.abort();
{
let mut primary_cache = PRIMARY_CACHE.write().await;
primary_cache.remove(&k1);
}
{
let mut secondary_cache = SECONDARY_CACHE.write().await;
secondary_cache.remove(&k1);
}
return Err(e);
}
let _ = prop_task.await;
Ok(())
}
pub async fn get_key(config: LypheDBConfig, key: &[u8]) -> Result<Option<Vec<u8>>, OperationError> {
let k1 = Arc::new(key.to_vec());
{
let k1 = k1.clone();
tokio::spawn(async move {
let mut access_counter = KEY_ACCESS_COUNTER.write().await;
let counter = access_counter.entry(k1.clone()).or_insert(AtomicU64::new(0));
if counter.load(Ordering::Relaxed) > 10000000 {
return;
}
counter.fetch_add(1, Ordering::SeqCst);
});
}
let disk_task = {
let k1 = k1.clone();
tokio::spawn(async move { get_key_disk(&config, &k1).await })
};
{
// read from cache
let cache = PRIMARY_CACHE.read().await;
if let Some(val) = cache.get(&k1) {
disk_task.abort();
return Ok(Some(val.to_vec()));
}
}
//debug!("cache miss");
if let Ok(result) = disk_task.await {
if let Ok(Some(val)) = &result {
let val = Arc::new(val.to_vec());
tokio::spawn(async move {
{
let mut cache = SECONDARY_CACHE.write().await;
cache.insert(k1.clone(), val.clone());
}
{
let secondary_cache = SECONDARY_CACHE.read().await;
let mut primary_cache = PRIMARY_CACHE.write().await;
*primary_cache = secondary_cache.clone();
}
//debug!("cache insert");
});
}
result
} else {
Err(OperationError::FilesystemPermissionError)
}
}
pub async fn delete_key(config: LypheDBConfig, key: &[u8]) -> Result<(), OperationError> {
let k1 = Arc::new(key.to_vec());
let disk_task = {
let k1 = k1.clone();
tokio::spawn(async move { delete_key_disk(&config, &k1).await })
};
let prop_task = {
let k1 = k1.clone();
tokio::spawn(async move {
{
let mut key_access_counter = KEY_ACCESS_COUNTER.write().await;
key_access_counter.remove(&k1);
}
{
let mut secondary_cache = SECONDARY_CACHE.write().await;
secondary_cache.remove(&k1);
}
let mut primary_cache = PRIMARY_CACHE.write().await;
let secondary_cache = SECONDARY_CACHE.read().await;
*primary_cache = secondary_cache.clone();
})
};
prop_task.await.expect("couldn't delete key");
disk_task.await.expect("couldn't delete key")
}

49
lyphedb/src/lib.rs Normal file
View file

@ -0,0 +1,49 @@
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum LDBNatsMessage {
Command(LypheDBCommand),
Entries(KVList),
Count(u64),
Success,
BadRequest,
NotFound,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum PropagationStrategy {
/// Reads will immediately be able to read this key's value as soon as it has been set
Immediate,
/// The value change will be queued along with others,
/// then in a period of either inactivity or a maximum time, all changes will be immediately
/// seen at once
Timeout,
/// The key will be removed from the cache, and thus the next read will cause a cache miss,
/// and the value will be loaded from disk
OnRead,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum LypheDBCommand {
SetKeys(KVList, PropagationStrategy),
GetKeys(KeyList),
CountKeys(KeyDirectory),
/// NOT RECURSIVE
GetKeyDirectory(KeyDirectory),
DeleteKeys(KeyList),
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct KVList {
pub kvs: Vec<(Vec<u8>, Vec<u8>)>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct KeyList {
pub keys: Vec<Vec<u8>>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct KeyDirectory {
pub key: Vec<u8>,
}

248
lyphedb/src/main.rs Normal file
View file

@ -0,0 +1,248 @@
/*
keys are stored on disk like this
<master>/<keyname>/.self
so if i stored the key "/this/is/a/key" with the data "hello world", it'd look like this
<master>/this/is/a/key/.self -> "hello world"
*/
use crate::config::{LypheDBConfig, load_config};
use crate::dbimpl::{KEY_ACCESS_COUNTER, PRIMARY_CACHE, SECONDARY_CACHE};
use futures::StreamExt;
use log::{debug, error, info, warn};
use lyphedb::{KVList, LDBNatsMessage, LypheDBCommand};
mod config;
mod dbimpl;
pub async fn gc_thread(config: LypheDBConfig) {
loop {
tokio::time::sleep(std::time::Duration::from_secs(61)).await;
{
let cache = PRIMARY_CACHE.read().await;
let cache_size = cache.len() * config.avg_entry_size;
if cache_size > config.gc_limit {
debug!("gc triggered, cache size: {} bytes", cache_size);
let keycount_to_remove = cache.len() - config.gc_limit / config.avg_entry_size;
drop(cache);
let mut least_freq_keys = vec![];
for (key, count) in KEY_ACCESS_COUNTER.read().await.iter() {
let count = count.load(std::sync::atomic::Ordering::Relaxed);
if least_freq_keys.len() < keycount_to_remove {
least_freq_keys.push((key.clone(), count));
} else {
for (other, oc) in least_freq_keys.iter_mut() {
if count < *oc {
*other = key.clone();
*oc = count;
break;
}
}
}
}
let mut cache = SECONDARY_CACHE.write().await;
for (key, _) in least_freq_keys.iter() {
cache.remove(key);
}
let mut primary_cache = PRIMARY_CACHE.write().await;
*primary_cache = cache.clone();
debug!(
"gc finished, cache size: {} bytes",
primary_cache.len() * config.avg_entry_size
);
}
}
}
}
pub async fn precache_keys_until_limit(config: LypheDBConfig) {
info!("precache started");
let mut precache_count = 0;
let mut precache_stack = dbimpl::get_keys_under_keydir_disk(&config, b"")
.await
.expect("couldn't get root keys");
while let Some(precache_key) = precache_stack.pop() {
{
let cache = PRIMARY_CACHE.read().await;
if cache.len() * config.avg_entry_size > config.gc_limit {
break;
}
if cache.len() % 10000 == 0 {
info!("precache size: {} mb", (cache.len() * config.avg_entry_size) / (1024*1024));
}
}
let children = dbimpl::get_keys_under_keydir_disk(&config, &precache_key)
.await
.expect("couldn't get children of key");
if !children.is_empty() {
precache_stack.extend(children);
} else {
let _value = dbimpl::get_key(config.clone(), &precache_key)
.await
.expect("couldn't get value of key");
precache_count += 1;
}
}
info!("precache finished, {} values precached", precache_count);
}
#[tokio::main]
async fn main() {
let config = load_config(&std::env::args().nth(1).expect("please specify config file"));
let logger = env_logger::builder()
.filter_level(match config.log.as_str() {
"debug" => log::LevelFilter::Debug,
"info" => log::LevelFilter::Info,
"warn" => log::LevelFilter::Warn,
"error" => log::LevelFilter::Error,
_ => unreachable!(),
})
.build();
log::set_boxed_logger(Box::new(logger)).unwrap();
log::set_max_level(match config.log.as_str() {
"debug" => log::LevelFilter::Debug,
"info" => log::LevelFilter::Info,
"warn" => log::LevelFilter::Warn,
"error" => log::LevelFilter::Error,
_ => unreachable!(),
});
info!("lyphedb started");
let nats = async_nats::ConnectOptions::new()
.add_client_certificate(
config.nats_cert.as_str().into(),
config.nats_key.as_str().into(),
)
.connect(config.nats_url.as_str())
.await;
if let Err(e) = nats {
error!("FATAL ERROR, COULDN'T CONNECT TO NATS: {}", e);
return;
}
let nats = nats.unwrap();
let mut subscriber = nats
.queue_subscribe(config.name.clone(), "lyphedb".to_string())
.await
.expect("couldn't subscribe to subject");
info!("nats connected");
tokio::spawn(precache_keys_until_limit(config.clone()));
tokio::spawn(gc_thread(config.clone()));
while let Some(msg) = subscriber.next().await {
let config = config.clone();
let nats = nats.clone();
tokio::spawn(async move {
async fn bad_request(nats: &async_nats::Client, replyto: async_nats::Subject) {
let reply = rmp_serde::to_vec(&LDBNatsMessage::BadRequest).unwrap();
if let Err(e) = nats.publish(replyto, reply.into()).await {
warn!("couldn't send reply: {:?}", e);
}
}
let data = msg.payload.to_vec();
let data = rmp_serde::from_slice::<LDBNatsMessage>(&data);
if let Err(e) = data {
warn!("couldn't deserialize message: {:?}", e);
if let Some(replyto) = msg.reply {
bad_request(&nats, replyto).await;
}
return;
}
let data = data.unwrap();
match data {
LDBNatsMessage::Command(cmd) => match cmd {
LypheDBCommand::SetKeys(kvlist, propstrat) => {
for (key, value) in kvlist.kvs {
if let Err(e) =
dbimpl::set_key(config.clone(), &key, &value, &propstrat).await
{
warn!("couldn't set key: {:?}", e);
}
}
let reply = rmp_serde::to_vec(&LDBNatsMessage::Success).unwrap();
if let Err(e) = nats.publish(msg.reply.unwrap(), reply.into()).await {
warn!("couldn't send reply: {:?}", e);
}
}
LypheDBCommand::GetKeys(klist) => {
let mut reply = Vec::new();
for key in klist.keys {
if let Ok(Some(value)) = dbimpl::get_key(config.clone(), &key).await {
reply.push((key, value));
} else {
warn!("couldn't get key: {:?}", key);
}
}
let reply =
rmp_serde::to_vec(&LDBNatsMessage::Entries(KVList { kvs: reply }))
.unwrap();
if let Err(e) = nats.publish(msg.reply.unwrap(), reply.into()).await {
warn!("couldn't send reply: {:?}", e);
}
}
LypheDBCommand::CountKeys(keydir) => {
let keys = dbimpl::get_keys_under_keydir_disk(&config, &keydir.key)
.await
.expect("couldn't get keys under keydir");
let mut count = 0;
for key in keys {
let value = dbimpl::get_key(config.clone(), &key)
.await
.expect("couldn't get value of key");
if value.is_some() {
count += 1;
}
}
let reply = rmp_serde::to_vec(&LDBNatsMessage::Count(count)).unwrap();
if let Err(e) = nats.publish(msg.reply.unwrap(), reply.into()).await {
warn!("couldn't send reply: {:?}", e);
}
}
LypheDBCommand::GetKeyDirectory(keydir) => {
let mut reply = Vec::new();
let keys = dbimpl::get_keys_under_keydir_disk(&config, &keydir.key)
.await
.expect("couldn't get keys under keydir");
for key in keys {
let value = dbimpl::get_key(config.clone(), &key)
.await
.expect("couldn't get value of key");
if let Some(value) = value {
reply.push((key, value));
}
}
let reply =
rmp_serde::to_vec(&LDBNatsMessage::Entries(KVList { kvs: reply }))
.unwrap();
if let Err(e) = nats.publish(msg.reply.unwrap(), reply.into()).await {
warn!("couldn't send reply: {:?}", e);
}
}
LypheDBCommand::DeleteKeys(klist) => {
for key in klist.keys {
if let Err(e) = dbimpl::delete_key(config.clone(), &key).await {
warn!("couldn't delete key: {:?}", e);
}
}
let reply = rmp_serde::to_vec(&LDBNatsMessage::Success).unwrap();
if let Err(e) = nats.publish(msg.reply.unwrap(), reply.into()).await {
warn!("couldn't send reply: {:?}", e);
}
}
},
_ => {
warn!("bad request, not command");
if let Some(replyto) = msg.reply {
bad_request(&nats, replyto).await;
}
}
}
});
}
info!("lyphedb shutting down");
}

1
vorebot/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
/target

28
vorebot/Cargo.toml Normal file
View file

@ -0,0 +1,28 @@
[package]
name = "vorebot"
version = "0.2.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
asklyphe-common = { path = "../asklyphe-common" }
tokio = { version = "1.0", features = ["full"] }
serde = { version = "1.0", features = ["derive"] }
rand = "0.8.5"
rmp-serde = "1.1.2"
base64 = "0.21.7"
image = "0.24.8"
isahc = "1.7.2"
ulid = "1.0.0"
async-nats = "0.38.0"
futures = "0.3.28"
chrono = "0.4.26"
once_cell = "1.18.0"
env_logger = "0.10.0"
log = "0.4.19"
mutex-timeouts = { version = "0.3.0", features = ["tokio"] }
prometheus_exporter = "0.8.5"
thirtyfour = "0.35.0"
stopwords = "0.1.1"
texting_robots = "0.2.2"

352
vorebot/src/main.rs Normal file
View file

@ -0,0 +1,352 @@
mod webparse;
use asklyphe_common::db;
use asklyphe_common::nats::vorebot::{
VOREBOT_NEWHOSTNAME_SERVICE, VOREBOT_SERVICE, VOREBOT_SUGGESTED_SERVICE,
};
use async_nats::jetstream;
use async_nats::jetstream::consumer::PullConsumer;
use async_nats::jetstream::stream::RetentionPolicy;
use chrono::TimeZone;
use futures::StreamExt;
use log::{debug, error, info, warn};
use mutex_timeouts::tokio::MutexWithTimeoutAuto as Mutex;
use once_cell::sync::Lazy;
use prometheus_exporter::prometheus::core::{AtomicF64, GenericGauge};
use prometheus_exporter::prometheus::{register_counter, register_gauge, Counter};
use std::cmp::max;
use std::collections::{BTreeMap, BTreeSet};
use std::hash::{DefaultHasher, Hasher};
use std::io::Read;
use std::iter::Iterator;
use std::str::FromStr;
use std::string::ToString;
use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
use std::sync::Arc;
use std::time::Duration;
use async_nats::jetstream::kv;
use stopwords::{Language, Spark, Stopwords};
use thirtyfour::{CapabilitiesHelper, DesiredCapabilities, Proxy, WebDriver};
use thirtyfour::common::capabilities::firefox::FirefoxPreferences;
use tokio::task::JoinHandle;
use asklyphe_common::ldb::DBConn;
use asklyphe_common::nats::vorebot::CrawlRequest;
use crate::webparse::web_parse;
pub static NATS_URL: Lazy<String> =
Lazy::new(|| std::env::var("NATS_URL").expect("NO NATS URL DEFINED"));
pub static NATS_CERT: Lazy<String> = Lazy::new(|| std::env::var("NATS_CERT").expect("NO NATS_CERT DEFINED"));
pub static NATS_KEY: Lazy<String> = Lazy::new(|| std::env::var("NATS_KEY").expect("NO NATS_KEY DEFINED"));
pub static BROWSER_THREADS: Lazy<Vec<String>> =
Lazy::new(|| std::env::var("BROWSER_THREADS").expect("PLEASE LIST BROWSER_THREADS").split(',').map(|v| v.to_string()).collect());
pub static DB_NAME: Lazy<String> =
Lazy::new(|| std::env::var("DB_NAME").expect("PLEASE ADD DB_NAME"));
pub static DOCUMENTS_CRAWLED: AtomicU64 = AtomicU64::new(0);
pub static LAST_MESSAGE: AtomicI64 = AtomicI64::new(0);
pub static LAST_TASK_COMPLETE: Lazy<Vec<Arc<AtomicI64>>> = Lazy::new(|| {
let max_threads: usize = BROWSER_THREADS.len();
let mut vals = vec![];
for i in 0..max_threads {
// let db = Database::default().expect("couldn't connect to foundation db!");
// DBS.lock().await.push(Arc::new(db));
vals.push(Arc::new(AtomicI64::new(chrono::Utc::now().timestamp())));
}
vals
});
pub static USER_AGENT: Lazy<String> = Lazy::new(|| {
format!(
"Vorebot/{} (compatible; Googlebot/2.1; +https://voremicrocomputers.com/crawler.html)",
env!("CARGO_PKG_VERSION")
)
});
#[tokio::main]
async fn main() {
mutex_timeouts::tokio::GLOBAL_TOKIO_TIMEOUT.store(72, Ordering::Relaxed);
env_logger::init();
info!("began at {}", chrono::Utc::now().to_string());
let nats = async_nats::ConnectOptions::new()
.add_client_certificate(NATS_CERT.as_str().into(), NATS_KEY.as_str().into())
.connect(NATS_URL.as_str())
.await;
if let Err(e) = nats {
error!("FATAL ERROR, COULDN'T CONNECT TO NATS: {}", e);
return;
}
let nats = nats.unwrap();
let dbconn = DBConn::new(nats.clone(), DB_NAME.to_string());
let nats = jetstream::new(nats);
let mut tasks: Vec<(JoinHandle<()>, String)> = vec![];
let mut available_browsers: Vec<String> = BROWSER_THREADS.clone();
{
loop {
while tasks.len() < BROWSER_THREADS.len() {
let nats = nats.clone();
let browser = available_browsers.pop().expect("NO BROWSERS LEFT, THIS IS A FATAL BUG!");
let db = dbconn.clone();
let b = browser.clone();
tasks.push((tokio::spawn(async move {
let browser = b;
info!("using {}", browser);
info!("crawler spawned");
/* normal priority */
let consumer: PullConsumer = nats.get_or_create_stream(jetstream::stream::Config {
name: VOREBOT_SERVICE.to_string(),
subjects: vec![VOREBOT_SERVICE.to_string()],
retention: RetentionPolicy::WorkQueue,
..Default::default()
}).await
.expect("FATAL! FAILED TO SUBSCRIBE TO NATS!")
.get_or_create_consumer("parser", jetstream::consumer::pull::Config {
durable_name: Some("parser".to_string()),
filter_subject: VOREBOT_SERVICE.to_string(),
..Default::default()
}).await.expect("FATAL! FAILED TO SUBSCRIBE TO NATS!");
let mut messages = consumer.messages().await.expect("FATAL! FAILED TO SUBSCRIBE TO NATS!");
/* higher priority (new hostnames) */
let higher_consumer: PullConsumer = nats.get_or_create_stream(jetstream::stream::Config {
name: VOREBOT_NEWHOSTNAME_SERVICE.to_string(),
subjects: vec![VOREBOT_NEWHOSTNAME_SERVICE.to_string()],
retention: RetentionPolicy::WorkQueue,
..Default::default()
}).await
.expect("FATAL! FAILED TO SUBSCRIBE TO NATS!")
.get_or_create_consumer("highparser", jetstream::consumer::pull::Config {
durable_name: Some("highparser".to_string()),
filter_subject: VOREBOT_NEWHOSTNAME_SERVICE.to_string(),
..Default::default()
}).await.expect("FATAL! FAILED TO SUBSCRIBE TO NATS!");
let mut high_messages = higher_consumer.messages().await.expect("FATAL! FAILED TO SUBSCRIBE TO NATS!");
/* highest priority (user-suggested) */
let highest_consumer: PullConsumer = nats.get_or_create_stream(jetstream::stream::Config {
name: VOREBOT_SUGGESTED_SERVICE.to_string(),
subjects: vec![VOREBOT_SUGGESTED_SERVICE.to_string()],
retention: RetentionPolicy::WorkQueue,
..Default::default()
}).await
.expect("FATAL! FAILED TO SUBSCRIBE TO NATS!")
.get_or_create_consumer("highestparser", jetstream::consumer::pull::Config {
durable_name: Some("highestparser".to_string()),
filter_subject: VOREBOT_SUGGESTED_SERVICE.to_string(),
..Default::default()
}).await.expect("FATAL! FAILED TO SUBSCRIBE TO NATS!");
let mut highest_messages = highest_consumer.messages().await.expect("FATAL! FAILED TO SUBSCRIBE TO NATS!");
let mut prefs = FirefoxPreferences::new();
prefs.set_user_agent(USER_AGENT.to_string()).unwrap();
let mut caps = DesiredCapabilities::firefox();
caps.set_preferences(prefs).unwrap();
let driver = WebDriver::new(&browser, caps).await.unwrap();
info!("crawler ready");
loop {
tokio::select! {
Some(highest) = StreamExt::next(&mut highest_messages) => {
if let Err(e) = highest {
warn!("error when recv js message! {e}");
} else {
let message = highest.unwrap();
if let Err(e) = message.ack().await {
warn!("failed acking message {e}");
}
let req = rmp_serde::from_slice::<CrawlRequest>(message.payload.as_ref());
if let Err(e) = req {
error!("BAD NATS REQUEST: {e}");
continue;
}
let req = req.unwrap();
info!("RECV USER SUGGESTION!");
let now = chrono::Utc::now().timestamp();
LAST_MESSAGE.store(now, Ordering::Relaxed);
let nats = nats.clone();
let mut bad = false;
driver.in_new_tab(|| async {
if web_parse(nats.clone(), db.clone(), &driver, &req.url, req.damping).await.is_err() {
warn!("temporary failure detected in parsing, requeuing");
nats.publish(VOREBOT_SERVICE.to_string(), rmp_serde::to_vec(&req).unwrap().into()).await.expect("FAILED TO REQUEUE");
bad = true;
}
Ok(())
}).await.unwrap();
if bad {
continue;
}
if DOCUMENTS_CRAWLED.load(Ordering::Relaxed) % 100 == 0 {
DOCUMENTS_CRAWLED.fetch_add(1, Ordering::Relaxed);
info!("crawled {} pages!", DOCUMENTS_CRAWLED.load(Ordering::Relaxed));
}
}
}
Some(high) = StreamExt::next(&mut high_messages) => {
if let Err(e) = high {
warn!("error when recv js message! {e}");
} else {
let message = high.unwrap();
if let Err(e) = message.ack().await {
warn!("failed acking message {e}");
}
let req = rmp_serde::from_slice::<CrawlRequest>(message.payload.as_ref());
if let Err(e) = req {
error!("BAD NATS REQUEST: {e}");
continue;
}
let req = req.unwrap();
info!("RECV HIGH PRIORITY!");
let now = chrono::Utc::now().timestamp();
LAST_MESSAGE.store(now, Ordering::Relaxed);
let nats = nats.clone();
let mut bad = false;
driver.in_new_tab(|| async {
if web_parse(nats.clone(), db.clone(), &driver, &req.url, req.damping).await.is_err() {
warn!("temporary failure detected in parsing, requeuing");
nats.publish(VOREBOT_SERVICE.to_string(), rmp_serde::to_vec(&req).unwrap().into()).await.expect("FAILED TO REQUEUE");
bad = true;
}
Ok(())
}).await.unwrap();
if bad {
continue;
}
if DOCUMENTS_CRAWLED.load(Ordering::Relaxed) % 100 == 0 {
DOCUMENTS_CRAWLED.fetch_add(1, Ordering::Relaxed);
info!("crawled {} pages!", DOCUMENTS_CRAWLED.load(Ordering::Relaxed));
}
}
}
Some(normal) = StreamExt::next(&mut messages) => {
if let Err(e) = normal {
warn!("error when recv js message! {e}");
} else {
let message = normal.unwrap();
if let Err(e) = message.ack().await {
warn!("failed acking message {e}");
}
let req = rmp_serde::from_slice::<CrawlRequest>(message.payload.as_ref());
if let Err(e) = req {
error!("BAD NATS REQUEST: {e}");
continue;
}
let req = req.unwrap();
let now = chrono::Utc::now().timestamp();
LAST_MESSAGE.store(now, Ordering::Relaxed);
let nats = nats.clone();
let mut hash = DefaultHasher::new();
hash.write(req.url.as_bytes());
let hash = hash.finish();
let dehomo_bucket = nats.get_key_value("dehomo").await;
let dehomo_bucket = if dehomo_bucket.is_err() {
let dehomo_bucket = nats.create_key_value(kv::Config {
bucket: "dehomo".to_string(),
description: "prevent the same url from being scraped again too quickly".to_string(),
max_age: Duration::from_secs(60*60),
..Default::default()
}).await;
if let Err(e) = dehomo_bucket {
panic!("FAILED TO CREATE DEHOMO BUCKET!!! {e}");
} else {
dehomo_bucket.unwrap()
}
} else {
dehomo_bucket.unwrap()
};
if dehomo_bucket.get(hash.to_string()).await.ok().flatten().map(|v| *v.first().unwrap_or(&0) == 1).unwrap_or(false) {
info!("too soon to scrape {}", req.url);
continue;
}
let mut bad = false;
driver.in_new_tab(|| async {
if web_parse(nats.clone(), db.clone(), &driver, &req.url, req.damping).await.is_err() {
warn!("temporary failure detected in parsing, requeuing");
nats.publish(VOREBOT_SERVICE.to_string(), rmp_serde::to_vec(&req).unwrap().into()).await.expect("FAILED TO REQUEUE");
bad = true;
}
Ok(())
}).await.unwrap();
if bad {
continue;
}
dehomo_bucket.put(hash.to_string(), vec![1u8].into()).await.expect("failed to store dehomo");
if DOCUMENTS_CRAWLED.load(Ordering::Relaxed) % 100 == 0 {
DOCUMENTS_CRAWLED.fetch_add(1, Ordering::Relaxed);
info!("crawled {} pages!", DOCUMENTS_CRAWLED.load(Ordering::Relaxed));
}
}
}
}
}
}),
browser
));
warn!("spawning new injest thread");
}
let mut tasks_to_remove = vec![];
for task in tasks.iter() {
if task.0.is_finished() {
tasks_to_remove.push(task.1.clone());
available_browsers.push(task.1.clone());
}
}
tasks.retain(|v| !tasks_to_remove.contains(&v.1));
tokio::time::sleep(Duration::from_secs(3)).await;
//while let Some(p) = initial.pop() {
// nats.publish(VOREBOT_SUGGESTED_SERVICE.to_string(), rmp_serde::to_vec(&p).unwrap().into()).await.unwrap();
//}
}
}
}
//#[tokio::main]
//async fn main() {
// mutex_timeouts::tokio::GLOBAL_TOKIO_TIMEOUT.store(72, Ordering::Relaxed);
//
// env_logger::init();
// info!("began at {}", chrono::Utc::now().to_string());
//
// let nats = async_nats::connect(NATS_URL.as_str()).await;
// if let Err(e) = nats {
// error!("FATAL ERROR, COULDN'T CONNECT TO NATS: {}", e);
// return;
// }
// let nats = nats.unwrap();
//
// let dbconn = DBConn::new(nats.clone(), "lyphedb-test");
//
// let nats = jetstream::new(nats);
//
// let mut prefs = FirefoxPreferences::new();
// prefs.set_user_agent(USER_AGENT.to_string()).unwrap();
// let mut caps = DesiredCapabilities::firefox();
// caps.set_preferences(prefs).unwrap();
// let driver = WebDriver::new(&BROWSER_THREADS[0], caps).await.unwrap();
//
// driver.in_new_tab(|| async {
// web_parse(nats.clone(), dbconn.clone(), &driver, "https://asklyphe.com/", 0.85).await.expect("Failed to run web parse");
//
// Ok(())
// }).await.unwrap();
//}

479
vorebot/src/webparse/mod.rs Normal file
View file

@ -0,0 +1,479 @@
use crate::USER_AGENT;
use asklyphe_common::ldb::{linkstore, metastore, sitestore, titlestore, wordstore, DBConn};
use async_nats::jetstream;
use async_nats::jetstream::kv;
use futures::AsyncReadExt;
use image::EncodableLayout;
use isahc::config::RedirectPolicy;
use isahc::prelude::Configurable;
use isahc::HttpClient;
use log::{debug, error, warn};
use std::collections::{BTreeMap, BTreeSet};
use std::hash::{DefaultHasher, Hasher};
use std::sync::atomic::AtomicBool;
use std::sync::{mpsc, Arc};
use std::time::Duration;
use stopwords::{Language, Spark, Stopwords};
use texting_robots::{get_robots_url, Robot};
use thirtyfour::{By, WebDriver};
use asklyphe_common::nats::vorebot::CrawlRequest;
use asklyphe_common::nats::vorebot::VOREBOT_SERVICE;
pub fn allowed_to_crawl(robotstxt: &[u8], url: &str) -> Result<bool, ()> {
let robot1 = Robot::new("Vorebot", robotstxt);
if let Err(e) = robot1 {
warn!(
"potentially malformed robots.txt ({}), not crawling {}",
e, url
);
return Err(());
}
let robot1 = robot1.unwrap();
Ok(robot1.allowed(url))
}
// returns Err if we cannot access a page, but the error associated with it seems temporary (i.e. it's worth trying again later)
// otherwise, returns Ok
pub async fn web_parse(
nats: jetstream::Context,
db: DBConn,
driver: &WebDriver,
url: &str,
damping: f64,
) -> Result<(), ()> {
driver.delete_all_cookies().await.map_err(|_| ())?;
let robots_bucket = nats.get_key_value("robots").await;
let robots_bucket = if robots_bucket.is_err() {
let robots_bucket = nats
.create_key_value(kv::Config {
bucket: "robots".to_string(),
description: "storage of robots.txt data for given hosts".to_string(),
..Default::default()
})
.await;
if let Err(e) = robots_bucket {
error!("could not create robots.txt bucket: {}", e);
None
} else {
Some(robots_bucket.unwrap())
}
} else {
robots_bucket.ok()
};
let hosts_bucket = nats.get_key_value("hosts").await;
let hosts_bucket = if hosts_bucket.is_err() {
let hosts_bucket = nats
.create_key_value(kv::Config {
bucket: "hosts".to_string(),
description: "prevent the same host from being scraped too quickly".to_string(),
max_age: Duration::from_secs(60 * 5),
..Default::default()
})
.await;
if let Err(e) = hosts_bucket {
error!("could not create hosts bucket: {}", e);
return Err(());
} else {
hosts_bucket.unwrap()
}
} else {
hosts_bucket.unwrap()
};
let robots_url = get_robots_url(url);
if robots_url.is_err() {
error!("could not get a robots.txt url from {}, not crawling", url);
return Ok(());
}
let robots_url = robots_url.unwrap();
let mut hash = DefaultHasher::new();
hash.write(robots_url.as_bytes());
let hash = hash.finish();
if let Ok(Some(host)) = hosts_bucket.get(hash.to_string()).await {
let count = *host.first().unwrap_or(&0);
if count > 100 {
warn!("scraping {} too quickly, avoiding for one minute", robots_url);
return Err(());
}
hosts_bucket.put(hash.to_string(), vec![count + 1].into()).await.expect("COULDN'T INSERT INTO HOSTS BUCKET!");
} else {
hosts_bucket.put(hash.to_string(), vec![1].into()).await.expect("COULDN'T INSERT INTO HOSTS BUCKET!");
}
let mut skip_robots_check = false;
if let Some(robots_bucket) = &robots_bucket {
if let Ok(Some(entry)) = robots_bucket.get(hash.to_string()).await {
if let Ok(res) = allowed_to_crawl(entry.as_bytes(), url) {
if !res {
debug!("robots.txt does not allow us to crawl {}", url);
return Ok(());
} else {
skip_robots_check = true;
}
}
}
}
if !skip_robots_check {
// check manually
debug!("checking new robots.txt \"{}\"", robots_url);
let client = HttpClient::builder()
.redirect_policy(RedirectPolicy::Limit(10))
.timeout(Duration::from_secs(60))
.build();
if let Err(e) = client {
error!("could not create new robots.txt httpclient: {}", e);
return Err(());
}
let client = client.unwrap();
let request = isahc::Request::get(&robots_url)
.header("user-agent", USER_AGENT.as_str())
.body(());
if let Err(e) = request {
error!("could not create robots.txt get request: {}", e);
return Ok(());
}
let request = request.unwrap();
let response = client.send_async(request).await;
if let Err(e) = response {
warn!("could not get robots.txt page: {}", e);
return Err(());
}
let mut response = response.unwrap();
if response.status() == 429 {
// too many requests
warn!("too many requests for {}", robots_url);
return Err(());
}
if response.status().is_server_error() {
// don't crawl at the moment
debug!("not crawling {} due to server error", robots_url);
return Err(());
}
let mut body = "".to_string();
if let Err(e) = response.body_mut().read_to_string(&mut body).await {
warn!("could not read from robots.txt response: {}", e);
return Err(());
}
if let Ok(res) = allowed_to_crawl(body.as_bytes(), url) {
if let Some(robots_bucket) = &robots_bucket {
if let Err(e) = robots_bucket
.put(hash.to_string(), body.as_bytes().to_vec().into())
.await
{
warn!("could not put robots.txt data: {}", e);
}
}
if !res {
debug!("robots.txt does not allow us to crawl {}", url);
return Ok(());
} else {
// we're allowed to crawl!
}
}
}
let start = std::time::Instant::now();
debug!("handling request for {}", url);
// check for bad status codes
// fixme: i hate this solution, can we get something that actually checks the browser's request?
let client = HttpClient::builder()
.redirect_policy(RedirectPolicy::Limit(10))
.timeout(Duration::from_secs(60))
.build();
if let Err(e) = client {
error!("could not create new badstatuscode httpclient: {}", e);
return Err(());
}
let client = client.unwrap();
let request = isahc::Request::get(url)
.header("user-agent", USER_AGENT.as_str())
.body(());
if let Err(e) = request {
error!("could not create badstatuscode get request: {}", e);
return Ok(());
}
let request = request.unwrap();
let response = client.send_async(request).await;
if let Err(e) = response {
warn!("could not get badstatuscode page: {}", e);
return Err(());
}
let mut response = response.unwrap();
if response.status() == 429 {
// too many requests
warn!("too many requests for {}", url);
return Err(());
}
if response.status().is_server_error() || response.status().is_client_error() {
// don't crawl at the moment
debug!("not crawling {} due to bad status code {}", url, response.status());
return Err(());
}
// i guess we're good
driver.goto(url).await.map_err(|_| ())?;
let meta_elements = driver.find_all(By::Tag("meta")).await.map_err(|_| ())?;
let title = driver.title().await.map_err(|_| ())?;
let mut description = None;
let mut keywords = vec![];
for elem in meta_elements {
if let Ok(Some(name)) = elem.attr("name").await {
match name.as_str() {
"description" => {
if let Ok(Some(content)) = elem.attr("content").await {
description = Some(content);
}
}
"keywords" => {
if let Ok(Some(content)) = elem.attr("content").await {
keywords = content
.split(',')
.map(|v| v.to_lowercase())
.filter(|v| !v.is_empty())
.collect();
}
}
_ => {}
}
}
}
let body = driver.find(By::Tag("body")).await.map_err(|_| ())?;
let raw_page_content = body.text().await.map_err(|_| ())?;
async fn gather_elements_with_multiplier(
driver: &WebDriver,
wordmap: &mut BTreeMap<String, f64>,
stops: &BTreeSet<&&str>,
elements: &[&str],
multiplier: f64,
) {
let mut elms = vec![];
for tag in elements {
elms.push(driver.find_all(By::Tag(*tag)).await);
}
let elms = elms.iter().flatten().flatten().collect::<Vec<_>>();
let mut sentences = vec![];
let mut sentence_set = BTreeSet::new();
debug!("processing elements...");
for node in elms {
let _ = node.scroll_into_view().await;
let boxmodel = node.rect().await;
if boxmodel.is_err() {
// not visible
continue;
}
let boxmodel = boxmodel.unwrap();
let current_text = node.text().await;
if current_text.is_err() {
// no text on this node
continue;
}
let current_text = current_text.unwrap().trim().to_string();
if current_text.is_empty() {
continue;
}
let sqs = (boxmodel.width * boxmodel.height).max(1.0); // no 0 divides pls (:
let ccount = current_text.chars().count() as f64;
let cssq = if ccount > 0.0 { sqs / ccount } else { 0.0 };
if sentence_set.contains(&current_text) {
continue;
}
sentence_set.insert(current_text.clone());
sentences.push((current_text, cssq));
}
for (sentence, cssq) in sentences {
let mut cssq = (cssq / 500.0).powi(2) * multiplier;
for word in sentence.split_whitespace() {
let word = word
.to_lowercase()
.trim_end_matches(|v: char| v.is_ascii_punctuation())
.to_string();
if stops.contains(&word.as_str()) {
// less valuable
cssq /= 100.0;
}
if let Some(wentry) = wordmap.get_mut(&word) {
*wentry += cssq;
} else {
if word.is_empty() {
continue;
}
wordmap.insert(word.to_string(), cssq);
}
}
}
}
let mut wordmap: BTreeMap<String, f64> = BTreeMap::new();
let stops: BTreeSet<_> = Spark::stopwords(Language::English)
.unwrap()
.iter()
.collect();
debug!("headers...");
gather_elements_with_multiplier(driver, &mut wordmap, &stops, &["h1","h2","h3","h4","h5","h6"], 3.0)
.await;
debug!("paragraphs...");
gather_elements_with_multiplier(driver, &mut wordmap, &stops, &["p","div"], 1.0).await;
let mut wordmap = wordmap.into_iter().collect::<Vec<_>>();
wordmap.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
let mut db_error_so_requeue_anyways = false;
let words = wordmap
.iter()
.map(|(word, _)| word.as_str())
.collect::<Vec<_>>();
#[allow(clippy::collapsible_if)]
if !words.is_empty() {
if wordstore::add_url_to_keywords(&db, &words, url)
.await
.is_err()
{
warn!("couldn't add {} to keywords!", url);
db_error_so_requeue_anyways = true;
}
}
let mut metawords = keywords.iter().map(|v| v.as_str()).collect::<Vec<_>>();
let desc2 = description.clone();
let desc2 = desc2.map(|v| {
v.to_lowercase()
.split_whitespace()
.map(String::from)
.collect::<Vec<_>>()
});
if let Some(description) = &desc2 {
for word in description {
let word = word.trim_end_matches(|v: char| v.is_ascii_punctuation());
if word.is_empty() {
continue;
}
metawords.push(word);
}
}
#[allow(clippy::collapsible_if)]
if !metawords.is_empty() {
if metastore::add_url_to_metawords(&db, &metawords, url)
.await
.is_err()
{
warn!("couldn't add {} to metawords!", url);
db_error_so_requeue_anyways = true;
}
}
let mut titlewords = vec![];
let title2 = title.clone();
let title2 = title2.to_lowercase();
for word in title2.split_whitespace() {
let word = word.trim_end_matches(|v: char| v.is_ascii_punctuation());
if word.is_empty() {
continue;
}
titlewords.push(word);
}
#[allow(clippy::collapsible_if)]
if !titlewords.is_empty() {
if titlestore::add_url_to_titlewords(&db, &titlewords, url)
.await
.is_err()
{
warn!("couldn't add {} to titlewords!", url);
db_error_so_requeue_anyways = true;
}
}
if sitestore::add_website(
&db,
url,
Some(title),
description,
if keywords.is_empty() {
None
} else {
Some(keywords)
},
&wordmap,
raw_page_content,
damping
)
.await
.is_err()
{
warn!("couldn't add {} to sitestore!", url);
db_error_so_requeue_anyways = true;
}
debug!("finished with main site stuff for {}", url);
let linkelms = driver.find_all(By::Tag("a")).await.map_err(|_| ())?;
for linkelm in linkelms {
if linkelm.scroll_into_view().await.is_err() {
debug!("couldn't scroll into view!");
}
let href = linkelm.prop("href").await.map_err(|_| ())?;
if href.is_none() {
debug!("no href!");
continue;
}
let href = href.unwrap();
if href.contains('#') {
continue;
}
let linktext = linkelm.text().await.map_err(|_| ())?.to_lowercase();
let linkimgs = linkelm.find_all(By::Tag("img")).await.map_err(|_| ())?;
let mut alts = "".to_string();
for img in linkimgs {
if let Ok(Some(alt)) = img.attr("alt").await {
alts.push_str(&alt);
alts.push(' ');
}
}
let alts = alts.trim().to_lowercase();
let mut linkwords = vec![];
for word in linktext.split_whitespace() {
let word = word.trim_end_matches(|v: char| v.is_ascii_punctuation());
linkwords.push(word);
}
for word in alts.split_whitespace() {
let word = word.trim_end_matches(|v: char| v.is_ascii_punctuation());
linkwords.push(word);
}
#[allow(clippy::collapsible_if)]
if !linkwords.is_empty() {
if linkstore::add_url_to_linkwords(&db, &linkwords, &href).await.is_err() {
warn!("couldn't add {} to linkwords!", url);
}
}
nats.publish(VOREBOT_SERVICE.to_string(), rmp_serde::to_vec(&CrawlRequest {
url: href,
damping: 0.85,
}).unwrap().into()).await.unwrap();
}
let elapsed = start.elapsed().as_secs_f64();
debug!("crawled {} in {} seconds", url, elapsed);
// todo: queue links to be crawled
Ok(())
}