bring over new crawler and lyphedb

This commit is contained in:
husky 2025-03-12 12:52:24 -07:00
parent b1202cc889
commit 906c2ed8df
25 changed files with 2919 additions and 15 deletions

594
Cargo.lock generated
View file

@ -133,6 +133,12 @@ version = "1.0.97"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f"
[[package]]
name = "arc-swap"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457"
[[package]] [[package]]
name = "argon2" name = "argon2"
version = "0.5.3" version = "0.5.3"
@ -151,6 +157,12 @@ version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "ascii"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16"
[[package]] [[package]]
name = "askama" name = "askama"
version = "0.12.1" version = "0.12.1"
@ -236,6 +248,8 @@ dependencies = [
"foundationdb", "foundationdb",
"futures", "futures",
"log", "log",
"lyphedb",
"percent-encoding",
"rand 0.8.5", "rand 0.8.5",
"rmp-serde", "rmp-serde",
"serde", "serde",
@ -406,7 +420,7 @@ version = "0.38.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76433c4de73442daedb3a59e991d94e85c14ebfc33db53dfcd347a21cd6ef4f8" checksum = "76433c4de73442daedb3a59e991d94e85c14ebfc33db53dfcd347a21cd6ef4f8"
dependencies = [ dependencies = [
"base64", "base64 0.22.1",
"bytes", "bytes",
"futures", "futures",
"memchr", "memchr",
@ -651,6 +665,12 @@ dependencies = [
"windows-targets 0.52.6", "windows-targets 0.52.6",
] ]
[[package]]
name = "base64"
version = "0.21.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
[[package]] [[package]]
name = "base64" name = "base64"
version = "0.22.1" version = "0.22.1"
@ -703,7 +723,7 @@ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
"regex", "regex",
"rustc-hash", "rustc-hash 1.1.0",
"shlex", "shlex",
"syn 2.0.99", "syn 2.0.99",
"which", "which",
@ -726,6 +746,12 @@ dependencies = [
"url_encoded_data", "url_encoded_data",
] ]
[[package]]
name = "bit_field"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61"
[[package]] [[package]]
name = "bitflags" name = "bitflags"
version = "1.3.2" version = "1.3.2"
@ -807,6 +833,17 @@ dependencies = [
"syn 2.0.99", "syn 2.0.99",
] ]
[[package]]
name = "bstr"
version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"
dependencies = [
"lazy_static",
"memchr",
"regex-automata 0.1.10",
]
[[package]] [[package]]
name = "bumpalo" name = "bumpalo"
version = "3.17.0" version = "3.17.0"
@ -835,6 +872,12 @@ dependencies = [
"syn 1.0.109", "syn 1.0.109",
] ]
[[package]]
name = "bytemuck"
version = "1.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540"
[[package]] [[package]]
name = "byteorder" name = "byteorder"
version = "1.5.0" version = "1.5.0"
@ -911,6 +954,12 @@ dependencies = [
"stacker", "stacker",
] ]
[[package]]
name = "chunked_transfer"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901"
[[package]] [[package]]
name = "clang-sys" name = "clang-sys"
version = "1.8.1" version = "1.8.1"
@ -962,6 +1011,12 @@ version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6"
[[package]]
name = "color_quant"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
[[package]] [[package]]
name = "colorchoice" name = "colorchoice"
version = "1.0.3" version = "1.0.3"
@ -983,6 +1038,26 @@ version = "0.9.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"
[[package]]
name = "const_format"
version = "0.2.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "126f97965c8ad46d6d9163268ff28432e8f6a1196a55578867832e3049df63dd"
dependencies = [
"const_format_proc_macros",
]
[[package]]
name = "const_format_proc_macros"
version = "0.2.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744"
dependencies = [
"proc-macro2",
"quote",
"unicode-xid",
]
[[package]] [[package]]
name = "cookie" name = "cookie"
version = "0.18.1" version = "0.18.1"
@ -1044,6 +1119,34 @@ version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
[[package]]
name = "crc32fast"
version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3"
dependencies = [
"cfg-if",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
dependencies = [
"crossbeam-utils",
]
[[package]] [[package]]
name = "crossbeam-queue" name = "crossbeam-queue"
version = "0.3.12" version = "0.3.12"
@ -1059,6 +1162,12 @@ version = "0.8.21"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
[[package]]
name = "crunchy"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929"
[[package]] [[package]]
name = "crypto-common" name = "crypto-common"
version = "0.1.6" version = "0.1.6"
@ -1308,7 +1417,7 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea3d894bbbab314476b265f9b2d46bf24b123a36dd0e96b06a1b49545b9d9dcc" checksum = "ea3d894bbbab314476b265f9b2d46bf24b123a36dd0e96b06a1b49545b9d9dcc"
dependencies = [ dependencies = [
"base64", "base64 0.22.1",
"memchr", "memchr",
] ]
@ -1424,6 +1533,21 @@ dependencies = [
"pin-project-lite", "pin-project-lite",
] ]
[[package]]
name = "exr"
version = "1.73.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f83197f59927b46c04a183a619b7c29df34e63e63c7869320862268c0ef687e0"
dependencies = [
"bit_field",
"half",
"lebe",
"miniz_oxide",
"rayon-core",
"smallvec",
"zune-inflate",
]
[[package]] [[package]]
name = "fastrand" name = "fastrand"
version = "1.9.0" version = "1.9.0"
@ -1439,12 +1563,31 @@ version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "fdeflate"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c"
dependencies = [
"simd-adler32",
]
[[package]] [[package]]
name = "fiat-crypto" name = "fiat-crypto"
version = "0.2.9" version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d"
[[package]]
name = "flate2"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc"
dependencies = [
"crc32fast",
"miniz_oxide",
]
[[package]] [[package]]
name = "flume" name = "flume"
version = "0.11.1" version = "0.11.1"
@ -1722,8 +1865,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"js-sys",
"libc", "libc",
"wasi 0.11.0+wasi-snapshot-preview1", "wasi 0.11.0+wasi-snapshot-preview1",
"wasm-bindgen",
] ]
[[package]] [[package]]
@ -1738,6 +1883,16 @@ dependencies = [
"windows-targets 0.52.6", "windows-targets 0.52.6",
] ]
[[package]]
name = "gif"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fb2d69b19215e18bb912fa30f7ce15846e301408695e44e0ef719f1da9e19f2"
dependencies = [
"color_quant",
"weezl",
]
[[package]] [[package]]
name = "gimli" name = "gimli"
version = "0.31.1" version = "0.31.1"
@ -1779,6 +1934,16 @@ dependencies = [
"url_encoded_data", "url_encoded_data",
] ]
[[package]]
name = "half"
version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888"
dependencies = [
"cfg-if",
"crunchy",
]
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.12.3" version = "0.12.3"
@ -2001,6 +2166,24 @@ dependencies = [
"want", "want",
] ]
[[package]]
name = "hyper-rustls"
version = "0.27.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2"
dependencies = [
"futures-util",
"http 1.2.0",
"hyper",
"hyper-util",
"rustls",
"rustls-pki-types",
"tokio",
"tokio-rustls",
"tower-service",
"webpki-roots",
]
[[package]] [[package]]
name = "hyper-tls" name = "hyper-tls"
version = "0.6.0" version = "0.6.0"
@ -2204,6 +2387,24 @@ dependencies = [
"icu_properties", "icu_properties",
] ]
[[package]]
name = "image"
version = "0.24.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5690139d2f55868e080017335e4b94cb7414274c74f1669c84fb5feba2c9f69d"
dependencies = [
"bytemuck",
"byteorder",
"color_quant",
"exr",
"gif",
"jpeg-decoder",
"num-traits",
"png",
"qoi",
"tiff",
]
[[package]] [[package]]
name = "indexmap" name = "indexmap"
version = "2.7.1" version = "2.7.1"
@ -2310,6 +2511,15 @@ version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
[[package]]
name = "jpeg-decoder"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0"
dependencies = [
"rayon",
]
[[package]] [[package]]
name = "js-sys" name = "js-sys"
version = "0.3.77" version = "0.3.77"
@ -2344,13 +2554,33 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]]
name = "ldbtesttool"
version = "0.1.0"
dependencies = [
"async-nats",
"futures",
"lyphedb",
"rand 0.9.0",
"rmp-serde",
"serde",
"tokio",
"toml",
]
[[package]]
name = "lebe"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8"
[[package]] [[package]]
name = "lettre" name = "lettre"
version = "0.11.14" version = "0.11.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d476fe7a4a798f392ce34947aa7d53d981127e37523c5251da3c927f7fa901f" checksum = "5d476fe7a4a798f392ce34947aa7d53d981127e37523c5251da3c927f7fa901f"
dependencies = [ dependencies = [
"base64", "base64 0.22.1",
"chumsky", "chumsky",
"email-encoding", "email-encoding",
"email_address", "email_address",
@ -2470,6 +2700,24 @@ dependencies = [
"prost-types", "prost-types",
] ]
[[package]]
name = "lyphedb"
version = "0.1.0"
dependencies = [
"async-nats",
"chrono",
"env_logger 0.11.6",
"futures",
"log",
"once_cell",
"percent-encoding",
"rand 0.9.0",
"rmp-serde",
"serde",
"tokio",
"toml",
]
[[package]] [[package]]
name = "mac" name = "mac"
version = "0.1.1" version = "0.1.1"
@ -2558,6 +2806,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5"
dependencies = [ dependencies = [
"adler2", "adler2",
"simd-adler32",
] ]
[[package]] [[package]]
@ -2588,6 +2837,16 @@ dependencies = [
"version_check", "version_check",
] ]
[[package]]
name = "mutex-timeouts"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2e9a1502e1bb4bbb96a448f684d77f17023e5a14ff49fc87c3161df6aaac2d9"
dependencies = [
"once_cell",
"tokio",
]
[[package]] [[package]]
name = "native-tls" name = "native-tls"
version = "0.2.14" version = "0.2.14"
@ -3061,6 +3320,19 @@ version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
[[package]]
name = "png"
version = "0.17.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526"
dependencies = [
"bitflags 1.3.2",
"crc32fast",
"fdeflate",
"flate2",
"miniz_oxide",
]
[[package]] [[package]]
name = "polling" name = "polling"
version = "2.8.0" version = "2.8.0"
@ -3182,6 +3454,34 @@ dependencies = [
"yansi", "yansi",
] ]
[[package]]
name = "prometheus"
version = "0.13.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1"
dependencies = [
"cfg-if",
"fnv",
"lazy_static",
"memchr",
"parking_lot",
"thiserror 1.0.69",
]
[[package]]
name = "prometheus_exporter"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "caf17cbebe0bfdf4f279ef84eeefe0d50468b0b7116f078acf41d456e48fe81a"
dependencies = [
"ascii",
"lazy_static",
"log",
"prometheus",
"thiserror 1.0.69",
"tiny_http",
]
[[package]] [[package]]
name = "prost" name = "prost"
version = "0.13.5" version = "0.13.5"
@ -3243,6 +3543,67 @@ dependencies = [
"syn 1.0.109", "syn 1.0.109",
] ]
[[package]]
name = "qoi"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001"
dependencies = [
"bytemuck",
]
[[package]]
name = "quinn"
version = "0.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef"
dependencies = [
"bytes",
"pin-project-lite",
"quinn-proto",
"quinn-udp",
"rustc-hash 2.1.1",
"rustls",
"socket2",
"thiserror 2.0.12",
"tokio",
"tracing",
]
[[package]]
name = "quinn-proto"
version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d"
dependencies = [
"bytes",
"getrandom 0.2.15",
"rand 0.8.5",
"ring",
"rustc-hash 2.1.1",
"rustls",
"rustls-pki-types",
"slab",
"thiserror 2.0.12",
"tinyvec",
"tracing",
"web-time",
]
[[package]]
name = "quinn-udp"
version = "0.5.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e46f3055866785f6b92bc6164b76be02ca8f2eb4b002c0354b28cf4c119e5944"
dependencies = [
"cfg_aliases",
"libc",
"once_cell",
"socket2",
"tracing",
"windows-sys 0.59.0",
]
[[package]] [[package]]
name = "quote" name = "quote"
version = "1.0.39" version = "1.0.39"
@ -3324,6 +3685,26 @@ dependencies = [
"getrandom 0.3.1", "getrandom 0.3.1",
] ]
[[package]]
name = "rayon"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
dependencies = [
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
dependencies = [
"crossbeam-deque",
"crossbeam-utils",
]
[[package]] [[package]]
name = "redox_syscall" name = "redox_syscall"
version = "0.5.10" version = "0.5.10"
@ -3392,7 +3773,7 @@ version = "0.12.12"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da" checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da"
dependencies = [ dependencies = [
"base64", "base64 0.22.1",
"bytes", "bytes",
"futures-core", "futures-core",
"futures-util", "futures-util",
@ -3400,6 +3781,7 @@ dependencies = [
"http-body", "http-body",
"http-body-util", "http-body-util",
"hyper", "hyper",
"hyper-rustls",
"hyper-tls", "hyper-tls",
"hyper-util", "hyper-util",
"ipnet", "ipnet",
@ -3410,19 +3792,24 @@ dependencies = [
"once_cell", "once_cell",
"percent-encoding", "percent-encoding",
"pin-project-lite", "pin-project-lite",
"quinn",
"rustls",
"rustls-pemfile", "rustls-pemfile",
"rustls-pki-types",
"serde", "serde",
"serde_json", "serde_json",
"serde_urlencoded", "serde_urlencoded",
"sync_wrapper", "sync_wrapper",
"tokio", "tokio",
"tokio-native-tls", "tokio-native-tls",
"tokio-rustls",
"tower", "tower",
"tower-service", "tower-service",
"url", "url",
"wasm-bindgen", "wasm-bindgen",
"wasm-bindgen-futures", "wasm-bindgen-futures",
"web-sys", "web-sys",
"webpki-roots",
"windows-registry", "windows-registry",
] ]
@ -3539,6 +3926,12 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
[[package]]
name = "rustc-hash"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
[[package]] [[package]]
name = "rustc_version" name = "rustc_version"
version = "0.4.1" version = "0.4.1"
@ -3627,6 +4020,9 @@ name = "rustls-pki-types"
version = "1.11.0" version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c"
dependencies = [
"web-time",
]
[[package]] [[package]]
name = "rustls-webpki" name = "rustls-webpki"
@ -3938,9 +4334,9 @@ dependencies = [
[[package]] [[package]]
name = "serde_bytes" name = "serde_bytes"
version = "0.11.16" version = "0.11.17"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "364fec0df39c49a083c9a8a18a23a6bcfd9af130fe9fe321d18520a0d113e09e" checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96"
dependencies = [ dependencies = [
"serde", "serde",
] ]
@ -3962,6 +4358,7 @@ version = "1.0.140"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373"
dependencies = [ dependencies = [
"indexmap",
"itoa", "itoa",
"memchr", "memchr",
"ryu", "ryu",
@ -3998,6 +4395,15 @@ dependencies = [
"syn 2.0.99", "syn 2.0.99",
] ]
[[package]]
name = "serde_spanned"
version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1"
dependencies = [
"serde",
]
[[package]] [[package]]
name = "serde_urlencoded" name = "serde_urlencoded"
version = "0.7.1" version = "0.7.1"
@ -4087,6 +4493,12 @@ dependencies = [
"rand_core 0.6.4", "rand_core 0.6.4",
] ]
[[package]]
name = "simd-adler32"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
[[package]] [[package]]
name = "simdutf8" name = "simdutf8"
version = "0.1.5" version = "0.1.5"
@ -4270,7 +4682,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4560278f0e00ce64938540546f59f590d60beee33fffbd3b9cd47851e5fff233" checksum = "4560278f0e00ce64938540546f59f590d60beee33fffbd3b9cd47851e5fff233"
dependencies = [ dependencies = [
"atoi", "atoi",
"base64", "base64 0.22.1",
"bigdecimal", "bigdecimal",
"bitflags 2.9.0", "bitflags 2.9.0",
"byteorder", "byteorder",
@ -4317,7 +4729,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5b98a57f363ed6764d5b3a12bfedf62f07aa16e1856a7ddc2a0bb190a959613" checksum = "c5b98a57f363ed6764d5b3a12bfedf62f07aa16e1856a7ddc2a0bb190a959613"
dependencies = [ dependencies = [
"atoi", "atoi",
"base64", "base64 0.22.1",
"bigdecimal", "bigdecimal",
"bitflags 2.9.0", "bitflags 2.9.0",
"byteorder", "byteorder",
@ -4404,6 +4816,16 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "stopwords"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e4508a6e132e6ea159112d42ed1f29927460dd45a118eed298d7666c81b713e"
dependencies = [
"lazy_static",
"thiserror 1.0.69",
]
[[package]] [[package]]
name = "string_cache" name = "string_cache"
version = "0.8.8" version = "0.8.8"
@ -4429,6 +4851,15 @@ dependencies = [
"quote", "quote",
] ]
[[package]]
name = "stringmatch"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6aadc0801d92f0cdc26127c67c4b8766284f52a5ba22894f285e3101fa57d05d"
dependencies = [
"regex",
]
[[package]] [[package]]
name = "stringprep" name = "stringprep"
version = "0.1.5" version = "0.1.5"
@ -4540,6 +4971,61 @@ dependencies = [
"winapi-util", "winapi-util",
] ]
[[package]]
name = "texting_robots"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b82a718a28dda2e67ad6e0464597b58eae39e2e4d0451e03d1028d71e81bb4a"
dependencies = [
"anyhow",
"bstr",
"lazy_static",
"nom 7.1.3",
"percent-encoding",
"regex",
"thiserror 1.0.69",
"url",
]
[[package]]
name = "thirtyfour"
version = "0.35.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44d1dd5271076f9cf5cb6a673497d1476cda2633e6cf2523edd00ac16218d651"
dependencies = [
"arc-swap",
"async-trait",
"base64 0.22.1",
"bytes",
"cfg-if",
"const_format",
"futures-util",
"http 1.2.0",
"indexmap",
"paste",
"reqwest",
"serde",
"serde_json",
"serde_repr",
"stringmatch",
"thirtyfour-macros",
"thiserror 1.0.69",
"tokio",
"tracing",
"url",
]
[[package]]
name = "thirtyfour-macros"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5cf0ffc3ba4368e99597bd6afd83f4ff6febad66d9ae541ab46e697d32285fc0"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.99",
]
[[package]] [[package]]
name = "thiserror" name = "thiserror"
version = "1.0.69" version = "1.0.69"
@ -4590,6 +5076,17 @@ dependencies = [
"once_cell", "once_cell",
] ]
[[package]]
name = "tiff"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba1310fcea54c6a9a4fd1aad794ecc02c31682f6bfbecdf460bf19533eed1e3e"
dependencies = [
"flate2",
"jpeg-decoder",
"weezl",
]
[[package]] [[package]]
name = "time" name = "time"
version = "0.3.39" version = "0.3.39"
@ -4621,6 +5118,19 @@ dependencies = [
"time-core", "time-core",
] ]
[[package]]
name = "tiny_http"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5f8734c6d6943ad6df6b588d228a87b4af184998bcffa268ceddf05c2055a8c"
dependencies = [
"ascii",
"chunked_transfer",
"log",
"time",
"url",
]
[[package]] [[package]]
name = "tinystr" name = "tinystr"
version = "0.7.6" version = "0.7.6"
@ -4725,7 +5235,7 @@ version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f591660438b3038dd04d16c938271c79e7e06260ad2ea2885a4861bfb238605d" checksum = "f591660438b3038dd04d16c938271c79e7e06260ad2ea2885a4861bfb238605d"
dependencies = [ dependencies = [
"base64", "base64 0.22.1",
"bytes", "bytes",
"futures-core", "futures-core",
"futures-sink", "futures-sink",
@ -4740,11 +5250,26 @@ dependencies = [
"tokio-util", "tokio-util",
] ]
[[package]]
name = "toml"
version = "0.8.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148"
dependencies = [
"serde",
"serde_spanned",
"toml_datetime",
"toml_edit",
]
[[package]] [[package]]
name = "toml_datetime" name = "toml_datetime"
version = "0.6.8" version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
dependencies = [
"serde",
]
[[package]] [[package]]
name = "toml_edit" name = "toml_edit"
@ -4753,6 +5278,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474"
dependencies = [ dependencies = [
"indexmap", "indexmap",
"serde",
"serde_spanned",
"toml_datetime", "toml_datetime",
"winnow", "winnow",
] ]
@ -4982,6 +5509,12 @@ version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af"
[[package]]
name = "unicode-xid"
version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
[[package]] [[package]]
name = "unit_converter" name = "unit_converter"
version = "0.1.0" version = "0.1.0"
@ -5080,6 +5613,32 @@ version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "vorebot"
version = "0.2.0"
dependencies = [
"asklyphe-common",
"async-nats",
"base64 0.21.7",
"chrono",
"env_logger 0.10.2",
"futures",
"image",
"isahc",
"log",
"mutex-timeouts",
"once_cell",
"prometheus_exporter",
"rand 0.8.5",
"rmp-serde",
"serde",
"stopwords",
"texting_robots",
"thirtyfour",
"tokio",
"ulid",
]
[[package]] [[package]]
name = "waker-fn" name = "waker-fn"
version = "1.2.0" version = "1.2.0"
@ -5216,6 +5775,12 @@ dependencies = [
"rustls-pki-types", "rustls-pki-types",
] ]
[[package]]
name = "weezl"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082"
[[package]] [[package]]
name = "which" name = "which"
version = "4.4.2" version = "4.4.2"
@ -5636,3 +6201,12 @@ dependencies = [
"quote", "quote",
"syn 2.0.99", "syn 2.0.99",
] ]
[[package]]
name = "zune-inflate"
version = "0.2.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02"
dependencies = [
"simd-adler32",
]

View file

@ -1,2 +1,2 @@
[workspace] [workspace]
members = ["asklyphe-common", "asklyphe-frontend", "searchservice", "asklyphe-auth-frontend", "unit_converter", "authservice", "authservice/migration", "authservice/entity", "bingservice", "googleservice"] members = ["asklyphe-common", "asklyphe-frontend", "searchservice", "asklyphe-auth-frontend", "unit_converter", "authservice", "authservice/migration", "authservice/entity", "bingservice", "googleservice", "vorebot", "lyphedb", "lyphedb/ldbtesttool"]

View file

@ -11,10 +11,12 @@ license-file = "LICENSE"
tokio = { version = "1.0", features = ["full"] } tokio = { version = "1.0", features = ["full"] }
chrono = "0.4.31" chrono = "0.4.31"
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
lyphedb = { path = "../lyphedb" }
foundationdb = { version = "0.8.0", features = ["embedded-fdb-include"] } foundationdb = { version = "0.8.0", features = ["embedded-fdb-include"] }
log = "0.4.20" log = "0.4.20"
rmp-serde = "1.1.2" rmp-serde = "1.1.2"
futures = "0.3.30" futures = "0.3.30"
async-nats = "0.38.0" async-nats = "0.38.0"
ulid = "1.1.0" ulid = "1.1.0"
rand = "0.8.5" rand = "0.8.5"
percent-encoding = "2.3.1"

View file

@ -0,0 +1,63 @@
use log::{error, warn};
use lyphedb::{KVList, KeyDirectory, LDBNatsMessage, LypheDBCommand, PropagationStrategy};
use crate::ldb::{construct_path, hash, DBConn};
pub const LINKSTORE: &str = "linkstore";
pub async fn add_url_to_linkwords(db: &DBConn, linkwords: &[&str], url: &str) -> Result<(), ()> {
let mut key_sets = Vec::new();
for linkword in linkwords {
let path = construct_path(&[LINKSTORE, linkword, &hash(url)]).as_bytes().to_vec();
key_sets.push((path, url.as_bytes().to_vec()));
}
let cmd = LDBNatsMessage::Command(LypheDBCommand::SetKeys(KVList {
kvs: key_sets,
}, PropagationStrategy::OnRead));
match db.query(cmd).await {
LDBNatsMessage::Success => {
Ok(())
}
LDBNatsMessage::BadRequest => {
error!("bad request for add_url_to_linkwords");
Err(())
}
LDBNatsMessage::NotFound => {
error!("not found for add_url_to_linkwords");
Err(())
}
_ => {
warn!("lyphedb sent weird message as response, treating as error");
Err(())
}
}
}
pub async fn get_urls_from_linkword(db: &DBConn, keyword: &str) -> Result<Vec<String>, ()> {
let path = construct_path(&[LINKSTORE, keyword]).as_bytes().to_vec();
let cmd = LDBNatsMessage::Command(LypheDBCommand::GetKeyDirectory(KeyDirectory {
key: path,
}));
match db.query(cmd).await {
LDBNatsMessage::Entries(kvs) => {
Ok(kvs.kvs.into_iter().map(|v| String::from_utf8_lossy(&v.1).to_string()).collect())
}
LDBNatsMessage::Success => {
warn!("lyphedb responded with \"success\" to get_urls_from_linkwords, treating as error");
Err(())
}
LDBNatsMessage::BadRequest => {
warn!("bad request for get_urls_from_linkwords");
Err(())
}
LDBNatsMessage::NotFound => {
Ok(vec![])
}
_ => {
warn!("lyphedb sent weird message as response, treating as error");
Err(())
}
}
}

View file

@ -0,0 +1,63 @@
use log::{error, warn};
use lyphedb::{KVList, KeyDirectory, LDBNatsMessage, LypheDBCommand, PropagationStrategy};
use crate::ldb::{construct_path, hash, DBConn};
pub const METASTORE: &str = "metastore";
pub async fn add_url_to_metawords(db: &DBConn, metawords: &[&str], url: &str) -> Result<(), ()> {
let mut key_sets = Vec::new();
for metaword in metawords {
let path = construct_path(&[METASTORE, metaword, &hash(url)]).as_bytes().to_vec();
key_sets.push((path, url.as_bytes().to_vec()));
}
let cmd = LDBNatsMessage::Command(LypheDBCommand::SetKeys(KVList {
kvs: key_sets,
}, PropagationStrategy::OnRead));
match db.query(cmd).await {
LDBNatsMessage::Success => {
Ok(())
}
LDBNatsMessage::BadRequest => {
error!("bad request for add_url_to_metawords");
Err(())
}
LDBNatsMessage::NotFound => {
error!("not found for add_url_to_metawords");
Err(())
}
_ => {
warn!("lyphedb sent weird message as response, treating as error");
Err(())
}
}
}
pub async fn get_urls_from_metaword(db: &DBConn, metaword: &str) -> Result<Vec<String>, ()> {
let path = construct_path(&[METASTORE, metaword]).as_bytes().to_vec();
let cmd = LDBNatsMessage::Command(LypheDBCommand::GetKeyDirectory(KeyDirectory {
key: path,
}));
match db.query(cmd).await {
LDBNatsMessage::Entries(kvs) => {
Ok(kvs.kvs.into_iter().map(|v| String::from_utf8_lossy(&v.1).to_string()).collect())
}
LDBNatsMessage::Success => {
warn!("lyphedb responded with \"success\" to get_urls_from_metawords, treating as error");
Err(())
}
LDBNatsMessage::BadRequest => {
warn!("bad request for get_urls_from_metawords");
Err(())
}
LDBNatsMessage::NotFound => {
Ok(vec![])
}
_ => {
warn!("lyphedb sent weird message as response, treating as error");
Err(())
}
}
}

View file

@ -0,0 +1,62 @@
pub mod wordstore;
pub mod sitestore;
pub mod linkstore;
pub mod metastore;
pub mod titlestore;
use std::hash::{DefaultHasher, Hasher};
use std::sync::Arc;
use std::sync::atomic::AtomicU64;
use futures::StreamExt;
use log::warn;
use lyphedb::LDBNatsMessage;
use percent_encoding::{percent_encode, AsciiSet, CONTROLS, NON_ALPHANUMERIC};
static NEXT_REPLY_ID: AtomicU64 = AtomicU64::new(0);
pub const NOT_ALLOWED_ASCII: AsciiSet = CONTROLS.add(b' ').add(b'/').add(b'.').add(b'\\');
pub fn hash(str: &str) -> String {
let mut hasher = DefaultHasher::new();
hasher.write(str.as_bytes());
hasher.finish().to_string()
}
pub fn construct_path(path_elements: &[&str]) -> String {
let mut buf = String::new();
buf.push_str("ASKLYPHE/");
for el in path_elements {
buf.push_str(&percent_encode(el.as_bytes(), &NOT_ALLOWED_ASCII).to_string());
buf.push('/');
}
buf.pop();
buf
}
#[derive(Clone)]
pub struct DBConn {
nats: async_nats::Client,
name: String,
}
impl DBConn {
pub fn new(nats: async_nats::Client, name: impl ToString) -> DBConn {
DBConn { nats, name: name.to_string() }
}
pub async fn query(&self, message: LDBNatsMessage) -> LDBNatsMessage {
let data = rmp_serde::to_vec(&message).unwrap();
let replyto = format!("ldb-reply-{}", NEXT_REPLY_ID.fetch_add(1, std::sync::atomic::Ordering::SeqCst));
let mut subscriber = self.nats.subscribe(replyto.clone()).await.expect("NATS ERROR");
self.nats.publish_with_reply(self.name.clone(), replyto, data.into()).await.expect("NATS ERROR");
if let Some(reply) = subscriber.next().await {
let reply = rmp_serde::from_slice::<LDBNatsMessage>(&reply.payload);
if reply.is_err() {
warn!("DECODED BAD MESSAGE FROM LYPHEDB: {}", reply.err().unwrap());
return LDBNatsMessage::NotFound;
}
return reply.unwrap();
}
LDBNatsMessage::NotFound
}
}

View file

@ -0,0 +1,178 @@
use crate::ldb::{construct_path, hash, DBConn};
use log::{error, warn};
use lyphedb::{KVList, KeyDirectory, KeyList, LDBNatsMessage, LypheDBCommand, PropagationStrategy};
pub const SITESTORE: &str = "sitestore";
pub const TITLE: &str = "title";
pub const DESCRIPTION: &str = "desc";
pub const KEYWORDS: &str = "keywords";
pub const PAGE_TEXT_RANKING: &str = "ptranks";
pub const PAGE_TEXT_RAW: &str = "textraw";
pub const DAMPING: &str = "damping";
pub async fn add_website(
db: &DBConn,
url: &str,
title: Option<String>,
description: Option<String>,
keywords: Option<Vec<String>>,
page_text_ranking: &[(String, f64)],
page_text_raw: String,
damping: f64,
) -> Result<(), ()> {
let keyurl = hash(url);
let mut kvs = vec![
(
construct_path(&[SITESTORE, &keyurl]).as_bytes().to_vec(),
url.as_bytes().to_vec(),
),
(
construct_path(&[SITESTORE, &keyurl, PAGE_TEXT_RANKING])
.as_bytes()
.to_vec(),
rmp_serde::to_vec(page_text_ranking).unwrap(),
),
(
construct_path(&[SITESTORE, &keyurl, PAGE_TEXT_RAW])
.as_bytes()
.to_vec(),
page_text_raw.as_bytes().to_vec(),
),
(
construct_path(&[SITESTORE, &keyurl, DAMPING])
.as_bytes()
.to_vec(),
damping.to_be_bytes().to_vec(),
),
];
if let Some(title) = title {
kvs.push((
construct_path(&[SITESTORE, &keyurl, TITLE]).as_bytes().to_vec(),
title.as_bytes().to_vec(),
))
}
if let Some(description) = description {
kvs.push((
construct_path(&[SITESTORE, &keyurl, DESCRIPTION])
.as_bytes()
.to_vec(),
description.as_bytes().to_vec(),
))
}
if let Some(keywords) = keywords {
kvs.push((
construct_path(&[SITESTORE, &keyurl, KEYWORDS])
.as_bytes()
.to_vec(),
rmp_serde::to_vec(&keywords).unwrap(),
))
}
let cmd = LDBNatsMessage::Command(LypheDBCommand::SetKeys(
KVList { kvs },
PropagationStrategy::OnRead,
));
match db.query(cmd).await {
LDBNatsMessage::Success => Ok(()),
LDBNatsMessage::BadRequest => {
error!("bad request for add_website");
Err(())
}
LDBNatsMessage::NotFound => {
error!("not found for add_website");
Err(())
}
_ => {
warn!("lyphedb sent weird message as response, treating as error");
Err(())
}
}
}
#[derive(Default, Debug, Clone)]
pub struct WebsiteData {
pub title: Option<String>,
pub description: Option<String>,
pub keywords: Option<Vec<String>>,
pub page_text_ranking: Vec<(String, f64)>,
pub page_text_raw: String,
pub damping: f64,
}
pub async fn get_website(db: &DBConn, url: &str) -> Result<WebsiteData, ()> {
let keyurl = hash(url);
let keys = [
construct_path(&[SITESTORE, &keyurl, TITLE]).as_bytes().to_vec(),
construct_path(&[SITESTORE, &keyurl, DESCRIPTION]).as_bytes().to_vec(),
construct_path(&[SITESTORE, &keyurl, KEYWORDS]).as_bytes().to_vec(),
construct_path(&[SITESTORE, &keyurl, PAGE_TEXT_RANKING]).as_bytes().to_vec(),
construct_path(&[SITESTORE, &keyurl, PAGE_TEXT_RAW]).as_bytes().to_vec(),
construct_path(&[SITESTORE, &keyurl, DAMPING]).as_bytes().to_vec(),
].to_vec();
let cmd = LDBNatsMessage::Command(LypheDBCommand::GetKeys(KeyList { keys }));
match db.query(cmd).await {
LDBNatsMessage::Entries(kvlist) => {
let mut data = WebsiteData::default();
for (key, value) in kvlist.kvs {
let key = String::from_utf8_lossy(&key).to_string();
match key.as_str() {
_ if key.ends_with(TITLE) => {
data.title = Some(String::from_utf8_lossy(&value).to_string());
}
_ if key.ends_with(DESCRIPTION) => {
data.description = Some(String::from_utf8_lossy(&value).to_string());
}
_ if key.ends_with(KEYWORDS) => {
let deser = rmp_serde::from_slice::<Vec<String>>(&value);
if let Err(e) = deser {
error!("bad keywords entry for {}, deser error: {:?}", key, e);
} else {
data.keywords = Some(deser.unwrap());
}
}
_ if key.ends_with(PAGE_TEXT_RANKING) => {
let deser = rmp_serde::from_slice::<Vec<(String, f64)>>(&value);
if let Err(e) = deser {
error!("bad page_text_ranking entry for {}, deser error: {:?}", key, e);
} else {
data.page_text_ranking = deser.unwrap();
}
}
_ if key.ends_with(PAGE_TEXT_RAW) => {
data.page_text_raw = String::from_utf8_lossy(&value).to_string();
}
_ if key.ends_with(DAMPING) => {
data.damping = f64::from_be_bytes(value.try_into().unwrap_or(0.85f64.to_be_bytes()));
}
_ => {
warn!("encountered weird returned key for get_website");
}
}
}
Ok(data)
}
LDBNatsMessage::Success => {
warn!("lyphedb responded with \"success\" to get_website, treating as error");
Err(())
}
LDBNatsMessage::BadRequest => {
error!("bad request for get_website");
Err(())
}
LDBNatsMessage::NotFound => {
warn!("not found for get_website");
Err(())
}
_ => {
warn!("lyphedb sent weird message as response, treating as error");
Err(())
}
}
}

View file

@ -0,0 +1,63 @@
use log::{error, warn};
use lyphedb::{KVList, KeyDirectory, LDBNatsMessage, LypheDBCommand, PropagationStrategy};
use crate::ldb::{construct_path, hash, DBConn};
pub const TITLESTORE: &str = "titlestore";
pub async fn add_url_to_titlewords(db: &DBConn, titlewords: &[&str], url: &str) -> Result<(), ()> {
let mut key_sets = Vec::new();
for titleword in titlewords {
let path = construct_path(&[TITLESTORE, titleword, &hash(url)]).as_bytes().to_vec();
key_sets.push((path, url.as_bytes().to_vec()));
}
let cmd = LDBNatsMessage::Command(LypheDBCommand::SetKeys(KVList {
kvs: key_sets,
}, PropagationStrategy::OnRead));
match db.query(cmd).await {
LDBNatsMessage::Success => {
Ok(())
}
LDBNatsMessage::BadRequest => {
error!("bad request for add_url_to_titlewords");
Err(())
}
LDBNatsMessage::NotFound => {
error!("not found for add_url_to_titlewords");
Err(())
}
_ => {
warn!("lyphedb sent weird message as response, treating as error");
Err(())
}
}
}
pub async fn get_urls_from_titleword(db: &DBConn, titleword: &str) -> Result<Vec<String>, ()> {
let path = construct_path(&[TITLESTORE, titleword]).as_bytes().to_vec();
let cmd = LDBNatsMessage::Command(LypheDBCommand::GetKeyDirectory(KeyDirectory {
key: path,
}));
match db.query(cmd).await {
LDBNatsMessage::Entries(kvs) => {
Ok(kvs.kvs.into_iter().map(|v| String::from_utf8_lossy(&v.1).to_string()).collect())
}
LDBNatsMessage::Success => {
warn!("lyphedb responded with \"success\" to get_urls_from_titlewords, treating as error");
Err(())
}
LDBNatsMessage::BadRequest => {
warn!("bad request for get_urls_from_titlewords");
Err(())
}
LDBNatsMessage::NotFound => {
Ok(vec![])
}
_ => {
warn!("lyphedb sent weird message as response, treating as error");
Err(())
}
}
}

View file

@ -0,0 +1,64 @@
use log::{error, warn};
use lyphedb::{KVList, KeyDirectory, LDBNatsMessage, LypheDBCommand, PropagationStrategy};
use crate::ldb::{construct_path, hash, DBConn};
pub const WORDSTORE: &str = "wordstore";
pub async fn add_url_to_keywords(db: &DBConn, keywords: &[&str], url: &str) -> Result<(), ()> {
let mut key_sets = Vec::new();
for keyword in keywords {
let path = construct_path(&[WORDSTORE, keyword, &hash(url)]).as_bytes().to_vec();
let data = url.as_bytes().to_vec();
key_sets.push((path, data));
}
let cmd = LDBNatsMessage::Command(LypheDBCommand::SetKeys(KVList {
kvs: key_sets,
}, PropagationStrategy::OnRead));
match db.query(cmd).await {
LDBNatsMessage::Success => {
Ok(())
}
LDBNatsMessage::BadRequest => {
error!("bad request for add_url_to_keywords");
Err(())
}
LDBNatsMessage::NotFound => {
error!("not found for add_url_to_keywords");
Err(())
}
_ => {
warn!("lyphedb sent weird message as response, treating as error");
Err(())
}
}
}
pub async fn get_urls_from_keyword(db: &DBConn, keyword: &str) -> Result<Vec<String>, ()> {
let path = construct_path(&[WORDSTORE, keyword]).as_bytes().to_vec();
let cmd = LDBNatsMessage::Command(LypheDBCommand::GetKeyDirectory(KeyDirectory {
key: path,
}));
match db.query(cmd).await {
LDBNatsMessage::Entries(kvs) => {
Ok(kvs.kvs.into_iter().map(|v| String::from_utf8_lossy(&v.1).to_string()).collect())
}
LDBNatsMessage::Success => {
warn!("lyphedb responded with \"success\" to get_urls_from_keywords, treating as error");
Err(())
}
LDBNatsMessage::BadRequest => {
warn!("bad request for get_urls_from_keywords");
Err(())
}
LDBNatsMessage::NotFound => {
Ok(vec![])
}
_ => {
warn!("lyphedb sent weird message as response, treating as error");
Err(())
}
}
}

View file

@ -13,7 +13,9 @@
pub mod nats; pub mod nats;
pub mod db; pub mod db;
pub mod ldb;
pub use lyphedb;
pub use foundationdb; pub use foundationdb;
pub fn add(left: usize, right: usize) -> usize { pub fn add(left: usize, right: usize) -> usize {

View file

@ -18,7 +18,7 @@ pub const VOREBOT_NEWHOSTNAME_SERVICE: &str = "websiteparse_highpriority";
pub const VOREBOT_SUGGESTED_SERVICE: &str = "websiteparse_highestpriority"; pub const VOREBOT_SUGGESTED_SERVICE: &str = "websiteparse_highestpriority";
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WebParseRequest { pub struct CrawlRequest {
pub url: String, pub url: String,
pub damping_factor: f32, pub damping: f64,
} }

30
lyphedb/.gitignore vendored Normal file
View file

@ -0,0 +1,30 @@
# ---> Rust
# Generated by Cargo
# will have compiled files and executables
debug/
target/
# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
Cargo.lock
# These are backup files generated by rustfmt
**/*.rs.bk
# MSVC Windows builds of rustc generate these, which store debugging information
*.pdb
# RustRover
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
# Added by cargo
/target
# don't commit employee test configs (:
husky_config.toml

18
lyphedb/Cargo.toml Normal file
View file

@ -0,0 +1,18 @@
[package]
name = "lyphedb"
version = "0.1.0"
edition = "2024"
[dependencies]
log = "0.4.20"
rmp-serde = "1.1.2"
futures = "0.3.30"
async-nats = "0.38.0"
tokio = { version = "1.0", features = ["full"] }
chrono = "0.4.31"
serde = { version = "1.0", features = ["derive"] }
rand = "0.9.0"
toml = "0.8.20"
env_logger = "0.11.6"
once_cell = "1.20.3"
percent-encoding = "2.3.1"

10
lyphedb/config.toml Normal file
View file

@ -0,0 +1,10 @@
name = "lyphedb-test"
write = true
master = "/lyphedb_master"
ram_limit = "1mb" # supported suffixes: b,kb,mb,gb
gc_limit = "512kb"
avg_entry_size = "1kb"
log = "debug"
nats_cert = "nats/nats.cert"
nats_key = "nats/nats.pem"
nats_url = "127.0.0.1:4222"

View file

@ -0,0 +1,14 @@
[package]
name = "ldbtesttool"
version = "0.1.0"
edition = "2024"
[dependencies]
rmp-serde = "1.1.2"
futures = "0.3.30"
async-nats = "0.38.0"
tokio = { version = "1.0", features = ["full"] }
serde = { version = "1.0", features = ["derive"] }
lyphedb = { path = "../" }
toml = "0.8.20"
rand = "0.9.0"

View file

@ -0,0 +1,40 @@
use serde::Deserialize;
#[derive(Deserialize)]
struct ConfigFile {
pub name: String,
pub nats_cert: String,
pub nats_key: String,
pub nats_url: String,
}
#[derive(Clone)]
pub struct LypheDBConfig {
pub name: String,
pub nats_cert: String,
pub nats_key: String,
pub nats_url: String,
}
pub fn load_config(path: &str) -> LypheDBConfig {
let s = std::fs::read_to_string(path);
if let Err(e) = s {
panic!("failed to read config file: {}", e);
}
let s = s.unwrap();
let cnf = toml::from_str::<ConfigFile>(&s);
if let Err(e) = cnf {
panic!("failed to parse config file: {}", e);
}
let cnf = cnf.unwrap();
// quick checks and conversions
let mut config = LypheDBConfig {
name: cnf.name,
nats_cert: cnf.nats_cert,
nats_key: cnf.nats_key,
nats_url: cnf.nats_url,
};
config
}

View file

@ -0,0 +1,114 @@
use crate::config::load_config;
use lyphedb::{KVList, KeyList, LDBNatsMessage, LypheDBCommand, PropagationStrategy};
use std::collections::BTreeMap;
use async_nats::Message;
use futures::StreamExt;
use tokio::sync::mpsc;
mod config;
#[tokio::main]
async fn main() {
let config = load_config(&std::env::args().nth(1).expect("please specify config file"));
let nats = async_nats::ConnectOptions::new()
.add_client_certificate(
config.nats_cert.as_str().into(),
config.nats_key.as_str().into(),
)
.connect(config.nats_url.as_str())
.await;
if let Err(e) = nats {
eprintln!("FATAL ERROR, COULDN'T CONNECT TO NATS: {}", e);
return;
}
let nats = nats.unwrap();
// test 1: create 10000 keys, send in chunks of 10
println!("test 1: create 10_000 keys, send in chunks of 100");
let (key_tx, mut key_rx) = mpsc::unbounded_channel();
let start = std::time::Instant::now();
let mut tasks = vec![];
for _ in 0..(10_000 / 100) {
let name = config.name.clone();
let nats = nats.clone();
let key_tx = key_tx.clone();
tasks.push(tokio::spawn(async move {
let mut keys = BTreeMap::new();
for _ in 0..100 {
let key = rand::random::<[u8; 16]>();
let data = rand::random::<[u8; 16]>();
key_tx.send((key, data)).unwrap();
keys.insert(key, data);
}
let data = rmp_serde::to_vec(&LDBNatsMessage::Command(LypheDBCommand::SetKeys(
KVList {
kvs: keys
.into_iter()
.map(|(k, v)| (k.to_vec(), v.to_vec()))
.collect(),
},
PropagationStrategy::Immediate,
)))
.unwrap();
let replyto_sub = "lyphedb_test_1".to_string();
let mut subscriber = nats.subscribe(replyto_sub.clone()).await.unwrap();
nats.publish_with_reply(name.clone(), replyto_sub, data.into()).await.unwrap();
if let Some(reply) = subscriber.next().await {
let reply = rmp_serde::from_slice::<LDBNatsMessage>(&reply.payload).unwrap();
if let LDBNatsMessage::Success = reply {} else {
eprintln!("failure");
}
}
}));
}
for task in tasks {
task.await.unwrap();
}
let end = std::time::Instant::now();
println!("test 1: {}ms", end.duration_since(start).as_millis());
let mut our_copy = BTreeMap::new();
let mut buffer = vec![];
key_rx.recv_many(&mut buffer, 10_000).await;
for (k, v) in buffer {
our_copy.insert(k, v);
}
// test 2: read back all keys and check for accuracy
println!("test 2: read back all keys and check for accuracy");
let kv: Vec<_> = our_copy.into_iter().map(|(k, v)| (k.to_vec(), v.to_vec())).collect();
let start = std::time::Instant::now();
let mut tasks = vec![];
for i in 0..100 {
let batch = kv[i * 100..((i + 1) * 100).min(kv.len())].to_vec();
let name = config.name.clone();
let nats = nats.clone();
tasks.push(tokio::spawn(async move {
let data = rmp_serde::to_vec(&LDBNatsMessage::Command(LypheDBCommand::GetKeys(
KeyList {
keys: batch.iter().map(|(k, _)| k.to_vec()).collect()
}
))).unwrap();
let replyto_sub = format!("lyphedb_test_2_{}", i);
let mut subscriber = nats.subscribe(replyto_sub.clone()).await.unwrap();
nats.publish_with_reply(name.clone(), replyto_sub, data.into()).await.unwrap();
if let Some(reply) = subscriber.next().await {
let reply = rmp_serde::from_slice::<LDBNatsMessage>(&reply.payload).unwrap();
if let LDBNatsMessage::Entries(kvlist) = reply {
// check all keys
for (k1, v1) in batch {
let v2 = kvlist.kvs.iter().find(|(k, v)| k == &k1).expect("key not found");
assert_eq!(v1, v2.1);
}
} else {
eprintln!("failure");
}
}
}));
}
for task in tasks {
task.await.unwrap();
}
let end = std::time::Instant::now();
println!("test 2: {}ms", end.duration_since(start).as_millis());
}

129
lyphedb/src/config.rs Normal file
View file

@ -0,0 +1,129 @@
use serde::Deserialize;
#[derive(Deserialize)]
struct ConfigFile {
pub name: String,
pub write: bool,
pub master: String,
pub ram_limit: String,
pub gc_limit: String,
pub avg_entry_size: String,
pub log: String,
pub nats_cert: String,
pub nats_key: String,
pub nats_url: String,
}
#[derive(Clone)]
pub struct LypheDBConfig {
pub name: String,
pub write: bool,
pub master: String,
pub ram_limit: usize,
pub gc_limit: usize,
pub avg_entry_size: usize,
pub log: String,
pub nats_cert: String,
pub nats_key: String,
pub nats_url: String,
}
pub fn load_config(path: &str) -> LypheDBConfig {
let s = std::fs::read_to_string(path);
if let Err(e) = s {
panic!("failed to read config file: {}", e);
}
let s = s.unwrap();
let cnf = toml::from_str::<ConfigFile>(&s);
if let Err(e) = cnf {
panic!("failed to parse config file: {}", e);
}
let cnf = cnf.unwrap();
// quick checks and conversions
let mut config = LypheDBConfig {
name: cnf.name,
write: cnf.write,
master: cnf.master,
ram_limit: 0,
gc_limit: 0,
avg_entry_size: 0,
log: cnf.log,
nats_cert: cnf.nats_cert,
nats_key: cnf.nats_key,
nats_url: cnf.nats_url,
};
if !(cnf.ram_limit.ends_with("b") &&
(
cnf.ram_limit.trim_end_matches("b").ends_with("k") ||
cnf.ram_limit.trim_end_matches("b").ends_with("m") ||
cnf.ram_limit.trim_end_matches("b").ends_with("g")
)
) {
panic!("invalid ram limit");
}
config.ram_limit = if cnf.ram_limit.ends_with("gb") {
cnf.ram_limit.trim_end_matches("gb").parse::<usize>().unwrap() * 1024 * 1024 * 1024
} else if cnf.ram_limit.ends_with("mb") {
cnf.ram_limit.trim_end_matches("mb").parse::<usize>().unwrap() * 1024 * 1024
} else if cnf.ram_limit.ends_with("kb") {
cnf.ram_limit.trim_end_matches("kb").parse::<usize>().unwrap() * 1024
} else {
cnf.ram_limit.trim_end_matches("b").parse::<usize>().unwrap()
};
if !(cnf.gc_limit.ends_with("b") &&
(
cnf.gc_limit.trim_end_matches("b").ends_with("k") ||
cnf.gc_limit.trim_end_matches("b").ends_with("m") ||
cnf.gc_limit.trim_end_matches("b").ends_with("g")
)
) {
panic!("invalid ram limit");
}
config.gc_limit = if cnf.gc_limit.ends_with("gb") {
cnf.gc_limit.trim_end_matches("gb").parse::<usize>().unwrap() * 1024 * 1024 * 1024
} else if cnf.gc_limit.ends_with("mb") {
cnf.gc_limit.trim_end_matches("mb").parse::<usize>().unwrap() * 1024 * 1024
} else if cnf.gc_limit.ends_with("kb") {
cnf.gc_limit.trim_end_matches("kb").parse::<usize>().unwrap() * 1024
} else {
cnf.gc_limit.trim_end_matches("b").parse::<usize>().unwrap()
};
if !(cnf.avg_entry_size.ends_with("b") &&
(
cnf.avg_entry_size.trim_end_matches("b").ends_with("k") ||
cnf.avg_entry_size.trim_end_matches("b").ends_with("m") ||
cnf.avg_entry_size.trim_end_matches("b").ends_with("g")
)
) {
panic!("invalid ram limit");
}
config.avg_entry_size = if cnf.avg_entry_size.ends_with("gb") {
cnf.avg_entry_size.trim_end_matches("gb").parse::<usize>().unwrap() * 1024 * 1024 * 1024
} else if cnf.avg_entry_size.ends_with("mb") {
cnf.avg_entry_size.trim_end_matches("mb").parse::<usize>().unwrap() * 1024 * 1024
} else if cnf.avg_entry_size.ends_with("kb") {
cnf.avg_entry_size.trim_end_matches("kb").parse::<usize>().unwrap() * 1024
} else {
cnf.avg_entry_size.trim_end_matches("b").parse::<usize>().unwrap()
};
if config.avg_entry_size > config.ram_limit {
panic!("avg entry size is larger than ram limit");
}
if config.gc_limit > config.ram_limit {
panic!("gc limit is larger than ram limit");
}
if config.log.is_empty() {
config.log = "info".to_string();
}
if config.log != "debug" && config.log != "info" && config.log != "warn" && config.log != "error" {
panic!("invalid log level");
}
config
}

321
lyphedb/src/dbimpl/mod.rs Normal file
View file

@ -0,0 +1,321 @@
use crate::config::LypheDBConfig;
use log::*;
use lyphedb::PropagationStrategy;
use once_cell::sync::Lazy;
use std::borrow::Cow;
use std::collections::BTreeMap;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use percent_encoding::{percent_decode_str, percent_encode, AsciiSet, CONTROLS};
use tokio::sync::RwLock;
pub const NOT_ALLOWED_ASCII: AsciiSet = CONTROLS.add(b' ').add(b'/').add(b'.').add(b'\\');
pub type LDBCache = BTreeMap<Arc<Vec<u8>>, Arc<Vec<u8>>>;
/// Reads will read from this cache
pub static PRIMARY_CACHE: Lazy<RwLock<LDBCache>> =
Lazy::new(|| RwLock::new(BTreeMap::new()));
/// Writes will be written to this cache, and then depending on the propagation method, the
/// Primary cache will be set to a copy
pub static SECONDARY_CACHE: Lazy<RwLock<LDBCache>> =
Lazy::new(|| RwLock::new(BTreeMap::new()));
/// how often are keys accessed? this will influence the garbage collector
pub static KEY_ACCESS_COUNTER: Lazy<RwLock<BTreeMap<Arc<Vec<u8>>, AtomicU64>>> =
Lazy::new(|| RwLock::new(BTreeMap::new()));
pub fn key_to_path(config: &LypheDBConfig, key: &[u8]) -> PathBuf {
let mut path = PathBuf::new();
path.push(&config.master);
let mut fnbuf = Vec::new();
for (i, byte) in key.iter().enumerate() {
if *byte == b'/' {
let encoded = percent_encode(&fnbuf, &NOT_ALLOWED_ASCII).to_string();
path.push(encoded);
fnbuf.clear();
} else {
fnbuf.push(*byte);
}
}
if !fnbuf.is_empty() {
let encoded = percent_encode(&fnbuf, &NOT_ALLOWED_ASCII).to_string();
path.push(encoded);
fnbuf.clear();
}
path.push(".self");
path
}
#[derive(Debug)]
pub enum OperationError {
KeyCannotBeEmpty,
FilesystemPermissionError,
BadFilesystemEntry,
}
pub async fn set_key_disk(
config: &LypheDBConfig,
key: &[u8],
value: &[u8],
) -> Result<(), OperationError> {
if key.is_empty() {
return Err(OperationError::KeyCannotBeEmpty);
}
let path = key_to_path(config, key);
let directory = path.parent().ok_or(OperationError::KeyCannotBeEmpty)?;
if let Ok(directory_exists) = directory.try_exists() {
if !directory_exists {
std::fs::create_dir_all(directory).map_err(|e| {
warn!("couldn't create directory: {:?}", e);
OperationError::FilesystemPermissionError
})?;
}
} else {
return Err(OperationError::FilesystemPermissionError);
}
std::fs::write(path, value).map_err(|e| {
warn!("couldn't write file: {:?}", e);
OperationError::FilesystemPermissionError
})?;
Ok(())
}
pub async fn delete_key_disk(
config: &LypheDBConfig,
key: &[u8],
) -> Result<(), OperationError> {
if key.is_empty() {
return Err(OperationError::KeyCannotBeEmpty);
}
let path = key_to_path(config, key);
if let Ok(exists) = path.try_exists() {
if !exists {
return Ok(());
}
}
std::fs::remove_file(path).map_err(|e| {
warn!("couldn't remove file: {:?}", e);
OperationError::FilesystemPermissionError
})
}
pub async fn get_key_disk(
config: &LypheDBConfig,
key: &[u8],
) -> Result<Option<Vec<u8>>, OperationError> {
if key.is_empty() {
return Err(OperationError::KeyCannotBeEmpty);
}
let path = key_to_path(config, key);
if let Ok(exists) = path.try_exists() {
if !exists {
return Ok(None);
}
}
let data = std::fs::read(path).map_err(|e| {
warn!("couldn't read file: {:?}", e);
OperationError::FilesystemPermissionError
})?;
Ok(Some(data))
}
/// this function allows empty keys, so you can get all keys under root by doing
/// get_keys_under_keydir_disk(..., b"")
pub async fn get_keys_under_keydir_disk(
config: &LypheDBConfig,
keydir: &[u8],
) -> Result<Vec<Vec<u8>>, OperationError> {
let path = key_to_path(config, keydir);
let path = path.parent().expect("bad master");
let mut keys = Vec::new();
for entry in std::fs::read_dir(path).map_err(|e| {
warn!("couldn't read directory: {:?}", e);
OperationError::FilesystemPermissionError
})? {
let entry = entry.map_err(|e| {
warn!("couldn't read directory entry: {:?}", e);
OperationError::FilesystemPermissionError
})?;
let path = entry.path();
let filename = path
.to_str()
.ok_or(OperationError::FilesystemPermissionError)?;
if filename.ends_with(".self") {
// this is a value file, ignore
continue;
}
let filename = filename.trim_start_matches(&config.master);
let key = percent_decode_str(filename).collect();
keys.push(key);
}
Ok(keys)
}
pub async fn set_key(
config: LypheDBConfig,
key: &[u8],
value: &[u8],
strat: &PropagationStrategy,
) -> Result<(), OperationError> {
let k1 = Arc::new(key.to_vec());
let v1 = Arc::new(value.to_vec());
let disk_task = {
let k1 = k1.clone();
let v1 = v1.clone();
tokio::spawn(async move { set_key_disk(&config, &k1, &v1).await })
};
let prop_task = match strat {
PropagationStrategy::Immediate => {
let k1 = k1.clone();
let v1 = v1.clone();
tokio::spawn(async move {
let mut secondary_cache = SECONDARY_CACHE.write().await;
secondary_cache.insert(k1, v1);
let mut primary_cache = PRIMARY_CACHE.write().await;
*primary_cache = secondary_cache.clone();
})
}
PropagationStrategy::Timeout => {
let k1 = k1.clone();
let v1 = v1.clone();
tokio::spawn(async move {
tokio::spawn(async move {
{
let mut secondary_cache = SECONDARY_CACHE.write().await;
secondary_cache.insert(k1, v1);
}
tokio::spawn(async move {
let start = std::time::Instant::now();
loop {
if start.elapsed().as_secs() > 60 {
break;
}
let pc = PRIMARY_CACHE.try_write();
if pc.is_err() {
tokio::time::sleep(std::time::Duration::from_secs(10)).await;
continue;
} else {
break;
}
}
let mut primary_cache = PRIMARY_CACHE.write().await;
let secondary_cache = SECONDARY_CACHE.read().await;
*primary_cache = secondary_cache.clone();
});
});
})
}
PropagationStrategy::OnRead => {
let k1 = k1.clone();
tokio::spawn(async move {
{
let mut secondary_cache = SECONDARY_CACHE.write().await;
secondary_cache.remove(&k1);
}
{
let mut primary_cache = PRIMARY_CACHE.write().await;
primary_cache.remove(&k1);
}
})
}
};
if let Ok(Err(e)) = disk_task.await {
error!("couldn't set key on disk: {:?} ({:?})", e, key);
// undo propagation
prop_task.abort();
{
let mut primary_cache = PRIMARY_CACHE.write().await;
primary_cache.remove(&k1);
}
{
let mut secondary_cache = SECONDARY_CACHE.write().await;
secondary_cache.remove(&k1);
}
return Err(e);
}
let _ = prop_task.await;
Ok(())
}
pub async fn get_key(config: LypheDBConfig, key: &[u8]) -> Result<Option<Vec<u8>>, OperationError> {
let k1 = Arc::new(key.to_vec());
{
let k1 = k1.clone();
tokio::spawn(async move {
let mut access_counter = KEY_ACCESS_COUNTER.write().await;
let counter = access_counter.entry(k1.clone()).or_insert(AtomicU64::new(0));
if counter.load(Ordering::Relaxed) > 10000000 {
return;
}
counter.fetch_add(1, Ordering::SeqCst);
});
}
let disk_task = {
let k1 = k1.clone();
tokio::spawn(async move { get_key_disk(&config, &k1).await })
};
{
// read from cache
let cache = PRIMARY_CACHE.read().await;
if let Some(val) = cache.get(&k1) {
disk_task.abort();
return Ok(Some(val.to_vec()));
}
}
//debug!("cache miss");
if let Ok(result) = disk_task.await {
if let Ok(Some(val)) = &result {
let val = Arc::new(val.to_vec());
tokio::spawn(async move {
{
let mut cache = SECONDARY_CACHE.write().await;
cache.insert(k1.clone(), val.clone());
}
{
let secondary_cache = SECONDARY_CACHE.read().await;
let mut primary_cache = PRIMARY_CACHE.write().await;
*primary_cache = secondary_cache.clone();
}
//debug!("cache insert");
});
}
result
} else {
Err(OperationError::FilesystemPermissionError)
}
}
pub async fn delete_key(config: LypheDBConfig, key: &[u8]) -> Result<(), OperationError> {
let k1 = Arc::new(key.to_vec());
let disk_task = {
let k1 = k1.clone();
tokio::spawn(async move { delete_key_disk(&config, &k1).await })
};
let prop_task = {
let k1 = k1.clone();
tokio::spawn(async move {
{
let mut key_access_counter = KEY_ACCESS_COUNTER.write().await;
key_access_counter.remove(&k1);
}
{
let mut secondary_cache = SECONDARY_CACHE.write().await;
secondary_cache.remove(&k1);
}
let mut primary_cache = PRIMARY_CACHE.write().await;
let secondary_cache = SECONDARY_CACHE.read().await;
*primary_cache = secondary_cache.clone();
})
};
prop_task.await.expect("couldn't delete key");
disk_task.await.expect("couldn't delete key")
}

49
lyphedb/src/lib.rs Normal file
View file

@ -0,0 +1,49 @@
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum LDBNatsMessage {
Command(LypheDBCommand),
Entries(KVList),
Count(u64),
Success,
BadRequest,
NotFound,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum PropagationStrategy {
/// Reads will immediately be able to read this key's value as soon as it has been set
Immediate,
/// The value change will be queued along with others,
/// then in a period of either inactivity or a maximum time, all changes will be immediately
/// seen at once
Timeout,
/// The key will be removed from the cache, and thus the next read will cause a cache miss,
/// and the value will be loaded from disk
OnRead,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum LypheDBCommand {
SetKeys(KVList, PropagationStrategy),
GetKeys(KeyList),
CountKeys(KeyDirectory),
/// NOT RECURSIVE
GetKeyDirectory(KeyDirectory),
DeleteKeys(KeyList),
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct KVList {
pub kvs: Vec<(Vec<u8>, Vec<u8>)>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct KeyList {
pub keys: Vec<Vec<u8>>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct KeyDirectory {
pub key: Vec<u8>,
}

248
lyphedb/src/main.rs Normal file
View file

@ -0,0 +1,248 @@
/*
keys are stored on disk like this
<master>/<keyname>/.self
so if i stored the key "/this/is/a/key" with the data "hello world", it'd look like this
<master>/this/is/a/key/.self -> "hello world"
*/
use crate::config::{LypheDBConfig, load_config};
use crate::dbimpl::{KEY_ACCESS_COUNTER, PRIMARY_CACHE, SECONDARY_CACHE};
use futures::StreamExt;
use log::{debug, error, info, warn};
use lyphedb::{KVList, LDBNatsMessage, LypheDBCommand};
mod config;
mod dbimpl;
pub async fn gc_thread(config: LypheDBConfig) {
loop {
tokio::time::sleep(std::time::Duration::from_secs(61)).await;
{
let cache = PRIMARY_CACHE.read().await;
let cache_size = cache.len() * config.avg_entry_size;
if cache_size > config.gc_limit {
debug!("gc triggered, cache size: {} bytes", cache_size);
let keycount_to_remove = cache.len() - config.gc_limit / config.avg_entry_size;
drop(cache);
let mut least_freq_keys = vec![];
for (key, count) in KEY_ACCESS_COUNTER.read().await.iter() {
let count = count.load(std::sync::atomic::Ordering::Relaxed);
if least_freq_keys.len() < keycount_to_remove {
least_freq_keys.push((key.clone(), count));
} else {
for (other, oc) in least_freq_keys.iter_mut() {
if count < *oc {
*other = key.clone();
*oc = count;
break;
}
}
}
}
let mut cache = SECONDARY_CACHE.write().await;
for (key, _) in least_freq_keys.iter() {
cache.remove(key);
}
let mut primary_cache = PRIMARY_CACHE.write().await;
*primary_cache = cache.clone();
debug!(
"gc finished, cache size: {} bytes",
primary_cache.len() * config.avg_entry_size
);
}
}
}
}
pub async fn precache_keys_until_limit(config: LypheDBConfig) {
info!("precache started");
let mut precache_count = 0;
let mut precache_stack = dbimpl::get_keys_under_keydir_disk(&config, b"")
.await
.expect("couldn't get root keys");
while let Some(precache_key) = precache_stack.pop() {
{
let cache = PRIMARY_CACHE.read().await;
if cache.len() * config.avg_entry_size > config.gc_limit {
break;
}
if cache.len() % 10000 == 0 {
info!("precache size: {} mb", (cache.len() * config.avg_entry_size) / (1024*1024));
}
}
let children = dbimpl::get_keys_under_keydir_disk(&config, &precache_key)
.await
.expect("couldn't get children of key");
if !children.is_empty() {
precache_stack.extend(children);
} else {
let _value = dbimpl::get_key(config.clone(), &precache_key)
.await
.expect("couldn't get value of key");
precache_count += 1;
}
}
info!("precache finished, {} values precached", precache_count);
}
#[tokio::main]
async fn main() {
let config = load_config(&std::env::args().nth(1).expect("please specify config file"));
let logger = env_logger::builder()
.filter_level(match config.log.as_str() {
"debug" => log::LevelFilter::Debug,
"info" => log::LevelFilter::Info,
"warn" => log::LevelFilter::Warn,
"error" => log::LevelFilter::Error,
_ => unreachable!(),
})
.build();
log::set_boxed_logger(Box::new(logger)).unwrap();
log::set_max_level(match config.log.as_str() {
"debug" => log::LevelFilter::Debug,
"info" => log::LevelFilter::Info,
"warn" => log::LevelFilter::Warn,
"error" => log::LevelFilter::Error,
_ => unreachable!(),
});
info!("lyphedb started");
let nats = async_nats::ConnectOptions::new()
.add_client_certificate(
config.nats_cert.as_str().into(),
config.nats_key.as_str().into(),
)
.connect(config.nats_url.as_str())
.await;
if let Err(e) = nats {
error!("FATAL ERROR, COULDN'T CONNECT TO NATS: {}", e);
return;
}
let nats = nats.unwrap();
let mut subscriber = nats
.queue_subscribe(config.name.clone(), "lyphedb".to_string())
.await
.expect("couldn't subscribe to subject");
info!("nats connected");
tokio::spawn(precache_keys_until_limit(config.clone()));
tokio::spawn(gc_thread(config.clone()));
while let Some(msg) = subscriber.next().await {
let config = config.clone();
let nats = nats.clone();
tokio::spawn(async move {
async fn bad_request(nats: &async_nats::Client, replyto: async_nats::Subject) {
let reply = rmp_serde::to_vec(&LDBNatsMessage::BadRequest).unwrap();
if let Err(e) = nats.publish(replyto, reply.into()).await {
warn!("couldn't send reply: {:?}", e);
}
}
let data = msg.payload.to_vec();
let data = rmp_serde::from_slice::<LDBNatsMessage>(&data);
if let Err(e) = data {
warn!("couldn't deserialize message: {:?}", e);
if let Some(replyto) = msg.reply {
bad_request(&nats, replyto).await;
}
return;
}
let data = data.unwrap();
match data {
LDBNatsMessage::Command(cmd) => match cmd {
LypheDBCommand::SetKeys(kvlist, propstrat) => {
for (key, value) in kvlist.kvs {
if let Err(e) =
dbimpl::set_key(config.clone(), &key, &value, &propstrat).await
{
warn!("couldn't set key: {:?}", e);
}
}
let reply = rmp_serde::to_vec(&LDBNatsMessage::Success).unwrap();
if let Err(e) = nats.publish(msg.reply.unwrap(), reply.into()).await {
warn!("couldn't send reply: {:?}", e);
}
}
LypheDBCommand::GetKeys(klist) => {
let mut reply = Vec::new();
for key in klist.keys {
if let Ok(Some(value)) = dbimpl::get_key(config.clone(), &key).await {
reply.push((key, value));
} else {
warn!("couldn't get key: {:?}", key);
}
}
let reply =
rmp_serde::to_vec(&LDBNatsMessage::Entries(KVList { kvs: reply }))
.unwrap();
if let Err(e) = nats.publish(msg.reply.unwrap(), reply.into()).await {
warn!("couldn't send reply: {:?}", e);
}
}
LypheDBCommand::CountKeys(keydir) => {
let keys = dbimpl::get_keys_under_keydir_disk(&config, &keydir.key)
.await
.expect("couldn't get keys under keydir");
let mut count = 0;
for key in keys {
let value = dbimpl::get_key(config.clone(), &key)
.await
.expect("couldn't get value of key");
if value.is_some() {
count += 1;
}
}
let reply = rmp_serde::to_vec(&LDBNatsMessage::Count(count)).unwrap();
if let Err(e) = nats.publish(msg.reply.unwrap(), reply.into()).await {
warn!("couldn't send reply: {:?}", e);
}
}
LypheDBCommand::GetKeyDirectory(keydir) => {
let mut reply = Vec::new();
let keys = dbimpl::get_keys_under_keydir_disk(&config, &keydir.key)
.await
.expect("couldn't get keys under keydir");
for key in keys {
let value = dbimpl::get_key(config.clone(), &key)
.await
.expect("couldn't get value of key");
if let Some(value) = value {
reply.push((key, value));
}
}
let reply =
rmp_serde::to_vec(&LDBNatsMessage::Entries(KVList { kvs: reply }))
.unwrap();
if let Err(e) = nats.publish(msg.reply.unwrap(), reply.into()).await {
warn!("couldn't send reply: {:?}", e);
}
}
LypheDBCommand::DeleteKeys(klist) => {
for key in klist.keys {
if let Err(e) = dbimpl::delete_key(config.clone(), &key).await {
warn!("couldn't delete key: {:?}", e);
}
}
let reply = rmp_serde::to_vec(&LDBNatsMessage::Success).unwrap();
if let Err(e) = nats.publish(msg.reply.unwrap(), reply.into()).await {
warn!("couldn't send reply: {:?}", e);
}
}
},
_ => {
warn!("bad request, not command");
if let Some(replyto) = msg.reply {
bad_request(&nats, replyto).await;
}
}
}
});
}
info!("lyphedb shutting down");
}

1
vorebot/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
/target

28
vorebot/Cargo.toml Normal file
View file

@ -0,0 +1,28 @@
[package]
name = "vorebot"
version = "0.2.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
asklyphe-common = { path = "../asklyphe-common" }
tokio = { version = "1.0", features = ["full"] }
serde = { version = "1.0", features = ["derive"] }
rand = "0.8.5"
rmp-serde = "1.1.2"
base64 = "0.21.7"
image = "0.24.8"
isahc = "1.7.2"
ulid = "1.0.0"
async-nats = "0.38.0"
futures = "0.3.28"
chrono = "0.4.26"
once_cell = "1.18.0"
env_logger = "0.10.0"
log = "0.4.19"
mutex-timeouts = { version = "0.3.0", features = ["tokio"] }
prometheus_exporter = "0.8.5"
thirtyfour = "0.35.0"
stopwords = "0.1.1"
texting_robots = "0.2.2"

352
vorebot/src/main.rs Normal file
View file

@ -0,0 +1,352 @@
mod webparse;
use asklyphe_common::db;
use asklyphe_common::nats::vorebot::{
VOREBOT_NEWHOSTNAME_SERVICE, VOREBOT_SERVICE, VOREBOT_SUGGESTED_SERVICE,
};
use async_nats::jetstream;
use async_nats::jetstream::consumer::PullConsumer;
use async_nats::jetstream::stream::RetentionPolicy;
use chrono::TimeZone;
use futures::StreamExt;
use log::{debug, error, info, warn};
use mutex_timeouts::tokio::MutexWithTimeoutAuto as Mutex;
use once_cell::sync::Lazy;
use prometheus_exporter::prometheus::core::{AtomicF64, GenericGauge};
use prometheus_exporter::prometheus::{register_counter, register_gauge, Counter};
use std::cmp::max;
use std::collections::{BTreeMap, BTreeSet};
use std::hash::{DefaultHasher, Hasher};
use std::io::Read;
use std::iter::Iterator;
use std::str::FromStr;
use std::string::ToString;
use std::sync::atomic::{AtomicI64, AtomicU64, Ordering};
use std::sync::Arc;
use std::time::Duration;
use async_nats::jetstream::kv;
use stopwords::{Language, Spark, Stopwords};
use thirtyfour::{CapabilitiesHelper, DesiredCapabilities, Proxy, WebDriver};
use thirtyfour::common::capabilities::firefox::FirefoxPreferences;
use tokio::task::JoinHandle;
use asklyphe_common::ldb::DBConn;
use asklyphe_common::nats::vorebot::CrawlRequest;
use crate::webparse::web_parse;
pub static NATS_URL: Lazy<String> =
Lazy::new(|| std::env::var("NATS_URL").expect("NO NATS URL DEFINED"));
pub static NATS_CERT: Lazy<String> = Lazy::new(|| std::env::var("NATS_CERT").expect("NO NATS_CERT DEFINED"));
pub static NATS_KEY: Lazy<String> = Lazy::new(|| std::env::var("NATS_KEY").expect("NO NATS_KEY DEFINED"));
pub static BROWSER_THREADS: Lazy<Vec<String>> =
Lazy::new(|| std::env::var("BROWSER_THREADS").expect("PLEASE LIST BROWSER_THREADS").split(',').map(|v| v.to_string()).collect());
pub static DB_NAME: Lazy<String> =
Lazy::new(|| std::env::var("DB_NAME").expect("PLEASE ADD DB_NAME"));
pub static DOCUMENTS_CRAWLED: AtomicU64 = AtomicU64::new(0);
pub static LAST_MESSAGE: AtomicI64 = AtomicI64::new(0);
pub static LAST_TASK_COMPLETE: Lazy<Vec<Arc<AtomicI64>>> = Lazy::new(|| {
let max_threads: usize = BROWSER_THREADS.len();
let mut vals = vec![];
for i in 0..max_threads {
// let db = Database::default().expect("couldn't connect to foundation db!");
// DBS.lock().await.push(Arc::new(db));
vals.push(Arc::new(AtomicI64::new(chrono::Utc::now().timestamp())));
}
vals
});
pub static USER_AGENT: Lazy<String> = Lazy::new(|| {
format!(
"Vorebot/{} (compatible; Googlebot/2.1; +https://voremicrocomputers.com/crawler.html)",
env!("CARGO_PKG_VERSION")
)
});
#[tokio::main]
async fn main() {
mutex_timeouts::tokio::GLOBAL_TOKIO_TIMEOUT.store(72, Ordering::Relaxed);
env_logger::init();
info!("began at {}", chrono::Utc::now().to_string());
let nats = async_nats::ConnectOptions::new()
.add_client_certificate(NATS_CERT.as_str().into(), NATS_KEY.as_str().into())
.connect(NATS_URL.as_str())
.await;
if let Err(e) = nats {
error!("FATAL ERROR, COULDN'T CONNECT TO NATS: {}", e);
return;
}
let nats = nats.unwrap();
let dbconn = DBConn::new(nats.clone(), DB_NAME.to_string());
let nats = jetstream::new(nats);
let mut tasks: Vec<(JoinHandle<()>, String)> = vec![];
let mut available_browsers: Vec<String> = BROWSER_THREADS.clone();
{
loop {
while tasks.len() < BROWSER_THREADS.len() {
let nats = nats.clone();
let browser = available_browsers.pop().expect("NO BROWSERS LEFT, THIS IS A FATAL BUG!");
let db = dbconn.clone();
let b = browser.clone();
tasks.push((tokio::spawn(async move {
let browser = b;
info!("using {}", browser);
info!("crawler spawned");
/* normal priority */
let consumer: PullConsumer = nats.get_or_create_stream(jetstream::stream::Config {
name: VOREBOT_SERVICE.to_string(),
subjects: vec![VOREBOT_SERVICE.to_string()],
retention: RetentionPolicy::WorkQueue,
..Default::default()
}).await
.expect("FATAL! FAILED TO SUBSCRIBE TO NATS!")
.get_or_create_consumer("parser", jetstream::consumer::pull::Config {
durable_name: Some("parser".to_string()),
filter_subject: VOREBOT_SERVICE.to_string(),
..Default::default()
}).await.expect("FATAL! FAILED TO SUBSCRIBE TO NATS!");
let mut messages = consumer.messages().await.expect("FATAL! FAILED TO SUBSCRIBE TO NATS!");
/* higher priority (new hostnames) */
let higher_consumer: PullConsumer = nats.get_or_create_stream(jetstream::stream::Config {
name: VOREBOT_NEWHOSTNAME_SERVICE.to_string(),
subjects: vec![VOREBOT_NEWHOSTNAME_SERVICE.to_string()],
retention: RetentionPolicy::WorkQueue,
..Default::default()
}).await
.expect("FATAL! FAILED TO SUBSCRIBE TO NATS!")
.get_or_create_consumer("highparser", jetstream::consumer::pull::Config {
durable_name: Some("highparser".to_string()),
filter_subject: VOREBOT_NEWHOSTNAME_SERVICE.to_string(),
..Default::default()
}).await.expect("FATAL! FAILED TO SUBSCRIBE TO NATS!");
let mut high_messages = higher_consumer.messages().await.expect("FATAL! FAILED TO SUBSCRIBE TO NATS!");
/* highest priority (user-suggested) */
let highest_consumer: PullConsumer = nats.get_or_create_stream(jetstream::stream::Config {
name: VOREBOT_SUGGESTED_SERVICE.to_string(),
subjects: vec![VOREBOT_SUGGESTED_SERVICE.to_string()],
retention: RetentionPolicy::WorkQueue,
..Default::default()
}).await
.expect("FATAL! FAILED TO SUBSCRIBE TO NATS!")
.get_or_create_consumer("highestparser", jetstream::consumer::pull::Config {
durable_name: Some("highestparser".to_string()),
filter_subject: VOREBOT_SUGGESTED_SERVICE.to_string(),
..Default::default()
}).await.expect("FATAL! FAILED TO SUBSCRIBE TO NATS!");
let mut highest_messages = highest_consumer.messages().await.expect("FATAL! FAILED TO SUBSCRIBE TO NATS!");
let mut prefs = FirefoxPreferences::new();
prefs.set_user_agent(USER_AGENT.to_string()).unwrap();
let mut caps = DesiredCapabilities::firefox();
caps.set_preferences(prefs).unwrap();
let driver = WebDriver::new(&browser, caps).await.unwrap();
info!("crawler ready");
loop {
tokio::select! {
Some(highest) = StreamExt::next(&mut highest_messages) => {
if let Err(e) = highest {
warn!("error when recv js message! {e}");
} else {
let message = highest.unwrap();
if let Err(e) = message.ack().await {
warn!("failed acking message {e}");
}
let req = rmp_serde::from_slice::<CrawlRequest>(message.payload.as_ref());
if let Err(e) = req {
error!("BAD NATS REQUEST: {e}");
continue;
}
let req = req.unwrap();
info!("RECV USER SUGGESTION!");
let now = chrono::Utc::now().timestamp();
LAST_MESSAGE.store(now, Ordering::Relaxed);
let nats = nats.clone();
let mut bad = false;
driver.in_new_tab(|| async {
if web_parse(nats.clone(), db.clone(), &driver, &req.url, req.damping).await.is_err() {
warn!("temporary failure detected in parsing, requeuing");
nats.publish(VOREBOT_SERVICE.to_string(), rmp_serde::to_vec(&req).unwrap().into()).await.expect("FAILED TO REQUEUE");
bad = true;
}
Ok(())
}).await.unwrap();
if bad {
continue;
}
if DOCUMENTS_CRAWLED.load(Ordering::Relaxed) % 100 == 0 {
DOCUMENTS_CRAWLED.fetch_add(1, Ordering::Relaxed);
info!("crawled {} pages!", DOCUMENTS_CRAWLED.load(Ordering::Relaxed));
}
}
}
Some(high) = StreamExt::next(&mut high_messages) => {
if let Err(e) = high {
warn!("error when recv js message! {e}");
} else {
let message = high.unwrap();
if let Err(e) = message.ack().await {
warn!("failed acking message {e}");
}
let req = rmp_serde::from_slice::<CrawlRequest>(message.payload.as_ref());
if let Err(e) = req {
error!("BAD NATS REQUEST: {e}");
continue;
}
let req = req.unwrap();
info!("RECV HIGH PRIORITY!");
let now = chrono::Utc::now().timestamp();
LAST_MESSAGE.store(now, Ordering::Relaxed);
let nats = nats.clone();
let mut bad = false;
driver.in_new_tab(|| async {
if web_parse(nats.clone(), db.clone(), &driver, &req.url, req.damping).await.is_err() {
warn!("temporary failure detected in parsing, requeuing");
nats.publish(VOREBOT_SERVICE.to_string(), rmp_serde::to_vec(&req).unwrap().into()).await.expect("FAILED TO REQUEUE");
bad = true;
}
Ok(())
}).await.unwrap();
if bad {
continue;
}
if DOCUMENTS_CRAWLED.load(Ordering::Relaxed) % 100 == 0 {
DOCUMENTS_CRAWLED.fetch_add(1, Ordering::Relaxed);
info!("crawled {} pages!", DOCUMENTS_CRAWLED.load(Ordering::Relaxed));
}
}
}
Some(normal) = StreamExt::next(&mut messages) => {
if let Err(e) = normal {
warn!("error when recv js message! {e}");
} else {
let message = normal.unwrap();
if let Err(e) = message.ack().await {
warn!("failed acking message {e}");
}
let req = rmp_serde::from_slice::<CrawlRequest>(message.payload.as_ref());
if let Err(e) = req {
error!("BAD NATS REQUEST: {e}");
continue;
}
let req = req.unwrap();
let now = chrono::Utc::now().timestamp();
LAST_MESSAGE.store(now, Ordering::Relaxed);
let nats = nats.clone();
let mut hash = DefaultHasher::new();
hash.write(req.url.as_bytes());
let hash = hash.finish();
let dehomo_bucket = nats.get_key_value("dehomo").await;
let dehomo_bucket = if dehomo_bucket.is_err() {
let dehomo_bucket = nats.create_key_value(kv::Config {
bucket: "dehomo".to_string(),
description: "prevent the same url from being scraped again too quickly".to_string(),
max_age: Duration::from_secs(60*60),
..Default::default()
}).await;
if let Err(e) = dehomo_bucket {
panic!("FAILED TO CREATE DEHOMO BUCKET!!! {e}");
} else {
dehomo_bucket.unwrap()
}
} else {
dehomo_bucket.unwrap()
};
if dehomo_bucket.get(hash.to_string()).await.ok().flatten().map(|v| *v.first().unwrap_or(&0) == 1).unwrap_or(false) {
info!("too soon to scrape {}", req.url);
continue;
}
let mut bad = false;
driver.in_new_tab(|| async {
if web_parse(nats.clone(), db.clone(), &driver, &req.url, req.damping).await.is_err() {
warn!("temporary failure detected in parsing, requeuing");
nats.publish(VOREBOT_SERVICE.to_string(), rmp_serde::to_vec(&req).unwrap().into()).await.expect("FAILED TO REQUEUE");
bad = true;
}
Ok(())
}).await.unwrap();
if bad {
continue;
}
dehomo_bucket.put(hash.to_string(), vec![1u8].into()).await.expect("failed to store dehomo");
if DOCUMENTS_CRAWLED.load(Ordering::Relaxed) % 100 == 0 {
DOCUMENTS_CRAWLED.fetch_add(1, Ordering::Relaxed);
info!("crawled {} pages!", DOCUMENTS_CRAWLED.load(Ordering::Relaxed));
}
}
}
}
}
}),
browser
));
warn!("spawning new injest thread");
}
let mut tasks_to_remove = vec![];
for task in tasks.iter() {
if task.0.is_finished() {
tasks_to_remove.push(task.1.clone());
available_browsers.push(task.1.clone());
}
}
tasks.retain(|v| !tasks_to_remove.contains(&v.1));
tokio::time::sleep(Duration::from_secs(3)).await;
//while let Some(p) = initial.pop() {
// nats.publish(VOREBOT_SUGGESTED_SERVICE.to_string(), rmp_serde::to_vec(&p).unwrap().into()).await.unwrap();
//}
}
}
}
//#[tokio::main]
//async fn main() {
// mutex_timeouts::tokio::GLOBAL_TOKIO_TIMEOUT.store(72, Ordering::Relaxed);
//
// env_logger::init();
// info!("began at {}", chrono::Utc::now().to_string());
//
// let nats = async_nats::connect(NATS_URL.as_str()).await;
// if let Err(e) = nats {
// error!("FATAL ERROR, COULDN'T CONNECT TO NATS: {}", e);
// return;
// }
// let nats = nats.unwrap();
//
// let dbconn = DBConn::new(nats.clone(), "lyphedb-test");
//
// let nats = jetstream::new(nats);
//
// let mut prefs = FirefoxPreferences::new();
// prefs.set_user_agent(USER_AGENT.to_string()).unwrap();
// let mut caps = DesiredCapabilities::firefox();
// caps.set_preferences(prefs).unwrap();
// let driver = WebDriver::new(&BROWSER_THREADS[0], caps).await.unwrap();
//
// driver.in_new_tab(|| async {
// web_parse(nats.clone(), dbconn.clone(), &driver, "https://asklyphe.com/", 0.85).await.expect("Failed to run web parse");
//
// Ok(())
// }).await.unwrap();
//}

479
vorebot/src/webparse/mod.rs Normal file
View file

@ -0,0 +1,479 @@
use crate::USER_AGENT;
use asklyphe_common::ldb::{linkstore, metastore, sitestore, titlestore, wordstore, DBConn};
use async_nats::jetstream;
use async_nats::jetstream::kv;
use futures::AsyncReadExt;
use image::EncodableLayout;
use isahc::config::RedirectPolicy;
use isahc::prelude::Configurable;
use isahc::HttpClient;
use log::{debug, error, warn};
use std::collections::{BTreeMap, BTreeSet};
use std::hash::{DefaultHasher, Hasher};
use std::sync::atomic::AtomicBool;
use std::sync::{mpsc, Arc};
use std::time::Duration;
use stopwords::{Language, Spark, Stopwords};
use texting_robots::{get_robots_url, Robot};
use thirtyfour::{By, WebDriver};
use asklyphe_common::nats::vorebot::CrawlRequest;
use asklyphe_common::nats::vorebot::VOREBOT_SERVICE;
pub fn allowed_to_crawl(robotstxt: &[u8], url: &str) -> Result<bool, ()> {
let robot1 = Robot::new("Vorebot", robotstxt);
if let Err(e) = robot1 {
warn!(
"potentially malformed robots.txt ({}), not crawling {}",
e, url
);
return Err(());
}
let robot1 = robot1.unwrap();
Ok(robot1.allowed(url))
}
// returns Err if we cannot access a page, but the error associated with it seems temporary (i.e. it's worth trying again later)
// otherwise, returns Ok
pub async fn web_parse(
nats: jetstream::Context,
db: DBConn,
driver: &WebDriver,
url: &str,
damping: f64,
) -> Result<(), ()> {
driver.delete_all_cookies().await.map_err(|_| ())?;
let robots_bucket = nats.get_key_value("robots").await;
let robots_bucket = if robots_bucket.is_err() {
let robots_bucket = nats
.create_key_value(kv::Config {
bucket: "robots".to_string(),
description: "storage of robots.txt data for given hosts".to_string(),
..Default::default()
})
.await;
if let Err(e) = robots_bucket {
error!("could not create robots.txt bucket: {}", e);
None
} else {
Some(robots_bucket.unwrap())
}
} else {
robots_bucket.ok()
};
let hosts_bucket = nats.get_key_value("hosts").await;
let hosts_bucket = if hosts_bucket.is_err() {
let hosts_bucket = nats
.create_key_value(kv::Config {
bucket: "hosts".to_string(),
description: "prevent the same host from being scraped too quickly".to_string(),
max_age: Duration::from_secs(60 * 5),
..Default::default()
})
.await;
if let Err(e) = hosts_bucket {
error!("could not create hosts bucket: {}", e);
return Err(());
} else {
hosts_bucket.unwrap()
}
} else {
hosts_bucket.unwrap()
};
let robots_url = get_robots_url(url);
if robots_url.is_err() {
error!("could not get a robots.txt url from {}, not crawling", url);
return Ok(());
}
let robots_url = robots_url.unwrap();
let mut hash = DefaultHasher::new();
hash.write(robots_url.as_bytes());
let hash = hash.finish();
if let Ok(Some(host)) = hosts_bucket.get(hash.to_string()).await {
let count = *host.first().unwrap_or(&0);
if count > 100 {
warn!("scraping {} too quickly, avoiding for one minute", robots_url);
return Err(());
}
hosts_bucket.put(hash.to_string(), vec![count + 1].into()).await.expect("COULDN'T INSERT INTO HOSTS BUCKET!");
} else {
hosts_bucket.put(hash.to_string(), vec![1].into()).await.expect("COULDN'T INSERT INTO HOSTS BUCKET!");
}
let mut skip_robots_check = false;
if let Some(robots_bucket) = &robots_bucket {
if let Ok(Some(entry)) = robots_bucket.get(hash.to_string()).await {
if let Ok(res) = allowed_to_crawl(entry.as_bytes(), url) {
if !res {
debug!("robots.txt does not allow us to crawl {}", url);
return Ok(());
} else {
skip_robots_check = true;
}
}
}
}
if !skip_robots_check {
// check manually
debug!("checking new robots.txt \"{}\"", robots_url);
let client = HttpClient::builder()
.redirect_policy(RedirectPolicy::Limit(10))
.timeout(Duration::from_secs(60))
.build();
if let Err(e) = client {
error!("could not create new robots.txt httpclient: {}", e);
return Err(());
}
let client = client.unwrap();
let request = isahc::Request::get(&robots_url)
.header("user-agent", USER_AGENT.as_str())
.body(());
if let Err(e) = request {
error!("could not create robots.txt get request: {}", e);
return Ok(());
}
let request = request.unwrap();
let response = client.send_async(request).await;
if let Err(e) = response {
warn!("could not get robots.txt page: {}", e);
return Err(());
}
let mut response = response.unwrap();
if response.status() == 429 {
// too many requests
warn!("too many requests for {}", robots_url);
return Err(());
}
if response.status().is_server_error() {
// don't crawl at the moment
debug!("not crawling {} due to server error", robots_url);
return Err(());
}
let mut body = "".to_string();
if let Err(e) = response.body_mut().read_to_string(&mut body).await {
warn!("could not read from robots.txt response: {}", e);
return Err(());
}
if let Ok(res) = allowed_to_crawl(body.as_bytes(), url) {
if let Some(robots_bucket) = &robots_bucket {
if let Err(e) = robots_bucket
.put(hash.to_string(), body.as_bytes().to_vec().into())
.await
{
warn!("could not put robots.txt data: {}", e);
}
}
if !res {
debug!("robots.txt does not allow us to crawl {}", url);
return Ok(());
} else {
// we're allowed to crawl!
}
}
}
let start = std::time::Instant::now();
debug!("handling request for {}", url);
// check for bad status codes
// fixme: i hate this solution, can we get something that actually checks the browser's request?
let client = HttpClient::builder()
.redirect_policy(RedirectPolicy::Limit(10))
.timeout(Duration::from_secs(60))
.build();
if let Err(e) = client {
error!("could not create new badstatuscode httpclient: {}", e);
return Err(());
}
let client = client.unwrap();
let request = isahc::Request::get(url)
.header("user-agent", USER_AGENT.as_str())
.body(());
if let Err(e) = request {
error!("could not create badstatuscode get request: {}", e);
return Ok(());
}
let request = request.unwrap();
let response = client.send_async(request).await;
if let Err(e) = response {
warn!("could not get badstatuscode page: {}", e);
return Err(());
}
let mut response = response.unwrap();
if response.status() == 429 {
// too many requests
warn!("too many requests for {}", url);
return Err(());
}
if response.status().is_server_error() || response.status().is_client_error() {
// don't crawl at the moment
debug!("not crawling {} due to bad status code {}", url, response.status());
return Err(());
}
// i guess we're good
driver.goto(url).await.map_err(|_| ())?;
let meta_elements = driver.find_all(By::Tag("meta")).await.map_err(|_| ())?;
let title = driver.title().await.map_err(|_| ())?;
let mut description = None;
let mut keywords = vec![];
for elem in meta_elements {
if let Ok(Some(name)) = elem.attr("name").await {
match name.as_str() {
"description" => {
if let Ok(Some(content)) = elem.attr("content").await {
description = Some(content);
}
}
"keywords" => {
if let Ok(Some(content)) = elem.attr("content").await {
keywords = content
.split(',')
.map(|v| v.to_lowercase())
.filter(|v| !v.is_empty())
.collect();
}
}
_ => {}
}
}
}
let body = driver.find(By::Tag("body")).await.map_err(|_| ())?;
let raw_page_content = body.text().await.map_err(|_| ())?;
async fn gather_elements_with_multiplier(
driver: &WebDriver,
wordmap: &mut BTreeMap<String, f64>,
stops: &BTreeSet<&&str>,
elements: &[&str],
multiplier: f64,
) {
let mut elms = vec![];
for tag in elements {
elms.push(driver.find_all(By::Tag(*tag)).await);
}
let elms = elms.iter().flatten().flatten().collect::<Vec<_>>();
let mut sentences = vec![];
let mut sentence_set = BTreeSet::new();
debug!("processing elements...");
for node in elms {
let _ = node.scroll_into_view().await;
let boxmodel = node.rect().await;
if boxmodel.is_err() {
// not visible
continue;
}
let boxmodel = boxmodel.unwrap();
let current_text = node.text().await;
if current_text.is_err() {
// no text on this node
continue;
}
let current_text = current_text.unwrap().trim().to_string();
if current_text.is_empty() {
continue;
}
let sqs = (boxmodel.width * boxmodel.height).max(1.0); // no 0 divides pls (:
let ccount = current_text.chars().count() as f64;
let cssq = if ccount > 0.0 { sqs / ccount } else { 0.0 };
if sentence_set.contains(&current_text) {
continue;
}
sentence_set.insert(current_text.clone());
sentences.push((current_text, cssq));
}
for (sentence, cssq) in sentences {
let mut cssq = (cssq / 500.0).powi(2) * multiplier;
for word in sentence.split_whitespace() {
let word = word
.to_lowercase()
.trim_end_matches(|v: char| v.is_ascii_punctuation())
.to_string();
if stops.contains(&word.as_str()) {
// less valuable
cssq /= 100.0;
}
if let Some(wentry) = wordmap.get_mut(&word) {
*wentry += cssq;
} else {
if word.is_empty() {
continue;
}
wordmap.insert(word.to_string(), cssq);
}
}
}
}
let mut wordmap: BTreeMap<String, f64> = BTreeMap::new();
let stops: BTreeSet<_> = Spark::stopwords(Language::English)
.unwrap()
.iter()
.collect();
debug!("headers...");
gather_elements_with_multiplier(driver, &mut wordmap, &stops, &["h1","h2","h3","h4","h5","h6"], 3.0)
.await;
debug!("paragraphs...");
gather_elements_with_multiplier(driver, &mut wordmap, &stops, &["p","div"], 1.0).await;
let mut wordmap = wordmap.into_iter().collect::<Vec<_>>();
wordmap.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap());
let mut db_error_so_requeue_anyways = false;
let words = wordmap
.iter()
.map(|(word, _)| word.as_str())
.collect::<Vec<_>>();
#[allow(clippy::collapsible_if)]
if !words.is_empty() {
if wordstore::add_url_to_keywords(&db, &words, url)
.await
.is_err()
{
warn!("couldn't add {} to keywords!", url);
db_error_so_requeue_anyways = true;
}
}
let mut metawords = keywords.iter().map(|v| v.as_str()).collect::<Vec<_>>();
let desc2 = description.clone();
let desc2 = desc2.map(|v| {
v.to_lowercase()
.split_whitespace()
.map(String::from)
.collect::<Vec<_>>()
});
if let Some(description) = &desc2 {
for word in description {
let word = word.trim_end_matches(|v: char| v.is_ascii_punctuation());
if word.is_empty() {
continue;
}
metawords.push(word);
}
}
#[allow(clippy::collapsible_if)]
if !metawords.is_empty() {
if metastore::add_url_to_metawords(&db, &metawords, url)
.await
.is_err()
{
warn!("couldn't add {} to metawords!", url);
db_error_so_requeue_anyways = true;
}
}
let mut titlewords = vec![];
let title2 = title.clone();
let title2 = title2.to_lowercase();
for word in title2.split_whitespace() {
let word = word.trim_end_matches(|v: char| v.is_ascii_punctuation());
if word.is_empty() {
continue;
}
titlewords.push(word);
}
#[allow(clippy::collapsible_if)]
if !titlewords.is_empty() {
if titlestore::add_url_to_titlewords(&db, &titlewords, url)
.await
.is_err()
{
warn!("couldn't add {} to titlewords!", url);
db_error_so_requeue_anyways = true;
}
}
if sitestore::add_website(
&db,
url,
Some(title),
description,
if keywords.is_empty() {
None
} else {
Some(keywords)
},
&wordmap,
raw_page_content,
damping
)
.await
.is_err()
{
warn!("couldn't add {} to sitestore!", url);
db_error_so_requeue_anyways = true;
}
debug!("finished with main site stuff for {}", url);
let linkelms = driver.find_all(By::Tag("a")).await.map_err(|_| ())?;
for linkelm in linkelms {
if linkelm.scroll_into_view().await.is_err() {
debug!("couldn't scroll into view!");
}
let href = linkelm.prop("href").await.map_err(|_| ())?;
if href.is_none() {
debug!("no href!");
continue;
}
let href = href.unwrap();
if href.contains('#') {
continue;
}
let linktext = linkelm.text().await.map_err(|_| ())?.to_lowercase();
let linkimgs = linkelm.find_all(By::Tag("img")).await.map_err(|_| ())?;
let mut alts = "".to_string();
for img in linkimgs {
if let Ok(Some(alt)) = img.attr("alt").await {
alts.push_str(&alt);
alts.push(' ');
}
}
let alts = alts.trim().to_lowercase();
let mut linkwords = vec![];
for word in linktext.split_whitespace() {
let word = word.trim_end_matches(|v: char| v.is_ascii_punctuation());
linkwords.push(word);
}
for word in alts.split_whitespace() {
let word = word.trim_end_matches(|v: char| v.is_ascii_punctuation());
linkwords.push(word);
}
#[allow(clippy::collapsible_if)]
if !linkwords.is_empty() {
if linkstore::add_url_to_linkwords(&db, &linkwords, &href).await.is_err() {
warn!("couldn't add {} to linkwords!", url);
}
}
nats.publish(VOREBOT_SERVICE.to_string(), rmp_serde::to_vec(&CrawlRequest {
url: href,
damping: 0.85,
}).unwrap().into()).await.unwrap();
}
let elapsed = start.elapsed().as_secs_f64();
debug!("crawled {} in {} seconds", url, elapsed);
// todo: queue links to be crawled
Ok(())
}