Spaces:
Running
Running
Zsombor Gegesy
commited on
Commit
·
e69126c
1
Parent(s):
1e7805c
Fix git rebase problems, and the failing doctest
Browse files- Cargo.lock +109 -0
- src/cache/cacher.rs +10 -6
- src/cache/error.rs +1 -1
- src/config/parser.rs +0 -6
- src/lib.rs +1 -1
- websurfx/config.lua +0 -1
Cargo.lock
CHANGED
|
@@ -446,6 +446,12 @@ version = "3.14.0"
|
|
| 446 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 447 |
checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
|
| 448 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 449 |
[[package]]
|
| 450 |
name = "byteorder"
|
| 451 |
version = "1.4.3"
|
|
@@ -478,6 +484,37 @@ dependencies = [
|
|
| 478 |
"bytes 1.5.0",
|
| 479 |
]
|
| 480 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 481 |
[[package]]
|
| 482 |
name = "cast"
|
| 483 |
version = "0.3.0"
|
|
@@ -973,6 +1010,15 @@ dependencies = [
|
|
| 973 |
"libc",
|
| 974 |
]
|
| 975 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 976 |
[[package]]
|
| 977 |
name = "error-stack"
|
| 978 |
version = "0.4.1"
|
|
@@ -1256,6 +1302,12 @@ version = "0.28.0"
|
|
| 1256 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 1257 |
checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0"
|
| 1258 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1259 |
[[package]]
|
| 1260 |
name = "governor"
|
| 1261 |
version = "0.5.1"
|
|
@@ -1834,6 +1886,21 @@ dependencies = [
|
|
| 1834 |
"unicase",
|
| 1835 |
]
|
| 1836 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1837 |
[[package]]
|
| 1838 |
name = "miniz_oxide"
|
| 1839 |
version = "0.7.1"
|
|
@@ -2383,6 +2450,17 @@ dependencies = [
|
|
| 2383 |
"url 2.4.1",
|
| 2384 |
]
|
| 2385 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 2386 |
[[package]]
|
| 2387 |
name = "quanta"
|
| 2388 |
version = "0.9.3"
|
|
@@ -2900,6 +2978,9 @@ name = "semver"
|
|
| 2900 |
version = "1.0.18"
|
| 2901 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 2902 |
checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918"
|
|
|
|
|
|
|
|
|
|
| 2903 |
|
| 2904 |
[[package]]
|
| 2905 |
name = "semver-parser"
|
|
@@ -3020,6 +3101,21 @@ version = "0.3.11"
|
|
| 3020 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 3021 |
checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d"
|
| 3022 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3023 |
[[package]]
|
| 3024 |
name = "slab"
|
| 3025 |
version = "0.4.9"
|
|
@@ -3197,6 +3293,12 @@ dependencies = [
|
|
| 3197 |
"libc",
|
| 3198 |
]
|
| 3199 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3200 |
[[package]]
|
| 3201 |
name = "tempfile"
|
| 3202 |
version = "3.8.0"
|
|
@@ -3553,6 +3655,12 @@ dependencies = [
|
|
| 3553 |
"once_cell",
|
| 3554 |
]
|
| 3555 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3556 |
[[package]]
|
| 3557 |
name = "try-lock"
|
| 3558 |
version = "0.2.4"
|
|
@@ -3815,6 +3923,7 @@ dependencies = [
|
|
| 3815 |
"log",
|
| 3816 |
"md5",
|
| 3817 |
"mimalloc",
|
|
|
|
| 3818 |
"mlua",
|
| 3819 |
"once_cell",
|
| 3820 |
"rand 0.8.5",
|
|
|
|
| 446 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 447 |
checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
|
| 448 |
|
| 449 |
+
[[package]]
|
| 450 |
+
name = "bytecount"
|
| 451 |
+
version = "0.6.3"
|
| 452 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 453 |
+
checksum = "2c676a478f63e9fa2dd5368a42f28bba0d6c560b775f38583c8bbaa7fcd67c9c"
|
| 454 |
+
|
| 455 |
[[package]]
|
| 456 |
name = "byteorder"
|
| 457 |
version = "1.4.3"
|
|
|
|
| 484 |
"bytes 1.5.0",
|
| 485 |
]
|
| 486 |
|
| 487 |
+
[[package]]
|
| 488 |
+
name = "camino"
|
| 489 |
+
version = "1.1.6"
|
| 490 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 491 |
+
checksum = "c59e92b5a388f549b863a7bea62612c09f24c8393560709a54558a9abdfb3b9c"
|
| 492 |
+
dependencies = [
|
| 493 |
+
"serde",
|
| 494 |
+
]
|
| 495 |
+
|
| 496 |
+
[[package]]
|
| 497 |
+
name = "cargo-platform"
|
| 498 |
+
version = "0.1.3"
|
| 499 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 500 |
+
checksum = "2cfa25e60aea747ec7e1124f238816749faa93759c6ff5b31f1ccdda137f4479"
|
| 501 |
+
dependencies = [
|
| 502 |
+
"serde",
|
| 503 |
+
]
|
| 504 |
+
|
| 505 |
+
[[package]]
|
| 506 |
+
name = "cargo_metadata"
|
| 507 |
+
version = "0.14.2"
|
| 508 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 509 |
+
checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa"
|
| 510 |
+
dependencies = [
|
| 511 |
+
"camino",
|
| 512 |
+
"cargo-platform",
|
| 513 |
+
"semver 1.0.18",
|
| 514 |
+
"serde",
|
| 515 |
+
"serde_json",
|
| 516 |
+
]
|
| 517 |
+
|
| 518 |
[[package]]
|
| 519 |
name = "cast"
|
| 520 |
version = "0.3.0"
|
|
|
|
| 1010 |
"libc",
|
| 1011 |
]
|
| 1012 |
|
| 1013 |
+
[[package]]
|
| 1014 |
+
name = "error-chain"
|
| 1015 |
+
version = "0.12.4"
|
| 1016 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 1017 |
+
checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc"
|
| 1018 |
+
dependencies = [
|
| 1019 |
+
"version_check",
|
| 1020 |
+
]
|
| 1021 |
+
|
| 1022 |
[[package]]
|
| 1023 |
name = "error-stack"
|
| 1024 |
version = "0.4.1"
|
|
|
|
| 1302 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 1303 |
checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0"
|
| 1304 |
|
| 1305 |
+
[[package]]
|
| 1306 |
+
name = "glob"
|
| 1307 |
+
version = "0.3.1"
|
| 1308 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 1309 |
+
checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
|
| 1310 |
+
|
| 1311 |
[[package]]
|
| 1312 |
name = "governor"
|
| 1313 |
version = "0.5.1"
|
|
|
|
| 1886 |
"unicase",
|
| 1887 |
]
|
| 1888 |
|
| 1889 |
+
[[package]]
|
| 1890 |
+
name = "mini-moka"
|
| 1891 |
+
version = "0.10.2"
|
| 1892 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 1893 |
+
checksum = "23e0b72e7c9042467008b10279fc732326bd605459ae03bda88825909dd19b56"
|
| 1894 |
+
dependencies = [
|
| 1895 |
+
"crossbeam-channel",
|
| 1896 |
+
"crossbeam-utils 0.8.16",
|
| 1897 |
+
"dashmap",
|
| 1898 |
+
"skeptic",
|
| 1899 |
+
"smallvec 1.11.0",
|
| 1900 |
+
"tagptr",
|
| 1901 |
+
"triomphe",
|
| 1902 |
+
]
|
| 1903 |
+
|
| 1904 |
[[package]]
|
| 1905 |
name = "miniz_oxide"
|
| 1906 |
version = "0.7.1"
|
|
|
|
| 2450 |
"url 2.4.1",
|
| 2451 |
]
|
| 2452 |
|
| 2453 |
+
[[package]]
|
| 2454 |
+
name = "pulldown-cmark"
|
| 2455 |
+
version = "0.9.3"
|
| 2456 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 2457 |
+
checksum = "77a1a2f1f0a7ecff9c31abbe177637be0e97a0aef46cf8738ece09327985d998"
|
| 2458 |
+
dependencies = [
|
| 2459 |
+
"bitflags 1.3.2",
|
| 2460 |
+
"memchr",
|
| 2461 |
+
"unicase",
|
| 2462 |
+
]
|
| 2463 |
+
|
| 2464 |
[[package]]
|
| 2465 |
name = "quanta"
|
| 2466 |
version = "0.9.3"
|
|
|
|
| 2978 |
version = "1.0.18"
|
| 2979 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 2980 |
checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918"
|
| 2981 |
+
dependencies = [
|
| 2982 |
+
"serde",
|
| 2983 |
+
]
|
| 2984 |
|
| 2985 |
[[package]]
|
| 2986 |
name = "semver-parser"
|
|
|
|
| 3101 |
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 3102 |
checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d"
|
| 3103 |
|
| 3104 |
+
[[package]]
|
| 3105 |
+
name = "skeptic"
|
| 3106 |
+
version = "0.13.7"
|
| 3107 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 3108 |
+
checksum = "16d23b015676c90a0f01c197bfdc786c20342c73a0afdda9025adb0bc42940a8"
|
| 3109 |
+
dependencies = [
|
| 3110 |
+
"bytecount",
|
| 3111 |
+
"cargo_metadata",
|
| 3112 |
+
"error-chain",
|
| 3113 |
+
"glob",
|
| 3114 |
+
"pulldown-cmark",
|
| 3115 |
+
"tempfile",
|
| 3116 |
+
"walkdir",
|
| 3117 |
+
]
|
| 3118 |
+
|
| 3119 |
[[package]]
|
| 3120 |
name = "slab"
|
| 3121 |
version = "0.4.9"
|
|
|
|
| 3293 |
"libc",
|
| 3294 |
]
|
| 3295 |
|
| 3296 |
+
[[package]]
|
| 3297 |
+
name = "tagptr"
|
| 3298 |
+
version = "0.2.0"
|
| 3299 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 3300 |
+
checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
|
| 3301 |
+
|
| 3302 |
[[package]]
|
| 3303 |
name = "tempfile"
|
| 3304 |
version = "3.8.0"
|
|
|
|
| 3655 |
"once_cell",
|
| 3656 |
]
|
| 3657 |
|
| 3658 |
+
[[package]]
|
| 3659 |
+
name = "triomphe"
|
| 3660 |
+
version = "0.1.9"
|
| 3661 |
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
| 3662 |
+
checksum = "0eee8098afad3fb0c54a9007aab6804558410503ad676d4633f9c2559a00ac0f"
|
| 3663 |
+
|
| 3664 |
[[package]]
|
| 3665 |
name = "try-lock"
|
| 3666 |
version = "0.2.4"
|
|
|
|
| 3923 |
"log",
|
| 3924 |
"md5",
|
| 3925 |
"mimalloc",
|
| 3926 |
+
"mini-moka",
|
| 3927 |
"mlua",
|
| 3928 |
"once_cell",
|
| 3929 |
"rand 0.8.5",
|
src/cache/cacher.rs
CHANGED
|
@@ -8,7 +8,7 @@ use mini_moka::sync::Cache as MokaCache;
|
|
| 8 |
use std::time::Duration;
|
| 9 |
use tokio::sync::Mutex;
|
| 10 |
|
| 11 |
-
use crate::{config::parser::Config,
|
| 12 |
|
| 13 |
use super::error::PoolError;
|
| 14 |
#[cfg(feature = "redis-cache")]
|
|
@@ -29,9 +29,9 @@ pub enum Cache {
|
|
| 29 |
|
| 30 |
impl Cache {
|
| 31 |
/// Builds the cache from the given configuration.
|
| 32 |
-
pub async fn build(
|
| 33 |
#[cfg(feature = "redis-cache")]
|
| 34 |
-
if let Some(url) = &
|
| 35 |
log::info!("Using Redis running at {} for caching", &url);
|
| 36 |
return Cache::new(
|
| 37 |
RedisCache::new(url, 5)
|
|
@@ -40,12 +40,15 @@ impl Cache {
|
|
| 40 |
);
|
| 41 |
}
|
| 42 |
#[cfg(feature = "memory-cache")]
|
| 43 |
-
|
| 44 |
log::info!("Using an in-memory cache");
|
| 45 |
return Cache::new_in_memory();
|
| 46 |
}
|
| 47 |
-
|
| 48 |
-
|
|
|
|
|
|
|
|
|
|
| 49 |
}
|
| 50 |
|
| 51 |
/// Creates a new cache, which wraps the given RedisCache.
|
|
@@ -117,6 +120,7 @@ impl Cache {
|
|
| 117 |
|
| 118 |
/// A structure to efficiently share the cache between threads - as it is protected by a Mutex.
|
| 119 |
pub struct SharedCache {
|
|
|
|
| 120 |
cache: Mutex<Cache>,
|
| 121 |
}
|
| 122 |
|
|
|
|
| 8 |
use std::time::Duration;
|
| 9 |
use tokio::sync::Mutex;
|
| 10 |
|
| 11 |
+
use crate::{config::parser::Config, models::aggregation_models::SearchResults};
|
| 12 |
|
| 13 |
use super::error::PoolError;
|
| 14 |
#[cfg(feature = "redis-cache")]
|
|
|
|
| 29 |
|
| 30 |
impl Cache {
|
| 31 |
/// Builds the cache from the given configuration.
|
| 32 |
+
pub async fn build(_config: &Config) -> Self {
|
| 33 |
#[cfg(feature = "redis-cache")]
|
| 34 |
+
if let Some(url) = &_config.redis_url {
|
| 35 |
log::info!("Using Redis running at {} for caching", &url);
|
| 36 |
return Cache::new(
|
| 37 |
RedisCache::new(url, 5)
|
|
|
|
| 40 |
);
|
| 41 |
}
|
| 42 |
#[cfg(feature = "memory-cache")]
|
| 43 |
+
{
|
| 44 |
log::info!("Using an in-memory cache");
|
| 45 |
return Cache::new_in_memory();
|
| 46 |
}
|
| 47 |
+
#[cfg(not(feature = "memory-cache"))]
|
| 48 |
+
{
|
| 49 |
+
log::info!("Caching is disabled");
|
| 50 |
+
Cache::Disabled
|
| 51 |
+
}
|
| 52 |
}
|
| 53 |
|
| 54 |
/// Creates a new cache, which wraps the given RedisCache.
|
|
|
|
| 120 |
|
| 121 |
/// A structure to efficiently share the cache between threads - as it is protected by a Mutex.
|
| 122 |
pub struct SharedCache {
|
| 123 |
+
/// The internal cache protected from concurrent access by a mutex
|
| 124 |
cache: Mutex<Cache>,
|
| 125 |
}
|
| 126 |
|
src/cache/error.rs
CHANGED
|
@@ -16,7 +16,7 @@ pub enum PoolError {
|
|
| 16 |
PoolExhaustionWithConnectionDropError,
|
| 17 |
/// Whenever serialization or deserialization fails during communication with the cache.
|
| 18 |
SerializationError,
|
| 19 |
-
/// Returned when the value is missing.
|
| 20 |
MissingValue,
|
| 21 |
}
|
| 22 |
|
|
|
|
| 16 |
PoolExhaustionWithConnectionDropError,
|
| 17 |
/// Whenever serialization or deserialization fails during communication with the cache.
|
| 18 |
SerializationError,
|
| 19 |
+
/// Returned when the value is missing.
|
| 20 |
MissingValue,
|
| 21 |
}
|
| 22 |
|
src/config/parser.rs
CHANGED
|
@@ -20,8 +20,6 @@ pub struct Config {
|
|
| 20 |
/// It stores the redis connection url address on which the redis
|
| 21 |
/// client should connect.
|
| 22 |
pub redis_url: Option<String>,
|
| 23 |
-
/// enable/disable the in-memory cache. Only checked, when no redis_url is provided.
|
| 24 |
-
pub in_memory_cache: bool,
|
| 25 |
/// It stores the option to whether enable or disable production use.
|
| 26 |
pub aggregator: AggregatorConfig,
|
| 27 |
/// It stores the option to whether enable or disable logs.
|
|
@@ -102,10 +100,6 @@ impl Config {
|
|
| 102 |
globals.get::<_, String>("colorscheme")?,
|
| 103 |
),
|
| 104 |
redis_url: globals.get::<_, String>("redis_url").ok(),
|
| 105 |
-
in_memory_cache: globals
|
| 106 |
-
.get::<_, bool>("in_memory_cache")
|
| 107 |
-
.ok()
|
| 108 |
-
.unwrap_or(false),
|
| 109 |
aggregator: AggregatorConfig {
|
| 110 |
random_delay: globals.get::<_, bool>("production_use")?,
|
| 111 |
},
|
|
|
|
| 20 |
/// It stores the redis connection url address on which the redis
|
| 21 |
/// client should connect.
|
| 22 |
pub redis_url: Option<String>,
|
|
|
|
|
|
|
| 23 |
/// It stores the option to whether enable or disable production use.
|
| 24 |
pub aggregator: AggregatorConfig,
|
| 25 |
/// It stores the option to whether enable or disable logs.
|
|
|
|
| 100 |
globals.get::<_, String>("colorscheme")?,
|
| 101 |
),
|
| 102 |
redis_url: globals.get::<_, String>("redis_url").ok(),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
aggregator: AggregatorConfig {
|
| 104 |
random_delay: globals.get::<_, bool>("production_use")?,
|
| 105 |
},
|
src/lib.rs
CHANGED
|
@@ -40,7 +40,7 @@ use handler::paths::{file_path, FileType};
|
|
| 40 |
///
|
| 41 |
/// ```rust
|
| 42 |
/// use std::net::TcpListener;
|
| 43 |
-
/// use websurfx::{config::parser::Config, run};
|
| 44 |
///
|
| 45 |
/// let config = Config::parse(true).unwrap();
|
| 46 |
/// let listener = TcpListener::bind("127.0.0.1:8080").expect("Failed to bind address");
|
|
|
|
| 40 |
///
|
| 41 |
/// ```rust
|
| 42 |
/// use std::net::TcpListener;
|
| 43 |
+
/// use websurfx::{config::parser::Config, run, cache::cacher::Cache};
|
| 44 |
///
|
| 45 |
/// let config = Config::parse(true).unwrap();
|
| 46 |
/// let listener = TcpListener::bind("127.0.0.1:8080").expect("Failed to bind address");
|
websurfx/config.lua
CHANGED
|
@@ -47,7 +47,6 @@ theme = "simple" -- the theme name which should be used for the website
|
|
| 47 |
|
| 48 |
-- ### Caching ###
|
| 49 |
redis_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on.
|
| 50 |
-
in_memory_cache = true
|
| 51 |
|
| 52 |
-- ### Search Engines ###
|
| 53 |
upstream_search_engines = {
|
|
|
|
| 47 |
|
| 48 |
-- ### Caching ###
|
| 49 |
redis_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on.
|
|
|
|
| 50 |
|
| 51 |
-- ### Search Engines ###
|
| 52 |
upstream_search_engines = {
|