Spaces:
Running
Running
neon_arch
commited on
Commit
·
4afc07f
1
Parent(s):
f2d2068
:zap: perf: reduce the usage of clones (#603)
Browse files- src/results/aggregator.rs +15 -22
src/results/aggregator.rs
CHANGED
|
@@ -14,7 +14,6 @@ use futures::stream::FuturesUnordered;
|
|
| 14 |
use regex::Regex;
|
| 15 |
use reqwest::{Client, ClientBuilder};
|
| 16 |
use std::sync::Arc;
|
| 17 |
-
use std::time::{SystemTime, UNIX_EPOCH};
|
| 18 |
use tokio::{
|
| 19 |
fs::File,
|
| 20 |
io::{AsyncBufReadExt, BufReader},
|
|
@@ -93,13 +92,6 @@ pub async fn aggregate(
|
|
| 93 |
|
| 94 |
let user_agent: &str = random_user_agent();
|
| 95 |
|
| 96 |
-
// Add a random delay before making the request.
|
| 97 |
-
if config.aggregator.random_delay || !config.debug {
|
| 98 |
-
let nanos = SystemTime::now().duration_since(UNIX_EPOCH)?.subsec_nanos() as f32;
|
| 99 |
-
let delay = ((nanos / 1_0000_0000 as f32).floor() as u64) + 1;
|
| 100 |
-
tokio::time::sleep(Duration::from_secs(delay)).await;
|
| 101 |
-
}
|
| 102 |
-
|
| 103 |
let mut names: Vec<&str> = Vec::with_capacity(0);
|
| 104 |
|
| 105 |
// create tasks for upstream result fetching
|
|
@@ -188,19 +180,21 @@ pub async fn aggregate(
|
|
| 188 |
drop(blacklist_map);
|
| 189 |
}
|
| 190 |
|
| 191 |
-
let mut results:
|
| 192 |
-
.
|
| 193 |
-
.map(|(_, value)| {
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
copy.calculate_relevance(query.as_str())
|
| 197 |
}
|
| 198 |
-
|
| 199 |
})
|
| 200 |
.collect();
|
| 201 |
sort_search_results(&mut results);
|
| 202 |
|
| 203 |
-
Ok(SearchResults::new(
|
|
|
|
|
|
|
|
|
|
| 204 |
}
|
| 205 |
|
| 206 |
/// Filters a map of search results using a list of regex patterns.
|
|
@@ -265,7 +259,6 @@ fn sort_search_results(results: &mut [SearchResult]) {
|
|
| 265 |
#[cfg(test)]
|
| 266 |
mod tests {
|
| 267 |
use super::*;
|
| 268 |
-
use smallvec::smallvec;
|
| 269 |
use std::io::Write;
|
| 270 |
use tempfile::NamedTempFile;
|
| 271 |
|
|
@@ -281,7 +274,7 @@ mod tests {
|
|
| 281 |
description: "This domain is for use in illustrative examples in documents."
|
| 282 |
.to_owned(),
|
| 283 |
relevance_score: 0.0,
|
| 284 |
-
engine:
|
| 285 |
},
|
| 286 |
));
|
| 287 |
map_to_be_filtered.push((
|
|
@@ -290,7 +283,7 @@ mod tests {
|
|
| 290 |
title: "Rust Programming Language".to_owned(),
|
| 291 |
url: "https://www.rust-lang.org/".to_owned(),
|
| 292 |
description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_owned(),
|
| 293 |
-
engine:
|
| 294 |
relevance_score:0.0
|
| 295 |
},)
|
| 296 |
);
|
|
@@ -331,7 +324,7 @@ mod tests {
|
|
| 331 |
url: "https://www.example.com".to_owned(),
|
| 332 |
description: "This domain is for use in illustrative examples in documents."
|
| 333 |
.to_owned(),
|
| 334 |
-
engine:
|
| 335 |
relevance_score: 0.0,
|
| 336 |
},
|
| 337 |
));
|
|
@@ -341,7 +334,7 @@ mod tests {
|
|
| 341 |
title: "Rust Programming Language".to_owned(),
|
| 342 |
url: "https://www.rust-lang.org/".to_owned(),
|
| 343 |
description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_owned(),
|
| 344 |
-
engine:
|
| 345 |
relevance_score:0.0
|
| 346 |
},
|
| 347 |
));
|
|
@@ -398,7 +391,7 @@ mod tests {
|
|
| 398 |
url: "https://www.example.com".to_owned(),
|
| 399 |
description: "This domain is for use in illustrative examples in documents."
|
| 400 |
.to_owned(),
|
| 401 |
-
engine:
|
| 402 |
relevance_score: 0.0,
|
| 403 |
},
|
| 404 |
));
|
|
|
|
| 14 |
use regex::Regex;
|
| 15 |
use reqwest::{Client, ClientBuilder};
|
| 16 |
use std::sync::Arc;
|
|
|
|
| 17 |
use tokio::{
|
| 18 |
fs::File,
|
| 19 |
io::{AsyncBufReadExt, BufReader},
|
|
|
|
| 92 |
|
| 93 |
let user_agent: &str = random_user_agent();
|
| 94 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 95 |
let mut names: Vec<&str> = Vec::with_capacity(0);
|
| 96 |
|
| 97 |
// create tasks for upstream result fetching
|
|
|
|
| 180 |
drop(blacklist_map);
|
| 181 |
}
|
| 182 |
|
| 183 |
+
let mut results: Box<[SearchResult]> = result_map
|
| 184 |
+
.into_iter()
|
| 185 |
+
.map(|(_, mut value)| {
|
| 186 |
+
if !value.url.contains("temu.com") {
|
| 187 |
+
value.calculate_relevance(query.as_str())
|
|
|
|
| 188 |
}
|
| 189 |
+
value
|
| 190 |
})
|
| 191 |
.collect();
|
| 192 |
sort_search_results(&mut results);
|
| 193 |
|
| 194 |
+
Ok(SearchResults::new(
|
| 195 |
+
results,
|
| 196 |
+
engine_errors_info.into_boxed_slice(),
|
| 197 |
+
))
|
| 198 |
}
|
| 199 |
|
| 200 |
/// Filters a map of search results using a list of regex patterns.
|
|
|
|
| 259 |
#[cfg(test)]
|
| 260 |
mod tests {
|
| 261 |
use super::*;
|
|
|
|
| 262 |
use std::io::Write;
|
| 263 |
use tempfile::NamedTempFile;
|
| 264 |
|
|
|
|
| 274 |
description: "This domain is for use in illustrative examples in documents."
|
| 275 |
.to_owned(),
|
| 276 |
relevance_score: 0.0,
|
| 277 |
+
engine: vec!["Google".to_owned(), "Bing".to_owned()],
|
| 278 |
},
|
| 279 |
));
|
| 280 |
map_to_be_filtered.push((
|
|
|
|
| 283 |
title: "Rust Programming Language".to_owned(),
|
| 284 |
url: "https://www.rust-lang.org/".to_owned(),
|
| 285 |
description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_owned(),
|
| 286 |
+
engine: vec!["Google".to_owned(), "DuckDuckGo".to_owned()],
|
| 287 |
relevance_score:0.0
|
| 288 |
},)
|
| 289 |
);
|
|
|
|
| 324 |
url: "https://www.example.com".to_owned(),
|
| 325 |
description: "This domain is for use in illustrative examples in documents."
|
| 326 |
.to_owned(),
|
| 327 |
+
engine: vec!["Google".to_owned(), "Bing".to_owned()],
|
| 328 |
relevance_score: 0.0,
|
| 329 |
},
|
| 330 |
));
|
|
|
|
| 334 |
title: "Rust Programming Language".to_owned(),
|
| 335 |
url: "https://www.rust-lang.org/".to_owned(),
|
| 336 |
description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_owned(),
|
| 337 |
+
engine: vec!["Google".to_owned(), "DuckDuckGo".to_owned()],
|
| 338 |
relevance_score:0.0
|
| 339 |
},
|
| 340 |
));
|
|
|
|
| 391 |
url: "https://www.example.com".to_owned(),
|
| 392 |
description: "This domain is for use in illustrative examples in documents."
|
| 393 |
.to_owned(),
|
| 394 |
+
engine: vec!["Google".to_owned(), "Bing".to_owned()],
|
| 395 |
relevance_score: 0.0,
|
| 396 |
},
|
| 397 |
));
|