Spaces:
Running
Running
neon_arch
commited on
Commit
·
fe74f2e
1
Parent(s):
b42adaa
♻️ refactor: code to take prebuilt client to request search results (#384)
Browse files- src/engines/brave.rs +3 -3
- src/engines/duckduckgo.rs +3 -2
- src/engines/searx.rs +3 -2
src/engines/brave.rs
CHANGED
|
@@ -4,7 +4,7 @@
|
|
| 4 |
|
| 5 |
use std::collections::HashMap;
|
| 6 |
|
| 7 |
-
use reqwest::header::HeaderMap;
|
| 8 |
use scraper::Html;
|
| 9 |
|
| 10 |
use crate::models::aggregation_models::SearchResult;
|
|
@@ -42,7 +42,7 @@ impl SearchEngine for Brave {
|
|
| 42 |
query: &str,
|
| 43 |
page: u32,
|
| 44 |
user_agent: &str,
|
| 45 |
-
|
| 46 |
safe_search: u8,
|
| 47 |
) -> Result<HashMap<String, SearchResult>, EngineError> {
|
| 48 |
let url = format!("https://search.brave.com/search?q={query}&offset={page}");
|
|
@@ -68,7 +68,7 @@ impl SearchEngine for Brave {
|
|
| 68 |
.change_context(EngineError::UnexpectedError)?;
|
| 69 |
|
| 70 |
let document: Html = Html::parse_document(
|
| 71 |
-
&Brave::fetch_html_from_upstream(self, &url, header_map,
|
| 72 |
);
|
| 73 |
|
| 74 |
if let Some(no_result_msg) = self.parser.parse_for_no_results(&document).nth(0) {
|
|
|
|
| 4 |
|
| 5 |
use std::collections::HashMap;
|
| 6 |
|
| 7 |
+
use reqwest::{header::HeaderMap, Client};
|
| 8 |
use scraper::Html;
|
| 9 |
|
| 10 |
use crate::models::aggregation_models::SearchResult;
|
|
|
|
| 42 |
query: &str,
|
| 43 |
page: u32,
|
| 44 |
user_agent: &str,
|
| 45 |
+
client: &Client,
|
| 46 |
safe_search: u8,
|
| 47 |
) -> Result<HashMap<String, SearchResult>, EngineError> {
|
| 48 |
let url = format!("https://search.brave.com/search?q={query}&offset={page}");
|
|
|
|
| 68 |
.change_context(EngineError::UnexpectedError)?;
|
| 69 |
|
| 70 |
let document: Html = Html::parse_document(
|
| 71 |
+
&Brave::fetch_html_from_upstream(self, &url, header_map, client).await?,
|
| 72 |
);
|
| 73 |
|
| 74 |
if let Some(no_result_msg) = self.parser.parse_for_no_results(&document).nth(0) {
|
src/engines/duckduckgo.rs
CHANGED
|
@@ -5,6 +5,7 @@
|
|
| 5 |
use std::collections::HashMap;
|
| 6 |
|
| 7 |
use reqwest::header::HeaderMap;
|
|
|
|
| 8 |
use scraper::Html;
|
| 9 |
|
| 10 |
use crate::models::aggregation_models::SearchResult;
|
|
@@ -44,7 +45,7 @@ impl SearchEngine for DuckDuckGo {
|
|
| 44 |
query: &str,
|
| 45 |
page: u32,
|
| 46 |
user_agent: &str,
|
| 47 |
-
|
| 48 |
_safe_search: u8,
|
| 49 |
) -> Result<HashMap<String, SearchResult>, EngineError> {
|
| 50 |
// Page number can be missing or empty string and so appropriate handling is required
|
|
@@ -76,7 +77,7 @@ impl SearchEngine for DuckDuckGo {
|
|
| 76 |
.change_context(EngineError::UnexpectedError)?;
|
| 77 |
|
| 78 |
let document: Html = Html::parse_document(
|
| 79 |
-
&DuckDuckGo::fetch_html_from_upstream(self, &url, header_map,
|
| 80 |
);
|
| 81 |
|
| 82 |
if self.parser.parse_for_no_results(&document).next().is_some() {
|
|
|
|
| 5 |
use std::collections::HashMap;
|
| 6 |
|
| 7 |
use reqwest::header::HeaderMap;
|
| 8 |
+
use reqwest::Client;
|
| 9 |
use scraper::Html;
|
| 10 |
|
| 11 |
use crate::models::aggregation_models::SearchResult;
|
|
|
|
| 45 |
query: &str,
|
| 46 |
page: u32,
|
| 47 |
user_agent: &str,
|
| 48 |
+
client: &Client,
|
| 49 |
_safe_search: u8,
|
| 50 |
) -> Result<HashMap<String, SearchResult>, EngineError> {
|
| 51 |
// Page number can be missing or empty string and so appropriate handling is required
|
|
|
|
| 77 |
.change_context(EngineError::UnexpectedError)?;
|
| 78 |
|
| 79 |
let document: Html = Html::parse_document(
|
| 80 |
+
&DuckDuckGo::fetch_html_from_upstream(self, &url, header_map, client).await?,
|
| 81 |
);
|
| 82 |
|
| 83 |
if self.parser.parse_for_no_results(&document).next().is_some() {
|
src/engines/searx.rs
CHANGED
|
@@ -3,6 +3,7 @@
|
|
| 3 |
//! number if provided.
|
| 4 |
|
| 5 |
use reqwest::header::HeaderMap;
|
|
|
|
| 6 |
use scraper::Html;
|
| 7 |
use std::collections::HashMap;
|
| 8 |
|
|
@@ -40,7 +41,7 @@ impl SearchEngine for Searx {
|
|
| 40 |
query: &str,
|
| 41 |
page: u32,
|
| 42 |
user_agent: &str,
|
| 43 |
-
|
| 44 |
mut safe_search: u8,
|
| 45 |
) -> Result<HashMap<String, SearchResult>, EngineError> {
|
| 46 |
// Page number can be missing or empty string and so appropriate handling is required
|
|
@@ -68,7 +69,7 @@ impl SearchEngine for Searx {
|
|
| 68 |
.change_context(EngineError::UnexpectedError)?;
|
| 69 |
|
| 70 |
let document: Html = Html::parse_document(
|
| 71 |
-
&Searx::fetch_html_from_upstream(self, &url, header_map,
|
| 72 |
);
|
| 73 |
|
| 74 |
if let Some(no_result_msg) = self.parser.parse_for_no_results(&document).nth(1) {
|
|
|
|
| 3 |
//! number if provided.
|
| 4 |
|
| 5 |
use reqwest::header::HeaderMap;
|
| 6 |
+
use reqwest::Client;
|
| 7 |
use scraper::Html;
|
| 8 |
use std::collections::HashMap;
|
| 9 |
|
|
|
|
| 41 |
query: &str,
|
| 42 |
page: u32,
|
| 43 |
user_agent: &str,
|
| 44 |
+
client: &Client,
|
| 45 |
mut safe_search: u8,
|
| 46 |
) -> Result<HashMap<String, SearchResult>, EngineError> {
|
| 47 |
// Page number can be missing or empty string and so appropriate handling is required
|
|
|
|
| 69 |
.change_context(EngineError::UnexpectedError)?;
|
| 70 |
|
| 71 |
let document: Html = Html::parse_document(
|
| 72 |
+
&Searx::fetch_html_from_upstream(self, &url, header_map, client).await?,
|
| 73 |
);
|
| 74 |
|
| 75 |
if let Some(no_result_msg) = self.parser.parse_for_no_results(&document).nth(1) {
|