Spaces:
Running
Running
Merge pull request #125 from MilimTheTrueOne/rolling
Browse files- CONTRIBUTING.md +1 -1
- PULL_REQUEST_TEMPLATE.md +1 -1
- README.md +2 -2
- public/static/pagination.js +2 -2
- src/config_parser/parser.rs +33 -40
- src/config_parser/parser_models.rs +1 -1
- src/engines/duckduckgo.rs +2 -2
- src/engines/engine_models.rs +1 -1
- src/search_results_handler/aggregator.rs +1 -1
- src/server/routes.rs +9 -33
CONTRIBUTING.md
CHANGED
|
@@ -14,7 +14,7 @@ Know how to fix or improve a github action?. Consider Submitting a Pull request
|
|
| 14 |
|
| 15 |
## Source Code
|
| 16 |
|
| 17 |
-
You should know
|
| 18 |
|
| 19 |
- Rust basics
|
| 20 |
- Actix-web crate basics
|
|
|
|
| 14 |
|
| 15 |
## Source Code
|
| 16 |
|
| 17 |
+
You should know at least one of the things below to start contributing:
|
| 18 |
|
| 19 |
- Rust basics
|
| 20 |
- Actix-web crate basics
|
PULL_REQUEST_TEMPLATE.md
CHANGED
|
@@ -16,7 +16,7 @@
|
|
| 16 |
|
| 17 |
## Author's checklist
|
| 18 |
|
| 19 |
-
<!-- additional notes for
|
| 20 |
|
| 21 |
## Related issues
|
| 22 |
|
|
|
|
| 16 |
|
| 17 |
## Author's checklist
|
| 18 |
|
| 19 |
+
<!-- additional notes for reviewers -->
|
| 20 |
|
| 21 |
## Related issues
|
| 22 |
|
README.md
CHANGED
|
@@ -59,7 +59,7 @@
|
|
| 59 |
- **Community**
|
| 60 |
- [π System Requirements](#system-requirements-)
|
| 61 |
- [π¨οΈ FAQ (Frequently Asked Questions)](#faq-frequently-asked-questions-)
|
| 62 |
-
- [π£ More
|
| 63 |
- [π Supporting Websurfx](#supporting-websurfx-)
|
| 64 |
- [π Documentation](#documentation-)
|
| 65 |
- [π£οΈ Roadmap](#roadmap-)
|
|
@@ -165,7 +165,7 @@ Websurfx is based on Rust due to its memory safety features, which prevents vuln
|
|
| 165 |
|
| 166 |
**[β¬οΈ Back to Top](#--)**
|
| 167 |
|
| 168 |
-
# More
|
| 169 |
|
| 170 |
We are looking for more willing contributors to help grow this project. For more information on how you can contribute, check out the [project board](https://github.com/neon-mmd/websurfx/projects?query=is%3Aopen) and the [CONTRIBUTING.md](CONTRIBUTING.md) file for guidelines and rules for making contributions.
|
| 171 |
|
|
|
|
| 59 |
- **Community**
|
| 60 |
- [π System Requirements](#system-requirements-)
|
| 61 |
- [π¨οΈ FAQ (Frequently Asked Questions)](#faq-frequently-asked-questions-)
|
| 62 |
+
- [π£ More Contributors Wanted](#more-contributors-wanted-)
|
| 63 |
- [π Supporting Websurfx](#supporting-websurfx-)
|
| 64 |
- [π Documentation](#documentation-)
|
| 65 |
- [π£οΈ Roadmap](#roadmap-)
|
|
|
|
| 165 |
|
| 166 |
**[β¬οΈ Back to Top](#--)**
|
| 167 |
|
| 168 |
+
# More Contributors Wanted π£
|
| 169 |
|
| 170 |
We are looking for more willing contributors to help grow this project. For more information on how you can contribute, check out the [project board](https://github.com/neon-mmd/websurfx/projects?query=is%3Aopen) and the [CONTRIBUTING.md](CONTRIBUTING.md) file for guidelines and rules for making contributions.
|
| 171 |
|
public/static/pagination.js
CHANGED
|
@@ -30,8 +30,8 @@ function navigate_backward() {
|
|
| 30 |
let page = parseInt(searchParams.get('page'));
|
| 31 |
|
| 32 |
if (isNaN(page)) {
|
| 33 |
-
page =
|
| 34 |
-
} else if (page >
|
| 35 |
page--;
|
| 36 |
}
|
| 37 |
|
|
|
|
| 30 |
let page = parseInt(searchParams.get('page'));
|
| 31 |
|
| 32 |
if (isNaN(page)) {
|
| 33 |
+
page = 0;
|
| 34 |
+
} else if (page > 0) {
|
| 35 |
page--;
|
| 36 |
}
|
| 37 |
|
src/config_parser/parser.rs
CHANGED
|
@@ -24,46 +24,35 @@ pub struct Config {
|
|
| 24 |
pub binding_ip_addr: String,
|
| 25 |
pub style: Style,
|
| 26 |
pub redis_connection_url: String,
|
| 27 |
-
pub aggregator:
|
| 28 |
pub logging: bool,
|
| 29 |
pub debug: bool,
|
| 30 |
}
|
| 31 |
|
| 32 |
/// Configuration options for the aggregator.
|
| 33 |
#[derive(Clone)]
|
| 34 |
-
pub struct
|
| 35 |
/// Whether to introduce a random delay before sending the request to the search engine.
|
| 36 |
pub random_delay: bool,
|
| 37 |
}
|
| 38 |
|
| 39 |
impl Config {
|
| 40 |
/// A function which parses the config.lua file and puts all the parsed options in the newly
|
| 41 |
-
///
|
| 42 |
///
|
| 43 |
/// # Error
|
| 44 |
///
|
| 45 |
/// Returns a lua parse error if parsing of the config.lua file fails or has a syntax error
|
| 46 |
-
/// or io error if the config.lua file doesn't exists otherwise it returns a newly
|
| 47 |
/// Config struct with all the parsed config options from the parsed config file.
|
| 48 |
pub fn parse() -> Result<Self, Box<dyn std::error::Error>> {
|
| 49 |
Lua::new().context(|context| -> Result<Self, Box<dyn std::error::Error>> {
|
| 50 |
let globals = context.globals();
|
| 51 |
|
| 52 |
context
|
| 53 |
-
.load(&fs::read_to_string(
|
| 54 |
-
Config::handle_different_config_file_path()?,
|
| 55 |
-
)?)
|
| 56 |
.exec()?;
|
| 57 |
|
| 58 |
-
let production_use = globals.get::<_, bool>("production_use")?;
|
| 59 |
-
let aggregator_config = if production_use {
|
| 60 |
-
AggreatorConfig { random_delay: true }
|
| 61 |
-
} else {
|
| 62 |
-
AggreatorConfig {
|
| 63 |
-
random_delay: false,
|
| 64 |
-
}
|
| 65 |
-
};
|
| 66 |
-
|
| 67 |
Ok(Config {
|
| 68 |
port: globals.get::<_, u16>("port")?,
|
| 69 |
binding_ip_addr: globals.get::<_, String>("binding_ip_addr")?,
|
|
@@ -72,7 +61,9 @@ impl Config {
|
|
| 72 |
globals.get::<_, String>("colorscheme")?,
|
| 73 |
),
|
| 74 |
redis_connection_url: globals.get::<_, String>("redis_connection_url")?,
|
| 75 |
-
aggregator:
|
|
|
|
|
|
|
| 76 |
logging: globals.get::<_, bool>("logging")?,
|
| 77 |
debug: globals.get::<_, bool>("debug")?,
|
| 78 |
})
|
|
@@ -90,35 +81,37 @@ impl Config {
|
|
| 90 |
/// one (3).
|
| 91 |
/// 3. `websurfx/` (under project folder ( or codebase in other words)) if it is not present
|
| 92 |
/// here then it returns an error as mentioned above.
|
| 93 |
-
fn
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
)
|
| 102 |
-
|
| 103 |
-
{
|
| 104 |
-
Ok(format!(
|
| 105 |
"{}/.config/{}/{}",
|
| 106 |
std::env::var("HOME").unwrap(),
|
| 107 |
COMMON_DIRECTORY_NAME,
|
| 108 |
CONFIG_FILE_NAME
|
| 109 |
-
))
|
| 110 |
-
}
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
.
|
| 114 |
-
{
|
| 115 |
-
Ok("/etc/xdg/websurfx/config.lua".to_string())
|
| 116 |
-
} else if Path::new(format!("./{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME).as_str())
|
| 117 |
.exists()
|
| 118 |
{
|
| 119 |
-
Ok("
|
| 120 |
-
} else {
|
| 121 |
-
Err("Config file not found!!".to_string().into())
|
| 122 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 123 |
}
|
| 124 |
}
|
|
|
|
| 24 |
pub binding_ip_addr: String,
|
| 25 |
pub style: Style,
|
| 26 |
pub redis_connection_url: String,
|
| 27 |
+
pub aggregator: AggregatorConfig,
|
| 28 |
pub logging: bool,
|
| 29 |
pub debug: bool,
|
| 30 |
}
|
| 31 |
|
| 32 |
/// Configuration options for the aggregator.
|
| 33 |
#[derive(Clone)]
|
| 34 |
+
pub struct AggregatorConfig {
|
| 35 |
/// Whether to introduce a random delay before sending the request to the search engine.
|
| 36 |
pub random_delay: bool,
|
| 37 |
}
|
| 38 |
|
| 39 |
impl Config {
|
| 40 |
/// A function which parses the config.lua file and puts all the parsed options in the newly
|
| 41 |
+
/// constructed Config struct and returns it.
|
| 42 |
///
|
| 43 |
/// # Error
|
| 44 |
///
|
| 45 |
/// Returns a lua parse error if parsing of the config.lua file fails or has a syntax error
|
| 46 |
+
/// or io error if the config.lua file doesn't exists otherwise it returns a newly constructed
|
| 47 |
/// Config struct with all the parsed config options from the parsed config file.
|
| 48 |
pub fn parse() -> Result<Self, Box<dyn std::error::Error>> {
|
| 49 |
Lua::new().context(|context| -> Result<Self, Box<dyn std::error::Error>> {
|
| 50 |
let globals = context.globals();
|
| 51 |
|
| 52 |
context
|
| 53 |
+
.load(&fs::read_to_string(Config::get_config_path()?)?)
|
|
|
|
|
|
|
| 54 |
.exec()?;
|
| 55 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 56 |
Ok(Config {
|
| 57 |
port: globals.get::<_, u16>("port")?,
|
| 58 |
binding_ip_addr: globals.get::<_, String>("binding_ip_addr")?,
|
|
|
|
| 61 |
globals.get::<_, String>("colorscheme")?,
|
| 62 |
),
|
| 63 |
redis_connection_url: globals.get::<_, String>("redis_connection_url")?,
|
| 64 |
+
aggregator: AggregatorConfig {
|
| 65 |
+
random_delay: globals.get::<_, bool>("production_use")?,
|
| 66 |
+
},
|
| 67 |
logging: globals.get::<_, bool>("logging")?,
|
| 68 |
debug: globals.get::<_, bool>("debug")?,
|
| 69 |
})
|
|
|
|
| 81 |
/// one (3).
|
| 82 |
/// 3. `websurfx/` (under project folder ( or codebase in other words)) if it is not present
|
| 83 |
/// here then it returns an error as mentioned above.
|
| 84 |
+
fn get_config_path() -> Result<String, Box<dyn std::error::Error>> {
|
| 85 |
+
// check user config
|
| 86 |
+
|
| 87 |
+
let path = format!(
|
| 88 |
+
"{}/.config/{}/config.lua",
|
| 89 |
+
std::env::var("HOME").unwrap(),
|
| 90 |
+
COMMON_DIRECTORY_NAME
|
| 91 |
+
);
|
| 92 |
+
if Path::new(path.as_str()).exists() {
|
| 93 |
+
return Ok(format!(
|
|
|
|
|
|
|
| 94 |
"{}/.config/{}/{}",
|
| 95 |
std::env::var("HOME").unwrap(),
|
| 96 |
COMMON_DIRECTORY_NAME,
|
| 97 |
CONFIG_FILE_NAME
|
| 98 |
+
));
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
// look for config in /etc/xdg
|
| 102 |
+
if Path::new(format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME).as_str())
|
|
|
|
|
|
|
|
|
|
| 103 |
.exists()
|
| 104 |
{
|
| 105 |
+
return Ok("/etc/xdg/websurfx/config.lua".to_string());
|
|
|
|
|
|
|
| 106 |
}
|
| 107 |
+
|
| 108 |
+
// use dev config
|
| 109 |
+
if Path::new(format!("./{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME).as_str()).exists()
|
| 110 |
+
{
|
| 111 |
+
return Ok("./websurfx/config.lua".to_string());
|
| 112 |
+
}
|
| 113 |
+
|
| 114 |
+
// if no of the configs above exist, return error
|
| 115 |
+
Err("Config file not found!!".to_string().into())
|
| 116 |
}
|
| 117 |
}
|
src/config_parser/parser_models.rs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
//! This module provides public models for handling, storing and serializing parsed config file
|
| 2 |
-
//! options from config.lua by grouping them
|
| 3 |
|
| 4 |
use serde::{Deserialize, Serialize};
|
| 5 |
|
|
|
|
| 1 |
//! This module provides public models for handling, storing and serializing parsed config file
|
| 2 |
+
//! options from config.lua by grouping them together.
|
| 3 |
|
| 4 |
use serde::{Deserialize, Serialize};
|
| 5 |
|
src/engines/duckduckgo.rs
CHANGED
|
@@ -36,7 +36,7 @@ pub async fn results(
|
|
| 36 |
user_agent: &str,
|
| 37 |
) -> Result<HashMap<String, RawSearchResult>, EngineError> {
|
| 38 |
// Page number can be missing or empty string and so appropriate handling is required
|
| 39 |
-
// so that upstream server
|
| 40 |
let url: String = match page {
|
| 41 |
1 => {
|
| 42 |
format!("https://html.duckduckgo.com/html/?q={query}&s=&dc=&v=1&o=json&api=/d.js")
|
|
@@ -86,7 +86,7 @@ pub async fn results(
|
|
| 86 |
let results: String = reqwest::Client::new()
|
| 87 |
.get(url)
|
| 88 |
.timeout(Duration::from_secs(5))
|
| 89 |
-
.headers(header_map) // add spoofed headers to emulate human
|
| 90 |
.send()
|
| 91 |
.await
|
| 92 |
.into_report()
|
|
|
|
| 36 |
user_agent: &str,
|
| 37 |
) -> Result<HashMap<String, RawSearchResult>, EngineError> {
|
| 38 |
// Page number can be missing or empty string and so appropriate handling is required
|
| 39 |
+
// so that upstream server receives valid page number.
|
| 40 |
let url: String = match page {
|
| 41 |
1 => {
|
| 42 |
format!("https://html.duckduckgo.com/html/?q={query}&s=&dc=&v=1&o=json&api=/d.js")
|
|
|
|
| 86 |
let results: String = reqwest::Client::new()
|
| 87 |
.get(url)
|
| 88 |
.timeout(Duration::from_secs(5))
|
| 89 |
+
.headers(header_map) // add spoofed headers to emulate human behavior
|
| 90 |
.send()
|
| 91 |
.await
|
| 92 |
.into_report()
|
src/engines/engine_models.rs
CHANGED
|
@@ -13,7 +13,7 @@ use std::fmt;
|
|
| 13 |
/// search engines.
|
| 14 |
/// * `UnexpectedError` - This variant handles all the errors which are unexpected or occur rarely
|
| 15 |
/// and are errors mostly related to failure in initialization of HeaderMap, Selector errors and
|
| 16 |
-
/// all other errors
|
| 17 |
#[derive(Debug)]
|
| 18 |
pub enum EngineError {
|
| 19 |
EmptyResultSet,
|
|
|
|
| 13 |
/// search engines.
|
| 14 |
/// * `UnexpectedError` - This variant handles all the errors which are unexpected or occur rarely
|
| 15 |
/// and are errors mostly related to failure in initialization of HeaderMap, Selector errors and
|
| 16 |
+
/// all other errors occurring within the code handling the `upstream search engines`.
|
| 17 |
#[derive(Debug)]
|
| 18 |
pub enum EngineError {
|
| 19 |
EmptyResultSet,
|
src/search_results_handler/aggregator.rs
CHANGED
|
@@ -17,7 +17,7 @@ use crate::engines::{duckduckgo, searx};
|
|
| 17 |
/// then removes duplicate results and if two results are found to be from two or more engines
|
| 18 |
/// then puts their names together to show the results are fetched from these upstream engines
|
| 19 |
/// and then removes all data from the HashMap and puts into a struct of all results aggregated
|
| 20 |
-
/// into a vector and also adds the query used into the struct this is
|
| 21 |
/// otherwise the search bar in search remains empty if searched from the query url
|
| 22 |
///
|
| 23 |
/// # Example:
|
|
|
|
| 17 |
/// then removes duplicate results and if two results are found to be from two or more engines
|
| 18 |
/// then puts their names together to show the results are fetched from these upstream engines
|
| 19 |
/// and then removes all data from the HashMap and puts into a struct of all results aggregated
|
| 20 |
+
/// into a vector and also adds the query used into the struct this is necessary because
|
| 21 |
/// otherwise the search bar in search remains empty if searched from the query url
|
| 22 |
///
|
| 23 |
/// # Example:
|
src/server/routes.rs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
//! This module provides the functionality to handle different routes of the `websurfx`
|
| 2 |
-
//! meta search engine website and provide
|
| 3 |
//! when requested.
|
| 4 |
|
| 5 |
use std::fs::read_to_string;
|
|
@@ -82,40 +82,16 @@ pub async fn search(
|
|
| 82 |
.insert_header(("location", "/"))
|
| 83 |
.finish())
|
| 84 |
} else {
|
| 85 |
-
let
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
let page = match params.page {
|
| 90 |
-
Some(page_number) => {
|
| 91 |
-
if page_number <= 1 {
|
| 92 |
-
page_url = format!(
|
| 93 |
-
"http://{}:{}/search?q={}&page={}",
|
| 94 |
-
config.binding_ip_addr, config.port, query, 1
|
| 95 |
-
);
|
| 96 |
-
1
|
| 97 |
-
} else {
|
| 98 |
-
page_url = format!(
|
| 99 |
-
"http://{}:{}/search?q={}&page={}",
|
| 100 |
-
config.binding_ip_addr, config.port, query, page_number
|
| 101 |
-
);
|
| 102 |
-
|
| 103 |
-
page_number
|
| 104 |
-
}
|
| 105 |
-
}
|
| 106 |
-
None => {
|
| 107 |
-
page_url = format!(
|
| 108 |
-
"http://{}:{}{}&page={}",
|
| 109 |
-
config.binding_ip_addr,
|
| 110 |
-
config.port,
|
| 111 |
-
req.uri(),
|
| 112 |
-
1
|
| 113 |
-
);
|
| 114 |
-
|
| 115 |
-
1
|
| 116 |
-
}
|
| 117 |
};
|
| 118 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 119 |
// fetch the cached results json.
|
| 120 |
let cached_results_json = redis_cache.cached_results_json(&page_url);
|
| 121 |
// check if fetched results was indeed fetched or it was an error and if so
|
|
|
|
| 1 |
//! This module provides the functionality to handle different routes of the `websurfx`
|
| 2 |
+
//! meta search engine website and provide appropriate response to each route/page
|
| 3 |
//! when requested.
|
| 4 |
|
| 5 |
use std::fs::read_to_string;
|
|
|
|
| 82 |
.insert_header(("location", "/"))
|
| 83 |
.finish())
|
| 84 |
} else {
|
| 85 |
+
let page = match ¶ms.page {
|
| 86 |
+
Some(page) => *page,
|
| 87 |
+
None => 0,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
};
|
| 89 |
|
| 90 |
+
let page_url = format!(
|
| 91 |
+
"http://{}:{}/search?q={}&page={}",
|
| 92 |
+
config.binding_ip_addr, config.port, query, page
|
| 93 |
+
);
|
| 94 |
+
|
| 95 |
// fetch the cached results json.
|
| 96 |
let cached_results_json = redis_cache.cached_results_json(&page_url);
|
| 97 |
// check if fetched results was indeed fetched or it was an error and if so
|