Improve configurability

src/client.rs

@@ -1,4 +1,4 @@
-use crate::bookmarks::TargetBookmark;
+use crate::{bookmarks::TargetBookmark, Config};
 use anyhow::anyhow;
 use chrono::{DateTime, Utc};
 use log::debug;
@@ -9,23 +9,19 @@ use std::{
 };
 use tokio::time::{self, Duration};
 
-/// The request timeout in milliseconds.
-const REQUEST_TIMEOUT: u64 = 60_000;
-
-/// The throttling between requests in milliseconds.
-const THROTTLING: u64 = 1_000;
-
 pub struct Client {
     client: ReqwestClient,
     throttler: Option<Throttler>,
 }
 
 impl Client {
-    pub fn new() -> Result<Self, anyhow::Error> {
+    pub fn new(config: &Config) -> Result<Self, anyhow::Error> {
+        let request_timeout = config.settings.request_timeout;
+        let request_throttling = config.settings.request_throttling;
         let client = ReqwestClient::builder()
-            .timeout(Duration::from_millis(REQUEST_TIMEOUT))
+            .timeout(Duration::from_millis(request_timeout))
             .build()?;
-        let throttler = Some(Throttler::new());
+        let throttler = Some(Throttler::new(request_throttling));
         Ok(Self { client, throttler })
     }
 
@@ -46,12 +42,14 @@ impl Client {
 #[derive(Debug)]
 struct Throttler {
     last_fetched: Mutex<HashMap<String, DateTime<Utc>>>,
+    request_throttling: u64,
 }
 
 impl Throttler {
-    pub fn new() -> Self {
+    pub fn new(request_throttling: u64) -> Self {
         Self {
             last_fetched: Mutex::new(HashMap::new()),
+            request_throttling,
         }
     }
 
@@ -63,15 +61,18 @@ impl Throttler {
         if let Some(last_fetched) = self.last_fetched(bookmark, now)? {
             let duration_since_last_fetched = now - last_fetched;
 
-            if duration_since_last_fetched < chrono::Duration::milliseconds(THROTTLING as i64 / 2) {
+            if duration_since_last_fetched
+                < chrono::Duration::milliseconds(self.request_throttling as i64 / 2)
+            {
                 debug!("Wait for bookmark {}", bookmark.url);
-                time::sleep(Duration::from_millis(THROTTLING)).await;
-            } else if chrono::Duration::milliseconds(THROTTLING as i64 / 2)
+                time::sleep(Duration::from_millis(self.request_throttling)).await;
+            } else if chrono::Duration::milliseconds(self.request_throttling as i64 / 2)
                 < duration_since_last_fetched
-                && duration_since_last_fetched < chrono::Duration::milliseconds(THROTTLING as i64)
+                && duration_since_last_fetched
+                    < chrono::Duration::milliseconds(self.request_throttling as i64)
             {
                 debug!("Wait for bookmark {}", bookmark.url);
-                time::sleep(Duration::from_millis(THROTTLING / 2)).await;
+                time::sleep(Duration::from_millis(self.request_throttling / 2)).await;
             }
         }
 
@@ -112,7 +113,8 @@ mod tests {
     async fn test_throttle() {
         tokio::time::pause();
         let now = Utc::now();
-        let throttler = Throttler::new();
+        let request_throttling = 1000;
+        let throttler = Throttler::new(request_throttling);
         let bookmark1 = TargetBookmark::new(
             "https://en.wikipedia.org/wiki/Applicative_functor",
             now,
@@ -132,14 +134,15 @@ mod tests {
         throttler.throttle(&bookmark2).await.unwrap();
         assert_eq!(
             Instant::now().duration_since(start_instant).as_millis(),
-            (THROTTLING + 1) as u128
+            (request_throttling + 1) as u128
         );
     }
 
     #[test]
     fn test_last_fetched() {
         let now = Utc::now();
-        let throttler = Throttler::new();
+        let request_throttling = 1000;
+        let throttler = Throttler::new(request_throttling);
         let bookmark1 = TargetBookmark::new(
             "https://en.wikipedia.org/wiki/Applicative_functor",
             now,

src/cmd/configure.rs

@@ -1,5 +1,5 @@
 use crate::{Config, ConfigArgs, SourceFile};
-use log::info;
+use log::{debug, info};
 use std::path::PathBuf;
 
 /// Configure the source files to import the bookmarks.
@@ -11,13 +11,15 @@ pub fn configure(mut config: Config, args: ConfigArgs) -> Result<(), anyhow::Err
     let settings_path = config.settings_path;
 
     if let Some(source) = args.set_source.source {
+        debug!("Set source");
         let source = PathBuf::from(source);
         let source_file = SourceFile::new(source, args.set_source.folders);
         config.settings.source_bookmark_files.push(source_file);
         config.settings.write(&settings_path)?;
     }
 
     if let Some(set_cache_mode) = args.set_cache_mode {
+        debug!("Set cache mode");
         config.settings.cache_mode = set_cache_mode;
         config.settings.write(&settings_path)?;
     }

src/cmd/fetch.rs

@@ -10,7 +10,7 @@ use std::path::Path;
 pub async fn fetch(config: &Config, args: &FetchArgs) -> Result<(), anyhow::Error> {
     let mut bookmarks = TargetBookmarks::read(config)?;
     let cache = Cache::init(config, &args.mode).await?;
-    let client = Client::new()?;
+    let client = Client::new(config)?;
 
     if args.all {
         fetch_and_replace_all(config, &client, &cache, &mut bookmarks).await?;
@@ -151,7 +151,7 @@ pub async fn fetch_diff(config: &Config, args: FetchArgs) -> Result<(), anyhow::
     debug!("Diff content for urls: {:#?}", args.urls);
     let target_bookmarks = TargetBookmarks::read(config)?;
     let cache = Cache::new(&config.cache_path, &args.mode)?;
-    let client = Client::new()?;
+    let client = Client::new(config)?;
 
     for url in args.urls {
         let bookmark = target_bookmarks.find(&url);

src/cmd/init.rs

@@ -5,7 +5,7 @@ use crate::{Cache, Client, Config, InitArgs, TargetBookmarks};
 pub async fn init(config: &Config, args: &InitArgs) -> Result<(), anyhow::Error> {
     let bookmarks = TargetBookmarks::init(config)?;
     let cache = Cache::init(config, &args.mode).await?;
-    let client = Client::new()?;
+    let client = Client::new(config)?;
     fetch_and_add_all(config, &client, &cache, &bookmarks).await?;
     Ok(())
 }

src/cmd/update.rs

@@ -7,7 +7,7 @@ use log::info;
 /// new bookmarks; delete cache for removed bookmarks.
 pub async fn update(config: &Config, args: &UpdateArgs) -> Result<(), anyhow::Error> {
     let cache = Cache::new(&config.cache_path, &args.mode)?;
-    let client = Client::new()?;
+    let client = Client::new(config)?;
 
     let mut source_bookmarks = SourceBookmarks::new();
     source_bookmarks.read(config)?;

src/main.rs

@@ -6,7 +6,7 @@ use env_logger::{Builder, Env};
 #[tokio::main]
 async fn main() -> Result<(), anyhow::Error> {
     // Default to INFO level logs if RUST_LOG is not set.
-    Builder::from_env(Env::default().default_filter_or("info")).init();
+    Builder::from_env(Env::default().default_filter_or("bogrep=info")).init();
 
     let args = Args::parse();
     let config = Config::init(&args)?;

src/settings.rs

@@ -7,9 +7,15 @@ use std::{
     path::{Path, PathBuf},
 };
 
-/// The maximal number of concurrent requests.
+/// The default `Settungs::max_concurrent_requests`.
 const MAX_CONCURRENT_REQUESTS_DEFAULT: usize = 100;
 
+/// The default for `Settings::request_timeout`.
+const REQUEST_TIMEOUT_DEFAULT: u64 = 60_000;
+
+/// The default for `Settings::request_throttling`.
+const REQUEST_THROTTLING_DEFAULT: u64 = 1_000;
+
 #[derive(Debug, Serialize, Deserialize)]
 pub struct Settings {
     /// The paths to the configured bookmark files.
@@ -21,6 +27,10 @@ pub struct Settings {
     pub cache_mode: CacheMode,
     /// The maximal number of concurrent requests.
     pub max_concurrent_requests: usize,
+    /// The request timeout in milliseconds.
+    pub request_timeout: u64,
+    /// The throttling between requests in milliseconds.
+    pub request_throttling: u64,
 }
 
 impl Default for Settings {
@@ -29,6 +39,8 @@ impl Default for Settings {
             source_bookmark_files: Vec::new(),
             cache_mode: CacheMode::default(),
             max_concurrent_requests: MAX_CONCURRENT_REQUESTS_DEFAULT,
+            request_timeout: REQUEST_TIMEOUT_DEFAULT,
+            request_throttling: REQUEST_THROTTLING_DEFAULT,
         }
     }
 }
@@ -38,11 +50,15 @@ impl Settings {
         source_bookmark_files: Vec<SourceFile>,
         cache_mode: CacheMode,
         max_concurrent_requests: usize,
+        request_timeout: u64,
+        request_throttling: u64,
     ) -> Self {
         Self {
             source_bookmark_files,
             cache_mode,
             max_concurrent_requests,
+            request_timeout,
+            request_throttling,
         }
     }