Better defaults / estimates

Author: junderw

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "hibp_downloader"
-version = "1.0.1"
+version = "1.0.2"
 license = "MIT"
 authors = [
     "Jonathan Underwood <[email protected]>"

Cargo.toml

@@ -12,7 +12,7 @@ pub struct Config {
     #[arg(long, default_value_t = default_workers())]
     pub workers: usize,
     /// The number of requests per worker
-    #[arg(long, default_value = "20")]
+    #[arg(long, default_value_t = default_multiplier())]
     pub multiplier: usize,
     /// Download NTLM hashes instead of SHA1 hashes
     #[arg(short, long)]
@@ -69,6 +69,12 @@ fn default_workers() -> usize {
         .get()
 }
 
+fn default_multiplier() -> usize {
+    // Cloudflare seems to throttle at 128 in-flight connections at a time
+    // (Note: available_parallelism is guaranteed to be non-zero)
+    128 / default_workers()
+}
+
 /// This function gives a static reference to a Config struct.
 pub fn get_config<'a>() -> &'a Config {
     static CONFIG: OnceLock<Config> = OnceLock::new();

src/config.rs

@@ -37,8 +37,8 @@ pub fn run_sort(input: &Path, output: &Path, temp_dir: &Path) -> anyhow::Result<
         assert!(len == 32 || len == 40);
         len
     };
-    // colon + average of 3 length number (max 8, min 1) + new line
-    let row_size = hash_size + 5;
+    // colon + average of 5 length number (max 8, min 1) + new line
+    let row_size = hash_size + 7;
     let rows_in_file = input_byte_size / row_size as u64;
 
     // This is a rough estimate.