various improvements

Author: snshn

Cargo.lock

@@ -26,20 +26,21 @@ license = "CC0-1.0"
 atty = "0.2.14"  # Used for highlighting network errors
 base64 = "0.22.1"  # Used for integrity attributes
 chrono = "0.4.40"  # Used for formatting timestamps
-clap = { version = "4.5.34", features = ["derive"], optional = true }  # Used for processing CLI arguments
+clap = { version = "4.5.35", features = ["derive"], optional = true }  # Used for processing CLI arguments
 cssparser = "0.35.0"  # Used for dealing with CSS
+directories = { version = "6.0.0", optional = true }  # Used for GUI
 druid = { version = "0.8.3", optional = true }  # Used for GUI
 encoding_rs = "0.8.35"  # Used for parsing and converting document charsets
-html5ever = "0.28.0"  # Used for all things DOM
-markup5ever_rcdom = "=0.4.0-unofficial"  # Used for manipulating DOM
+html5ever = "0.29.1"  # Used for all things DOM
+markup5ever_rcdom = "=0.5.0-unofficial"  # Used for manipulating DOM
 percent-encoding = "2.3.1"  # Used for encoding URLs
 sha2 = "0.10.8"  # Used for calculating checksums during integrity checks
 redb = "2.4.0"  # Used for on-disk caching of remote assets
 tempfile = { version = "3.19.1", optional = true }  # Used for on-disk caching of remote assets
 url = "2.5.4"  # Used for parsing URLs
 openssl = "0.10.72"  # Used for static linking of the OpenSSL library
 
-# Used for parsing srcset and NOSCRIPT
+# Used for unwrapping NOSCRIPT
 [dependencies.regex]
 version = "1.11.1"
 default-features = false
@@ -57,7 +58,7 @@ assert_cmd = "2.0.16"
 [features]
 default = ["cli", "vendored-openssl"]
 cli = ["clap", "tempfile"]  # Build a CLI tool that includes main() function
-gui = ["druid", "tempfile"]  # Build a GUI executable that includes main() function
+gui = ["directories", "druid", "tempfile"]  # Build a GUI executable that includes main() function
 vendored-openssl = ["openssl/vendored"]  # Compile and statically link a copy of OpenSSL
 
 [lib]

Cargo.toml

@@ -29,6 +29,7 @@ install:
 
 lint:
 	@cargo clippy --fix --allow-dirty --allow-staged
+# 	@cargo fix --allow-dirty --allow-staged
 .PHONY: lint
 
 lint_check:

Makefile

@@ -4,8 +4,6 @@
 [![Monolith Actor on Apify](https://apify.com/actor-badge?actor=netmilk/monolith)](https://apify.com/netmilk/monolith?fpr=monolith)
 
 
-
-
 ```
  _____    _____________   __________     ___________________    ___
 |     \  /             \ |          |   |                   |  |   |
@@ -225,6 +223,7 @@ monolith -I -d example.com -d www.example.com https://example.com -o example-onl
 monolith -I -B -d .googleusercontent.com -d googleanalytics.com -d .google.com https://example.com -o example-no-ads.html
 ```
 
+
 ---------------------------------------------------
 
 
@@ -242,6 +241,16 @@ chromium --headless --window-size=1920,1080 --run-all-compositor-stages-before-d
 ---------------------------------------------------
 
 
+## Authentication
+
+```console
+monolith https://username:[email protected] -o example-basic-auth.html
+```
+
+
+---------------------------------------------------
+
+
 ## Proxies
 
 Please set `https_proxy`, `http_proxy`, and `no_proxy` environment variables.

README.md

@@ -266,10 +266,9 @@ pub fn create_monolithic_document_from_data(
     }
 
     // Append noindex META-tag
-    if let meta_robots_content_value = get_robots(&dom.document).unwrap_or_default() {
-        if meta_robots_content_value.trim().is_empty() || meta_robots_content_value != "none" {
-            dom = set_robots(dom, "none");
-        }
+    let meta_robots_content_value = get_robots(&dom.document).unwrap_or_default();
+    if meta_robots_content_value.trim().is_empty() || meta_robots_content_value != "none" {
+        dom = set_robots(dom, "none");
     }
 
     // Save using specified charset, if given
@@ -523,11 +522,14 @@ pub fn domain_is_within_domain(domain: &str, domain_to_match_against: &str) -> b
     ok
 }
 
-pub fn format_output_path(destination: &str, document_title: &str) -> String {
+pub fn format_output_path(
+    path: &str,
+    document_title: &str,
+    output_format: MonolithOutputFormat,
+) -> String {
     let datetime: &str = &Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);
 
-    destination
-        .replace("%timestamp%", &datetime.replace(':', "_"))
+    path.replace("%timestamp%", &datetime.replace(':', "_"))
         .replace(
             "%title%",
             document_title
@@ -541,13 +543,22 @@ pub fn format_output_path(destination: &str, document_title: &str) -> String {
                 .replace('?', "")
                 .trim_start_matches('.'),
         )
-        .to_string()
-        .replace('<', "[")
-        .replace('>', "]")
-        .replace(':', " - ")
-        .replace('\"', "")
-        .replace('|', "-")
-        .replace('?', "")
+        .replace(
+            "%ext%",
+            if output_format == MonolithOutputFormat::HTML {
+                "html"
+            } else {
+                ""
+            },
+        )
+        .replace(
+            "%extension%",
+            if output_format == MonolithOutputFormat::HTML {
+                "html"
+            } else {
+                ""
+            },
+        )
         .to_string()
 }
 

src/core.rs

@@ -229,8 +229,10 @@ pub fn process_css<'a>(
                         result.push_str(format_quoted_string(EMPTY_IMAGE_DATA_URL).as_str());
                     } else {
                         let resolved_url: Url = resolve_url(document_url, value);
+
                         match retrieve_asset(cache, client, document_url, &resolved_url, options) {
                             Ok((data, final_url, media_type, charset)) => {
+                                // TODO: if it's @font-face, exclude definitions of non-woff/woff-2 fonts (if woff/woff-2 are present)
                                 let mut data_url =
                                     create_data_url(&media_type, &charset, &data, &final_url);
                                 data_url.set_fragment(resolved_url.fragment());