improve NamedFile and embed file detect text content type (#1210)

Author: chrislearn

Cargo.toml

@@ -52,8 +52,10 @@ bytes = "1"
 bcrypt = "0.17"
 cookie = "0.18"
 chacha20poly1305 = "0.10"
+chardetng = "0.1"
 chrono = "0.4"
 compact_str = { version = "0.9", features = ["serde"] }
+content_inspector = "0.2"
 encoding_rs = "0.8"
 email_address = "0.2"
 enumflags2 = "0.7"

crates/core/Cargo.toml

@@ -44,6 +44,8 @@ anyhow = { workspace = true, optional = true }
 async-trait = { workspace = true }
 base64 = { workspace = true }
 bytes = { workspace = true }
+chardetng = { workspace = true }
+content_inspector = { workspace = true }
 cookie = { workspace = true, features = ["percent-encode", "private", "signed"], optional = true }
 encoding_rs = { workspace = true, optional = true }
 enumflags2 = { workspace = true }

crates/core/src/fs/named_file.rs

@@ -11,12 +11,13 @@ use std::os::unix::fs::MetadataExt;
 use enumflags2::{BitFlags, bitflags};
 use headers::*;
 use tokio::fs::File;
+use tokio::io::AsyncReadExt;
 
 use super::{ChunkedFile, ChunkedState};
 use crate::http::header::{
     CONTENT_DISPOSITION, CONTENT_ENCODING, CONTENT_TYPE, IF_NONE_MATCH, RANGE,
 };
-use crate::http::{HttpRange, Mime, Request, Response, StatusCode, StatusError};
+use crate::http::{HttpRange, Mime, Request, Response, StatusCode, StatusError, detect_text_mime};
 use crate::{Depot, Error, Result, Writer, async_trait};
 
 const CHUNK_SIZE: u64 = 1024 * 1024;
@@ -171,21 +172,30 @@ impl NamedFileBuilder {
         } = self;
 
         let file = File::open(&path).await?;
-        let content_type = content_type.unwrap_or_else(|| {
-            let ct = mime_infer::from_path(&path).first_or_octet_stream();
-            let ftype = ct.type_();
-            let stype = ct.subtype();
-            if (ftype == mime::TEXT || stype == mime::JSON || stype == mime::JAVASCRIPT)
-                && ct.get_param(mime::CHARSET).is_none()
-            {
-                //TODO: auto detect charset
-                format!("{ct}; charset=utf-8")
-                    .parse::<mime::Mime>()
-                    .unwrap_or(ct)
+        let content_type =
+            if let Some(mime) = content_type.or_else(|| mime_infer::from_path(&path).first()) {
+                if mime == mime::TEXT_PLAIN {
+                    let mut buffer: Vec<u8> = vec![];
+                    let _ = file.take(1024).read(&mut buffer).await;
+                    if let Some(mime) = detect_text_mime(&buffer) {
+                        mime
+                    } else {
+                        mime
+                    }
+                } else {
+                    mime
+                }
             } else {
-                ct
-            }
-        });
+                let mut buffer: Vec<u8> = vec![];
+                let _ = file.take(1024).read(&mut buffer).await;
+                if let Some(mime) = detect_text_mime(&buffer) {
+                    mime
+                } else {
+                    mime::APPLICATION_OCTET_STREAM
+                }
+            };
+
+        let file = File::open(&path).await?;
         let metadata = file.metadata().await?;
         let modified = metadata.modified().ok();
         let content_encoding = match content_encoding {

crates/core/src/http/mod.rs

@@ -63,6 +63,28 @@ pub fn guess_accept_mime(req: &Request, default_type: Option<Mime>) -> Mime {
         .unwrap_or(dmime)
 }
 
+#[doc(hidden)]
+#[inline]
+#[must_use]
+pub fn detect_text_mime(buffer: &[u8]) -> Option<mime::Mime> {
+    let info = content_inspector::inspect(buffer);
+    if info.is_text() {
+        let mut detector = chardetng::EncodingDetector::new();
+        detector.feed(buffer, buffer.len() < 1024);
+
+        let (encoding, _) = detector.guess_assess(None, true);
+        if encoding.name().eq_ignore_ascii_case("utf-8") {
+            Some(mime::TEXT_PLAIN_UTF_8)
+        } else {
+            format!("text/plain; charset={}", encoding.name())
+                .parse::<mime::Mime>()
+                .ok()
+        }
+    } else {
+        None
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::header::*;

crates/serve-static/src/dir.rs

@@ -447,10 +447,7 @@ impl Handler for StaticDir {
             };
 
             let builder = {
-                let mut builder = NamedFile::builder(named_path).content_type(
-                    mime_infer::from_ext(ext.as_deref().unwrap_or_default())
-                        .first_or_octet_stream(),
-                );
+                let mut builder = NamedFile::builder(named_path);
                 if let Some(content_encoding) = content_encoding {
                     builder = builder.content_encoding(content_encoding);
                 }

crates/serve-static/src/embed.rs

@@ -8,7 +8,9 @@ use salvo_core::http::header::{
     ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_TYPE, ETAG, IF_NONE_MATCH, RANGE,
 };
 use salvo_core::http::headers::{ContentLength, ContentRange, HeaderMapExt};
-use salvo_core::http::{HeaderValue, HttpRange, Mime, Request, Response, StatusCode};
+use salvo_core::http::{
+    HeaderValue, HttpRange, Mime, Request, Response, StatusCode, detect_text_mime,
+};
 use salvo_core::{Depot, FlowCtrl, IntoVecString, async_trait};
 
 use super::{decode_url_path_safely, format_url_path_safely, join_path, redirect_to_dir_url};
@@ -63,14 +65,29 @@ fn render_embedded_data(
     metadata: &Metadata,
     req: &Request,
     res: &mut Response,
-    mime_override: Option<Mime>,
+    mime: Option<Mime>,
 ) {
     // Determine Content-Type once
-    let effective_mime = mime_override
-        .unwrap_or_else(|| mime_infer::from_path(req.uri().path()).first_or_octet_stream());
+    let content_type =
+        if let Some(mime) = mime.or_else(|| mime_infer::from_path(req.uri().path()).first()) {
+            if mime == mime::TEXT_PLAIN {
+                if let Some(mime) = detect_text_mime(&data) {
+                    mime
+                } else {
+                    mime
+                }
+            } else {
+                mime
+            }
+        } else if let Some(mime) = detect_text_mime(&data) {
+            mime
+        } else {
+            mime::APPLICATION_OCTET_STREAM
+        };
+
     res.headers_mut().insert(
         CONTENT_TYPE,
-        effective_mime
+        content_type
             .as_ref()
             .parse()
             .unwrap_or_else(|_| HeaderValue::from_static("application/octet-stream")),

examples/static-dir-list/static/boy/work_ansi.txt

@@ -2,4 +2,10 @@ Let me take it down
 
 An elephant said to a mouse ,"no doubt that you are the smallest znd most useless thing that Ihave e ver seen ."
 
-"Pless ,say it again .Let me take it down ."the mouse said ."I will tell a flea what I know.
+"Pless ,say it again .Let me take it down ."the mouse said ."I will tell a flea what I know.
+
+���Ұ���������
+
+һͷ�����һֻ����˵�����������ʣ������Ҽ�������С����û�õĶ�������
+
+������˵һ�顣���Ұ�����������������˵�����һ����֪���ĸ������顣��

examples/static-dir-list/static/boy/新文件.txt

@@ -0,0 +1,2 @@
+你好朋友
+abc

examples/static-embed-files/static/index.html

@@ -1,5 +1,8 @@
 <html>
     <body>
         Index page
+
+        <p><a href="test1.txt">test1.txt</a></p>
+        <p><a href="test2.txt">test2.txt</a></p>
     </body>
 </html>

examples/static-embed-files/static/test1.txt

@@ -1 +1,3 @@
-copy1
+copy1
+
+你好朋友