Improve file write performance by fwrite with cache. v5.0.133

Author: winlinvip

trunk/doc/CHANGELOG.md

@@ -8,6 +8,7 @@ The changelog for SRS.
 
 ## SRS 5.0 Changelog
 
+* v5.0, 2023-01-08, Merge [#3308](https://github.com/ossrs/srs/pull/3308): DVR: Improve file write performance by fwrite with cache. v5.0.133
 * v5.0, 2023-01-06, DVR: Support blackbox test based on hooks. v5.0.132
 * v5.0, 2023-01-06, FFmpeg: Support build with FFmpeg native opus. v5.0.131 (#3140)
 * v5.0, 2023-01-05, CORS: Refine HTTP CORS headers. v5.0.130

trunk/src/app/srs_app_dvr.cpp

@@ -26,6 +26,8 @@ using namespace std;
 #include <srs_kernel_mp4.hpp>
 #include <srs_app_fragment.hpp>
 
+#define SRS_FWRITE_CACHE_SIZE 65536
+
 SrsDvrSegmenter::SrsDvrSegmenter()
 {
     req = NULL;
@@ -95,8 +97,8 @@ srs_error_t SrsDvrSegmenter::open()
         return srs_error_wrap(err, "open file %s", path.c_str());
     }
 
-    // set libc file write cache buffer size
-    if ((err = fs->set_iobuf_size(65536)) != srs_success) {
+    // Set libc file write cache buffer size
+    if ((err = fs->set_iobuf_size(SRS_FWRITE_CACHE_SIZE)) != srs_success) {
         return srs_error_wrap(err, "set iobuf size for file %s", path.c_str());
     }
 

trunk/src/core/srs_core_version5.hpp

@@ -9,6 +9,6 @@
 
 #define VERSION_MAJOR       5
 #define VERSION_MINOR       0
-#define VERSION_REVISION    132
+#define VERSION_REVISION    133
 
 #endif

trunk/src/kernel/srs_kernel_file.cpp

@@ -46,7 +46,6 @@ SrsFileWriter::~SrsFileWriter()
     srs_freepa(buf_);
 }
 
-
 srs_error_t SrsFileWriter::set_iobuf_size(int size)
 {
     srs_error_t err = srs_success;
@@ -56,25 +55,15 @@ srs_error_t SrsFileWriter::set_iobuf_size(int size)
     }
 
     srs_freepa(buf_);
-
-    int ret;
-
-    if (size > 0) {
-        buf_ = new char[size];
-        ret = _srs_setvbuf_fn(fp_, buf_, _IOFBF, size);
-    }
-    else {
-        ret = _srs_setvbuf_fn(fp_, NULL, _IONBF, size);
-    }
-
-    if (ret != 0) {
-        return srs_error_new(ERROR_SYSTEM_FILE_SETVBUF, "file %s set vbuf error", path_.c_str());
+    buf_ = size > 0 ? new char[size] : NULL;
+    int r0 = _srs_setvbuf_fn(fp_, buf_, _IOFBF, size);
+    if (!r0) {
+        return srs_error_new(ERROR_SYSTEM_FILE_SETVBUF, "setvbuf err, file=%s, r0=%d", path_.c_str(), r0);
     }
 
     return err;
 }
 
-
 srs_error_t SrsFileWriter::open(string p)
 {
     srs_error_t err = srs_success;

trunk/src/utest/srs_utest_kernel.cpp

@@ -4328,7 +4328,6 @@ VOID TEST(KernelFileWriterTest, WriteSpecialCase)
 
 		EXPECT_EQ(f.tellg(), -1);
 	}
-
 }
 
 VOID TEST(KernelFileReaderTest, WriteSpecialCase)
@@ -4385,26 +4384,24 @@ VOID TEST(KernelFileReaderTest, WriteSpecialCase)
 	}
 }
 
-class MockFileRemover
+MockFileRemover::MockFileRemover(string p)
 {
-private:
-	string f;
-public:
-	MockFileRemover(string p) {
-		f = p;
-	}
-	virtual ~MockFileRemover() {
-		if (f != "") {
-			::unlink(f.c_str());
-		}
-	}
-};
+    path_ = p;
+}
+
+MockFileRemover::~MockFileRemover()
+{
+    // Only remove {_srs_tmp_file_prefix}*.log file.
+    if (path_.find(_srs_tmp_file_prefix) != 0) return;
+    if (path_.find(".log") == string::npos) return;
+    ::unlink(path_.c_str());
+}
 
 VOID TEST(KernelFileTest, ReadWriteCase)
 {
 	srs_error_t err;
 
-	string filepath = _srs_tmp_file_prefix + "kernel-file-read-write-case";
+	string filepath = _srs_tmp_file_prefix + "kernel-file-read-write-case.log";
 	MockFileRemover _mfr(filepath);
 
 	SrsFileWriter w;
@@ -4431,7 +4428,7 @@ VOID TEST(KernelFileTest, SeekCase)
 {
 	srs_error_t err;
 
-	string filepath = _srs_tmp_file_prefix + "kernel-file-read-write-case";
+	string filepath = _srs_tmp_file_prefix + "kernel-file-read-write-case.log";
 	MockFileRemover _mfr(filepath);
 
 	SrsFileWriter w;
@@ -4468,7 +4465,6 @@ VOID TEST(KernelFileTest, SeekCase)
 	EXPECT_EQ(5, nn);
 
 	EXPECT_STREQ("World", buf);
-
 }
 
 VOID TEST(KernelFLVTest, CoverAll)

trunk/src/utest/srs_utest_kernel.hpp

@@ -39,6 +39,15 @@ class MockSrsFile
     virtual srs_error_t lseek(off_t offset, int whence, off_t* seeked);
 };
 
+class MockFileRemover
+{
+private:
+    std::string path_;
+public:
+    MockFileRemover(std::string p);
+    virtual ~MockFileRemover();
+};
+
 class MockSrsFileWriter : public SrsFileWriter
 {
 public:

trunk/src/utest/srs_utest_kernel2.cpp

@@ -352,3 +352,62 @@ VOID TEST(KernelLogTest, LogLevelStringV2)
     EXPECT_EQ(srs_get_log_level_v2("off"),     SrsLogLevelDisabled);
 }
 
+VOID TEST(KernelFileWriterTest, RealfileTest)
+{
+    srs_error_t err;
+
+    string filename = "./test-realfile.log";
+    MockFileRemover disposer(filename);
+
+    if (true) {
+        SrsFileWriter f;
+        HELPER_EXPECT_SUCCESS(f.open(filename));
+        EXPECT_TRUE(f.is_open());
+        EXPECT_EQ(0, f.tellg());
+
+        HELPER_EXPECT_SUCCESS(f.write((void*) "HelloWorld", 10, NULL));
+        EXPECT_EQ(10, f.tellg());
+
+        f.seek2(5);
+        EXPECT_EQ(5, f.tellg());
+
+        HELPER_EXPECT_SUCCESS(f.write((void*) "HelloWorld", 10, NULL));
+        EXPECT_EQ(15, f.tellg());
+
+        off_t v = 0;
+        HELPER_EXPECT_SUCCESS(f.lseek(0, SEEK_CUR, &v));
+        EXPECT_EQ(15, v);
+
+        HELPER_EXPECT_SUCCESS(f.lseek(0, SEEK_SET, &v));
+        EXPECT_EQ(0, v);
+
+        HELPER_EXPECT_SUCCESS(f.lseek(10, SEEK_SET, &v));
+        EXPECT_EQ(10, v);
+
+        HELPER_EXPECT_SUCCESS(f.lseek(0, SEEK_END, &v));
+        EXPECT_EQ(15, v);
+
+        // There are 5 bytes empty lagging in file.
+        HELPER_EXPECT_SUCCESS(f.lseek(5, SEEK_END, &v));
+        EXPECT_EQ(20, v);
+
+        HELPER_EXPECT_SUCCESS(f.write((void*) "HelloWorld", 10, NULL));
+        EXPECT_EQ(30, f.tellg());
+
+        HELPER_EXPECT_SUCCESS(f.lseek(0, SEEK_SET, &v));
+        EXPECT_EQ(0, v);
+
+        HELPER_EXPECT_SUCCESS(f.write((void*) "HelloWorld", 10, NULL));
+        EXPECT_EQ(10, f.tellg());
+    }
+
+    SrsFileReader fr;
+    HELPER_ASSERT_SUCCESS(fr.open(filename));
+
+    // "HelloWorldWorld\0\0\0\0\0HelloWorld"
+    string str;
+    HELPER_ASSERT_SUCCESS(srs_ioutil_read_all(&fr, str));
+    EXPECT_STREQ("HelloWorldWorld", str.c_str());
+    EXPECT_STREQ("HelloWorld", str.substr(20).c_str());
+}
+