Add builtin_httpfs SQL option to allow keep using current in-build httpfs

lib/src/webdb.cc

@@ -32,6 +32,7 @@
 #include "duckdb/common/arrow/arrow.hpp"
 #include "duckdb/common/arrow/arrow_converter.hpp"
 #include "duckdb/common/file_system.hpp"
+#include "duckdb/common/http_util.hpp"
 #include "duckdb/common/types.hpp"
 #include "duckdb/common/types/data_chunk.hpp"
 #include "duckdb/common/types/vector.hpp"
@@ -70,6 +71,9 @@
 #include "rapidjson/writer.h"
 
 namespace duckdb {
+
+bool preloaded_httpfs{true};
+
 namespace web {
 
 static constexpr int64_t DEFAULT_QUERY_POLLING_INTERVAL = 100;
@@ -753,6 +757,9 @@ void WebDB::RegisterCustomExtensionOptions(shared_ptr<duckdb::DuckDB> database)
 
     // Register S3 Config parameters
     if (webfs) {
+        auto callback_builtin_httpfs = [](ClientContext& context, SetScope scope, Value& parameter) {
+            preloaded_httpfs = BooleanValue::Get(parameter);
+        };
         auto callback_s3_region = [](ClientContext& context, SetScope scope, Value& parameter) {
             auto webfs = io::WebFileSystem::Get();
             webfs->Config()->duckdb_config_options.s3_region = StringValue::Get(parameter);
@@ -784,6 +791,8 @@ void WebDB::RegisterCustomExtensionOptions(shared_ptr<duckdb::DuckDB> database)
             webfs->IncrementCacheEpoch();
         };
 
+        config.AddExtensionOption("builtin_httpfs", "Use built-in HTTPS support", LogicalType::BOOLEAN, true,
+                                  callback_builtin_httpfs);
         config.AddExtensionOption("s3_region", "S3 Region", LogicalType::VARCHAR, Value(), callback_s3_region);
         config.AddExtensionOption("s3_access_key_id", "S3 Access Key ID", LogicalType::VARCHAR, Value(),
                                   callback_s3_access_key_id);

packages/duckdb-wasm/src/bindings/runtime.ts

@@ -58,6 +58,11 @@ export enum FileFlags {
     FILE_FLAGS_FILE_CREATE_NEW = 1 << 4,
     //! Open file in append mode
     FILE_FLAGS_APPEND = 1 << 5,
+    FILE_FLAGS_PRIVATE = 1 << 6,
+    FILE_FLAGS_NULL_IF_NOT_EXISTS = 1 << 7,
+    FILE_FLAGS_PARALLEL_ACCESS = 1 << 8,
+    FILE_FLAGS_EXCLUSIVE_CREATE = 1 << 9,
+    FILE_FLAGS_NULL_IF_EXISTS = 1 << 10
 }
 
 /** Configuration for the AWS S3 Filesystem */

packages/duckdb-wasm/src/bindings/runtime_browser.ts

@@ -381,6 +381,11 @@ export const BROWSER_RUNTIME: DuckDBRuntime & {
                         return result;
                     }
 
+                    // Depending on file flags, return nullptr
+                    if (flags & FileFlags.FILE_FLAGS_NULL_IF_NOT_EXISTS) {
+                       return 0;
+                    }
+
                     // Fall back to empty buffered file in the browser
                     console.warn(`Buffering missing file: ${file.fileName}`);
                     const result = mod._malloc(2 * 8);

packages/duckdb-wasm/src/bindings/runtime_node.ts

@@ -76,6 +76,11 @@ export const NODE_RUNTIME: DuckDBRuntime & {
                 case DuckDBDataProtocol.NODE_FS: {
                     let fd = NODE_RUNTIME._files?.get(file.dataUrl!);
                     if (fd === null || fd === undefined) {
+                        // Depending on file flags, return nullptr
+                        if (flags & FileFlags.FILE_FLAGS_NULL_IF_NOT_EXISTS) {
+                            return 0;
+                        }
+
                         fd = fs.openSync(
                             file.dataUrl!,
                             fs.constants.O_CREAT | fs.constants.O_RDWR,

patches/duckdb/extension_install_rework.patch

@@ -1,15 +1,16 @@
 diff --git a/src/include/duckdb/main/database.hpp b/src/include/duckdb/main/database.hpp
-index d3c5fb9bd5..b3d0aaa09e 100644
+index d3c5fb9bd5..eace7b4af1 100644
 --- a/src/include/duckdb/main/database.hpp
 +++ b/src/include/duckdb/main/database.hpp
-@@ -100,6 +100,10 @@ private:
+@@ -100,6 +100,11 @@ private:
  	unique_ptr<ExternalFileCache> external_file_cache;
 
  	duckdb_ext_api_v1 (*create_api_v1)();
++
 +public:
-+	static void SetPreferredRepository(const string& extension, const string &repository);
-+	static string GetPreferredRepository(const string& extension);
-+	static unordered_map<string, string> extensionsRepos;
++       static void SetPreferredRepository(const string& extension, const string &repository);
++       static string GetPreferredRepository(const string& extension);
++       static unordered_map<string, string> extensionsRepos;
  };
 
  //! The database object. This object holds the catalog and all the
@@ -31,10 +32,24 @@ index 6ccd1a1156..8040f537b6 100644
  	//! Debugging repositories (target local, relative paths that are produced by DuckDB's build system)
  	static constexpr const char *BUILD_DEBUG_REPOSITORY_PATH = "./build/debug/repository";
 diff --git a/src/main/database.cpp b/src/main/database.cpp
-index db6e1ed445..d495aab058 100644
+index 773f8d967a..fa089a47ee 100644
 --- a/src/main/database.cpp
 +++ b/src/main/database.cpp
-@@ -356,6 +356,28 @@ DuckDB::DuckDB(DatabaseInstance &instance_p) : instance(instance_p.shared_from_t
+@@ -1,5 +1,4 @@
+ #include "duckdb/main/database.hpp"
+-
+ #include "duckdb/catalog/catalog.hpp"
+ #include "duckdb/common/virtual_file_system.hpp"
+ #include "duckdb/execution/index/index_type_set.hpp"
+@@ -35,6 +34,7 @@
+ #include "duckdb/common/thread.hpp"
+ #endif
+
++
+ namespace duckdb {
+
+ DBConfig::DBConfig() {
+@@ -357,6 +357,28 @@ DuckDB::DuckDB(DatabaseInstance &instance_p) : instance(instance_p.shared_from_t
  DuckDB::~DuckDB() {
  }
 
@@ -63,11 +78,11 @@ index db6e1ed445..d495aab058 100644
  SecretManager &DatabaseInstance::GetSecretManager() {
  	return *config.secret_manager;
  }
-@@ -507,6 +529,10 @@ idx_t DuckDB::NumberOfThreads() {
+@@ -508,6 +530,10 @@ idx_t DuckDB::NumberOfThreads() {
 
  bool DatabaseInstance::ExtensionIsLoaded(const std::string &name) {
  	auto extension_name = ExtensionHelper::GetExtensionName(name);
-+	if (extension_name == "httpfs") {
++	if (extension_name == "httpfs" && preloaded_httpfs) {
 +		ExtensionInstallInfo info;
 +		SetExtensionLoaded(extension_name, info);
 +	}

patches/duckdb/fix_load_database.patch

@@ -1,20 +1,23 @@
 diff --git a/src/storage/storage_manager.cpp b/src/storage/storage_manager.cpp
-index cb6c654e5f..a6b2af3b85 100644
+index 45d42174f1..10b3ce47b1 100644
 --- a/src/storage/storage_manager.cpp
 +++ b/src/storage/storage_manager.cpp
-@@ -160,9 +160,12 @@ void SingleFileStorageManager::LoadDatabase(StorageOptions storage_options) {
+@@ -162,9 +162,15 @@ void SingleFileStorageManager::LoadDatabase(StorageOptions storage_options) {
  			    row_group_size, STANDARD_VECTOR_SIZE);
  		}
  	}
 -	// Check if the database file already exists.
 -	// Note: a file can also exist if there was a ROLLBACK on a previous transaction creating that file.
 -	if (!read_only && !fs.FileExists(path)) {
-+    auto db_file_handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ | FileFlags::FILE_FLAGS_NULL_IF_NOT_EXISTS);
-+    bool is_empty_file = db_file_handle->GetFileSize() == 0;
-+    db_file_handle.reset();
++	bool is_empty_file = true;
++	auto db_file_handle = fs.OpenFile(path, FileFlags::FILE_FLAGS_READ | FileFlags::FILE_FLAGS_NULL_IF_NOT_EXISTS);
++	if (db_file_handle && db_file_handle->GetFileSize() != 0) {
++		is_empty_file = false;
++		db_file_handle.reset();
++	}
 +
-+ 	// first check if the database exists
-+	if (!read_only && ( !fs.FileExists(path) || ( options.use_direct_io && is_empty_file )) ) {
++	// first check if the database exists
++	if (!read_only && (!fs.FileExists(path) || (options.use_direct_io && is_empty_file))) {
  		// file does not exist and we are in read-write mode
  		// create a new file
 

patches/duckdb/preloaded_httpfs.patch

@@ -0,0 +1,14 @@
+diff --git a/src/include/duckdb/main/database.hpp b/src/include/duckdb/main/database.hpp
+index d3c5fb9bd5..adc2ac42e6 100644
+--- a/src/include/duckdb/main/database.hpp
++++ b/src/include/duckdb/main/database.hpp
+@@ -17,6 +17,9 @@
+ #include "duckdb/main/valid_checker.hpp"
+
+ namespace duckdb {
++
++extern bool preloaded_httpfs;
++
+ class BufferManager;
+ class DatabaseManager;
+ class StorageManager;