Merge branch 'http-range-header' of https://github.com/sfc-gh-rnarubin/fio

* 'http-range-header' of https://github.com/sfc-gh-rnarubin/fio:
  engines/http: Add support for range reads

Author: vincentkfu

HOWTO.rst

@@ -3058,6 +3058,24 @@ with the caveat that when used on the command line, they must come after the
 	turns on verbose logging from libcurl, 2 additionally enables
 	HTTP IO tracing. Default is **0**
 
+.. option:: http_object_mode=str : [http]
+
+        How to structure objects for HTTP IO: *block* or *range*.
+        Default is **block**.
+
+        In *block* mode, one object is created for every block. The HTTP engine
+        treats :option:`blocksize` as the size of the object to read or write,
+        and appends the block start/end offsets to the :option:`filename` to
+        create the target object path. Reads and writes operate on whole
+        objects at a time.
+
+        In *range* mode, one object is created for every file. The object path
+        is the filename directly for both read and write I/O. For read
+        requests, the :option:`blocksize` and :option:`offset` will be used to
+        set the "Range" header on read requests to issue partial reads of the
+        object. For write requests, blocksize is used to set the size of the
+        object, the same as in *block* mode.
+
 .. option:: uri=str : [nbd]
 
 	Specify the NBD URI of the server to test.  The string

engines/http.c

@@ -35,13 +35,16 @@
 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 
 enum {
-	FIO_HTTP_WEBDAV	    = 0,
-	FIO_HTTP_S3	    = 1,
-	FIO_HTTP_SWIFT	    = 2,
+	FIO_HTTP_WEBDAV		= 0,
+	FIO_HTTP_S3		= 1,
+	FIO_HTTP_SWIFT		= 2,
 
-	FIO_HTTPS_OFF	    = 0,
-	FIO_HTTPS_ON	    = 1,
-	FIO_HTTPS_INSECURE  = 2,
+	FIO_HTTPS_OFF		= 0,
+	FIO_HTTPS_ON		= 1,
+	FIO_HTTPS_INSECURE	= 2,
+
+	FIO_HTTP_OBJECT_BLOCK	= 0,
+	FIO_HTTP_OBJECT_RANGE	= 1,
 };
 
 struct http_data {
@@ -64,6 +67,7 @@ struct http_options {
 	char *swift_auth_token;
 	int verbose;
 	unsigned int mode;
+	unsigned int object_mode;
 };
 
 struct http_curl_stream {
@@ -239,6 +243,26 @@ static struct fio_option options[] = {
 		.category = FIO_OPT_C_ENGINE,
 		.group    = FIO_OPT_G_HTTP,
 	},
+	{
+		.name     = "http_object_mode",
+		.lname    = "Object mode to use",
+		.type     = FIO_OPT_STR,
+		.help     = "How to structure objects when issuing HTTP requests",
+		.off1     = offsetof(struct http_options, object_mode),
+		.def	  = "block",
+		.posval = {
+			  { .ival = "block",
+			    .oval = FIO_HTTP_OBJECT_BLOCK,
+			    .help = "One object per block",
+			  },
+			  { .ival = "range",
+			    .oval = FIO_HTTP_OBJECT_RANGE,
+			    .help = "One object per file, range reads per block",
+			  },
+		},
+		.category = FIO_OPT_C_ENGINE,
+		.group    = FIO_OPT_G_HTTP,
+	},
 	{
 		.name     = NULL,
 	},
@@ -611,6 +635,26 @@ static void _add_swift_header(CURL *curl, struct curl_slist *slist, struct http_
 	free(dsha);
 }
 
+static struct curl_slist* _append_range_header(struct curl_slist *slist, unsigned long long offset, unsigned long long length, unsigned long long file_size)
+{
+	char s[256];
+	unsigned long long end_byte;
+
+	/* Don't request beyond end of file */
+	if (offset >= file_size) {
+		return slist;
+	}
+
+	/* Calculate end byte, but cap it at file size - 1 because end range is inclusive */
+	end_byte = offset + length - 1;
+	if (end_byte >= file_size) {
+		end_byte = file_size - 1;
+	}
+
+	snprintf(s, sizeof(s), "Range: bytes=%llu-%llu", offset, end_byte);
+	return curl_slist_append(slist, s);
+}
+
 static void fio_http_cleanup(struct thread_data *td)
 {
 	struct http_data *http = td->io_ops_data;
@@ -667,30 +711,39 @@ static enum fio_q_status fio_http_queue(struct thread_data *td,
 	struct http_options *o = td->eo;
 	struct http_curl_stream _curl_stream;
 	struct curl_slist *slist = NULL;
-	char object[512];
+	char object_path_buf[512];
+	char *object_path;
 	char url[1024];
 	long status;
 	CURLcode res;
 
 	fio_ro_check(td, io_u);
 	memset(&_curl_stream, 0, sizeof(_curl_stream));
-	snprintf(object, sizeof(object), "%s_%llu_%llu", io_u->file->file_name,
-		io_u->offset, io_u->xfer_buflen);
+	if (o->object_mode == FIO_HTTP_OBJECT_BLOCK) {
+		snprintf(object_path_buf, sizeof(object_path_buf), "%s_%llu_%llu", io_u->file->file_name,
+			io_u->offset, io_u->xfer_buflen);
+		object_path = object_path_buf;
+	} else
+		object_path = io_u->file->file_name;
 	if (o->https == FIO_HTTPS_OFF)
-		snprintf(url, sizeof(url), "http://%s%s", o->host, object);
+		snprintf(url, sizeof(url), "http://%s%s", o->host, object_path);
 	else
-		snprintf(url, sizeof(url), "https://%s%s", o->host, object);
+		snprintf(url, sizeof(url), "https://%s%s", o->host, object_path);
+
 	curl_easy_setopt(http->curl, CURLOPT_URL, url);
 	_curl_stream.buf = io_u->xfer_buf;
 	_curl_stream.max = io_u->xfer_buflen;
 	curl_easy_setopt(http->curl, CURLOPT_SEEKDATA, &_curl_stream);
 	curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)io_u->xfer_buflen);
 
+	if (io_u->ddir == DDIR_READ && o->object_mode == FIO_HTTP_OBJECT_RANGE)
+		slist = _append_range_header(slist, io_u->offset, io_u->xfer_buflen, io_u->file->real_file_size);
+
 	if (o->mode == FIO_HTTP_S3)
-		_add_aws_auth_header(http->curl, slist, o, io_u->ddir, object,
+		_add_aws_auth_header(http->curl, slist, o, io_u->ddir, object_path,
 			io_u->xfer_buf, io_u->xfer_buflen);
 	else if (o->mode == FIO_HTTP_SWIFT)
-		_add_swift_header(http->curl, slist, o, io_u->ddir, object,
+		_add_swift_header(http->curl, slist, o, io_u->ddir, object_path,
 			io_u->xfer_buf, io_u->xfer_buflen);
 
 	if (io_u->ddir == DDIR_WRITE) {
@@ -712,7 +765,9 @@ static enum fio_q_status fio_http_queue(struct thread_data *td,
 		res = curl_easy_perform(http->curl);
 		if (res == CURLE_OK) {
 			curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
-			if (status == 200)
+			/* 206 "Partial Content" means success when using the
+			 * Range header */
+			if (status == 200 || (o->object_mode == FIO_HTTP_OBJECT_RANGE && status == 206))
 				goto out;
 			else if (status == 404) {
 				/* Object doesn't exist. Pretend we read

fio.1

@@ -2648,6 +2648,26 @@ Enable verbose requests from libcurl. Useful for debugging. 1 turns on
 verbose logging from libcurl, 2 additionally enables HTTP IO tracing.
 Default is \fB0\fR
 .TP
+.BI (http)http_object_mode \fR=\fPstr
+How to structure objects for HTTP IO: block or range. Default is \fBblock\fR.
+.RS
+.RS
+.TP
+.B block
+One object is created for every block. The HTTP engine treats \fBblocksize\fR
+as the size of the object to read or write, and appends the block start/end
+offsets to the \fBfilename\fR to create the target object path. Reads and
+writes operate on whole objects at a time.
+.TP
+.B range
+One object is created for every file. The object path is the filename directly
+for both read and write I/O. For read requests, the \fBblocksize\fR and
+\fBoffset\fR will be used to set the "Range" header on read requests to issue
+partial reads of the object. For write requests, blocksize is used to set the
+size of the object, the same as in \fBblock\fR mode.
+.RE
+.RE
+.TP
 .BI (mtd)skip_bad \fR=\fPbool
 Skip operations against known bad blocks.
 .TP