engines/http: Add support for range reads

The existing HTTP implementation treated blocks as individual objects
when reading and writing. This is useful in as far as it enables read
and write symmetry -- object stores generally don't allow writes to
individual ranges of objects like file systems do with files, so
treating blocks as objects is a practical way to replicate block reads
and writes.

Reading of object ranges, on the other hand, is widely supported by
object stores with the "Range" HTTP header. This change adds a parameter
which alters fio's object conventions to issue reads using the block
size and offset parameters more like file IO. When enabled, both reads
and writes will use the plain filename as the object path to issue IO.
Reads will add a "Range: bytes=<start>-<end>" header to the requests,
where the start and end positions are determined by the blocksize and
offset of the benchmark. Aside from the object path, writes are
unchanged for simplicity: the object size is determined by blocksize as
before.

Signed-off-by: Renar Narubin <[email protected]>

Author: sfc-gh-rnarubin

HOWTO.rst

@@ -3038,6 +3038,24 @@ with the caveat that when used on the command line, they must come after the
 	turns on verbose logging from libcurl, 2 additionally enables
 	HTTP IO tracing. Default is **0**
 
+.. option:: http_object_mode=str : [http]
+
+        How to structure objects for HTTP IO: *block* or *range*.
+        Default is **block**.
+
+        In *block* mode, one object is created for every block. The HTTP engine
+        treats :option:`blocksize` as the size of the object to read or write,
+        and appends the block start/end offsets to the :option:`filename` to
+        create the target object path. Reads and writes operate on whole
+        objects at a time.
+
+        In *range* mode, one object is created for every file. The object path
+        is the filename directly for both read and write I/O. For read
+        requests, the :option:`blocksize` and :option:`offset` will be used to
+        set the "Range" header on read requests to issue partial reads of the
+        object. For write requests, blocksize is used to set the size of the
+        object, the same as in *block* mode.
+
 .. option:: uri=str : [nbd]
 
 	Specify the NBD URI of the server to test.  The string

engines/http.c

@@ -35,13 +35,16 @@
 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
 
 enum {
-	FIO_HTTP_WEBDAV	    = 0,
-	FIO_HTTP_S3	    = 1,
-	FIO_HTTP_SWIFT	    = 2,
+	FIO_HTTP_WEBDAV		= 0,
+	FIO_HTTP_S3		= 1,
+	FIO_HTTP_SWIFT		= 2,
 
-	FIO_HTTPS_OFF	    = 0,
-	FIO_HTTPS_ON	    = 1,
-	FIO_HTTPS_INSECURE  = 2,
+	FIO_HTTPS_OFF		= 0,
+	FIO_HTTPS_ON		= 1,
+	FIO_HTTPS_INSECURE	= 2,
+
+	FIO_HTTP_OBJECT_BLOCK	= 0,
+	FIO_HTTP_OBJECT_RANGE	= 1,
 };
 
 struct http_data {
@@ -63,6 +66,7 @@ struct http_options {
 	char *swift_auth_token;
 	int verbose;
 	unsigned int mode;
+	unsigned int object_mode;
 };
 
 struct http_curl_stream {
@@ -228,6 +232,26 @@ static struct fio_option options[] = {
 		.category = FIO_OPT_C_ENGINE,
 		.group    = FIO_OPT_G_HTTP,
 	},
+	{
+		.name     = "http_object_mode",
+		.lname    = "Object mode to use",
+		.type     = FIO_OPT_STR,
+		.help     = "How to structure objects when issuing HTTP requests",
+		.off1     = offsetof(struct http_options, object_mode),
+		.def	  = "block",
+		.posval = {
+			  { .ival = "block",
+			    .oval = FIO_HTTP_OBJECT_BLOCK,
+			    .help = "One object per block",
+			  },
+			  { .ival = "range",
+			    .oval = FIO_HTTP_OBJECT_RANGE,
+			    .help = "One object per file, range reads per block",
+			  },
+		},
+		.category = FIO_OPT_C_ENGINE,
+		.group    = FIO_OPT_G_HTTP,
+	},
 	{
 		.name     = NULL,
 	},
@@ -580,6 +604,26 @@ static void _add_swift_header(CURL *curl, struct curl_slist *slist, struct http_
 	free(dsha);
 }
 
+static struct curl_slist* _append_range_header(struct curl_slist *slist, unsigned long long offset, unsigned long long length, unsigned long long file_size)
+{
+	char s[256];
+	unsigned long long end_byte;
+
+	/* Don't request beyond end of file */
+	if (offset >= file_size) {
+		return slist;
+	}
+
+	/* Calculate end byte, but cap it at file size - 1 because end range is inclusive */
+	end_byte = offset + length - 1;
+	if (end_byte >= file_size) {
+		end_byte = file_size - 1;
+	}
+
+	snprintf(s, sizeof(s), "Range: bytes=%llu-%llu", offset, end_byte);
+	return curl_slist_append(slist, s);
+}
+
 static void fio_http_cleanup(struct thread_data *td)
 {
 	struct http_data *http = td->io_ops_data;
@@ -636,30 +680,39 @@ static enum fio_q_status fio_http_queue(struct thread_data *td,
 	struct http_options *o = td->eo;
 	struct http_curl_stream _curl_stream;
 	struct curl_slist *slist = NULL;
-	char object[512];
+	char object_path_buf[512];
+	char *object_path;
 	char url[1024];
 	long status;
 	CURLcode res;
 
 	fio_ro_check(td, io_u);
 	memset(&_curl_stream, 0, sizeof(_curl_stream));
-	snprintf(object, sizeof(object), "%s_%llu_%llu", io_u->file->file_name,
-		io_u->offset, io_u->xfer_buflen);
+	if (o->object_mode == FIO_HTTP_OBJECT_BLOCK) {
+		snprintf(object_path_buf, sizeof(object_path_buf), "%s_%llu_%llu", io_u->file->file_name,
+			io_u->offset, io_u->xfer_buflen);
+		object_path = object_path_buf;
+	} else
+		object_path = io_u->file->file_name;
 	if (o->https == FIO_HTTPS_OFF)
-		snprintf(url, sizeof(url), "http://%s%s", o->host, object);
+		snprintf(url, sizeof(url), "http://%s%s", o->host, object_path);
 	else
-		snprintf(url, sizeof(url), "https://%s%s", o->host, object);
+		snprintf(url, sizeof(url), "https://%s%s", o->host, object_path);
+
 	curl_easy_setopt(http->curl, CURLOPT_URL, url);
 	_curl_stream.buf = io_u->xfer_buf;
 	_curl_stream.max = io_u->xfer_buflen;
 	curl_easy_setopt(http->curl, CURLOPT_SEEKDATA, &_curl_stream);
 	curl_easy_setopt(http->curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)io_u->xfer_buflen);
 
+	if (io_u->ddir == DDIR_READ && o->object_mode == FIO_HTTP_OBJECT_RANGE)
+		slist = _append_range_header(slist, io_u->offset, io_u->xfer_buflen, io_u->file->real_file_size);
+
 	if (o->mode == FIO_HTTP_S3)
-		_add_aws_auth_header(http->curl, slist, o, io_u->ddir, object,
+		_add_aws_auth_header(http->curl, slist, o, io_u->ddir, object_path,
 			io_u->xfer_buf, io_u->xfer_buflen);
 	else if (o->mode == FIO_HTTP_SWIFT)
-		_add_swift_header(http->curl, slist, o, io_u->ddir, object,
+		_add_swift_header(http->curl, slist, o, io_u->ddir, object_path,
 			io_u->xfer_buf, io_u->xfer_buflen);
 
 	if (io_u->ddir == DDIR_WRITE) {
@@ -681,7 +734,9 @@ static enum fio_q_status fio_http_queue(struct thread_data *td,
 		res = curl_easy_perform(http->curl);
 		if (res == CURLE_OK) {
 			curl_easy_getinfo(http->curl, CURLINFO_RESPONSE_CODE, &status);
-			if (status == 200)
+			/* 206 "Partial Content" means success when using the
+			 * Range header */
+			if (status == 200 || (o->object_mode == FIO_HTTP_OBJECT_RANGE && status == 206))
 				goto out;
 			else if (status == 404) {
 				/* Object doesn't exist. Pretend we read

fio.1

@@ -2623,6 +2623,26 @@ Enable verbose requests from libcurl. Useful for debugging. 1 turns on
 verbose logging from libcurl, 2 additionally enables HTTP IO tracing.
 Default is \fB0\fR
 .TP
+.BI (http)http_object_mode \fR=\fPstr
+How to structure objects for HTTP IO: block or range. Default is \fBblock\fR.
+.RS
+.RS
+.TP
+.B block
+One object is created for every block. The HTTP engine treats \fBblocksize\fR
+as the size of the object to read or write, and appends the block start/end
+offsets to the \fBfilename\fR to create the target object path. Reads and
+writes operate on whole objects at a time.
+.TP
+.B range
+One object is created for every file. The object path is the filename directly
+for both read and write I/O. For read requests, the \fBblocksize\fR and
+\fBoffset\fR will be used to set the "Range" header on read requests to issue
+partial reads of the object. For write requests, blocksize is used to set the
+size of the object, the same as in \fBblock\fR mode.
+.RE
+.RE
+.TP
 .BI (mtd)skip_bad \fR=\fPbool
 Skip operations against known bad blocks.
 .TP