Hex encoding and byte buf utilities (#328)

* Add secondary hex encode variant that respects current buffer position and doesn't 0-terminate
* Add helper byte buf reserve function that is relative to the current length of the buffer

Author: bretambrose

.cbmc-batch/jobs/aws_byte_buf_reserve_relative/Makefile

@@ -0,0 +1,30 @@
+# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may not use
+# this file except in compliance with the License. A copy of the License is
+# located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing permissions and
+# limitations under the License.
+
+###########
+CBMC_UNWINDSET =
+
+CBMCFLAGS +=
+
+DEPENDENCIES += $(HELPERDIR)/source/make_common_data_structures.c
+DEPENDENCIES += $(HELPERDIR)/source/proof_allocators.c
+DEPENDENCIES += $(HELPERDIR)/stubs/error.c
+DEPENDENCIES += $(HELPERDIR)/stubs/memcpy_override_no_op.c
+DEPENDENCIES += $(HELPERDIR)/stubs/memset_override_no_op.c
+DEPENDENCIES += $(SRCDIR)/source/byte_buf.c
+DEPENDENCIES += $(SRCDIR)/source/common.c
+
+ENTRY = aws_byte_buf_reserve_relative_harness
+###########
+
+include ../Makefile.common

.cbmc-batch/jobs/aws_byte_buf_reserve_relative/aws_byte_buf_reserve_relative_harness.c

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+#include <aws/common/byte_buf.h>
+#include <proof_helpers/make_common_data_structures.h>
+
+void aws_byte_buf_reserve_relative_harness() {
+    struct aws_byte_buf buf;
+    ensure_byte_buf_has_allocated_buffer_member(&buf);
+    __CPROVER_assume(aws_byte_buf_is_valid(&buf));
+
+    struct aws_byte_buf old = buf;
+    size_t requested_capacity;
+    int rval = aws_byte_buf_reserve_relative(&buf, requested_capacity);
+
+    if (rval == AWS_OP_SUCCESS) {
+        assert(buf.capacity >= (old.len + requested_capacity));
+    }
+    assert(aws_byte_buf_is_valid(&buf));
+    assert(is_byte_buf_expected_alloc(&buf));
+}

.cbmc-batch/jobs/aws_byte_buf_reserve_relative/cbmc-batch.yaml

@@ -0,0 +1,4 @@
+jobos: ubuntu16
+cbmcflags: "--bounds-check;--div-by-zero-check;--float-overflow-check;--nan-check;--pointer-check;--pointer-overflow-check;--signed-overflow-check;--undefined-shift-check;--unsigned-overflow-check;--unwinding-assertions;--unwind;1"
+goto: aws_byte_buf_reserve_relative_harness.goto
+expected: "SUCCESSFUL"

include/aws/common/byte_buf.h

@@ -348,6 +348,16 @@ int aws_byte_buf_append_dynamic(struct aws_byte_buf *to, const struct aws_byte_c
 AWS_COMMON_API
 int aws_byte_buf_reserve(struct aws_byte_buf *buffer, size_t requested_capacity);
 
+/**
+ * Convenience function that attempts to increase the capacity of a buffer relative to the current
+ * length.
+ *
+ *  aws_byte_buf_reserve_relative(buf, x) ~~ aws_byte_buf_reserve(buf, buf->len + x)
+ *
+ */
+AWS_COMMON_API
+int aws_byte_buf_reserve_relative(struct aws_byte_buf *buffer, size_t additional_length);
+
 /**
  * Concatenates a variable number of struct aws_byte_buf * into destination.
  * Number of args must be greater than 1. If dest is too small,

include/aws/common/encoding.h

@@ -34,11 +34,21 @@ int aws_hex_compute_encoded_len(size_t to_encode_len, size_t *encoded_length);
 
 /*
  * Base 16 (hex) encodes the contents of to_encode and stores the result in
- * output.
+ * output.  0 terminates the result.  Assumes the buffer is empty and does not resize on
+ * insufficient capacity.
  */
 AWS_COMMON_API
 int aws_hex_encode(const struct aws_byte_cursor *AWS_RESTRICT to_encode, struct aws_byte_buf *AWS_RESTRICT output);
 
+/*
+ * Base 16 (hex) encodes the contents of to_encode and appends the result in
+ * output.  Does not 0-terminate.  Grows the destination buffer dynamically if necessary.
+ */
+AWS_COMMON_API
+int aws_hex_encode_append_dynamic(
+    const struct aws_byte_cursor *AWS_RESTRICT to_encode,
+    struct aws_byte_buf *AWS_RESTRICT output);
+
 /*
  * computes the length necessary to store the result of aws_hex_decode().
  * returns -1 on failure, and 0 on success. decoded_len will be set on success.

source/byte_buf.c

@@ -539,6 +539,17 @@ int aws_byte_buf_reserve(struct aws_byte_buf *buffer, size_t requested_capacity)
     return AWS_OP_SUCCESS;
 }
 
+int aws_byte_buf_reserve_relative(struct aws_byte_buf *buffer, size_t additional_length) {
+    AWS_PRECONDITION(aws_byte_buf_is_valid(buffer));
+
+    size_t requested_capacity = 0;
+    if (AWS_UNLIKELY(aws_add_size_checked(buffer->len, additional_length, &requested_capacity))) {
+        return AWS_OP_ERR;
+    }
+
+    return aws_byte_buf_reserve(buffer, requested_capacity);
+}
+
 struct aws_byte_cursor aws_byte_cursor_right_trim_pred(
     const struct aws_byte_cursor *source,
     aws_byte_predicate_fn *predicate) {

source/encoding.c

@@ -115,6 +115,33 @@ int aws_hex_encode(const struct aws_byte_cursor *AWS_RESTRICT to_encode, struct
     return AWS_OP_SUCCESS;
 }
 
+int aws_hex_encode_append_dynamic(
+    const struct aws_byte_cursor *AWS_RESTRICT to_encode,
+    struct aws_byte_buf *AWS_RESTRICT output) {
+    assert(to_encode->ptr);
+    assert(aws_byte_buf_is_valid(output));
+
+    size_t encoded_len = 0;
+    if (AWS_UNLIKELY(aws_add_size_checked(to_encode->len, to_encode->len, &encoded_len))) {
+        return AWS_OP_ERR;
+    }
+
+    if (AWS_UNLIKELY(aws_byte_buf_reserve_relative(output, encoded_len))) {
+        return AWS_OP_ERR;
+    }
+
+    size_t written = output->len;
+    for (size_t i = 0; i < to_encode->len; ++i) {
+
+        output->buffer[written++] = HEX_CHARS[to_encode->ptr[i] >> 4 & 0x0f];
+        output->buffer[written++] = HEX_CHARS[to_encode->ptr[i] & 0x0f];
+    }
+
+    output->len += encoded_len;
+
+    return AWS_OP_SUCCESS;
+}
+
 static int s_hex_decode_char_to_int(char character, uint8_t *int_val) {
     if (character >= 'a' && character <= 'f') {
         *int_val = (uint8_t)(10 + (character - 'a'));

tests/CMakeLists.txt

@@ -95,6 +95,8 @@ add_test_case(hex_encoding_invalid_buffer_size_test)
 add_test_case(hex_encoding_highbyte_string_test)
 add_test_case(hex_encoding_overflow_test)
 add_test_case(hex_encoding_invalid_string_test)
+add_test_case(hex_encoding_append_dynamic_test_case_empty)
+add_test_case(hex_encoding_append_dynamic_test_case_fooba)
 add_test_case(base64_encoding_test_case_empty_test)
 add_test_case(base64_encoding_test_case_f_test)
 add_test_case(base64_encoding_test_case_fo_test)
@@ -217,6 +219,7 @@ add_test_case(test_byte_buf_append_dynamic)
 add_test_case(test_byte_buf_append_lookup_success)
 add_test_case(test_byte_buf_append_lookup_failure)
 add_test_case(test_byte_buf_reserve)
+add_test_case(test_byte_buf_reserve_relative)
 
 add_test_case(byte_swap_test)
 

tests/byte_buf_test.c

@@ -634,3 +634,27 @@ static int s_test_byte_buf_reserve(struct aws_allocator *allocator, void *ctx) {
     return 0;
 }
 AWS_TEST_CASE(test_byte_buf_reserve, s_test_byte_buf_reserve)
+
+static int s_test_byte_buf_reserve_relative(struct aws_allocator *allocator, void *ctx) {
+    (void)ctx;
+    (void)allocator;
+
+    struct aws_byte_buf buffer;
+    aws_byte_buf_init(&buffer, allocator, 1);
+
+    struct aws_byte_cursor prefix_cursor = aws_byte_cursor_from_string(s_reserve_test_prefix);
+
+    ASSERT_TRUE(aws_byte_buf_reserve_relative(&buffer, prefix_cursor.len) == AWS_OP_SUCCESS);
+    ASSERT_TRUE(aws_byte_buf_append(&buffer, &prefix_cursor) == AWS_OP_SUCCESS);
+
+    struct aws_byte_cursor suffix_cursor = aws_byte_cursor_from_string(s_reserve_test_suffix);
+    ASSERT_TRUE(aws_byte_buf_reserve_relative(&buffer, suffix_cursor.len) == AWS_OP_SUCCESS);
+    ASSERT_TRUE(aws_byte_buf_append(&buffer, &suffix_cursor) == AWS_OP_SUCCESS);
+
+    ASSERT_TRUE(aws_byte_buf_eq_c_str(&buffer, (char *)s_reserve_test_prefix_concatenated->bytes));
+
+    aws_byte_buf_clean_up(&buffer);
+
+    return 0;
+}
+AWS_TEST_CASE(test_byte_buf_reserve_relative, s_test_byte_buf_reserve_relative)

tests/encoding_test.c

@@ -168,6 +168,17 @@ static int s_hex_encoding_test_case_foobar(struct aws_allocator *allocator, void
 
 AWS_TEST_CASE(hex_encoding_test_case_foobar_test, s_hex_encoding_test_case_foobar)
 
+static int s_hex_encoding_append_test_case(struct aws_allocator *allocator, void *ctx) {
+    (void)ctx;
+
+    char test_data[] = "foobar";
+    char expected[] = "666f6f626172";
+
+    return s_run_hex_encoding_test_case(allocator, test_data, sizeof(test_data), expected, sizeof(expected) - 1);
+}
+
+AWS_TEST_CASE(hex_encoding_append_test_case, s_hex_encoding_append_test_case)
+
 static int s_hex_encoding_test_case_missing_leading_zero_fn(struct aws_allocator *allocator, void *ctx) {
     (void)allocator;
     (void)ctx;
@@ -878,3 +889,80 @@ static int s_uint16_buffer_signed_negative_test_fn(struct aws_allocator *allocat
 }
 
 AWS_TEST_CASE(uint16_buffer_signed_negative_test, s_uint16_buffer_signed_negative_test_fn)
+
+static int s_run_hex_encoding_append_dynamic_test_case(
+    struct aws_allocator *allocator,
+    const char *test_str,
+    const char *expected,
+    size_t initial_capacity,
+    size_t starting_offset) {
+
+    size_t output_size = 2 * strlen(test_str);
+
+    struct aws_byte_cursor to_encode = aws_byte_cursor_from_c_str(test_str);
+
+    struct aws_byte_buf dest;
+    ASSERT_SUCCESS(aws_byte_buf_init(&dest, allocator, initial_capacity));
+    memset(dest.buffer, 0xdd, dest.capacity);
+
+    dest.len = starting_offset;
+
+    ASSERT_SUCCESS(aws_hex_encode_append_dynamic(&to_encode, &dest), "encode call should have succeeded");
+
+    size_t expected_size = strlen(expected);
+
+    ASSERT_BIN_ARRAYS_EQUALS(
+        expected,
+        expected_size,
+        dest.buffer + starting_offset,
+        output_size,
+        "Encode output should have been {%s}, was {%s}.",
+        expected,
+        dest.buffer + starting_offset);
+    ASSERT_INT_EQUALS(output_size, dest.len - starting_offset);
+
+    for (size_t i = 0; i < starting_offset; ++i) {
+        ASSERT_INT_EQUALS(
+            (unsigned char)*(dest.buffer + i),
+            (unsigned char)0xdd,
+            "Write should not have occurred before the the encoding's starting position.");
+    }
+
+    for (size_t i = starting_offset + output_size; i < dest.capacity; ++i) {
+        ASSERT_INT_EQUALS(
+            (unsigned char)*(dest.buffer + i),
+            (unsigned char)0xdd,
+            "Write should not have occurred after the encoding's final position.");
+    }
+
+    aws_byte_buf_clean_up(&dest);
+    return 0;
+}
+
+static int s_hex_encoding_append_dynamic_test_case_fooba(struct aws_allocator *allocator, void *ctx) {
+    (void)ctx;
+
+    char test_data[] = "fooba";
+    char expected[] = "666f6f6261";
+
+    ASSERT_TRUE(s_run_hex_encoding_append_dynamic_test_case(allocator, test_data, expected, 5, 3) == AWS_OP_SUCCESS);
+    ASSERT_TRUE(s_run_hex_encoding_append_dynamic_test_case(allocator, test_data, expected, 50, 3) == AWS_OP_SUCCESS);
+
+    return 0;
+}
+
+AWS_TEST_CASE(hex_encoding_append_dynamic_test_case_fooba, s_hex_encoding_append_dynamic_test_case_fooba)
+
+static int s_hex_encoding_append_dynamic_test_case_empty(struct aws_allocator *allocator, void *ctx) {
+    (void)ctx;
+
+    char test_data[] = "";
+    char expected[] = "";
+
+    ASSERT_TRUE(s_run_hex_encoding_append_dynamic_test_case(allocator, test_data, expected, 5, 3) == AWS_OP_SUCCESS);
+    ASSERT_TRUE(s_run_hex_encoding_append_dynamic_test_case(allocator, test_data, expected, 50, 3) == AWS_OP_SUCCESS);
+
+    return 0;
+}
+
+AWS_TEST_CASE(hex_encoding_append_dynamic_test_case_empty, s_hex_encoding_append_dynamic_test_case_empty)