libexpr: Implement small string optimization for Value

This attempts to store small strings inline in the Value
struct. This is possible to do for 64 bit systems that are
little endian or on other systems where pointer tagging optimization is not used.

Another reason for this change is to start storing the string
length explicitly (at least for the small string case for now).

Author: xokdvium

src/libexpr/eval.cc

@@ -824,6 +824,12 @@ DebugTraceStacker::DebugTraceStacker(EvalState & evalState, DebugTrace t)
 
 void Value::mkString(std::string_view s)
 {
+    if constexpr (ValueStorage::maxSmallStringSize > 0) {
+        if (s.size() <= ValueStorage::maxSmallStringSize) {
+            ValueStorage::setSmallString(s);
+            return;
+        }
+    }
     mkStringNoCopy(makeImmutableString(s));
 }
 
@@ -843,7 +849,12 @@ static const char ** encodeContext(const NixStringContext & context)
 
 void Value::mkString(std::string_view s, const NixStringContext & context)
 {
-    mkStringNoCopy(makeImmutableString(s), encodeContext(context));
+    auto encodedContext = encodeContext(context);
+    if (encodedContext == nullptr) {
+        mkString(s);
+        return;
+    }
+    mkStringNoCopy(makeImmutableString(s), encodedContext);
 }
 
 void Value::mkStringMove(const char * s, const NixStringContext & context)
@@ -1958,8 +1969,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
     /* c_str() is not str().c_str() because we want to create a string
        Value. allocating a GC'd string directly and moving it into a
        Value lets us avoid an allocation and copy. */
-    const auto c_str = [&] {
-        char * result = allocString(sSize + 1);
+    const auto c_str = [&](char * result) {
         char * tmp = result;
         for (const auto & part : s) {
             memcpy(tmp, part->data(), part->size());
@@ -2041,8 +2051,23 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
                 .withFrame(env, *this)
                 .debugThrow();
         v.mkPath(state.rootPath(CanonPath(str())));
-    } else
-        v.mkStringMove(c_str(), context);
+    } else {
+        if (sSize == 0) {
+            v.mkStringMove("", context);
+            return;
+        }
+
+        if (sSize <= Value::maxSmallStringSize) {
+            /* +1 is required for the NUL terminator. */
+            std::array<char, Value::maxSmallStringSize + 1> result;
+            v.mkString(c_str(result.data()), context);
+            return;
+        }
+
+        char * result = allocString(sSize + 1);
+        v.mkStringMove(c_str(result), context);
+        result[sSize] = 0;
+    }
 }
 
 void ExprPos::eval(EvalState & state, Env & env, Value & v)

src/libexpr/include/nix/expr/symbol-table.hh

@@ -123,6 +123,8 @@ public:
         const auto & [v, idx] = key.store.add(SymbolValue{});
         if (size == 0) {
             v.mkStringNoCopy("", nullptr);
+        } else if (size <= Value::maxSmallStringSize) {
+            v.mkString(key.s);
         } else {
             auto s = key.alloc.allocate(size + 1);
             memcpy(s, key.s.data(), size);

src/libexpr/include/nix/expr/value.hh

@@ -5,6 +5,7 @@
 #include <span>
 #include <type_traits>
 #include <concepts>
+#include <bit>
 
 #include "nix/expr/eval-gc.hh"
 #include "nix/expr/value/context.hh"
@@ -47,6 +48,7 @@ typedef enum {
     /* layout: Single untaggable field */
     tListN,
     tString,
+    tSmallString,
     tPath,
 } InternalType;
 
@@ -323,14 +325,23 @@ inline constexpr InternalType payloadTypeToInternalType = PayloadTypeToInternalT
 template<std::size_t ptrSize, typename Enable = void>
 class ValueStorage : public detail::ValueBase
 {
+    static constexpr std::size_t smallStringStorageSize = std::max({
+#define NIX_VALUE_STORAGE_FIELD_SIZE(T, FIELD_NAME, DISCRIMINATOR) sizeof(T),
+        NIX_VALUE_STORAGE_FOR_EACH_FIELD(NIX_VALUE_STORAGE_FIELD_SIZE)
+#undef NIX_VALUE_STORAGE_FIELD_SIZE
+    });
+
 protected:
     using Payload = union
     {
 #define NIX_VALUE_STORAGE_DEFINE_FIELD(T, FIELD_NAME, DISCRIMINATOR) T FIELD_NAME;
         NIX_VALUE_STORAGE_FOR_EACH_FIELD(NIX_VALUE_STORAGE_DEFINE_FIELD)
 #undef NIX_VALUE_STORAGE_DEFINE_FIELD
+        std::array<char, smallStringStorageSize> smallString;
     };
 
+    static constexpr std::size_t maxSmallStringSize = smallStringStorageSize - 1;
+
 private:
     InternalType internalType = tUninitialized;
     Payload payload;
@@ -357,6 +368,30 @@ protected:
 #undef NIX_VALUE_STORAGE_GET_IMPL
 #undef NIX_VALUE_STORAGE_FOR_EACH_FIELD
 
+    void setSmallString(std::string_view s)
+    {
+        assert(s.size() <= maxSmallStringSize);
+        internalType = tSmallString;
+        payload.smallString = {};
+        /* Trick is the same as in Facebook's Folly string. Use the last byte
+           of the string to store the remaining capacity. This was it naturally
+           becomes the null terminator when string has the size (smallStringStorageSize - 1). */
+        payload.smallString.back() = maxSmallStringSize - s.size();
+        std::memcpy(payload.smallString.data(), s.data(), s.size());
+    }
+
+    std::size_t getSmallStringSize() const
+    {
+        std::size_t remainingCapacity = payload.smallString.back();
+        return maxSmallStringSize - remainingCapacity;
+    }
+
+    const char * getSmallStringData() const
+    {
+        /* This string is null terminated. See setSmallString. */
+        return payload.smallString.data();
+    }
+
     /** Get internal type currently occupying the storage. */
     InternalType getInternalType() const noexcept
     {
@@ -434,6 +469,7 @@ class ValueStorage<ptrSize, std::enable_if_t<detail::useBitPackedValueStorage<pt
         /* The order of these enumations must be the same as in InternalType. */
         pdListN, //< layout: Single untaggable field.
         pdString,
+        pdSmallString,
         pdPath,
         pdPairOfPointers, //< layout: Pair of pointers payload
     };
@@ -513,6 +549,7 @@ protected:
         /* The order must match that of the enumerations defined in InternalType. */
         case pdListN:
         case pdString:
+        case pdSmallString:
         case pdPath:
             return static_cast<InternalType>(tListN + (pd - pdListN));
         case pdPairOfPointers:
@@ -643,6 +680,56 @@ protected:
     {
         setUntaggablePayload<pdPath>(path.accessor, path.path);
     }
+
+    /**
+     * Pointer tagging doesn't play well with big endian systems (because the tag will be in the middle
+     * of the array), so we don't do this optimization on big endian systems.
+     *
+     * 14 = 8 + 8 - 1 (the type tag) - 1 (string size + null terminator)
+     */
+    static constexpr std::size_t maxSmallStringSize = std::endian::native == std::endian::little ? 14 : 0;
+
+    void setSmallString(std::string_view s)
+    {
+        assert(s.size() <= maxSmallStringSize);
+
+        std::size_t remainingCapacity = maxSmallStringSize - s.size();
+        payload = {pdSmallString, remainingCapacity << 56};
+
+        /* 7 - we are skipping the first tag byte (it's stored in the 3 least significant bits). */
+        {
+            auto firstDWord = s.substr(0, 7);
+            std::size_t bitPos = 8;
+            for (auto c : firstDWord) {
+                payload[0] |= (PackedPointer{static_cast<unsigned char>(c)} << bitPos);
+                bitPos += 8;
+            }
+
+            s.remove_prefix(firstDWord.size());
+        }
+
+        {
+            auto secondDWord = s;
+            assert(secondDWord.size() <= 7);
+            std::size_t bitPos = 0;
+            for (auto c : secondDWord) {
+                payload[1] |= (PackedPointer{static_cast<unsigned char>(c)} << bitPos);
+                bitPos += 8;
+            }
+        }
+    }
+
+    std::size_t getSmallStringSize() const
+    {
+        std::size_t remainingCapacity = payload[1] >> 56;
+        return maxSmallStringSize - remainingCapacity;
+    }
+
+    const char * getSmallStringData() const
+    {
+        /* Skip the type tag byte. */
+        return reinterpret_cast<const char *>(payload.data()) + 1;
+    }
 };
 
 /**
@@ -849,6 +936,10 @@ struct Value : public ValueStorage<sizeof(void *)>
     }
 
 public:
+    /**
+     * Maximum size of a string that can be stored inline without allocations.
+     */
+    using ValueStorage::maxSmallStringSize;
 
     /**
      * Never modify the backing `Value` object!
@@ -907,6 +998,7 @@ public:
         case tBool:
             return nBool;
         case tString:
+        case tSmallString:
             return nString;
         case tPath:
             return nPath;
@@ -1071,16 +1163,28 @@ public:
 
     std::string_view string_view() const noexcept
     {
+        if constexpr (maxSmallStringSize > 0) {
+            if (isa<tSmallString>())
+                return std::string_view{getSmallStringData(), getSmallStringSize()};
+        }
         return std::string_view(getStorage<StringWithContext>().c_str);
     }
 
     const char * c_str() const noexcept
     {
+        if constexpr (maxSmallStringSize > 0) {
+            if (isa<tSmallString>())
+                return getSmallStringData();
+        }
         return getStorage<StringWithContext>().c_str;
     }
 
     const char ** context() const noexcept
     {
+        if constexpr (maxSmallStringSize > 0) {
+            if (isa<tSmallString>())
+                return nullptr;
+        }
         return getStorage<StringWithContext>().context;
     }