fix PRGN warnings + make room for more secrets

boazsegev · boazsegev · commit 0d54feb713dc · 2025-04-19T02:37:29.000-04:00
diff --git a/fio-stl.h b/fio-stl.h
@@ -3308,7 +3308,7 @@ Defining a Pseudo-Random Number Generator Function (deterministic / not)
       0x4bb8d885a0fe47d5ULL + seed_offset,                                     \
       0x95561f0927ad7ecdULL,                                                   \
       0};                                                                      \
-  extern void name##_reset(void) {                                             \
+  extern __attribute__((unused)) void name##_reset(void) {                     \
     name##___state[0] = 0x9c65875be1fce7b9ULL + seed_offset;                   \
     name##___state[1] = 0x7cc568e838f6a40dULL;                                 \
     name##___state[2] = 0x4bb8d885a0fe47d5ULL + seed_offset;                   \
@@ -3335,12 +3335,12 @@ Defining a Pseudo-Random Number Generator Function (deterministic / not)
     }                                                                          \
   }                                                                            \
   /** Re-seeds the PNGR so forked processes don't match. */                    \
-  extern void name##_on_fork(void *is_null) {                                  \
+  extern __attribute__((unused)) void name##_on_fork(void *is_null) {          \
     (void)is_null;                                                             \
     name##___state_reseed(name##___state);                                     \
   }                                                                            \
   /** Returns a 128 bit pseudo-random number. */                               \
-  extern fio_u128 name##128(void) {                                            \
+  extern __attribute__((unused)) fio_u128 name##128(void) {                    \
     fio_u256 r;                                                                \
     if (!((name##___state[4]++) & ((1ULL << reseed_log) - 1)) &&               \
         ((size_t)(reseed_log - 1) < 63))                                       \
@@ -3375,15 +3375,15 @@ Defining a Pseudo-Random Number Generator Function (deterministic / not)
     return r.u128[0];                                                          \
   }                                                                            \
   /** Returns a 64 bit pseudo-random number. */                                \
-  extern uint64_t name##64(void) {                                             \
+  extern __attribute__((unused)) uint64_t name##64(void) {                     \
     static size_t counter;                                                     \
     static fio_u128 r;                                                         \
     if (!((counter++) & 1))                                                    \
       r = name##128();                                                         \
     return r.u64[counter & 1];                                                 \
   }                                                                            \
   /** Fills the `dest` buffer with pseudo-random noise. */                     \
-  extern void name##_bytes(void *dest, size_t len) {                           \
+  extern __attribute__((unused)) void name##_bytes(void *dest, size_t len) {   \
     if (!dest || !len)                                                         \
       return;                                                                  \
     uint8_t *d = (uint8_t *)dest;                                              \
@@ -24500,6 +24500,12 @@ SFUNC fio_u512 fio_secret(void);
 /** Sets a (possibly shared) secret and stores its SHA512 hash. */
 SFUNC void fio_secret_set(char *str, size_t len, bool is_random);
 
+/** Sets a (possibly shared) secret and stores its SHA512 hash. */
+SFUNC void fio_secret_set_at(fio_u512 *secret, char *str, size_t len);
+
+/** Gets the SHA512 of a (possibly shared) secret. */
+SFUNC fio_u512 fio_secret_at(fio_u512 *secret);
+
 /* *****************************************************************************
 Module Implementation - possibly externed functions.
 ***************************************************************************** */
@@ -24509,6 +24515,8 @@ static fio_u512 fio___secret;
 static bool fio___secret_is_random;
 static uint64_t fio___secret_masker;
 
+FIO_DEFINE_RANDOM128_FN(static, fio___secret_rand, 1, 0)
+
 /** Returns true if the secret was randomly generated. */
 SFUNC bool fio_secret_is_random(void) { return fio___secret_is_random; }
 
@@ -24522,10 +24530,21 @@ SFUNC fio_u512 fio_secret(void) {
 /** Sets a (possibly shared) secret and stores its SHA512 hash. */
 SFUNC void fio_secret_set(char *str, size_t len, bool is_random) {
   if (!str || !len)
+    is_random = 1;
+  fio_secret_set_at(&fio___secret, str, len);
+  fio___secret_is_random = is_random;
+}
+
+/** Sets a (possibly shared) secret and stores its SHA512 hash. */
+SFUNC void fio_secret_set_at(fio_u512 *secret, char *str, size_t len) {
+  if (!secret)
     return;
+  fio_u512 random_buffer = {0};
   fio_u512 zero = {0};
   size_t i = 0;
   FIO_STR_INFO_TMP_VAR(from_hex, 4096);
+  if (!str)
+    len = 0;
   if (len > 8191)
     goto done;
   /* convert any Hex data to bytes */
@@ -24551,37 +24570,45 @@ SFUNC void fio_secret_set(char *str, size_t len, bool is_random) {
     str = from_hex.buf;
     len = from_hex.len;
   }
+  if (!len) {
+    str = (char *)random_buffer.u8;
+    len = sizeof(random_buffer);
+    fio___secret_rand_bytes(random_buffer.u8, sizeof(random_buffer));
+  }
 
 done:
-  fio___secret_is_random = is_random;
-  fio___secret = fio_sha512(str, len);
-  if (fio_u512_is_eq(&zero, &fio___secret)) {
-    fio___secret.u64[0] = len;
-    fio___secret = fio_sha512(fio___secret.u8, sizeof(fio___secret));
+
+  *secret = fio_sha512(str, len);
+  if (fio_u512_is_eq(&zero, secret)) {
+    secret->u64[0] = len;
+    secret[0] = fio_sha512(secret->u8, sizeof(*secret));
   }
-  while (!(fio___secret_masker = fio_rand64()))
+  while (!(fio___secret_masker = fio___secret_rand64()))
     ;
-  fio_u512_cxor64(&fio___secret, &fio___secret, fio___secret_masker);
+  fio_u512_cxor64(secret, secret, fio___secret_masker);
+}
+
+/** Gets the SHA512 of a (possibly shared) secret. */
+SFUNC fio_u512 fio_secret_at(fio_u512 *secret) {
+  fio_u512 r;
+  fio_u512_cxor64(&r, secret, fio___secret_masker);
+  return r;
 }
 
 FIO_CONSTRUCTOR(fio___secret_constructor) {
   char *str = NULL;
   size_t len = 0;
-  uint64_t fallback_secret = 0;
-  bool is_random = 0;
   if ((str = getenv("SECRET"))) {
     const char *secret_length = getenv("SECRET_LENGTH");
     len = secret_length ? fio_atol((char **)&secret_length) : 0;
     if (!len)
       len = strlen(str);
-  } else {
-    fallback_secret = fio_rand64();
-    str = (char *)&fallback_secret;
-    len = sizeof(fallback_secret);
-    is_random = 1;
+    if (!len)
+      str = NULL;
   }
-  fio_secret_set(str, len, is_random);
+  fio_secret_set(str, len, 0);
 }
+
 /* *****************************************************************************
 Module Cleanup
 ***************************************************************************** */
diff --git a/fio-stl.md b/fio-stl.md
@@ -6986,39 +6986,71 @@ Secrets should be kept secret.
 
 Secrets should **not** be kept in source files, which too often end up end up exposed.
 
-Additionally, secrets should **not** be used directly. It is better to use a hashed value of the secret, possible with some time based salt or spice.
+Secrets should **not** be logged (such as in case of a crash or a core dump).
 
-This way, if somehow information leaks regarding the secret, what is exposed is actually the hashed value and not the secret itself.
+Additionally, secrets should **not** be used directly if possible. It is better to use a hashed value of the secret, possible with some time based salt or spice. This way, if somehow information leaks regarding the secret, what is exposed is actually the hashed value and not the secret itself.
 
 For this reason, the most common place to place a secret is as a hashed value in the OS environment (often as a Hex encoded String).
 
-The following helper functions are defined:
+**Note**: some secrets, such as TLS certificates, are often stored as system files somewhere separate from the source code.
 
+To help with managing a program wide secret, the following helper functions are defined:
 
-#### `fio_secret_is_random`
+
+#### `fio_secret_set_at`
 
 ```c
-bool fio_secret_is_random(void);
+void fio_secret_set_at(fio_u512 *secret, char *str, size_t len);
 ```
 
-Returns true if the secret was randomly generated.
+Sets a (possibly shared) secret and stores its (masked) SHA512 hash in `secret`.
+
+**Note**: the SHA512 hash in `secret` is masked before it is stored, so that the final secret isn't logged in case of a core dump.
+
+#### `fio_secret_at`
+
+```c
+fio_u512 fio_secret_at(fio_u512 *secret);
+```
+
+Gets the SHA512 of a (possibly shared) masked secret stored in `secret`.
+
+Please store the returned value on the stack or not at all. The secret is stored masked in memory and unmasked copies should be temporary with short life-spans.
 
 #### `fio_secret`
 
 ```c
 fio_u512 fio_secret(void);
 ```
 
-Gets the SHA512 of a (possibly shared) secret.
+Returns the SHA512 of a (possibly shared) secret.
+
+Unless updated using `fio_secret_set`, this is either a random secret or the one derived from the `SECRET` environment variable.
+
+Please store the returned value on the stack or not at all. The secret is stored masked in memory and unmasked copies should be temporary with short life-spans.
 
 #### `fio_secret_set`
 
 ```c
-void fio_secret_set(char *str, size_t len, bool is_random);
+void fio_secret_set(char *secret, size_t len, bool is_random);
 ```
 
 Sets a (possibly shared) secret and stores its SHA512 hash.
 
+If `secret` is Hex encoded, it will be decoded before it is hashed and white spaces will be ignored.
+
+**Note**: the SHA512 hash is masked before it is stored, so that the final secret isn't logged in case of a core dump.
+
+
+#### `fio_secret_is_random`
+
+```c
+bool fio_secret_is_random(void);
+```
+
+Returns true if the secret was randomly generated.
+
+
 -------------------------------------------------------------------------------
 ## Dynamic Strings
 
diff --git a/fio-stl/000 core.h b/fio-stl/000 core.h
@@ -3271,7 +3271,7 @@ Defining a Pseudo-Random Number Generator Function (deterministic / not)
       0x4bb8d885a0fe47d5ULL + seed_offset,                                     \
       0x95561f0927ad7ecdULL,                                                   \
       0};                                                                      \
-  extern void name##_reset(void) {                                             \
+  extern __attribute__((unused)) void name##_reset(void) {                     \
     name##___state[0] = 0x9c65875be1fce7b9ULL + seed_offset;                   \
     name##___state[1] = 0x7cc568e838f6a40dULL;                                 \
     name##___state[2] = 0x4bb8d885a0fe47d5ULL + seed_offset;                   \
@@ -3298,12 +3298,12 @@ Defining a Pseudo-Random Number Generator Function (deterministic / not)
     }                                                                          \
   }                                                                            \
   /** Re-seeds the PNGR so forked processes don't match. */                    \
-  extern void name##_on_fork(void *is_null) {                                  \
+  extern __attribute__((unused)) void name##_on_fork(void *is_null) {          \
     (void)is_null;                                                             \
     name##___state_reseed(name##___state);                                     \
   }                                                                            \
   /** Returns a 128 bit pseudo-random number. */                               \
-  extern fio_u128 name##128(void) {                                            \
+  extern __attribute__((unused)) fio_u128 name##128(void) {                    \
     fio_u256 r;                                                                \
     if (!((name##___state[4]++) & ((1ULL << reseed_log) - 1)) &&               \
         ((size_t)(reseed_log - 1) < 63))                                       \
@@ -3338,15 +3338,15 @@ Defining a Pseudo-Random Number Generator Function (deterministic / not)
     return r.u128[0];                                                          \
   }                                                                            \
   /** Returns a 64 bit pseudo-random number. */                                \
-  extern uint64_t name##64(void) {                                             \
+  extern __attribute__((unused)) uint64_t name##64(void) {                     \
     static size_t counter;                                                     \
     static fio_u128 r;                                                         \
     if (!((counter++) & 1))                                                    \
       r = name##128();                                                         \
     return r.u64[counter & 1];                                                 \
   }                                                                            \
   /** Fills the `dest` buffer with pseudo-random noise. */                     \
-  extern void name##_bytes(void *dest, size_t len) {                           \
+  extern __attribute__((unused)) void name##_bytes(void *dest, size_t len) {   \
     if (!dest || !len)                                                         \
       return;                                                                  \
     uint8_t *d = (uint8_t *)dest;                                              \
diff --git a/fio-stl/160 secret.h b/fio-stl/160 secret.h
@@ -28,6 +28,12 @@ SFUNC fio_u512 fio_secret(void);
 /** Sets a (possibly shared) secret and stores its SHA512 hash. */
 SFUNC void fio_secret_set(char *str, size_t len, bool is_random);
 
+/** Sets a (possibly shared) secret and stores its SHA512 hash. */
+SFUNC void fio_secret_set_at(fio_u512 *secret, char *str, size_t len);
+
+/** Gets the SHA512 of a (possibly shared) secret. */
+SFUNC fio_u512 fio_secret_at(fio_u512 *secret);
+
 /* *****************************************************************************
 Module Implementation - possibly externed functions.
 ***************************************************************************** */
@@ -37,6 +43,8 @@ static fio_u512 fio___secret;
 static bool fio___secret_is_random;
 static uint64_t fio___secret_masker;
 
+FIO_DEFINE_RANDOM128_FN(static, fio___secret_rand, 1, 0)
+
 /** Returns true if the secret was randomly generated. */
 SFUNC bool fio_secret_is_random(void) { return fio___secret_is_random; }
 
@@ -50,10 +58,21 @@ SFUNC fio_u512 fio_secret(void) {
 /** Sets a (possibly shared) secret and stores its SHA512 hash. */
 SFUNC void fio_secret_set(char *str, size_t len, bool is_random) {
   if (!str || !len)
+    is_random = 1;
+  fio_secret_set_at(&fio___secret, str, len);
+  fio___secret_is_random = is_random;
+}
+
+/** Sets a (possibly shared) secret and stores its SHA512 hash. */
+SFUNC void fio_secret_set_at(fio_u512 *secret, char *str, size_t len) {
+  if (!secret)
     return;
+  fio_u512 random_buffer = {0};
   fio_u512 zero = {0};
   size_t i = 0;
   FIO_STR_INFO_TMP_VAR(from_hex, 4096);
+  if (!str)
+    len = 0;
   if (len > 8191)
     goto done;
   /* convert any Hex data to bytes */
@@ -79,37 +98,45 @@ SFUNC void fio_secret_set(char *str, size_t len, bool is_random) {
     str = from_hex.buf;
     len = from_hex.len;
   }
+  if (!len) {
+    str = (char *)random_buffer.u8;
+    len = sizeof(random_buffer);
+    fio___secret_rand_bytes(random_buffer.u8, sizeof(random_buffer));
+  }
 
 done:
-  fio___secret_is_random = is_random;
-  fio___secret = fio_sha512(str, len);
-  if (fio_u512_is_eq(&zero, &fio___secret)) {
-    fio___secret.u64[0] = len;
-    fio___secret = fio_sha512(fio___secret.u8, sizeof(fio___secret));
+
+  *secret = fio_sha512(str, len);
+  if (fio_u512_is_eq(&zero, secret)) {
+    secret->u64[0] = len;
+    secret[0] = fio_sha512(secret->u8, sizeof(*secret));
   }
-  while (!(fio___secret_masker = fio_rand64()))
+  while (!(fio___secret_masker = fio___secret_rand64()))
     ;
-  fio_u512_cxor64(&fio___secret, &fio___secret, fio___secret_masker);
+  fio_u512_cxor64(secret, secret, fio___secret_masker);
+}
+
+/** Gets the SHA512 of a (possibly shared) secret. */
+SFUNC fio_u512 fio_secret_at(fio_u512 *secret) {
+  fio_u512 r;
+  fio_u512_cxor64(&r, secret, fio___secret_masker);
+  return r;
 }
 
 FIO_CONSTRUCTOR(fio___secret_constructor) {
   char *str = NULL;
   size_t len = 0;
-  uint64_t fallback_secret = 0;
-  bool is_random = 0;
   if ((str = getenv("SECRET"))) {
     const char *secret_length = getenv("SECRET_LENGTH");
     len = secret_length ? fio_atol((char **)&secret_length) : 0;
     if (!len)
       len = strlen(str);
-  } else {
-    fallback_secret = fio_rand64();
-    str = (char *)&fallback_secret;
-    len = sizeof(fallback_secret);
-    is_random = 1;
+    if (!len)
+      str = NULL;
   }
-  fio_secret_set(str, len, is_random);
+  fio_secret_set(str, len, 0);
 }
+
 /* *****************************************************************************
 Module Cleanup
 ***************************************************************************** */
diff --git a/fio-stl/160 secret.md b/fio-stl/160 secret.md
