Cache locality improvement for deflate State.lookahead

Author: brianpane

zlib-rs/src/deflate.rs

@@ -316,7 +316,6 @@ pub fn init(stream: &mut z_stream, config: DeflateConfig) -> ReturnCode {
 
         // window
         w_size,
-        w_mask: w_size - 1,
 
         // allocated values
         window,
@@ -374,7 +373,7 @@ pub fn init(stream: &mut z_stream, config: DeflateConfig) -> ReturnCode {
         _cache_line_1: (),
         _cache_line_2: (),
         _cache_line_3: (),
-        _padding_0: 0,
+        _padding_0: [0; 16],
     };
 
     unsafe { state_allocation.as_ptr().write(state) }; // FIXME: write is stable for NonNull since 1.80.0
@@ -667,7 +666,6 @@ pub fn copy<'a>(
         static_len: source_state.static_len,
         insert: source_state.insert,
         w_size: source_state.w_size,
-        w_mask: source_state.w_mask,
         lookahead: source_state.lookahead,
         prev,
         head,
@@ -1253,7 +1251,8 @@ pub(crate) struct State<'a> {
 
     pub(crate) window: Window<'a>,
     pub(crate) w_size: usize, /* LZ77 window size (32K by default) */
-    pub(crate) w_mask: usize, /* w_size - 1 */
+
+    pub(crate) lookahead: usize, /* number of valid bytes ahead in window */
 
     _cache_line_0: (),
 
@@ -1334,15 +1333,13 @@ pub(crate) struct State<'a> {
     /// bytes at end of window left to insert
     pub(crate) insert: usize,
 
-    pub(crate) lookahead: usize, /* number of valid bytes ahead in window */
-
     ///  hash index of string to be inserted
     pub(crate) ins_h: u32,
 
     gzhead: Option<&'a mut gz_header>,
     gzindex: usize,
 
-    _padding_0: usize,
+    _padding_0: [u8; 16],
 
     _cache_line_3: (),
 
@@ -1394,6 +1391,10 @@ impl<'a> State<'a> {
         self.w_size.trailing_zeros()
     }
 
+    pub(crate) fn w_mask(&self) -> usize {
+        self.w_size - 1
+    }
+
     pub(crate) fn max_dist(&self) -> usize {
         self.w_size - MIN_LOOKAHEAD
     }

zlib-rs/src/deflate/hash_calc.rs

@@ -42,7 +42,7 @@ impl StandardHashCalc {
 
         let head = state.head.as_slice()[hm];
         if head != string as u16 {
-            state.prev.as_mut_slice()[string & state.w_mask] = head;
+            state.prev.as_mut_slice()[string & state.w_mask()] = head;
             state.head.as_mut_slice()[hm] = string as u16;
         }
 
@@ -56,6 +56,7 @@ impl StandardHashCalc {
         // .take(count) generates worse assembly
         let slice = &slice[..Ord::min(slice.len(), count + 3)];
 
+        let w_mask = state.w_mask();
         for (i, w) in slice.windows(4).enumerate() {
             let idx = string as u16 + i as u16;
 
@@ -65,7 +66,7 @@ impl StandardHashCalc {
 
             let head = state.head.as_slice()[hm];
             if head != idx {
-                state.prev.as_mut_slice()[idx as usize & state.w_mask] = head;
+                state.prev.as_mut_slice()[idx as usize & w_mask] = head;
                 state.head.as_mut_slice()[hm] = idx;
             }
         }
@@ -98,7 +99,7 @@ impl RollHashCalc {
 
         let head = state.head.as_slice()[hm];
         if head != string as u16 {
-            state.prev.as_mut_slice()[string & state.w_mask] = head;
+            state.prev.as_mut_slice()[string & state.w_mask()] = head;
             state.head.as_mut_slice()[hm] = string as u16;
         }
 
@@ -108,6 +109,7 @@ impl RollHashCalc {
     pub fn insert_string(state: &mut State, string: usize, count: usize) {
         let slice = &state.window.filled()[string + Self::HASH_CALC_OFFSET..][..count];
 
+        let w_mask = state.w_mask();
         for (i, val) in slice.iter().copied().enumerate() {
             let idx = string as u16 + i as u16;
 
@@ -117,7 +119,7 @@ impl RollHashCalc {
 
             let head = state.head.as_slice()[hm];
             if head != idx {
-                state.prev.as_mut_slice()[idx as usize & state.w_mask] = head;
+                state.prev.as_mut_slice()[idx as usize & w_mask] = head;
                 state.head.as_mut_slice()[hm] = idx;
             }
         }

zlib-rs/src/deflate/longest_match.rs

@@ -19,7 +19,7 @@ fn longest_match_help<const SLOW: bool>(
     let mut match_start = state.match_start;
 
     let strstart = state.strstart;
-    let wmask = state.w_mask;
+    let wmask = state.w_mask();
     let window = state.window.filled();
     let scan = &window[strstart..];
     let mut limit: Pos;