hint: Optimize byte ops in i/o and patch applying (#338)

Use bytearray to avoid repeatedly copying buffers when concatenating chunks for
every single patch. Store vocabulary as bytes to avoid doing UTF-8 encoding on
every patch with replacement ("v"). Remove ifs from the hint loading inner
loop.

Measurements show 5-10% speedup on big inputs (>10MB).

Author: emaxx-google

clang_delta/tests/test_clang_delta.py

@@ -3,7 +3,7 @@
 import re
 import subprocess
 import tempfile
-from typing import Union
+from typing import Optional, Union
 import unittest
 
 
@@ -38,19 +38,19 @@ def run_clang_delta(testcase: str, arguments: str) -> str:
     return subprocess.check_output(cmd, shell=True, encoding='utf8')
 
 
-def run_apply_hints(hints_file: Path, begin_index: int, end_index: int, testcase: str) -> str:
+def run_apply_hints(hints_file: Path, begin_index: Optional[int], end_index: Optional[int], testcase: str) -> str:
     hints_tool = Path(__file__).parent.parent.parent / 'cvise-cli.py'
     cmd = [
         hints_tool,
         '--action=apply-hints',
         '--hints-file',
         hints_file,
-        '--hint-begin-index',
-        str(begin_index),
-        '--hint-end-index',
-        str(end_index),
         get_testcase_path(testcase),
     ]
+    if begin_index is not None:
+        cmd += ['--hint-begin-index', str(begin_index)]
+    if end_index is not None:
+        cmd += ['--hint-end-index', str(end_index)]
     return subprocess.check_output(cmd, encoding='utf-8')
 
 
@@ -1165,8 +1165,8 @@ def test_replace_function_def_with_decl_simple_preserve_foo(self):
         self.check_clang_delta_hints(
             'replace-function-def-with-decl/simple.cpp',
             '--transformation=replace-function-def-with-decl --preserve-routine=Moo::foo',
-            begin_index=0,
-            end_index=9999,
+            begin_index=None,
+            end_index=None,
             output_file='replace-function-def-with-decl/simple.cpp.preserve_foo.output',
         )
 

cvise-cli.py

@@ -544,11 +544,11 @@ def do_reduce(args):
 def do_apply_hints(args):
     if args.hints_file is None:
         sys.exit('--hints-file is mandatory for --action=apply-hints')
-    if args.hint_begin_index is None:
-        sys.exit('--hint-begin-index is mandatory for --action=apply-hints')
-    if args.hint_end_index is None:
-        sys.exit('--hint-end-index is mandatory for --action=apply-hints')
-    if args.hint_begin_index >= args.hint_end_index:
+    if (
+        args.hint_begin_index is not None
+        and args.hint_end_index is not None
+        and args.hint_begin_index >= args.hint_end_index
+    ):
         sys.exit('HINT_BEGIN_INDEX must be smaller than HINT_END_INDEX')
     if len(args.test_cases) > 1:
         sys.exit('exactly one TEST_CASE must be supplied')

cvise/passes/balanced.py

@@ -36,7 +36,7 @@ def generate_hints(self, test_case: Path, *args, **kwargs):
         vocabulary = []
         if config.replacement:
             assert config.to_delete == Deletion.ALL
-            vocabulary.append(config.replacement)
+            vocabulary.append(config.replacement.encode())
 
         contents = test_case.read_bytes()
         prefixes = (

cvise/passes/blank.py

@@ -8,14 +8,14 @@
 
 class BlankPass(HintBasedPass):
     PATTERNS = {
-        'blankline': rb'^\s*$',
-        'hashline': rb'^#',
+        b'blankline': rb'^\s*$',
+        b'hashline': rb'^#',
     }
 
     def check_prerequisites(self):
         return True
 
-    def output_hint_types(self) -> List[str]:
+    def output_hint_types(self) -> List[bytes]:
         return list(self.PATTERNS.keys())
 
     def generate_hints(self, test_case: Path, *args, **kwargs):

cvise/passes/clanghints.py

@@ -139,7 +139,7 @@ def _generate_hints_for_standard(
         stdout = iter(stdout.splitlines())
         vocab_line = next(stdout, None)
         vocab_decoder = msgspec.json.Decoder(type=List[str])
-        vocab = vocab_decoder.decode(vocab_line) if vocab_line else []
+        vocab = [s.encode() for s in vocab_decoder.decode(vocab_line)] if vocab_line else []
 
         hints = []
         hint_decoder = msgspec.json.Decoder(type=Hint)

cvise/passes/clangmodulemap.py

@@ -17,11 +17,11 @@
 @unique
 class _Vocab(Enum):
     # Items must be listed in the index order; indices must be contiguous and start from zero.
-    MAKE_HEADER_NON_MODULAR = (0, 'make-header-non-modular')
-    DELETE_USE_DECL = (1, 'delete-use-decl')
-    DELETE_EMPTY_SUBMODULE = (2, 'delete-empty-submodule')
-    INLINE_SUBMODULE_CONTENTS = (3, 'inline-submodule-contents')
-    DELETE_LINE = (4, 'delete-line')
+    MAKE_HEADER_NON_MODULAR = (0, b'make-header-non-modular')
+    DELETE_USE_DECL = (1, b'delete-use-decl')
+    DELETE_EMPTY_SUBMODULE = (2, b'delete-empty-submodule')
+    INLINE_SUBMODULE_CONTENTS = (3, b'inline-submodule-contents')
+    DELETE_LINE = (4, b'delete-line')
 
 
 class ClangModuleMapPass(HintBasedPass):
@@ -36,19 +36,20 @@ def check_prerequisites(self):
     def supports_dir_test_cases(self):
         return True
 
-    def output_hint_types(self) -> List[str]:
+    def output_hint_types(self) -> List[bytes]:
         return [v.value[1] for v in _Vocab]
 
     def generate_hints(self, test_case: Path, *args, **kwargs):
         paths = list(test_case.rglob('*')) if test_case.is_dir() else [test_case]
         interesting_paths = [p for p in paths if _interesting_file(p)]
 
-        vocab: List[str] = [v.value[1] for v in _Vocab]  # collect all strings used in hints
+        vocab: List[bytes] = [v.value[1] for v in _Vocab]  # collect all strings used in hints
         hints: List[Hint] = []
         for path in interesting_paths:
             file = _parse_file(path)
 
-            vocab.append(str(path.relative_to(test_case)))
+            rel_path = path.relative_to(test_case)
+            vocab.append(str(rel_path).encode())
             file_id = len(vocab) - 1
 
             for mod in file.modules:

cvise/passes/clexhints.py

@@ -35,7 +35,7 @@ def generate_hints(self, test_case: Path, process_event_notifier: ProcessEventNo
             work_dir = test_case
             paths = [p.relative_to(test_case) for p in test_case.rglob('*') if not p.is_dir()]
             stdin = b'\n'.join(bytes(p) for p in paths)
-            files_vocab = [str(p) for p in paths]
+            files_vocab = [str(p).encode() for p in paths]
             cmd_arg = '--'
         else:
             work_dir = '.'
@@ -52,7 +52,7 @@ def generate_hints(self, test_case: Path, process_event_notifier: ProcessEventNo
         stdout = iter(stdout.splitlines())
         vocab_line = next(stdout, None)
         vocab_decoder = msgspec.json.Decoder(type=List[str])
-        orig_vocab = vocab_decoder.decode(vocab_line) if vocab_line else []
+        orig_vocab = [s.encode() for s in vocab_decoder.decode(vocab_line)] if vocab_line else []
 
         hints = []
         hint_decoder = msgspec.json.Decoder(type=Hint)

cvise/passes/comments.py

@@ -8,7 +8,7 @@
 
 class CommentsPass(HintBasedPass):
     # The hints vocabulary - strings used by our hint.
-    INITIAL_VOCAB = ('multi-line', 'single-line')
+    INITIAL_VOCAB = (b'multi-line', b'single-line')
     # The indices must match the order in INITIAL_VOCAB.
     MULTI_LINE_VOCAB_ID = 0
     SINGLE_LINE_VOCAB_ID = 1
@@ -19,7 +19,7 @@ def check_prerequisites(self):
     def supports_dir_test_cases(self):
         return True
 
-    def output_hint_types(self) -> List[str]:
+    def output_hint_types(self) -> List[bytes]:
         return list(self.INITIAL_VOCAB)
 
     def generate_hints(self, test_case: Path, *args, **kwargs):
@@ -28,7 +28,8 @@ def generate_hints(self, test_case: Path, *args, **kwargs):
         if test_case.is_dir():
             for path in test_case.rglob('*'):
                 if not path.is_dir():
-                    vocab.append(str(path.relative_to(test_case)))
+                    rel_path = path.relative_to(test_case)
+                    vocab.append(str(rel_path).encode())
                     file_id = len(vocab) - 1
                     hints += self._generate_hints_for_file(path, file_id)
         else:

cvise/passes/hint_based.py

@@ -25,7 +25,7 @@ class PerTypeHintState:
     """
 
     # A hint type for which this state is for; an empty string if a hint doesn't explicitly specify types.
-    type: str
+    type: bytes
     # Only the base name, not a full path - it's a small optimization (and we anyway need to store the tmp dir in the
     # HintState).
     hints_file_name: Path
@@ -62,7 +62,7 @@ class SpecialHintState:
     another - hence there's no underlying_state here.
     """
 
-    type: str
+    type: bytes
     hints_file_name: Path
     hint_count: int
 
@@ -97,10 +97,10 @@ def __repr__(self):
         parts = []
         for i, s in enumerate(self.per_type_states):
             mark = '[*]' if i == self.ptr and len(self.per_type_states) > 1 else ''
-            type_s = s.type + ': ' if s.type else ''
+            type_s = s.type.decode() + ': ' if s.type else ''
             parts.append(f'{mark}{type_s}{s.underlying_state.compact_repr()}')
         for s in self.special_hints:
-            parts.append(f'{s.type}: {s.hint_count}')
+            parts.append(f'{s.type.decode()}: {s.hint_count}')
         return f'HintState({", ".join(parts)})'
 
     def real_chunk(self) -> int:
@@ -134,7 +134,7 @@ def advance(self) -> Union[HintState, None]:
             special_hints=self.special_hints,
         )
 
-    def advance_on_success(self, type_to_bundle: Dict[str, HintBundle]):
+    def advance_on_success(self, type_to_bundle: Dict[bytes, HintBundle]):
         sub_states = []
         # Advance all previously present hint types' substates. We ignore any newly appearing hint types because it's
         # nontrivial to distinguish geniunely new hints from those that we (unsuccessfully) checked.
@@ -152,7 +152,7 @@ def advance_on_success(self, type_to_bundle: Dict[str, HintBundle]):
             tmp_dir=self.tmp_dir, per_type_states=tuple(sub_states), ptr=0, special_hints=self.special_hints
         )
 
-    def hint_bundle_paths(self) -> Dict[str, Path]:
+    def hint_bundle_paths(self) -> Dict[bytes, Path]:
         return {
             substate.type: self.tmp_dir / substate.hints_file_name
             for substate in self.per_type_states + self.special_hints
@@ -186,7 +186,7 @@ def generate_hints(
     ) -> HintBundle:
         raise NotImplementedError(f"Class {type(self).__name__} has not implemented 'generate_hints'!")
 
-    def input_hint_types(self) -> List[str]:
+    def input_hint_types(self) -> List[bytes]:
         """Declares hint types that are consumed by this pass as inputs.
 
         Intended to be overridden by subclasses, in cases where dependencies between passes need to be implemented:
@@ -196,7 +196,7 @@ def input_hint_types(self) -> List[str]:
         """
         return []
 
-    def output_hint_types(self) -> List[str]:
+    def output_hint_types(self) -> List[bytes]:
         """Declares hint types that are produced by this pass.
 
         A pass must override this method if it produces hints with a nonempty type (the "t" field).
@@ -304,16 +304,16 @@ def advance_on_success_from_hints(self, bundle: HintBundle, state: HintState) ->
         store_hints_per_type(state.tmp_dir, type_to_bundle)
         return state.advance_on_success(type_to_bundle)
 
-    def backfill_pass_names(self, type_to_bundle: Dict[str, HintBundle]) -> None:
+    def backfill_pass_names(self, type_to_bundle: Dict[bytes, HintBundle]) -> None:
         for bundle in type_to_bundle.values():
             if not bundle.pass_name:
                 bundle.pass_name = repr(self)
 
 
-def store_hints_per_type(tmp_dir: Path, type_to_bundle: Dict[str, HintBundle]) -> Dict[str, Path]:
+def store_hints_per_type(tmp_dir: Path, type_to_bundle: Dict[bytes, HintBundle]) -> Dict[bytes, Path]:
     type_to_file_name = {}
     for type, sub_bundle in type_to_bundle.items():
-        file_name = Path(HINTS_FILE_NAME_TEMPLATE.format(type=type))
+        file_name = Path(HINTS_FILE_NAME_TEMPLATE.format(type=type.decode()))
         store_hints(sub_bundle, tmp_dir / file_name)
         type_to_file_name[type] = file_name
     return type_to_file_name

cvise/passes/lines.py

@@ -21,7 +21,8 @@ def generate_hints(self, test_case: Path, process_event_notifier: ProcessEventNo
         if test_case.is_dir():
             for path in test_case.rglob('*'):
                 if not path.is_dir():
-                    vocab.append(str(path.relative_to(test_case)))
+                    rel_path = path.relative_to(test_case)
+                    vocab.append(str(rel_path).encode())
                     file_id = len(vocab) - 1
                     hints += self._generate_hints_for_file(path, decoder, process_event_notifier, file_id)
         else: