AutoBatch cudnn.benchmark=True fix (#9448)

* AutoBatch `cudnn.benchmark=True` fix

May resolve #9287

Signed-off-by: Glenn Jocher <[email protected]>

* Update autobatch.py

Signed-off-by: Glenn Jocher <[email protected]>

* Update autobatch.py

Signed-off-by: Glenn Jocher <[email protected]>

* Update general.py

Signed-off-by: Glenn Jocher <[email protected]>

Signed-off-by: Glenn Jocher <[email protected]>

Author: glenn-jocher

utils/autobatch.py

@@ -33,6 +33,9 @@ def autobatch(model, imgsz=640, fraction=0.8, batch_size=16):
     if device.type == 'cpu':
         LOGGER.info(f'{prefix}CUDA not detected, using default CPU batch-size {batch_size}')
         return batch_size
+    if torch.backends.cudnn.benchmark:
+        LOGGER.info(f'{prefix} ⚠️ Requires torch.backends.cudnn.benchmark=False, using default batch-size {batch_size}')
+        return batch_size
 
     # Inspect CUDA memory
     gb = 1 << 30  # bytes to GiB (1024 ** 3)

utils/general.py

@@ -223,7 +223,7 @@ def init_seeds(seed=0, deterministic=False):
     torch.manual_seed(seed)
     torch.cuda.manual_seed(seed)
     torch.cuda.manual_seed_all(seed)  # for Multi-GPU, exception safe
-    torch.backends.cudnn.benchmark = True  # for faster training
+    # torch.backends.cudnn.benchmark = True  # AutoBatch problem https://github.com/ultralytics/yolov5/issues/9287
     if deterministic and check_version(torch.__version__, '1.12.0'):  # https://github.com/ultralytics/yolov5/pull/8213
         torch.use_deterministic_algorithms(True)
         torch.backends.cudnn.deterministic = True