Single writer option (#79)

* save_checkpoint perf monitoring

* Disable checkpoint save on exit

* local rank arg

* Single writer option

Author: tjruwase

fast_io/model_checkpoint/deepspeed_save_model.py

@@ -43,6 +43,7 @@ def _get_ds_config(args, writer_type):
             "io_buffer_size": args.io_buffer_mb * (1024**2),
             "io_buffer_double": not args.single_io_buffer,
             "show_statistics": not args.no_statistics,
+            "data_parallel": not args.single_writer
         }
 
     return ds_config

fast_io/model_checkpoint/save_model_utils.py

@@ -118,6 +118,9 @@ def parse_arguments():
                         action='store_true',
                         help='Disable double buffering of i/o buffer.')
 
+
+    parser.add_argument('--single_writer', action='store_true', help='Disable parallel rank writes of data parallel (replicated) state')
+
     args = parser.parse_args()
     print(f'args = {args}')
     return args