Fix Accuracy Difference during Benchmark and Quantization on TF Examples (#1632)

Signed-off-by: zehao-intel <[email protected]>

Author: zehao-intel

examples/tensorflow/graph_networks/graphsage/quantization/ptq/main.py

@@ -138,7 +138,7 @@ def eval_func(size, output_tensor, minibatch, test):
             labels.append(batch_labels)
             iter_num += 1
             total_time += time_consume
-            if iteration and iter_num >= iteration:
+            if iteration != -1 and iter_num >= iteration:
                 break
         tf_logging.warn('\n---> Stop iteration {0}'.format(str(iter_num)))
         val_preds = np.vstack(val_preds)

examples/tensorflow/image_recognition/tensorflow_models/vision_transformer/quantization/ptq/main.py

@@ -52,7 +52,7 @@
 arg_parser.add_argument('--int8', dest='int8', action='store_true', help='whether to use int8 model for benchmark')
 args = arg_parser.parse_args()
 
-def evaluate(model, eval_dataloader, metric, postprocess=None):
+def evaluate(model, eval_dataloader, postprocess=None):
     """Custom evaluate function to estimate the accuracy of the model.
 
     Args:
@@ -61,12 +61,14 @@ def evaluate(model, eval_dataloader, metric, postprocess=None):
     Returns:
         accuracy (float): evaluation result, the larger is better.
     """
+    from neural_compressor import METRICS
     from neural_compressor.model import Model
     model = Model(model)
     input_tensor = model.input_tensor
     output_tensor = model.output_tensor if len(model.output_tensor)>1 else \
                         model.output_tensor[0]
     iteration = -1
+    metric = METRICS('tensorflow')['topk']()
     if args.benchmark and args.mode == 'performance':
         iteration = args.iters
 
@@ -136,9 +138,6 @@ def run(self):
                                            accuracy_criterion = AccuracyCriterion(tolerable_loss=0.01),
                                            op_type_dict={'conv2d':{ 'weight':{'dtype':['fp32']}, 'activation':{'dtype':['fp32']} }}
                                            )
-            from neural_compressor import METRICS
-            metrics = METRICS('tensorflow')
-            top1 = metrics['topk']()
             from tensorflow.core.protobuf import saved_model_pb2
             sm = saved_model_pb2.SavedModel()
             with tf.io.gfile.GFile(args.input_graph, "rb") as f:
@@ -147,10 +146,9 @@ def run(self):
             from neural_compressor.data import TensorflowShiftRescale
             postprocess = TensorflowShiftRescale()
             def eval(model):
-                return evaluate(model, eval_dataloader, top1, postprocess)
-            q_model = quantization.fit(graph_def, conf=conf, calib_dataloader=calib_dataloader,
-                        # eval_dataloader=eval_dataloader, eval_metric=top1)
-                        eval_func=eval)
+                return evaluate(model, eval_dataloader, postprocess)
+            q_model = quantization.fit(graph_def, conf=conf, eval_func=eval,
+                                            calib_dataloader=calib_dataloader)
             q_model.save(args.output_graph)
 
         if args.benchmark:
@@ -163,9 +161,6 @@ def eval(model):
                 'filter': None
             }
             dataloader = create_dataloader('tensorflow', dataloader_args)
-            from neural_compressor import METRICS
-            metrics = METRICS('tensorflow')
-            top1 = metrics['topk']()
 
             if args.int8 or args.input_graph.endswith("-tune.pb"):
                 input_graph = args.input_graph
@@ -180,7 +175,7 @@ def eval(model):
             from neural_compressor.data import TensorflowShiftRescale
             postprocess = TensorflowShiftRescale()
             def eval(model):
-                return evaluate(model, dataloader, top1, postprocess)
+                return evaluate(model, dataloader, postprocess)
 
             if args.mode == 'performance':
                 from neural_compressor.benchmark import fit

examples/tensorflow/nlp/bert_large_squad/quantization/ptq/tune_squad.py

@@ -51,7 +51,7 @@
 flags.DEFINE_integer("iters", 100, "The iteration used for benchmark.")
 
 
-def evaluate(model, dataloader, metric, postprocess):
+def evaluate(model, dataloader, postprocess):
     """Custom evaluate function to estimate the accuracy of the bert model.
 
     Args:
@@ -60,6 +60,7 @@ def evaluate(model, dataloader, metric, postprocess):
     Returns:
         accuracy (float): evaluation result, the larger is better.
     """
+    from neural_compressor.metric import SquadF1
     from neural_compressor.adaptor.tf_utils.util import iterator_sess_run
     from neural_compressor.objective import Performance
     from neural_compressor.model import Model, BaseModel
@@ -70,12 +71,12 @@ def evaluate(model, dataloader, metric, postprocess):
     input_tensor = model.input_tensor
     output_tensor = model.output_tensor if len(model.output_tensor)>1 else \
                         model.output_tensor[0]
+    warmup = 5
     iteration = -1
+    metric = SquadF1()
+    measurer = Performance()
     if FLAGS.benchmark and FLAGS.mode == 'performance':
         iteration = FLAGS.iters
-    measurer = Performance()
-
-    warmup = 5
     for idx, (inputs, labels) in enumerate(dataloader):
         # dataloader should keep the order and len of inputs same with input_tensor
         assert len(input_tensor) == len(inputs), \
@@ -125,8 +126,6 @@ def strip_iterator(graph_def):
 
 def main(_):
     tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.INFO)
-    from neural_compressor.metric import SquadF1
-    metric = SquadF1()
     from neural_compressor.utils.create_obj_from_config import create_dataloader
     data_path = os.path.join(FLAGS.dataset_location, 'eval.tf_record')
     label_path = os.path.join(FLAGS.dataset_location, 'dev-v1.1.json')
@@ -142,7 +141,7 @@ def main(_):
     from neural_compressor.data import TFSquadV1PostTransform
     postprocess = TFSquadV1PostTransform(label_file=label_path, vocab_file=vocab_path)
     def eval(model):
-        return evaluate(model, dataloader, metric, postprocess)
+        return evaluate(model, dataloader, postprocess)
     if FLAGS.benchmark:
         if FLAGS.mode == 'performance':
             from neural_compressor.benchmark import fit