Add keras saved_model examples (#1201)

Author: zehao-intel

examples/.config/model_params_tensorflow.json

@@ -36,6 +36,15 @@
       "batch_size": 100,
       "new_benchmark": true
     },
+    "resnet101_keras": {
+      "model_src_dir": "image_recognition/keras_models/resnet101/quantization/ptq",
+      "dataset_location": "/tf_dataset/dataset/imagenet",
+      "input_model": "/tf_dataset2/models/tensorflow/resnet101_keras/saved_model/",
+      "yaml": "resnet101.yaml",
+      "strategy": "basic",
+      "batch_size": 1,
+      "new_benchmark": true
+    },
     "resnet_v1_50_slim": {
       "model_src_dir": "image_recognition/tensorflow_models/quantization/ptq/slim",
       "dataset_location": "/tf_dataset/dataset/imagenet",
@@ -108,6 +117,15 @@
       "batch_size": 100,
       "new_benchmark": true
     },
+    "inception_v3_keras": {
+      "model_src_dir": "image_recognition/keras_models/inception_v3/quantization/ptq",
+      "dataset_location": "/tf_dataset/dataset/imagenet",
+      "input_model": "/tf_dataset2/models/tensorflow/inception_v3_keras/saved_model/",
+      "yaml": "inception_v3.yaml",
+      "strategy": "basic",
+      "batch_size": 1,
+      "new_benchmark": true
+    },
     "inception_v3_slim": {
       "model_src_dir": "image_recognition/tensorflow_models/quantization/ptq/slim",
       "dataset_location": "/tf_dataset/dataset/imagenet",
@@ -144,6 +162,15 @@
       "batch_size": 100,
       "new_benchmark": true
     },
+    "inception_resnet_v2_keras": {
+      "model_src_dir": "image_recognition/keras_models/inception_resnet_v2/quantization/ptq",
+      "dataset_location": "/tf_dataset/dataset/imagenet",
+      "input_model": "/tf_dataset2/models/tensorflow/inception_resnet_v2_keras/saved_model/",
+      "yaml": "inception_resnet_v2.yaml",
+      "strategy": "basic",
+      "batch_size": 1,
+      "new_benchmark": true
+    },
     "vgg16": {
       "model_src_dir": "image_recognition/tensorflow_models/quantization/ptq",
       "dataset_location": "/tf_dataset/dataset/imagenet",
@@ -261,6 +288,15 @@
       "batch_size": 100,
       "new_benchmark": true
     },
+    "resnetv2_50_keras": {
+      "model_src_dir": "image_recognition/keras_models/resnetv2_50/quantization/ptq",
+      "dataset_location": "/tf_dataset/dataset/imagenet",
+      "input_model": "/tf_dataset2/models/tensorflow/resnetv2_50_keras/saved_model",
+      "yaml": "resnetv2_50.yaml",
+      "strategy": "basic",
+      "batch_size": 1,
+      "new_benchmark": true
+    },
     "resnetv2_101": {
       "model_src_dir": "image_recognition/tensorflow_models/quantization/ptq",
       "dataset_location": "/tf_dataset/dataset/imagenet",
@@ -279,6 +315,15 @@
       "batch_size": 100,
       "new_benchmark": true
     },
+    "resnetv2_101_keras": {
+      "model_src_dir": "image_recognition/keras_models/resnetv2_101/quantization/ptq",
+      "dataset_location": "/tf_dataset/dataset/imagenet",
+      "input_model": "/tf_dataset2/models/tensorflow/resnetv2_101_keras/saved_model",
+      "yaml": "resnetv2_101.yaml",
+      "strategy": "basic",
+      "batch_size": 1,
+      "new_benchmark": true
+    },
     "resnetv2_152": {
       "model_src_dir": "image_recognition/tensorflow_models/quantization/ptq",
       "dataset_location": "/tf_dataset/dataset/imagenet",
@@ -315,6 +360,15 @@
       "batch_size": 100,
       "new_benchmark": true
     },
+    "mobilenetv2_keras": {
+      "model_src_dir": "image_recognition/keras_models/mobilenet_v2/quantization/ptq",
+      "dataset_location": "/tf_dataset/dataset/imagenet",
+      "input_model": "/tf_dataset2/models/tensorflow/mobilenet_v2_keras/saved_model/",
+      "yaml": "mobilenet_v2.yaml",
+      "strategy": "basic",
+      "batch_size": 1,
+      "new_benchmark": true
+    },
     "mobilenetv3": {
       "model_src_dir": "image_recognition/tensorflow_models/quantization/ptq",
       "dataset_location": "/tf_dataset/dataset/imagenet",
@@ -2412,6 +2466,15 @@
       "strategy": "basic",
       "batch_size": 64,
       "new_benchmark": false
+    },
+    "xception": {
+      "model_src_dir": "image_recognition/keras_models/xception/quantization/ptq",
+      "dataset_location": "/tf_dataset/dataset/imagenet",
+      "input_model": "/tf_dataset2/models/tensorflow/xception_keras/saved_model/",
+      "yaml": "xception.yaml",
+      "strategy": "basic",
+      "batch_size": 1,
+      "new_benchmark": true
     }
   }
 }

examples/tensorflow/image_recognition/keras_models/inception_resnet_v2/quantization/ptq/README.md

@@ -0,0 +1,33 @@
+Step-by-Step
+============
+
+This document is used to enable Tensorflow Keras models using Intel® Neural Compressor.
+
+
+## Prerequisite
+
+### 1. Installation
+```shell
+# Install Intel® Neural Compressor
+pip install neural-compressor
+```
+### 2. Install Intel Tensorflow
+```shell
+pip install intel-tensorflow
+```
+> Note: Supported Tensorflow [Version](../../../../../../../README.md).
+
+### 3. Prepare Pretrained model
+
+The pretrained model is provided by [Keras Applications](https://keras.io/api/applications/). prepare the model, Run as follow: 
+ ```
+python prepare_model.py   --output_model=/path/to/model
+ ```
+`--output_model ` the model should be saved as SavedModel format or H5 format.
+
+## Run Command
+  ```shell
+  bash run_tuning.sh --config=inception_resnet_v2.yaml --input_model=./path/to/model --output_model=./result --eval_data=/path/to/evaluation/dataset --calib_data=/path/to/calibration/dataset
+  bash run_benchmark.sh --config=inception_resnet_v2.yaml --input_model=./path/to/model --mode=performance --eval_data=/path/to/evaluation/dataset
+  ```
+

examples/tensorflow/image_recognition/keras_models/inception_resnet_v2/quantization/ptq/inception_resnet_v2.yaml

@@ -0,0 +1,42 @@
+#
+# Copyright (c) 2021 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+version: 1.0
+
+model:                                               # mandatory. used to specify model specific information.
+  name: inception_resnet_v2
+  framework: tensorflow                              # mandatory. supported values are tensorflow, pytorch, pytorch_ipex, onnxrt_integer, onnxrt_qlinear or mxnet; allow new framework backend extension.
+
+quantization:                                        # optional. tuning constraints on model-wise for advance user to reduce tuning space.
+  calibration:
+    sampling_size: 50, 100                           # optional. default value is 100. used to set how many samples should be used in calibration.
+  model_wise:                                        # optional. tuning constraints on model-wise for advance user to reduce tuning space.
+    activation:
+      algorithm: minmax
+
+evaluation:                                          # optional. required if user doesn't provide eval_func in neural_compressor.Quantization.
+  accuracy:
+  performance:                                       # optional. used to benchmark performance of passing model.
+    iteration: 100
+    configs:
+      cores_per_instance: 4
+      num_of_instance: 7
+
+tuning:
+  accuracy_criterion:
+    relative:  0.01                                  # optional. default value is relative, other value is absolute. this example allows relative accuracy loss: 1%.
+  exit_policy:
+    timeout: 0                                       # optional. tuning timeout (seconds). default value is 0 which means early stop. combine with max_trials field to decide when to exit.
+  random_seed: 9527                                  # optional. random seed for deterministic tuning.

examples/tensorflow/image_recognition/keras_models/inception_resnet_v2/quantization/ptq/main.py

@@ -0,0 +1,128 @@
+#
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2018 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import time
+import shutil
+import numpy as np
+from argparse import ArgumentParser
+from neural_compressor import data
+import tensorflow as tf
+tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
+
+flags = tf.compat.v1.flags
+FLAGS = flags.FLAGS
+
+## Required parameters
+flags.DEFINE_string(
+    'input_model', None, 'Run inference with specified pb graph.')
+
+flags.DEFINE_string(
+    'output_model', None, 'The output model of the quantized model.')
+
+flags.DEFINE_string(
+    'mode', 'performance', 'define benchmark mode for accuracy or performance')
+
+flags.DEFINE_bool(
+    'tune', False, 'whether to tune the model')
+
+flags.DEFINE_bool(
+    'benchmark', False, 'whether to benchmark the model')
+
+flags.DEFINE_string(
+    'config', 'bert.yaml', 'yaml configuration of the model')
+
+flags.DEFINE_string(
+    'calib_data', None, 'location of calibration dataset')
+
+flags.DEFINE_string(
+    'eval_data', None, 'location of evaluate dataset')
+
+from neural_compressor.experimental.metric.metric import TensorflowTopK
+from neural_compressor.experimental.data.transforms.transform import ComposeTransform
+from neural_compressor.experimental.data.datasets.dataset import TensorflowImageRecord
+from neural_compressor.experimental.data.transforms.imagenet_transform import LabelShift
+from neural_compressor.experimental.data.dataloaders.default_dataloader import DefaultDataLoader
+from neural_compressor.data.transforms.imagenet_transform import BilinearImagenetTransform
+
+eval_dataset = TensorflowImageRecord(root=FLAGS.eval_data, transform=ComposeTransform(transform_list= \
+  [BilinearImagenetTransform(height=299, width=299)]))
+if FLAGS.benchmark and FLAGS.mode == 'performance':
+  eval_dataloader = DefaultDataLoader(dataset=eval_dataset, batch_size=1)
+else:
+  eval_dataloader = DefaultDataLoader(dataset=eval_dataset, batch_size=32)
+if FLAGS.calib_data:
+  calib_dataset = TensorflowImageRecord(root=FLAGS.calib_data, transform=ComposeTransform(transform_list= \
+    [BilinearImagenetTransform(height=299, width=299)]))
+  calib_dataloader = DefaultDataLoader(dataset=calib_dataset, batch_size=10)
+
+def evaluate(model, measurer=None):
+  """
+  Custom Evaluate function to inference the model for specified metric on validation dataset.
+
+  Args:
+      model ([tf.saved_model.load]): The model will be the class of tf.saved_model.load(quantized_model_path).
+      measurer (object, optional): for precise benchmark measurement.
+
+  Returns:
+      [float]: evaluation result, the larger is better.
+  """
+  infer = model.signatures["serving_default"]
+  output_dict_keys = infer.structured_outputs.keys()
+  output_name = list(output_dict_keys )[0]
+  postprocess = LabelShift(label_shift=1)
+  metric = TensorflowTopK(k=1)
+  
+  def eval_func(dataloader, metric):
+      results = []
+      for idx, (inputs, labels) in enumerate(dataloader):
+          inputs = np.array(inputs)
+          input_tensor = tf.constant(inputs)
+          if measurer:
+            measurer.start()
+          predictions = infer(input_tensor)[output_name]
+          if measurer:
+            measurer.end()
+          predictions = predictions.numpy()
+          predictions, labels = postprocess((predictions, labels))
+          metric.update(predictions, labels)
+      return results
+
+  results = eval_func(eval_dataloader, metric)
+  acc = metric.result()
+  return acc
+
+def main(_):
+  if FLAGS.tune:
+    from neural_compressor.experimental import Quantization, common
+    quantizer = Quantization(FLAGS.config)
+    quantizer.model = common.Model(FLAGS.input_model)
+    quantizer.eval_func = evaluate
+    quantizer.calib_dataloader = calib_dataloader
+    q_model = quantizer.fit()
+    q_model.save(FLAGS.output_model)
+
+
+  if FLAGS.benchmark:
+    from neural_compressor.experimental import Benchmark, common
+    evaluator = Benchmark(FLAGS.config)
+    evaluator.model = common.Model(FLAGS.input_model)
+    evaluator.b_func = evaluate
+    evaluator.b_dataloader = eval_dataloader
+    evaluator(FLAGS.mode)
+
+if __name__ == "__main__":
+    tf.compat.v1.app.run()

examples/tensorflow/image_recognition/keras_models/inception_resnet_v2/quantization/ptq/prepare_model.py

@@ -0,0 +1,17 @@
+import argparse
+import tensorflow as tf
+def get_inception_resnet_v2_model(saved_path):
+    model = tf.keras.applications.InceptionResNetV2(weights='imagenet')
+    model.save(saved_path)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+    description='Export pretained keras model',
+    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument(
+        '--output_model',
+        type=str,
+        help='path to exported model file')
+
+    args = parser.parse_args()
+    get_inception_resnet_v2_model(args.output_model)

examples/tensorflow/image_recognition/keras_models/inception_resnet_v2/quantization/ptq/run_benchmark.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+set -x
+
+function main {
+
+  init_params "$@"
+  run_benchmark
+
+}
+
+# init params
+function init_params {
+  for var in "$@"
+  do
+    case $var in
+      --config=*)
+          config=$(echo $var |cut -f2 -d=)
+      ;;
+      --input_model=*)
+          input_model=$(echo $var |cut -f2 -d=)
+      ;;
+      --mode=*)
+          mode=$(echo $var |cut -f2 -d=)    
+      ;;
+      --eval_data=*)
+          eval_data=$(echo $var |cut -f2 -d=)    
+      ;;
+    esac
+  done
+
+}
+
+# run_tuning
+function run_benchmark {
+
+    python main.py \
+            --input_model ${input_model} \
+            --config ${config} \
+            --benchmark \
+            --mode ${mode} \
+            --eval_data ${eval_data}
+}
+
+main "$@"