Issues with ONNX model ReduceMax API not supported by ONNXRT TensorRT EP (but ok with CPU and Cuda EP)

[datinje]

Describe the issue

Onnxrunme tensort execution provider is crashing on loading model when parsing ReduceMax instruction
see more detailed below in ho wto reproduce it.

I used ONNX opset 17 and 16 with pytorch 2.0
Same issue with onnxruntime 1.13 to 1.15

using
TensorRT-8.6.1.6
cudnn 8.9.3
cuda 11.8
python 3.10
pytorch 2.0

same issue with C++ ort APIs

Note that because of #16883
I had to use a released package (1.13.1)

To reproduce

1. get a standard faster-rcnn model from Facebookresearch detectron2 model zoo in func setup()
2. convert the torch model to onnx using function export_tracing()
3. prepare a sample input with function get_sample_inputs() to be used for inferencing with torch and inferencing with onnxruntime tensorrt EP
4. check the resultant onnx model with check_onnx_model() : we see that model is valid but graph is not (don't bother these is a bug in the onnx checker functions (!)
5. do the inference with torch and onnxruntime and verify results match
6. => here we get and error with tensorrt EP . while inference is ok with CPU EP and Cuda EP - results match with these.
   as traces show : the problem is with ReduceMax operator which is not well interpreetd by onnxruntime EP , not tensorrt (when I run the model to native tensorrt there is no problem - after converting the ONNX to trt format (note that I need a special version of nvidia graph-surgeon to convert to trt)

Note

the following code demonstate the error:

#!/usr/bin/env python
# Copyright (c) Facebook, Inc. and its affiliates.
# this is an adaptation of detectron2/tools/deploy/export_model.py
# it does export of a faster-rcnn model to onnx and test it vs the original detectron2 model
# requires any RGB input image (jpg or png)
import argparse
import os
from typing import Dict, List, Tuple
import torch
from torch import Tensor, nn

import detectron2.data.transforms as T
from detectron2.checkpoint import DetectionCheckpointer
from detectron2.config import get_cfg
from detectron2.data import build_detection_test_loader, detection_utils
from detectron2.evaluation import COCOEvaluator, inference_on_dataset, print_csv_format
from detectron2 import model_zoo
""" 
# cannot use detectron2 export lib since it depends on Caffe2 which is not provided anymore with pytorch dist
from detectron2.export import (
    STABLE_ONNX_OPSET_VERSION,
    TracingAdapter,
    dump_torchscript_IR,
    scripting_with_instances,
)
"""
# # use export lib stripped out from caffe2 (/detectron2/export/__init__.py)
from lib.export import (
    TracingAdapter,
    dump_torchscript_IR,
    scripting_with_instances,
)
from detectron2.modeling import GeneralizedRCNN, RetinaNet, build_model
from detectron2.modeling.postprocessing import detector_postprocess
from detectron2.projects.point_rend import add_pointrend_config
from detectron2.structures import Boxes
from detectron2.utils.env import TORCH_VERSION
from detectron2.utils.file_io import PathManager
from detectron2.utils.logger import setup_logger

import onnx
import onnxruntime as ort
import numpy as np
import cv2 as cv2

def setup_cfg(args):   
    cfg = get_cfg()
    
    #use detectron2 satndard faster rcnn

    cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.5  # set threshold for this model
    cfg.MODEL.WEIGHTS = model_zoo.get_checkpoint_url("COCO-Detection/faster_rcnn_R_50_FPN_1x.yaml") 
    cfg.MODEL.DEVICE = 'cuda'   

    # cuda context is initialized before creating dataloader, so we don't fork anymore
    cfg.DATALOADER.NUM_WORKERS = 0
    add_pointrend_config(cfg)
    cfg.merge_from_file(model_zoo.get_config_file("COCO-Detection/faster_rcnn_R_50_FPN_1x.yaml"))
    cfg.merge_from_list(args.opts)
    cfg.freeze()
        
    return cfg

# experimental. API not yet final
def export_tracing(torch_model, inputs):
    assert TORCH_VERSION >= (1, 8)
    image = inputs[0]["image"]
    inputs = [{"image": image}]  # remove other unused keys

    inference=None
    """
    if isinstance(torch_model, GeneralizedRCNN):

        def inference(model, inputs):
            # use do_postprocess=False so it returns ROI mask
            inst = model.inference(inputs, do_postprocess=False)[0]
            return [{"instances": inst}]

    else:
        inference = None  # assume that we just call the model directly
    """
    
    traceable_model = TracingAdapter(torch_model, inputs, inference)
    
    with PathManager.open(os.path.join(args.output, "faster_rcnn_fpn.onnx"), "wb") as f:
        torch.onnx.export(
              traceable_model, 
              (image,), 
              f, 
              do_constant_folding=True,
              export_params=True,
              input_names=["image"], # the model's input names
              output_names=["boxes", "labels", "scores", "image_dims"], # the model's output names
              dynamic_axes={
                "image"      : {1: "height", 2: "width"},
                "boxes"      : {0: "findings"}, # boxes is a tensor of shape [number of findings, 4] 
                "labels"     : {0: "findings"},
                "scores"     : {0: "findings"}
                },
              verbose=True, 
              opset_version=17) #issue is same with opset 16 and opset 18 is not validated for pytorch 2.0
              
    logger.info("Inputs schema: " + str(traceable_model.inputs_schema))
    logger.info("Outputs schema: " + str(traceable_model.outputs_schema))

    onnx_model_path = os.path.join(args.output, "faster_rcnn_fpn.onnx")
    onnx_model = onnx.load(onnx_model_path)
    
    return onnx_model


def get_sample_inputs(args):

    if args.sample_image is None:
        # get a first batch from dataset
        data_loader = build_detection_test_loader(cfg, cfg.DATASETS.TEST[0])
        first_batch = next(iter(data_loader))
        return first_batch
    else:
        # get a sample data
        original_image = cv2.imread("./input.jpg")
        print ("original_image input shape :", original_image.shape)
         
        # Do same preprocessing as DefaultPredictor
        aug = T.ResizeShortestEdge(
            [cfg.INPUT.MIN_SIZE_TEST, cfg.INPUT.MIN_SIZE_TEST], cfg.INPUT.MAX_SIZE_TEST
        )
        image_with_different_size = aug.get_transform(original_image).apply_image(original_image)
        cv2.imwrite("./inputExpanded.jpg", image_with_different_size)
        
        image = original_image
        height, width = original_image.shape[:2]
        image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1)) # need chanel first for onnx
        print ("image input shape :", image.shape)
                      
        inputs = {"image": image, "height": height, "width": width}

        # Sample ready
        sample_inputs = [inputs]
        return sample_inputs

def check_onnx_model (onnx_model):
  # Check the model
  try:
    onnx.checker.check_model(onnx_model, full_check=True)
  except onnx.checker.ValidationError as e:
    print("The model is invalid: %s" % e)
  else:
    print("The model is valid!")
    
  # check the onnx graph
  try:
     graph = onnx_model.graph
     onnx.checker.check_graph(graph)
  except onnx.checker.ValidationError as e:
    print("The graph is invalid: %s" % e)
  else:
    print("The graph is valid!")
    
  input_shapes = [[d.dim_value for d in _input.type.tensor_type.shape.dim] for _input in onnx_model.graph.input]
  print ('onnx model input shapes', input_shapes)
   
  return None

def to_numpy(tensor):
    return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy()
  
   
def eval_onnx_model (torch_model, onnx_model, sample_inputs, args): 
  # get D2 results
  torch_model.eval()
  torch_outputs = torch_model(sample_inputs)
  print ('torch_outputs: ', torch_outputs)
  print ('torch size of outputs: ', len(torch_outputs))

  t_outputs_scores =   to_numpy(torch_outputs[0]['instances'].scores)
  print('d2_torch_scores: ', t_outputs_scores)
  t_outputs_boxes =   to_numpy(torch_outputs[0]['instances'].pred_boxes.tensor)
  print('d2_torch_boxes: ', t_outputs_boxes)
  t_outputs_classes =   to_numpy(torch_outputs[0]['instances'].pred_classes)
  print('d2_torch_classes: ', t_outputs_classes)
  print('')
  
  # get ONNXRT results
  onnx_model_path = os.path.join(args.output, "faster_rcnn_fpn.onnx")
  providers = [('TensorrtExecutionProvider')]
  #providers = [('CUDAExecutionProvider')] # works !
       
  sess_opt = ort.SessionOptions()
  sess = ort.InferenceSession(onnx_model_path, sess_options=sess_opt, providers=providers)
  
  input_name = sess.get_inputs()[0].name
  print("input name", input_name)
  input_shape = sess.get_inputs()[0].shape
  print("input shape", input_shape)
  input_type = sess.get_inputs()[0].type
  print("input type", input_type)

  output_name = sess.get_outputs()[0].name
  print("output name", output_name)
  output_shape = sess.get_outputs()[0].shape
  print("output shape", output_shape)
  output_type = sess.get_outputs()[0].type
  print("output type", output_type)
 
  image = sample_inputs[0]['image']
  np_image  = image.cpu().numpy()
  
  # compute ONNX Runtime output prediction
  ort_inputs = {sess.get_inputs()[0].name: np_image}
  ort_outputs = sess.run(None, ort_inputs)

  print ('ort_outputs: ', ort_outputs)
  print('ort_outputs number: ', len(ort_outputs))
  print('')
  
  boxes = ort_outputs[0]
  classes =  ort_outputs[1]
  scores = ort_outputs[2]
 
  print ('ort_boxes : ', boxes)
  print ('ort scores : ', scores)
  print ('ort classes : ', classes)
  print('')
  
  # eval torch and onnxrt outputs
  np.testing.assert_allclose(t_outputs_boxes, boxes, rtol=1e-03, atol=1e-05)
  np.testing.assert_allclose(t_outputs_scores, scores, rtol=1e-03, atol=1e-05)
  np.testing.assert_allclose(t_outputs_classes, classes, rtol=1e-03, atol=1e-05)
  print('detectron2 torch and onnx models results match!')
  print('')
  
  return None
  
if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Export a model for deployment.")
    parser.add_argument("--sample-image", default=None, type=str, help="sample image for input")
    parser.add_argument("--output", help="output directory for the converted model")
    parser.add_argument(
        "opts",
        help="Modify config options using the command-line",
        default=None,
        nargs=argparse.REMAINDER,
    )
    args = parser.parse_args()
    logger = setup_logger()
    logger.info("Command line arguments: " + str(args))
    
    PathManager.mkdirs(args.output)

    cfg = setup_cfg(args)

    # create a torch model
    torch_model = build_model(cfg)
    DetectionCheckpointer(torch_model).resume_or_load(cfg.MODEL.WEIGHTS)
    torch_model.eval()

    # convert and save model
    sample_inputs = get_sample_inputs(args)
    onnx_model = export_tracing(torch_model, sample_inputs)
    
    check_onnx_model (onnx_model)
    
    eval_onnx_model(torch_model, onnx_model, sample_inputs, args)

    logger.info("Success.")

What exact command you run:
python3 export_model.py --output onnx_output --sample-image input.jpg

Full logs or other relevant observations:

[04/04 16:14:53 detectron2]: Command line arguments: Namespace(sample_image='input.jpg', output='onnx_output', opts=[])
original_image input shape : (480, 640, 3)
image input shape : torch.Size([3, 480, 640])

  %/model/ReduceMax_output_0 : Long(2, strides=[1], requires_grad=0, device=cpu) = **onnx::ReduceMax[axes=[0],** keepdims=0, onnx_name="/model/ReduceMax"](%/model/Concat_1_output_0), scope: lib.export.flatten.TracingAdapter::/detectron2.modeling.meta_arch.rcnn.GeneralizedRCNN::model # /usr/local/lib/python3.10/dist-packages/**detectron2**/structures/image_list.py:83:0


      %max_coordinate.3 : Float(device=cpu) = **onnx::ReduceMax[keepdims=0]**(%/model/roi_heads/Cast_9_output_0) # /usr/local/lib/python3.10/dist-packages/**torchvision**/ops/boxes.py:91:21


============= Diagnostic Run torch.onnx.export version 2.0.0+cu118 =============
verbose: False, log level: Level.ERROR
======================= 0 NONE 0 NOTE 0 WARNING 0 ERROR ========================

[04/04 16:15:02 detectron2]: Inputs schema: TupleSchema(schemas=[ListSchema(schemas=[DictSchema(schemas=[IdentitySchema()], sizes=[1], keys=['image'])], sizes=[1])], sizes=[1])
[04/04 16:15:02 detectron2]: Outputs schema: ListSchema(schemas=[DictSchema(schemas=[InstancesSchema(schemas=[TensorWrapSchema(class_name='detectron2.structures.Boxes'), IdentitySchema(), IdentitySchema()], sizes=[1, 1, 1], keys=['pred_boxes', 'pred_classes', 'scores'])], sizes=[4], keys=['instances'])], sizes=[4])
The model is valid!

The graph is invalid: Unrecognized attribute: axes for operator ReduceMax
==> Context: Bad node spec for node. Name: /model/ReduceMax OpType: ReduceMax
onnx model input shapes [[3, 0, 0]]

2023-04-04 16:37:52.173723690 [E:onnxruntime:Default, tensorrt_execution_provider.h:61 log] [2023-04-04 16:37:52   ERROR] **ReduceMax_1597: at least 1 dimensions are required for input.**
2**023-04-04 16:37:52.324418966 [E:onnxruntime:, inference_session.cc:1532 operator()] Exception during initialization: /onnxruntime_src/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc:897 SubGraphCollection_t onnxruntime::TensorrtExecutionProvider::GetSupportedList(SubGraphCollection_t, int, int, const onnxruntime::GraphViewer&, bool*) const [ONNXRuntimeError] : 1 : FAIL : TensorRT input: /model/proposal_generator/GatherND_2_output_0 has no shape specified. Please run shape inference on the onnx model first. Details can be found in https://onnxruntime.ai/docs/execution-providers/TensorRT-ExecutionProvider.html#shape-inference-for-tensorrt-subgraphs**

Traceback (most recent call last):
  File "/cad-engine/export_model.py", line 264, in <module>
    eval_onnx_model(torch_model, onnx_model, sample_inputs, args)
  File "/cad-engine/export_model.py", line 190, in eval_onnx_model
    sess = ort.InferenceSession(onnx_model_path, sess_options=sess_opt, providers=providers)
  File "/usr/local/lib/python3.10/dist-packages/onnxruntime/capi/onnxruntime_inference_collection.py", line 360, in __init__
    self._create_inference_session(providers, provider_options, disabled_optimizers)
  File "/usr/local/lib/python3.10/dist-packages/onnxruntime/capi/onnxruntime_inference_collection.py", line 408, in _create_inference_session
    sess.initialize_session(providers, provider_options, disabled_optimizers)
onnxruntime.capi.onnxruntime_pybind11_state.RuntimeException: [ONNXRuntimeError] : 6 : RUNTIME_EXCEPTION : Exception duri

Urgency

I am blocked with onnxrt and need to revert to tensorrt native APIs which defeats our portability strategy.

Platform

Linux

OS Version

SLES15 SP4

ONNX Runtime Installation

Released Package

ONNX Runtime Version or Commit ID

1.13.1

ONNX Runtime API

Python

Architecture

X64

Execution Provider

TensorRT

Execution Provider Library Version

TensorRT-8.6.1.6

[jywu-msft]

Thanks for providing these details.
In step 6 , you mention that the model works fine when converting to native TRT format but you needed to use a special version of nvidia graphsurgeon. can you provide more details on that and share the steps for that conversion?
tensorrt ep relies on a component from nvidia, onnx-tensorrt parser to help us convert from onnx to tensorrt network, so it would be helpful for us to know what graph transform graphsurgeon is doing.

[datinje]

You need to load nvidia onnx graphsurgeon for tensorrt
(tensor rt is more strict than onnx runtime for onnx model to run : in the faster-rcnn model from detectron2 model zoo , it does not like an if-then-else branch to return a tensor with different dimensions , so the script below just fixes this. After that you can run trtexec.

run as : graph_surgeon_faster_rcnn.py --onnx faster_rcnn_fpn.onnx --output onnx_output
then run : trtexec --onnx=faster_rcnn_fpn-graph_surgeon.onnx --verbose --saveEngine=faster_rcnn_fpn-graph_surgeon.trt

Before that , you need to install the following from nvidia

# INSTALL PYTHON MODULES polygraphy, tensorrt, onnx_graphsurgeon
python -m pip install colored polygraph --trusted-host pypi.ngc.nvidia.com --extra-index-url https://pypi.ngc.nvidia.com
# and build onnx-graphsurgeon since the pip installer did not work for me 
# pip install onnx_graphsurgeon>=0.3.21 --trusted-host pypi.ngc.nvidia.com  --extra-index-url=https://pypi.ngc.nvidia.com
git clone https://github.com/NVIDIA/TensorRT.git
cd TensorRT/tools/onnx-graphsurgeon && make build && python3 -m pip install dist/onnx_graphsurgeon-*-py2.py3-none-any.whl

here is the graph_surgeon_faster_rcnn.py (tested on python 3.10)
(I provide the reference to the nice person from nvidia who helped me)

#! /usr/bin/env python3
## ---------------------------------------------------------------------------
##
## File: graph_surgeon_faster_rcnn.py for detectron2 faster-rcnn model zoo
##
## Created by Zhijin Li
## E-mail:   <zhijinl@nvidia.com>
##
## Started on  Mon Jul 17 15:53:26 2023 Zhijin Li
## Last update Tue Jul 18 02:08:41 2023 Zhijin Li
## Modified    Sun Jul 30                2023 JC Datin
## ---------------------------------------------------------------------------
import onnx
import onnx_graphsurgeon as gs

import os
import urllib
import argparse
import numpy as np

from detectron2 import model_zoo
from detectron2.config import get_cfg
from detectron2.utils.logger import setup_logger

def parse_arguments():
  """
  Parse input command line arguments.

  Returns
  ----------
  Parsed argument object.

  """
  parser = argparse.ArgumentParser(
    description='graph surgeon an ONNX model for tensorRT conversion.')

  parser.add_argument(
    '--onnx',
    type=str,
    required=True,
    help='Path to ONNX model')

  parser.add_argument(
    '--output',
    type=str,
    default='./output',
    help='output directory for the graph surgeoned model')

  parser.add_argument(
    'opts',
    help='Modify config options using the command-line',
    default=None,
    nargs=argparse.REMAINDER,
  )

  args = parser.parse_args()
  return args

def sanitize(graph):
  """
  Sanitize the graph by cleaning any unconnected nodes, do a topological resort, and fold constant inputs values.
  When possible, run shape inference on the ONNX graph to determine tensor shapes.
  """

  for i in range(3):

    count_before = len(graph.nodes)
    graph.cleanup().toposort()

    try:
      for node in graph.nodes:
        for o in node.outputs:
          o.shape = None

      model = gs.export_onnx(graph)
      model = shape_inference.infer_shapes(model)
      graph = gs.import_onnx(model)

    except Exception as e:
      log.info("Shape inference could not be performed at this time:\n{}".format(e))

    try:
      graph.fold_constants(fold_shapes=True)
    except TypeError as e:
      log.error("This version of ONNX GraphSurgeon does not support folding shapes, please upgrade your "
                "onnx_graphsurgeon module. Error:\n{}".format(e))
      raise

    count_after = len(graph.nodes)
    if count_before == count_after:
      # No new folding occurred in this iteration, so we can stop for now.
      break


if __name__ == '__main__':

  logger = setup_logger()
  args = parse_arguments()

   # Load graph

  logger.info("loading ONNX model: {}".format(args.onnx))
  original_model = args.onnx
  graph = gs.import_onnx(onnx.load(original_model))
  surgeoned_model_path = original_model.rsplit('.', 1)[0]
  surgeoned_model_name = surgeoned_model_path.rsplit('/', 1)[-1] 
  surgeoned_model= surgeoned_model_name + "-graph_surgeon" + ".onnx"
  print(surgeoned_model_path)
  print(surgeoned_model_name)
  print(surgeoned_model)

  graph = graph.cleanup().toposort()
  graph = graph.fold_constants().cleanup()

  # print(graph)

  # for node in graph.nodes:
  #   print(node)

  for node in graph.nodes:
    for o in node.outputs:
      o.shape = None

  model = gs.export_onnx(graph)
  model = onnx.shape_inference.infer_shapes(model)
  graph = gs.import_onnx(model)

  # print(graph)

  ###### FIX: /model/roi_heads/pooler/level_poolers.0/If
  for node in graph.nodes:
    if node.name == '/model/roi_heads/box_pooler/level_poolers.0/If':

      else_branch_graph = node.attrs['else_branch']

      squeeze_axes = gs.Constant(
        name='axes',
        values=np.array([1]))
      squeeze_node = gs.Node(
        "Squeeze",
        name="squeeze_fix")
      squeeze_node.inputs = [*else_branch_graph.outputs, squeeze_axes]
      squeeze_node.outputs = [gs.Variable('squeeze_fix_output', dtype=np.float32)]

      node.attrs['else_branch'].nodes = [*node.attrs['else_branch'].nodes, squeeze_node]
      node.attrs['else_branch'].outputs = squeeze_node.outputs
      
    if node.name == '/model/roi_heads/box_pooler/level_poolers.1/If':

      else_branch_graph = node.attrs['else_branch']

      squeeze_axes = gs.Constant(
        name='axes',
        values=np.array([1]))
      squeeze_node = gs.Node(
        "Squeeze",
        name="squeeze_fix")
      squeeze_node.inputs = [*else_branch_graph.outputs, squeeze_axes]
      squeeze_node.outputs = [gs.Variable('squeeze_fix_output', dtype=np.float32)]

      node.attrs['else_branch'].nodes = [*node.attrs['else_branch'].nodes, squeeze_node]
      node.attrs['else_branch'].outputs = squeeze_node.outputs

    if node.name == '/model/roi_heads/box_pooler/level_poolers.2/If':

      else_branch_graph = node.attrs['else_branch']

      squeeze_axes = gs.Constant(
        name='axes',
        values=np.array([1]))
      squeeze_node = gs.Node(
        "Squeeze",
        name="squeeze_fix")
      squeeze_node.inputs = [*else_branch_graph.outputs, squeeze_axes]
      squeeze_node.outputs = [gs.Variable('squeeze_fix_output', dtype=np.float32)]

      node.attrs['else_branch'].nodes = [*node.attrs['else_branch'].nodes, squeeze_node]
      node.attrs['else_branch'].outputs = squeeze_node.outputs
      
    if node.name == '/model/roi_heads/box_pooler/level_poolers.3/If':

      else_branch_graph = node.attrs['else_branch']

      squeeze_axes = gs.Constant(
        name='axes',
        values=np.array([1]))
      squeeze_node = gs.Node(
        "Squeeze",
        name="squeeze_fix")
      squeeze_node.inputs = [*else_branch_graph.outputs, squeeze_axes]
      squeeze_node.outputs = [gs.Variable('squeeze_fix_output', dtype=np.float32)]

      node.attrs['else_branch'].nodes = [*node.attrs['else_branch'].nodes, squeeze_node]
      node.attrs['else_branch'].outputs = squeeze_node.outputs

  ###### FIX: If_810
  for node in graph.nodes:
    if node.name == 'If_782':

      sub_graph = node.attrs['else_branch']

      for sub_node in sub_graph.nodes:
        if sub_node.name == 'If_810':

          else_branch_graph_810 = sub_node.attrs['else_branch']

          squeeze_axes = gs.Constant(
            name='axes',
            values=np.array([1]))
          squeeze_node = gs.Node(
            "Squeeze",
            name="squeeze_fix_if_810")
          squeeze_node.inputs = [*else_branch_graph_810.outputs, squeeze_axes]
          squeeze_node.outputs = [gs.Variable('squeeze_810_fix_output', dtype=np.int64)]

          sub_node.attrs['else_branch'].nodes = [*sub_node.attrs['else_branch'].nodes, squeeze_node]
          sub_node.attrs['else_branch'].outputs = squeeze_node.outputs


  graph = graph.cleanup().toposort()
  graph = graph.fold_constants().cleanup()

  model = gs.export_onnx(graph)
  model = onnx.shape_inference.infer_shapes(model)
  graph = gs.import_onnx(model)

  for node in graph.nodes:
    if node.name == '/model/roi_heads/box_pooler/level_poolers.0/If':
       print (node)

  save_path = os.path.join(args.output, surgeoned_model)
  onnx.save(model, save_path)

[datinje]

sorry , closed by mistake after entering previous comment

[datinje]

to be clear : issue is still open : I guess we are waiting for the ort team to check it is running with native TRT but not on ORT + TRT EP demonstrating the pb is on ONRT TRT EP.

[yf711]

@datinje Thanks for providing the script that using onnx-graphsurgeon.

Have you seen any inference failure after running trtexec --onnx=faster_rcnn_fpn-graph_surgeon.onnx --verbose --saveEngine=faster_rcnn_fpn-graph_surgeon.trt?
The native trt inference failed at:

[08/02/2023-19:15:46] [I] Starting inference
[08/02/2023-19:15:46] [E] Error[7]: [shapeMachine.cpp::executeContinuation::864] Error Code 7: Internal Error (If_1530_OutputLayer: dimensions not compatible for if-conditional outputs Condition '==' violated: 0 != 6. Instruction: CHECK_EQUAL 0 6.)
[08/02/2023-19:15:46] [E] Error occurred during inference

Or would it be convenient to share faster_rcnn_fpn-graph_surgeon.onnx that you had tested?

[datinje]

My mistake , I did not realize trtexec also failed on a similar node issue on the demonstrator after all node surgeon -albeit towards the end. trtexec worked on my real project once itself surgeoned.
If_1595_OutputLayer: dimensions not compatible for if-conditional outputs Condition '==' violated: 0 != 6. Instruction: CHECK_EQUAL 0 6.)
I have to find a way to surgeon this node : according to Netron one of the if node block is also missing a fix to get the same dimensions on all blocks.
Meanwhile : I will propose a much simpler reduceMax demonstrater.

[jywu-msft]

My mistake , I did not realize trtexec also failed on a similar node issue on the demonstrator after all node surgeon -albeit towards the end. trtexec worked on my real project once itself surgeoned. If_1595_OutputLayer: dimensions not compatible for if-conditional outputs Condition '==' violated: 0 != 6. Instruction: CHECK_EQUAL 0 6.) I have to find a way to surgeon this node : according to Netron one of the if node block is also missing a fix to get the same dimensions on all blocks. Meanwhile : I will propose a much simpler reduceMax demonstrater.

Thanks for confirming. We await your new repro test case. If an onnx model runs on native TensorRT (trtexec), it should also run with OnnxRuntime TensorRT EP. If it doesn't , that means there's a bug in OnnxRuntime TensorRT EP.

[chilo-ms]

There are at least three issues:

1. The ReduceMax node converted from the export_model.py you shared specified the opset_version to be 17. However, from the converted onnx model, the ReduceMax node doens't have axes input which is required by opset 17 (It's optional starts from opset 18).
   That might be the reason you saw this:

   2023-04-04 16:37:52.173723690 [E:onnxruntime:Default, tensorrt_execution_provider.h:61 log] [2023-04-04 16:37:52   ERROR] **ReduceMax_1597: at least 1 dimensions are required for input.**
   

   I think this issue is related to pytorch converter. But you also mentioned "issue is same with opset 16 and opset 18 is not validated for pytorch 2.0", that's a bit strange, since the axes input of ReduceMax is optional for opset 18.

2. The issue is related to some input tensor doesn't have shapes due to the main graph is being partitioned into TRT subgraphs and CUDA/CPU subgraphs where TRT requires all the graph input to have shape info. Typically, we ask user to run the shape inference script provided by us, but right now it has problmes running it. We are still investigating it.

   2023-04-04 16:37:52.324418966 [E:onnxruntime:, inference_session.cc:1532 operator()] Exception during initialization: /onnxruntime_src/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.cc:897 SubGraphCollection_t onnxruntime::TensorrtExecutionProvider::GetSupportedList(SubGraphCollection_t, int, int, const onnxruntime::GraphViewer&, bool*) const [ONNXRuntimeError] : 1 : FAIL : TensorRT input: /model/proposal_generator/GatherND_2_output_0 has no shape specified. Please run shape inference on the onnx model first. Details can be found in https://onnxruntime.ai/docs/execution-providers/TensorRT-ExecutionProvider.html#shape-inference-for-tensorrt-subgraphs**
   

3. The issue is because of "different output shapes of the conditionals (if node)" and it's the constraint of TensorRT where both branches for a conditional should have same output shape.

   [08/02/2023-19:15:46] [E] Error[7]: [shapeMachine.cpp::executeContinuation::864] Error Code 7: Internal Error (If_1530_OutputLayer: dimensions not compatible for if-conditional outputs Condition '==' violated: 0 != 6. Instruction: CHECK_EQUAL 0 6.)
   

   we discussed with Nvidia, right now TRT doesn't support conditionals with different output shapes.
   so yes, you need to find a way to surgeon this if node.

[datinje]

Thanks a lot for your investigation.
3. Yes I am working to fix the if then else branch to have the same output dimension. Actually this is fixed on my company model using Nvidia collaboration on the onnx- graphsurgeon tailored for the faster-rcnn we use.

1. Then pytorch onnx- converter has truly a problem since I am getting the problem for both opset16 and 17.
2. Could you tell me how to use your shape inference script when ready.
3. Still working on a simpler model for exercising reducemax. But I am in vacation and have no access to my computer now. Will be back on August 16.

[datinje]

Please find the small ReduceMax demonstrator model (thx to Nvidia Zhijin Li !)

When run, the onnxruntime + TRT EP passes on the resultant onnx file !
Similarly trtexec also passes.
Means that 1. pytorch generates the right API (no dim input required) , 2. onnxruntime TRT EP recognizes the ReduceMax API with no dim input (as of opset16). So does trtexec.

(side note : There is an error reported by the onnx checker tool which should not occur since opset16 is used. (problem seems to be on the onnx checker tools side).

However onnxruntime trt EP fails on the bigger faster-rcnn model from Detectron2 model zoo -above:

2023-08-08 15:39:49.852404881 [E:onnxruntime:Default, tensorrt_execution_provider.h:73 log] [2023-08-08 15:39:49   ERROR] ReduceMax_477: at least 1 dimensions are required for input.

So I would like to understand why onnxruntime trt EP this time requires a ReduceMax API that requires a dim input as of opset18 ?
Could it be because pytorch when generating the onnx file from the detectron2 model is generating at least 2 different ONNX blocks (this is what I found out when looking at the onnx file) : one block with opset16 ReduceMax API (no dim input parameter) and one with opset18 API (dim input parameter)?

Then I suspect that when onnxruntime EP code finds the opset18 ReduceMax API it switches to full opset18 and then when finding back the opset16 ReduceMax APi later in the onnx fle then it fails - by requesting the missing dim input.

Can you confirm the above ?
(if so then we have to fix the detectron2 model or fix the pytorch onnx exporter)

Here is the script full code (was made by Nvidia).

#! /usr/bin/env python3
## ---------------------------------------------------------------------------
##
## File: test_onnx_reducemax.py for project: GEHC Mammo
##
## Created by Zhijin Li
## E-mail:   <zhijinl@nvidia.com>
##
## Started on  Wed Jun 21 15:29:18 2023 Zhijin Li
## Last update Thu Jun 29 01:25:10 2023 Zhijin Li
## ---------------------------------------------------------------------------
import os
import torch
import numpy as np
import onnx
import onnxruntime as ort

ONNX_SAVE_PATH = './output/test-reducemax.onnx'

class ReduceMaxTest(torch.nn.Module):

  def forward(self, x):
    """
    X: list of 2D matrices.
    """
    # result = x.max(0).values
    result = torch.max(x, dim=0).values
    print('--> result:', result)

    return result

if __name__ == '__main__':

  if not os.path.exists('./output'):
    os.makedirs('./output', exist_ok=True)

  print('\n---------- ONNX VERSION: {}'.format(onnx.__version__))
  print('---------- ONNXRUNTIME VERSION: {}\n'.format(ort.__version__))

  # MUST set opset version to 16.
  # This will generate ReduceMax operator with `axes` as attribute.
  # Pytorch 2.0 currently does not support opset >= 17.
  # starting from opset 18, ReduceMax's attribute signature changes
  # by moving `axes` from attribute to input.
  #
  torch.onnx.export(
    ReduceMaxTest().eval(),
    torch.rand(64, 64),
    ONNX_SAVE_PATH,
    input_names=['inp'],
    output_names=['out'],
    dynamic_axes={
      'inp': [0,1]
    },
    verbose=True,
    opset_version=16
  )

  try:
    onnx_model = onnx.load(ONNX_SAVE_PATH)
    onnx.checker.check_model(onnx_model, full_check=True)
    print('---------- ONNX model check successful!')
  except Exception:
    print('\n!!!!! --> ONNX MODEL CHECK FAILED!\n')

  print(onnx.helper.printable_graph(onnx_model.graph))

  # Graph check should work for onnx 1.11 & 1.12, which supports
  # opset versions up to v16 and v17 respectively.
  # Starting from onnx version 1.13, which supports opset v18,
  # graph check will not work. This is probably a bug in onnx.
  #
  try:
    graph = onnx_model.graph
    onnx.checker.check_graph(graph)
    print('---------- ONNX graph check successful!')
  except Exception:
    print('\n!!!!! --> ONNX GRAPH CHECK FAILED!\n')

  # Run with TRT exec provider.
  providers = [(
    'TensorrtExecutionProvider',
    {
      'trt_fp16_enable': True,
      'trt_engine_cache_enable': True,
      'trt_engine_cache_path': './output/trt_engine/'
    })]
  # providers = [('CUDAExecutionProvider')]

  sess_opts = ort.SessionOptions()
  sess = ort.InferenceSession(ONNX_SAVE_PATH, sess_options=sess_opts, providers=providers)

  input = sess.get_inputs()[0]
  print('--> input name:', input.name)

  output = sess.get_outputs()[0]
  print('--> output name:', output.name)

  ort_inputs = {input.name: np.random.randn(57, 89).astype(np.float32)}
  ort_outputs = sess.run(None, ort_inputs)[0]

  print('--> ort_outputs value: ', ort_outputs)
  print('--> ort_outputs shape: ', ort_outputs.shape)
  print('\n---------- Runtime Execution Success.\n')

with regards,
JCD