[TensorRT EP] Refactor TRT version update logic & apply TRT 10.5

onnxruntime/python/tools/tensorrt/perf/build/build_image.py

@@ -17,8 +17,8 @@
 TRT_DOCKER_FILES = {
     "8.6.cuda_11_8_cudnn_8": "tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda11_8_tensorrt8_6",
     "8.6.cuda_12_3_cudnn_9": "tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda12_3_tensorrt8_6",
-    "10.4.cuda_11_8_cudnn_8": "tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda11_tensorrt10",
-    "10.4.cuda_12_5_cudnn_9": "tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda12_tensorrt10",
+    "10.5.cuda_11_8_cudnn_8": "tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda11_tensorrt10",
+    "10.5.cuda_12_5_cudnn_9": "tools/ci_build/github/linux/docker/Dockerfile.ubuntu_cuda12_tensorrt10",
     "BIN": "tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt_bin",
 }
 

tools/ci_build/build.py

@@ -2120,10 +2120,10 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs):
             if not args.disable_ml_ops and not args.use_tensorrt:
                 run_subprocess([sys.executable, "onnxruntime_test_python_mlops.py"], cwd=cwd, dll_path=dll_path)
 
-            # if args.use_tensorrt:
-            #     run_subprocess(
-            #         [sys.executable, "onnxruntime_test_python_nested_control_flow_op.py"], cwd=cwd, dll_path=dll_path
-            #     )
+            if args.use_tensorrt:
+                run_subprocess(
+                    [sys.executable, "onnxruntime_test_python_nested_control_flow_op.py"], cwd=cwd, dll_path=dll_path
+                )
 
             try:
                 import onnx  # noqa: F401

tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml

@@ -40,7 +40,6 @@ parameters:
   default: 0
 
 variables:
-  - template: templates/common-variables.yml
   - name: docker_base_image
     value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241020.1
   - name: linux_trt_version
@@ -123,7 +122,7 @@ stages:
                 --build_shared_lib \
                 --parallel \
                 --build_wheel \
-                --enable_onnx_tests --use_cuda --cuda_version=${{variables.common_cuda_version}} --cuda_home=/usr/local/cuda-${{variables.common_cuda_version}} --cudnn_home=/usr/local/cuda-${{variables.common_cuda_version}} \
+                --enable_onnx_tests --use_cuda --cuda_version=11.8 --cuda_home=/usr/local/cuda-11.8 --cudnn_home=/usr/local/cuda-11.8 \
                 --enable_cuda_profiling --enable_cuda_nhwc_ops \
                 --enable_pybind --build_java \
                 --use_cache \

tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml

@@ -77,13 +77,14 @@ resources:
     ref: 5eda9aded5462201e6310105728d33016e637ea7
 
 variables:
+- template: templates/common-variables.yml
 - name: ReleaseVersionSuffix
   value: ''
 - name: win_trt_version
   value: 11.8
 
 - name: win_trt_home
-  value: $(Agent.TempDirectory)\TensorRT-10.4.0.26.Windows10.x86_64.cuda-11.8
+  value: $(Agent.TempDirectory)\${{ variables.win_trt_folder_cuda11 }}
 - name: win_cuda_home
   value: $(Agent.TempDirectory)\v11.8
 

tools/ci_build/github/azure-pipelines/cuda-packaging-pipeline.yml

@@ -59,13 +59,14 @@ parameters:
       - 12.2
 
 variables:
+  - template: templates/common-variables.yml
   - name: ReleaseVersionSuffix
     value: ''
   - name: win_trt_home
     ${{ if eq(parameters.CudaVersion, '11.8') }}:
-      value: $(Agent.TempDirectory)\TensorRT-10.4.0.26.Windows10.x86_64.cuda-11.8
+      value: $(Agent.TempDirectory)\${{ variables.win_trt_folder_cuda11 }}
     ${{ if eq(parameters.CudaVersion, '12.2') }}:
-      value: $(Agent.TempDirectory)\TensorRT-10.4.0.26.Windows10.x86_64.cuda-12.6
+      value: $(Agent.TempDirectory)\${{ variables.win_trt_folder_cuda12 }}
   - name: win_cuda_home
     ${{ if eq(parameters.CudaVersion, '11.8') }}:
       value: $(Agent.TempDirectory)\v11.8

tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml

@@ -37,16 +37,17 @@ parameters:
       - 12.2
 
 variables:
+  - template: templates/common-variables.yml
   - name: docker_base_image
     ${{ if eq(parameters.CudaVersion, '11.8') }}:
       value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241020.1
     ${{ if eq(parameters.CudaVersion, '12.2') }}:
       value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241020.1
   - name: linux_trt_version
     ${{ if eq(parameters.CudaVersion, '11.8') }}:
-      value: 10.4.0.26-1.cuda11.8
+      value: ${{ variables.linux_trt_version_cuda11 }}
     ${{ if eq(parameters.CudaVersion, '12.2') }}:
-      value: 10.4.0.26-1.cuda12.6
+      value: ${{ variables.linux_trt_version_cuda12 }}
 
 jobs:
 - job: Linux_Build

tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-daily-perf-pipeline.yml

@@ -8,12 +8,12 @@ parameters:
 - name: TrtVersion
   displayName: TensorRT Version
   type: string
-  default: 10.4.cuda_12_5_cudnn_9
+  default: 10.5.cuda_12_5_cudnn_9
   values:
   - 8.6.cuda_11_8_cudnn_8
   - 8.6.cuda_12_3_cudnn_9
-  - 10.4.cuda_11_8_cudnn_8
-  - 10.4.cuda_12_5_cudnn_9
+  - 10.5.cuda_11_8_cudnn_8
+  - 10.5.cuda_12_5_cudnn_9
   - BIN
 
 - name: UseTensorrtOssParser

tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml

@@ -23,6 +23,7 @@ stages:
     pool: ${{ parameters.AgentPool }}
 
     variables:
+    - template: ../../templates/common-variables.yml
     - name: OnnxRuntimeBuildDirectory
       value: '$(Build.BinariesDirectory)'
 
@@ -52,7 +53,7 @@ stages:
       inputs:
         script: |
           ln -sf /data/models $(Build.BinariesDirectory)
-
+    # As for Debian installation, replace '-1.' by '-1+' when assigning trt version below
     - ${{if contains(parameters.StageSuffix , 'GPU') }}:
       - template: ../../templates/get-docker-image-steps.yml
         parameters:
@@ -61,7 +62,7 @@ stages:
           ${{ if eq(parameters.CudaVersion, '12.2') }}:
             DockerBuildArgs: "
             --build-arg BASEIMAGE=nvidia/cuda:12.2.2-devel-ubuntu20.04
-            --build-arg TRT_VERSION=10.4.0.26-1+cuda12.6
+            --build-arg TRT_VERSION=${{ replace(variables.linux_trt_version_cuda12, '-1.', '-1+') }}
             --build-arg BUILD_UID=$( id -u )
             "
           ${{ else }}:

tools/ci_build/github/azure-pipelines/post-merge-jobs.yml

@@ -1,3 +1,20 @@
+parameters:
+- name: CudaVersion
+  displayName: CUDA version
+  type: string
+  default: '12.2'
+  values:
+    - 11.8
+    - 12.2
+
+variables:
+  - template: templates/common-variables.yml
+  - name: win_trt_folder
+    ${{ if eq(parameters.CudaVersion, '11.8') }}:
+      value: ${{ variables.win_trt_folder_cuda11 }}
+    ${{ if eq(parameters.CudaVersion, '12.2') }}:
+      value: ${{ variables.win_trt_folder_cuda12 }}
+
 stages:
 - ${{ if or(startsWith(variables['System.CollectionUri'], 'https://dev.azure.com/aiinfra/'),startsWith(variables['System.CollectionUri'], 'https://aiinfra.visualstudio.com/')) }}:
   - template: templates/web-ci.yml
@@ -206,7 +223,7 @@ stages:
         BuildConfig: 'RelWithDebInfo'
         EnvSetupScript: setup_env_cuda.bat
         buildArch: x64
-        additionalBuildFlags: --enable_pybind --build_java --build_nodejs --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --enable_cuda_profiling --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86
+        additionalBuildFlags: --enable_pybind --build_java --build_nodejs --use_cuda --cuda_home="$(Agent.TempDirectory)\v${{ parameters.CudaVersion }}" --enable_cuda_profiling --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86
         msbuildPlatform: x64
         isX86: false
         job_name_suffix: x64_RelWithDebInfo
@@ -226,7 +243,7 @@ stages:
         BuildConfig: 'RelWithDebInfo'
         EnvSetupScript: setup_env_trt.bat
         buildArch: x64
-        additionalBuildFlags: --enable_pybind --build_java --build_nodejs --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --enable_cuda_profiling --use_tensorrt --tensorrt_home="$(Agent.TempDirectory)\TensorRT-10.4.0.26.Windows10.x86_64.cuda-11.8" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86
+        additionalBuildFlags: --enable_pybind --build_java --build_nodejs --use_cuda --cuda_home="$(Agent.TempDirectory)\v${{ parameters.CudaVersion }}" --enable_cuda_profiling --use_tensorrt --tensorrt_home="$(Agent.TempDirectory)\${{ variables.win_trt_folder }}" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86
         msbuildPlatform: x64
         isX86: false
         job_name_suffix: x64_RelWithDebInfo

tools/ci_build/github/azure-pipelines/py-cuda-package-test-pipeline.yml

@@ -19,6 +19,5 @@ stages:
           python_wheel_suffix: '_gpu'
           timeout: 480
           docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241020.1
-          trt_version: '10.4.0.26-1.cuda12.6'
           cuda_version: '12.2'
 

tools/ci_build/github/azure-pipelines/stages/jobs/py-linux-cuda-package-test-job.yml

@@ -42,16 +42,17 @@ jobs:
       dependsOn: [ ]
     timeoutInMinutes: ${{ parameters.timeout }}
     variables:
+      - template: ../../templates/common-variables.yml
       - name: docker_base_image
         ${{ if eq(parameters.CudaVersion, '11.8') }}:
-          value: nvidia/cuda:11.8.0-cudnn8-devel-ubi8
+          value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241020.1
         ${{ if eq(parameters.CudaVersion, '12.2') }}:
           value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241020.1
       - name: linux_trt_version
         ${{ if eq(parameters.CudaVersion, '11.8') }}:
-          value: 10.4.0.26-1.cuda11.8
+          value: ${{ variables.linux_trt_version_cuda11 }}
         ${{ if eq(parameters.CudaVersion, '12.2') }}:
-          value: 10.4.0.26-1.cuda12.6
+          value: ${{ variables.linux_trt_version_cuda12 }}
     pool: ${{ parameters.machine_pool }}
     steps:
       - checkout: self

tools/ci_build/github/azure-pipelines/stages/nuget-linux-cuda-packaging-stage.yml

@@ -68,19 +68,19 @@ stages:
     timeoutInMinutes: 180
     pool: 'onnxruntime-Ubuntu2204-AMD-CPU'
     variables:
+    - template: ../templates/common-variables.yml
     - name: CUDA_VERSION_MAJOR
       ${{ if eq(parameters.CudaVersion, '11.8') }}:
         value: '11'
       ${{ if eq(parameters.CudaVersion, '12.2') }}:
         value: '12'
     - name: CUDA_VERSION
       value: ${{ parameters.CudaVersion }}
-
     - name: linux_trt_version
       ${{ if eq(parameters.CudaVersion, '11.8') }}:
-        value: 10.4.0.26-1.cuda11.8
+        value: ${{ variables.linux_trt_version_cuda11 }}
       ${{ if eq(parameters.CudaVersion, '12.2') }}:
-        value: 10.4.0.26-1.cuda12.6
+        value: ${{ variables.linux_trt_version_cuda12 }}
     steps:
     - checkout: self
       clean: true
@@ -140,16 +140,17 @@ stages:
       clean: all
     pool: 'Onnxruntime-Linux-GPU'
     variables:
+    - template: ../templates/common-variables.yml
     - name: CUDA_VERSION_MAJOR
       ${{ if eq(parameters.CudaVersion, '11.8') }}:
         value: '11'
       ${{ if eq(parameters.CudaVersion, '12.2') }}:
         value: '12'
     - name: linux_trt_version
       ${{ if eq(parameters.CudaVersion, '11.8') }}:
-        value: 10.4.0.26-1.cuda11.8
+        value: ${{ variables.linux_trt_version_cuda11 }}
       ${{ if eq(parameters.CudaVersion, '12.2') }}:
-        value: 10.4.0.26-1.cuda12.6
+        value: ${{ variables.linux_trt_version_cuda12 }}
     steps:
     - checkout: self                           # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/onnxruntime
       submodules: false

tools/ci_build/github/azure-pipelines/stages/py-gpu-packaging-stage.yml

@@ -56,10 +56,8 @@ stages:
           PYTHON_VERSION: ${{ python_version }}
           EP_NAME: gpu
           CudaVersion: ${{ parameters.cuda_version }}
-          ${{ if eq(parameters.cuda_version, '11.8') }}:
-            EP_BUILD_FLAGS: --enable_lto --use_tensorrt --tensorrt_home=$(Agent.TempDirectory)\TensorRT-10.4.0.26.Windows10.x86_64.cuda-11.8 --cuda_home=$(Agent.TempDirectory)\v11.8  --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
-          ${{ if eq(parameters.cuda_version, '12.2') }}:
-            EP_BUILD_FLAGS: --enable_lto --use_tensorrt --tensorrt_home=$(Agent.TempDirectory)\TensorRT-10.4.0.26.Windows10.x86_64.cuda-12.6 --cuda_home=$(Agent.TempDirectory)\v12.2  --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
+          EP_BUILD_FLAGS: --enable_lto --cuda_home=$(Agent.TempDirectory)\v${{ parameters.cuda_version }} --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
+          use_tensorrt: True
 
   - ${{ if eq(parameters.enable_linux_cuda, true) }}:
       - template: py-linux-gpu-stage.yml
@@ -71,10 +69,8 @@ stages:
           cuda_version: ${{ parameters.cuda_version }}
           ${{ if eq(parameters.cuda_version, '11.8') }}:
             docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241020.1
-            trt_version: 10.4.0.26-1.cuda11.8
           ${{ if eq(parameters.cuda_version, '12.2') }}:
             docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241020.1
-            trt_version: 10.4.0.26-1.cuda12.6
 
   - ${{ if eq(parameters.enable_windows_dml, true) }}:
     - ${{ each python_version in parameters.PythonVersions }}:

tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml

@@ -20,12 +20,6 @@ parameters:
 - name: docker_base_image
   type: string
 
-- name: trt_version
-  type: string
-  default: '10.4.0.26-1.cuda11.8'
-  values:
-    - 10.4.0.26-1.cuda11.8
-    - 10.4.0.26-1.cuda12.6
 - name: cuda_version
   type: string
   default: '11.8'
@@ -71,6 +65,12 @@ stages:
           value: -x ${{ parameters.extra_build_arg }}
         ${{ if eq(parameters.extra_build_arg, '') }}:
           value: ''
+      - template: ../templates/common-variables.yml
+      - name: trt_version
+        ${{ if eq(parameters.cuda_version, '11.8') }}:
+          value: ${{ variables.linux_trt_version_cuda11 }}
+        ${{ if eq(parameters.cuda_version, '12.2') }}:
+          value: ${{ variables.linux_trt_version_cuda12 }}
     steps:
       - checkout: self
         clean: true
@@ -82,7 +82,7 @@ stages:
         parameters:
           Dockerfile: tools/ci_build/github/linux/docker/inference/${{ parameters.arch }}/python/cuda/Dockerfile
           Context: tools/ci_build/github/linux/docker/inference/${{ parameters.arch }}/python/cuda
-          DockerBuildArgs: "--build-arg BASEIMAGE=${{ parameters.docker_base_image }} --build-arg TRT_VERSION=${{ parameters.trt_version }} --build-arg BUILD_UID=$( id -u )"
+          DockerBuildArgs: "--build-arg BASEIMAGE=${{ parameters.docker_base_image }} --build-arg TRT_VERSION=${{ variables.trt_version }} --build-arg BUILD_UID=$( id -u )"
           Repository: onnxruntimecuda${{ replace(parameters.cuda_version, '.', '') }}xtrt86build${{ parameters.arch }}
 
 

tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml

@@ -37,6 +37,10 @@ parameters:
    - Release
    - RelWithDebInfo
    - MinSizeRel
+
+- name: use_tensorrt
+  type: boolean
+  default: false
 
 stages:
   - stage: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build
@@ -65,9 +69,23 @@ stages:
           targetPath: $(Build.ArtifactStagingDirectory)
           artifactName: win_${{ parameters.EP_NAME }}_wheel_${{ parameters.PYTHON_VERSION }}
       variables:
-        GRADLE_OPTS: '-Dorg.gradle.daemon=false'
-        VSGenerator: 'Visual Studio 17 2022'
-        CUDA_MODULE_LOADING: 'LAZY'
+      - template: ../templates/common-variables.yml
+      - name: GRADLE_OPTS
+        value: '-Dorg.gradle.daemon=false'
+      - name: VSGenerator
+        value: 'Visual Studio 17 2022'
+      - name: CUDA_MODULE_LOADING
+        value: 'LAZY'
+      - name: win_trt_folder
+        ${{ if eq(parameters.CudaVersion, '11.8') }}:
+          value: ${{ variables.win_trt_folder_cuda11 }}
+        ${{ if eq(parameters.CudaVersion, '12.2') }}:
+          value: ${{ variables.win_trt_folder_cuda12 }}
+      - name: trt_build_flag
+        ${{ if eq(parameters.use_tensorrt, true) }}:
+          value: '--use_tensorrt --tensorrt_home="$(Agent.TempDirectory)\${{ variables.win_trt_folder }}"'
+        ${{ if eq(parameters.use_tensorrt, false) }}:
+          value: ''
       steps:
           - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
             displayName: 'Clean Agent Directories'
@@ -91,19 +109,17 @@ stages:
             - template: ../templates/jobs/set-winenv.yml
               parameters:
                 EnvSetupScript: ${{ parameters.ENV_SETUP_SCRIPT }}
-                ${{ if or(contains(parameters.EP_BUILD_FLAGS, 'use_cuda'), contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt')) }}:
+                ${{ if or(contains(parameters.EP_BUILD_FLAGS, 'use_cuda'), eq(parameters.use_tensorrt, true)) }}:
                   DownloadCUDA: true
-                ${{ if contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt') }}:
-                  DownloadTRT: true
+                DownloadTRT: ${{ parameters.use_tensorrt }}
 
           - ${{ if eq(parameters.ENV_SETUP_SCRIPT, '') }}:
             - template: ../templates/jobs/download_win_gpu_library.yml
               parameters:
                 CudaVersion: ${{ parameters.CudaVersion }}
-                ${{ if or(contains(parameters.EP_BUILD_FLAGS, 'use_cuda'), contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt')) }}:
+                ${{ if or(contains(parameters.EP_BUILD_FLAGS, 'use_cuda'), eq(parameters.use_tensorrt, true)) }}:
                   DownloadCUDA: true
-                ${{ if contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt') }}:
-                  DownloadTRT: true
+                DownloadTRT: ${{ parameters.use_tensorrt }}
 
           - task: PythonScript@0
             displayName: 'Update deps.txt'
@@ -126,7 +142,7 @@ stages:
                 --enable_pybind
                 --enable_onnx_tests
                 --parallel --use_binskim_compliant_compile_flags --update --build
-                $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }}
+                $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }} ${{ variables.trt_build_flag }}
               workingDirectory: '$(Build.BinariesDirectory)'
 
           # Esrp signing

tools/ci_build/github/azure-pipelines/templates/common-variables.yml

@@ -1,3 +1,7 @@
 variables:
-  common_cuda_version: '11.8'
-  common_cuda_baseimg: 'nvidia/cuda:11.8.0-cudnn8-devel-ubi8'
+  common_trt_version: '10.5.0.18'
+  # As for Debian installation, replace '-1.' by '-1+' when assigning trt version below
+  linux_trt_version_cuda11: ${{ variables.common_trt_version }}-1.cuda11.8
+  linux_trt_version_cuda12: ${{ variables.common_trt_version }}-1.cuda12.6
+  win_trt_folder_cuda11: TensorRT-${{ variables.common_trt_version }}.Windows10.x86_64.cuda-11.8
+  win_trt_folder_cuda12: TensorRT-${{ variables.common_trt_version }}.Windows10.x86_64.cuda-12.6