refactor: Migrate Python bindings from pybind11 to nanobind

This commit migrates the Python bindings from pybind11 to nanobind with
intrusive reference counting for better memory management.

Key Changes:
- Updated CMakeLists.txt to use nanobind instead of pybind11
- Added intrusive_counter.cpp for nanobind's intrusive ref counting
- Migrated all Python bindings in python.cpp to use nanobind API
- Extracted wrapper classes to separate header files for modularity
- Added test coverage for all public APIs

Bug Fixes:
- Fixed tokenizer.encode() to use pure C API (avoid lifetime issues)
- Fixed NamedTensors.__setitem__ pointer dereference (was causing segfault)

Technical Details:
- Uses nb::intrusive_ptr for all Python-exposed C++ objects
- Follows pure C API pattern to avoid issues with non-copyable classes
- Intrusive reference counting eliminates shared_ptr overhead
- Compatible with nanobind's memory model

Files changed:
- cmake/deps.txt: Added nanobind dependency
- cmake/external/onnxruntime_external_deps.cmake: Fetch nanobind
- src/python/CMakeLists.txt: Build system changes
- src/python/python.cpp: Complete nanobind migration
- src/python/intrusive_counter.cpp: Reference counting support
- src/python/wrappers/*.h: Modular wrapper class headers
- test/python/test_onnxruntime_genai_api_coverage.py: API coverage tests

Author: ONNX GenAI Assistant

cmake/deps.txt

@@ -10,12 +10,12 @@
 #not affect built binaries.
 #
 # NOTE: You must run deps_update_and_upload.py and generate_cgmanifest.py when ready to test your changes in a CI.
-pybind11;https://github.com/pybind/pybind11/archive/refs/tags/v2.13.6.zip;f780292da9db273c8ef06ccf5fd4b623624143e9
+nanobind;https://github.com/wjakob/nanobind/archive/refs/tags/v2.9.2.zip;aeae2737eaac0013a806adb479a82e3b65ddab98
 googletest;https://github.com/google/googletest/archive/530d5c8c84abd2a46f38583ee817743c9b3a42b4.zip;5e3a61db2aa975cfd0f97ba92c818744e7fa7034
 microsoft_wil;https://github.com/microsoft/wil/archive/refs/tags/v1.0.230629.1.zip;e4a542a323c070376f7c2d1973d0f7ddbc1d2fa5
 directx_headers;https://github.com/microsoft/DirectX-Headers/archive/refs/tags/v1.613.1.zip;47653509a3371eabb156360f42faf582f314bf2e
 onnxruntime_extensions;https://github.com/microsoft/onnxruntime-extensions.git;68ee0397c002ef197ef5e9075c1c0f92b0db539a
 
 # These two dependencies are for the optional constrained decoding feature (USE_GUIDANCE)
-llguidance;https://github.com/microsoft/llguidance.git;94fa39128ef184ffeda33845f6d333f332a34b4d
+llguidance;https://github.com/microsoft/llguidance.git;2d2f1de3c87e3289528affc346f734f7471216d9
 corrosion;https://github.com/corrosion-rs/corrosion.git;b1fab721655c5c4b1b08a083d3cd29f163af75d0

cmake/external/onnxruntime_external_deps.cmake

@@ -20,18 +20,29 @@ endforeach()
 message("Loading Dependencies ...")
 
 if(ENABLE_PYTHON)
+  # Fetch robin_map dependency for nanobind (header-only library)
   FetchContent_Declare(
-    pybind11_project
-    URL ${DEP_URL_pybind11}
-    URL_HASH SHA1=${DEP_SHA1_pybind11}
-    FIND_PACKAGE_ARGS 2.6 NAMES pybind11
+    robin_map
+    GIT_REPOSITORY https://github.com/Tessil/robin-map.git
+    GIT_TAG v1.4.0
+    GIT_SHALLOW TRUE
   )
-  onnxruntime_fetchcontent_makeavailable(pybind11_project)
-
-  if(TARGET pybind11::module)
-    set(pybind11_lib pybind11::module)
-  else()
-    set(pybind11_dep pybind11::pybind11)
+  FetchContent_GetProperties(robin_map)
+  if(NOT robin_map_POPULATED)
+    FetchContent_Populate(robin_map)
+  endif()
+  
+  FetchContent_Declare(
+    nanobind
+    URL ${DEP_URL_nanobind}
+    URL_HASH SHA1=${DEP_SHA1_nanobind}
+  )
+  FetchContent_GetProperties(nanobind)
+  if(NOT nanobind_POPULATED)
+    FetchContent_Populate(nanobind)
+    # Copy robin_map into nanobind's ext directory before adding subdirectory
+    file(COPY ${robin_map_SOURCE_DIR}/ DESTINATION ${nanobind_SOURCE_DIR}/ext/robin_map)
+    add_subdirectory(${nanobind_SOURCE_DIR} ${nanobind_BINARY_DIR})
   endif()
 endif()
 

src/python/CMakeLists.txt

@@ -4,8 +4,13 @@ file(GLOB python_srcs CONFIGURE_DEPENDS
   "${CMAKE_CURRENT_SOURCE_DIR}/*.h"
   "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp"
 )
-find_package(Python COMPONENTS Interpreter Development)
-pybind11_add_module(python ${python_srcs})
+
+# Ensure intrusive_counter.cpp is included for nanobind intrusive reference counting
+list(APPEND python_srcs "${CMAKE_CURRENT_SOURCE_DIR}/intrusive_counter.cpp")
+
+# Find nanobind and create Python module
+find_package(Python COMPONENTS Interpreter Development.Module REQUIRED)
+nanobind_add_module(python STABLE_ABI NB_STATIC ${python_srcs})
 target_include_directories(python PRIVATE ${ORT_HEADER_DIR})
 target_link_directories(python PRIVATE ${ORT_LIB_DIR})
 target_link_libraries(python PRIVATE onnxruntime-genai)
@@ -103,7 +108,7 @@ if(BUILD_WHEEL)
 
   if(MANYLINUX)
     add_custom_target(PyPackageBuild
-      COMMAND ${PYTHON_EXECUTABLE} -m pip wheel --no-deps .
+      COMMAND ${Python_EXECUTABLE} -m pip wheel --no-deps .
       COMMAND ${CMAKE_COMMAND} -E remove ${WHEEL_TARGET_NAME}/onnxruntime_genai.cpython-*
       COMMAND auditwheel repair onnxruntime_genai*linux_x86_64.whl -w ${WHEEL_FILES_DIR} ${modified_exclude_list}
       WORKING_DIRECTORY "${WHEEL_FILES_DIR}"
@@ -112,7 +117,7 @@ if(BUILD_WHEEL)
     )
   else()
     add_custom_target(PyPackageBuild
-      COMMAND ${PYTHON_EXECUTABLE} -m pip wheel --no-deps .
+      COMMAND ${Python_EXECUTABLE} -m pip wheel --no-deps .
       WORKING_DIRECTORY "${WHEEL_FILES_DIR}"
       COMMENT "Building wheel on ${WHEEL_FILES_DIR}"
       EXCLUDE_FROM_ALL

src/python/intrusive_counter.cpp

@@ -0,0 +1,6 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+// This file must be compiled exactly once to provide the intrusive reference
+// counting implementation for nanobind
+#include <nanobind/intrusive/counter.inl>