Python: Release GIL during Processor::process()

dbaston · dbaston · commit 7c176c2369e7 · 2025-04-26T20:45:48.000-04:00
diff --git a/python/src/pybindings/feature_bindings.cpp b/python/src/pybindings/feature_bindings.cpp
@@ -42,59 +42,70 @@ class PyFeatureBase : public Feature
 
     double get_double(const std::string& name) const override
     {
+        py::gil_scoped_acquire gil;
         return get_py(name).cast<double>();
     }
 
     DoubleArray get_double_array(const std::string& name) const override
     {
+        py::gil_scoped_acquire gil;
         py::array_t<double> arr = get_py(name);
         return { arr.data(), static_cast<std::size_t>(arr.size()) };
     }
 
     std::int32_t get_int(const std::string& name) const override
     {
+        py::gil_scoped_acquire gil;
         return get_py(name).cast<int32_t>();
     }
 
     std::int64_t get_int64(const std::string& name) const override
     {
+        py::gil_scoped_acquire gil;
         return get_py(name).cast<int64_t>();
     }
 
     IntegerArray get_integer_array(const std::string& name) const override
     {
+        py::gil_scoped_acquire gil;
         py::array_t<std::int32_t> arr = get_py(name);
         return { arr.data(), static_cast<std::size_t>(arr.size()) };
     }
 
     Integer64Array get_integer64_array(const std::string& name) const override
     {
+        py::gil_scoped_acquire gil;
         py::array_t<std::int64_t> arr = get_py(name);
         return { arr.data(), static_cast<std::size_t>(arr.size()) };
     }
 
     std::string get_string(const std::string& name) const override
     {
+        py::gil_scoped_acquire gil;
         return get_py(name).cast<std::string>();
     }
 
     void set(const std::string& name, std::string value) override
     {
+        py::gil_scoped_acquire gil;
         set_py(name, py::str(value));
     }
 
     void set(const std::string& name, std::int32_t value) override
     {
+        py::gil_scoped_acquire gil;
         set_py(name, py::int_(value));
     }
 
     void set(const std::string& name, std::int64_t value) override
     {
+        py::gil_scoped_acquire gil;
         set_py(name, py::int_(value));
     }
 
     void set(const std::string& name, double value) override
     {
+        py::gil_scoped_acquire gil;
         set_py(name, py::float_(value));
     }
 
@@ -116,6 +127,8 @@ class PyFeatureBase : public Feature
     template<typename T>
     void set_array(const std::string& name, const T& value)
     {
+        py::gil_scoped_acquire gil;
+
         std::size_t shape[1]{ value.size };
         py::array_t<typename T::value_type> values(shape);
         auto x = values.template mutable_unchecked<1>();
@@ -127,6 +140,7 @@ class PyFeatureBase : public Feature
 
     ValueType field_type(const std::string& name) const override
     {
+        py::gil_scoped_acquire gil;
         py::object value = get_py(name);
 
         try {
@@ -139,6 +153,8 @@ class PyFeatureBase : public Feature
     const GEOSGeometry* geometry() const override
     {
         if (m_geom == nullptr) {
+            py::gil_scoped_acquire gil;
+
             py::object geom = geometry_py();
             if (py::isinstance<py::bytes>(geom)) {
                 py::buffer_info info = py::buffer(geom).request();
@@ -171,6 +187,8 @@ class PyFeatureBase : public Feature
             m_geom = nullptr;
         }
 
+        py::gil_scoped_acquire gil;
+
         if (other == nullptr) {
             set_geometry_py(py::none());
         } else {
@@ -195,6 +213,8 @@ class PyFeatureBase : public Feature
 
     void copy_to(Feature& other) const override
     {
+        py::gil_scoped_acquire gil;
+
         py::iterable field_list = fields();
         for (py::handle field : field_list) {
             std::string field_name = field.cast<std::string>();
diff --git a/python/src/pybindings/feature_source_bindings.cpp b/python/src/pybindings/feature_source_bindings.cpp
@@ -31,11 +31,13 @@ class PyFeatureSourceBase : public FeatureSource
 
     std::size_t count() const override
     {
+        py::gil_scoped_acquire gil;
         return py_count().cast<std::size_t>();
     }
 
     bool next() override
     {
+        py::gil_scoped_acquire gil;
         if (!m_initialized) {
             m_it = py_iter();
             m_initialized = true;
@@ -52,13 +54,15 @@ class PyFeatureSourceBase : public FeatureSource
 
     const Feature& feature() const override
     {
+        py::gil_scoped_acquire gil;
         return m_feature.cast<const Feature&>();
     }
 
     virtual py::object py_iter() = 0;
 
     virtual py::object py_srs_wkt() const
     {
+        py::gil_scoped_acquire gil;
         return py::none();
     }
 
diff --git a/python/src/pybindings/processor_bindings.cpp b/python/src/pybindings/processor_bindings.cpp
@@ -32,7 +32,10 @@ bind_processor(py::module& m)
       .def("add_operation", &Processor::add_operation)
       .def("add_col", &Processor::include_col)
       .def("add_geom", &Processor::include_geometry)
-      .def("process", &Processor::process)
+      .def("process", [](Processor& self) {
+          py::gil_scoped_release release;
+          self.process();
+      })
       .def("set_grid_compat_tol", &Processor::set_grid_compat_tol)
       .def("set_max_cells_in_memory", &Processor::set_max_cells_in_memory, py::arg("n"))
       .def("set_progress_fn", [](Processor& self, py::function fn) {
diff --git a/python/src/pybindings/raster_source_bindings.cpp b/python/src/pybindings/raster_source_bindings.cpp
@@ -30,18 +30,24 @@ class NumPyRaster : public AbstractRaster<T>
     NumPyRaster(py::array_t<T, py::array::c_style | py::array::forcecast> array,
                 const Grid<bounded_extent>& g)
       : AbstractRaster<T>(g)
-      , m_array(array)
+      , m_array(std::make_unique<py::array_t<T>>(array))
       , m_array_proxy(array.template unchecked<2>())
     {
     }
 
+    virtual ~NumPyRaster()
+    {
+        py::gil_scoped_acquire gil;
+        m_array.reset();
+    }
+
     T operator()(std::size_t row, std::size_t col) const override
     {
         return m_array_proxy(row, col);
     }
 
   private:
-    py::array_t<T> m_array;
+    std::unique_ptr<py::array_t<T>> m_array;
     Unchecked2DArrayProxy m_array_proxy;
 };
 
@@ -83,6 +89,8 @@ class PyRasterSourceBase : public RasterSource
 
     RasterVariant read_box(const Box& box) override
     {
+        py::gil_scoped_acquire gil;
+
         auto cropped_grid = grid().crop(box);
 
         auto x0 = cropped_grid.col_offset(grid());
@@ -119,6 +127,8 @@ class PyRasterSourceBase : public RasterSource
     const Grid<bounded_extent>& grid() const override
     {
         if (m_grid == nullptr) {
+            py::gil_scoped_acquire gil;
+
             py::sequence grid_ext = extent();
 
             if (grid_ext.size() != 4) {
diff --git a/python/src/pybindings/writer_bindings.cpp b/python/src/pybindings/writer_bindings.cpp
@@ -45,6 +45,8 @@ class PyWriter : public OutputWriter
 
     void write(const Feature& f) override
     {
+        py::gil_scoped_acquire gil;
+
         // https://github.com/pybind/pybind11/issues/2033#issuecomment-703177186
         py::object dummy = py::cast(f, py::return_value_policy::reference);
         PYBIND11_OVERLOAD_PURE(void, PyWriter, write, f);

Original file line number	Diff line number	Diff line change
`@@ -42,59 +42,70 @@ class PyFeatureBase : public Feature`
`42`	`42`
`43`	`43`	`double get_double(const std::string& name) const override`
`44`	`44`	`{`
	`45`	`+ py::gil_scoped_acquire gil;`
`45`	`46`	`return get_py(name).cast<double>();`
`46`	`47`	`}`
`47`	`48`
`48`	`49`	`DoubleArray get_double_array(const std::string& name) const override`
`49`	`50`	`{`
	`51`	`+ py::gil_scoped_acquire gil;`
`50`	`52`	`py::array_t<double> arr = get_py(name);`
`51`	`53`	`return { arr.data(), static_cast<std::size_t>(arr.size()) };`
`52`	`54`	`}`
`53`	`55`
`54`	`56`	`std::int32_t get_int(const std::string& name) const override`
`55`	`57`	`{`
	`58`	`+ py::gil_scoped_acquire gil;`
`56`	`59`	`return get_py(name).cast<int32_t>();`
`57`	`60`	`}`
`58`	`61`
`59`	`62`	`std::int64_t get_int64(const std::string& name) const override`
`60`	`63`	`{`
	`64`	`+ py::gil_scoped_acquire gil;`
`61`	`65`	`return get_py(name).cast<int64_t>();`
`62`	`66`	`}`
`63`	`67`
`64`	`68`	`IntegerArray get_integer_array(const std::string& name) const override`
`65`	`69`	`{`
	`70`	`+ py::gil_scoped_acquire gil;`
`66`	`71`	`py::array_t<std::int32_t> arr = get_py(name);`
`67`	`72`	`return { arr.data(), static_cast<std::size_t>(arr.size()) };`
`68`	`73`	`}`
`69`	`74`
`70`	`75`	`Integer64Array get_integer64_array(const std::string& name) const override`
`71`	`76`	`{`
	`77`	`+ py::gil_scoped_acquire gil;`
`72`	`78`	`py::array_t<std::int64_t> arr = get_py(name);`
`73`	`79`	`return { arr.data(), static_cast<std::size_t>(arr.size()) };`
`74`	`80`	`}`
`75`	`81`
`76`	`82`	`std::string get_string(const std::string& name) const override`
`77`	`83`	`{`
	`84`	`+ py::gil_scoped_acquire gil;`
`78`	`85`	`return get_py(name).cast<std::string>();`
`79`	`86`	`}`
`80`	`87`
`81`	`88`	`void set(const std::string& name, std::string value) override`
`82`	`89`	`{`
	`90`	`+ py::gil_scoped_acquire gil;`
`83`	`91`	`set_py(name, py::str(value));`
`84`	`92`	`}`
`85`	`93`
`86`	`94`	`void set(const std::string& name, std::int32_t value) override`
`87`	`95`	`{`
	`96`	`+ py::gil_scoped_acquire gil;`
`88`	`97`	`set_py(name, py::int_(value));`
`89`	`98`	`}`
`90`	`99`
`91`	`100`	`void set(const std::string& name, std::int64_t value) override`
`92`	`101`	`{`
	`102`	`+ py::gil_scoped_acquire gil;`
`93`	`103`	`set_py(name, py::int_(value));`
`94`	`104`	`}`
`95`	`105`
`96`	`106`	`void set(const std::string& name, double value) override`
`97`	`107`	`{`
	`108`	`+ py::gil_scoped_acquire gil;`
`98`	`109`	`set_py(name, py::float_(value));`
`99`	`110`	`}`
`100`	`111`
`@@ -116,6 +127,8 @@ class PyFeatureBase : public Feature`
`116`	`127`	`template<typename T>`
`117`	`128`	`void set_array(const std::string& name, const T& value)`
`118`	`129`	`{`
	`130`	`+ py::gil_scoped_acquire gil;`
	`131`	`+`
`119`	`132`	`std::size_t shape[1]{ value.size };`
`120`	`133`	`py::array_t<typename T::value_type> values(shape);`
`121`	`134`	`auto x = values.template mutable_unchecked<1>();`
`@@ -127,6 +140,7 @@ class PyFeatureBase : public Feature`
`127`	`140`
`128`	`141`	`ValueType field_type(const std::string& name) const override`
`129`	`142`	`{`
	`143`	`+ py::gil_scoped_acquire gil;`
`130`	`144`	`py::object value = get_py(name);`
`131`	`145`
`132`	`146`	`try {`
`@@ -139,6 +153,8 @@ class PyFeatureBase : public Feature`
`139`	`153`	`const GEOSGeometry* geometry() const override`
`140`	`154`	`{`
`141`	`155`	`if (m_geom == nullptr) {`
	`156`	`+ py::gil_scoped_acquire gil;`
	`157`	`+`
`142`	`158`	`py::object geom = geometry_py();`
`143`	`159`	`if (py::isinstance<py::bytes>(geom)) {`
`144`	`160`	`py::buffer_info info = py::buffer(geom).request();`
`@@ -171,6 +187,8 @@ class PyFeatureBase : public Feature`
`171`	`187`	`m_geom = nullptr;`
`172`	`188`	`}`
`173`	`189`
	`190`	`+ py::gil_scoped_acquire gil;`
	`191`	`+`
`174`	`192`	`if (other == nullptr) {`
`175`	`193`	`set_geometry_py(py::none());`
`176`	`194`	`} else {`
`@@ -195,6 +213,8 @@ class PyFeatureBase : public Feature`
`195`	`213`
`196`	`214`	`void copy_to(Feature& other) const override`
`197`	`215`	`{`
	`216`	`+ py::gil_scoped_acquire gil;`
	`217`	`+`
`198`	`218`	`py::iterable field_list = fields();`
`199`	`219`	`for (py::handle field : field_list) {`
`200`	`220`	`std::string field_name = field.cast<std::string>();`
Original file line number	Diff line number	Diff line change
`@@ -31,11 +31,13 @@ class PyFeatureSourceBase : public FeatureSource`
`31`	`31`
`32`	`32`	`std::size_t count() const override`
`33`	`33`	`{`
	`34`	`+ py::gil_scoped_acquire gil;`
`34`	`35`	`return py_count().cast<std::size_t>();`
`35`	`36`	`}`
`36`	`37`
`37`	`38`	`bool next() override`
`38`	`39`	`{`
	`40`	`+ py::gil_scoped_acquire gil;`
`39`	`41`	`if (!m_initialized) {`
`40`	`42`	`m_it = py_iter();`
`41`	`43`	`m_initialized = true;`
`@@ -52,13 +54,15 @@ class PyFeatureSourceBase : public FeatureSource`
`52`	`54`
`53`	`55`	`const Feature& feature() const override`
`54`	`56`	`{`
	`57`	`+ py::gil_scoped_acquire gil;`
`55`	`58`	`return m_feature.cast<const Feature&>();`
`56`	`59`	`}`
`57`	`60`
`58`	`61`	`virtual py::object py_iter() = 0;`
`59`	`62`
`60`	`63`	`virtual py::object py_srs_wkt() const`
`61`	`64`	`{`
	`65`	`+ py::gil_scoped_acquire gil;`
`62`	`66`	`return py::none();`
`63`	`67`	`}`
`64`	`68`
Original file line number	Diff line number	Diff line change
`@@ -45,6 +45,8 @@ class PyWriter : public OutputWriter`
`45`	`45`
`46`	`46`	`void write(const Feature& f) override`
`47`	`47`	`{`
	`48`	`+ py::gil_scoped_acquire gil;`
	`49`	`+`
`48`	`50`	`// https://github.com/pybind/pybind11/issues/2033#issuecomment-703177186`
`49`	`51`	`py::object dummy = py::cast(f, py::return_value_policy::reference);`
`50`	`52`	`PYBIND11_OVERLOAD_PURE(void, PyWriter, write, f);`