developmentseed · vincentsarago · Oct 17, 2024 · Oct 17, 2024
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -52,6 +52,41 @@ jobs:
           name: Python${{ matrix.python-version }}
           fail_ci_if_error: false
 
+  benchmark:
+    needs: [tests]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e ".[benchmark]"
+
+      - name: Run Benchmark
+        run: |
+          python -m pytest tests/benchmarks.py --benchmark-only --benchmark-columns 'min, max, mean, median' --benchmark-sort 'min' --benchmark-json output.json
+
+      - name: Store and Compare benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: morecantile Benchmarks
+          tool: 'pytest'
+          output-file-path: output.json
+          alert-threshold: '150%'
+          comment-on-alert: true
+          fail-on-alert: false
+          # GitHub API token to make a commit comment
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          gh-pages-branch: 'gh-benchmarks'
+          # Make a commit on `gh-pages` only if main
+          auto-push: ${{ github.ref == 'refs/heads/main' }}
+          benchmark-data-dir-path: dev/benchmarks
+
   publish:
     needs: [tests]
     runs-on: ubuntu-latest

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -24,6 +24,13 @@ This repo is set to use `pre-commit` to run *isort*, *flake8*, *pydocstring*, *b
 $ pre-commit install
 ```
 
+##### Performance tests
+
+```sh
+python -m pip install -e ".[benchmark]"
+python -m pytest tests/benchmarks.py --benchmark-only --benchmark-columns 'min, max, mean, median' --benchmark-sort 'min'
+```
+
 ### Docs
 
 ```bash

diff --git a/benchmarks/README.md b/benchmarks/README.md
@@ -0,0 +1,63 @@
+
+## Benchmark
+
+Compare `mercantile`, `utiles` and `morecantile`
+
+```sh
+python -m pip install -r requirements.txt
+python -m pytest benchmarks.py --benchmark-only --benchmark-columns 'min, max, mean, median' --benchmark-sort 'min'
+```
+
+```
+----------------------------------------------------- benchmark 'bounds': 21 tests ----------------------------------------------------
+Name (time in ns)                                   Min                     Max                   Mean                 Median
+---------------------------------------------------------------------------------------------------------------------------------------
+test_bounds[utiles-(486, 332, 30)]              83.0000 (1.0)       52,000.0000 (52.97)       195.3371 (1.48)        208.0000 (1.60)
+test_bounds[utiles-(0, 0, 0)]                   83.0000 (1.00)      31,167.0000 (31.75)       193.5957 (1.47)        208.0000 (1.60)
+test_bounds[utiles-(1, 0, 1)]                  125.0000 (1.51)         981.6700 (1.0)         132.0177 (1.0)         130.8300 (1.01)
+test_bounds[utiles-(1, 40, 7)]                 125.0000 (1.51)       8,606.2500 (8.77)        132.5298 (1.00)        130.0000 (1.0)
+test_bounds[utiles-(486, 332, 10)]             128.3300 (1.55)       1,600.8300 (1.63)        133.4046 (1.01)        131.6700 (1.01)
+test_bounds[utiles-(1, 1, 1)]                  129.5900 (1.56)      16,846.2500 (17.16)       141.9846 (1.08)        134.1700 (1.03)
+test_bounds[utiles-(486, 332, 20)]             138.3300 (1.67)       1,123.3300 (1.14)        142.5420 (1.08)        141.2500 (1.09)
+test_bounds[mercantile-(1, 40, 7)]           1,000.0000 (12.05)     92,416.0000 (94.14)     1,193.0112 (9.04)      1,167.0000 (8.98)
+test_bounds[mercantile-(1, 1, 1)]            1,075.0000 (12.95)     48,066.8000 (48.96)     1,154.8100 (8.75)      1,141.6000 (8.78)
+test_bounds[mercantile-(0, 0, 0)]            1,083.0000 (13.05)     21,041.0000 (21.43)     1,214.6049 (9.20)      1,208.0000 (9.29)
+test_bounds[mercantile-(1, 0, 1)]            1,083.0000 (13.05)     75,291.0000 (76.70)     1,213.7837 (9.19)      1,208.0000 (9.29)
+test_bounds[mercantile-(486, 332, 10)]       1,083.0000 (13.05)     55,750.0000 (56.79)     1,223.7598 (9.27)      1,208.0000 (9.29)
+test_bounds[mercantile-(486, 332, 20)]       1,083.0000 (13.05)     89,041.0000 (90.70)     1,248.4256 (9.46)      1,209.0000 (9.30)
+test_bounds[mercantile-(486, 332, 30)]       1,083.0000 (13.05)    194,958.0000 (198.60)    1,239.4339 (9.39)      1,209.0000 (9.30)
+test_bounds[morecantile-(1, 0, 1)]          20,083.0000 (241.96)   132,625.0000 (135.10)   20,893.1235 (158.26)   20,667.0000 (158.98)
+test_bounds[morecantile-(0, 0, 0)]          20,166.0000 (242.96)   120,333.0000 (122.58)   20,974.1961 (158.87)   20,708.0000 (159.29)
+test_bounds[morecantile-(1, 1, 1)]          20,334.0000 (244.99)    60,958.0000 (62.10)    21,236.2953 (160.86)   20,750.0000 (159.62)
+test_bounds[morecantile-(1, 40, 7)]         21,667.0000 (261.05)   173,458.0000 (176.70)   22,622.6537 (171.36)   22,333.0000 (171.79)
+test_bounds[morecantile-(486, 332, 10)]     22,166.0000 (267.06)   205,125.0000 (208.96)   23,512.1607 (178.10)   22,875.0000 (175.96)
+test_bounds[morecantile-(486, 332, 20)]     24,708.0000 (297.69)   129,250.0000 (131.66)   25,775.0796 (195.24)   25,375.0000 (195.19)
+test_bounds[morecantile-(486, 332, 30)]     71,292.0000 (858.94)   205,625.0000 (209.46)   73,569.4565 (557.27)   72,667.0000 (558.98)
+---------------------------------------------------------------------------------------------------------------------------------------
+
+---------------------------------------------------- benchmark 'xy_bounds': 21 tests -----------------------------------------------------
+Name (time in ns)                                      Min                     Max                   Mean                 Median
+------------------------------------------------------------------------------------------------------------------------------------------
+test_xy_bounds[utiles-(0, 0, 0)]                   83.0000 (1.0)       57,375.0000 (112.55)      145.9127 (1.57)        125.0000 (1.36)
+test_xy_bounds[utiles-(1, 1, 1)]                   88.3400 (1.06)       1,335.4200 (2.62)         93.1478 (1.00)         92.5000 (1.00)
+test_xy_bounds[utiles-(1, 40, 7)]                  88.7500 (1.07)       1,451.2500 (2.85)         93.1258 (1.0)          92.0900 (1.0)
+test_xy_bounds[utiles-(1, 0, 1)]                   89.5900 (1.08)         861.6600 (1.69)         93.6428 (1.01)         93.3300 (1.01)
+test_xy_bounds[utiles-(486, 332, 10)]              90.0000 (1.08)         509.7950 (1.0)          93.2818 (1.00)         92.9150 (1.01)
+test_xy_bounds[utiles-(486, 332, 20)]              90.4100 (1.09)         864.1700 (1.70)         94.2974 (1.01)         94.1600 (1.02)
+test_xy_bounds[utiles-(486, 332, 30)]              90.4100 (1.09)       1,377.5000 (2.70)         94.7366 (1.02)         94.1600 (1.02)
+test_xy_bounds[mercantile-(0, 0, 0)]              708.0000 (8.53)      42,250.0000 (82.88)       843.2292 (9.05)        833.0000 (9.05)
+test_xy_bounds[mercantile-(486, 332, 10)]         731.2500 (8.81)       5,393.7500 (10.58)       770.8331 (8.28)        764.6000 (8.30)
+test_xy_bounds[mercantile-(486, 332, 30)]         731.2500 (8.81)       7,316.6500 (14.35)       770.2423 (8.27)        764.6000 (8.30)
+test_xy_bounds[mercantile-(1, 0, 1)]              732.1429 (8.82)       9,404.8571 (18.45)       791.7721 (8.50)        785.7143 (8.53)
+test_xy_bounds[mercantile-(1, 40, 7)]             733.3000 (8.83)       5,950.0000 (11.67)       772.3273 (8.29)        766.7000 (8.33)
+test_xy_bounds[mercantile-(486, 332, 20)]         733.3500 (8.84)       7,408.3000 (14.53)       773.4718 (8.31)        768.7500 (8.35)
+test_xy_bounds[mercantile-(1, 1, 1)]              735.4000 (8.86)       4,762.5000 (9.34)        783.2100 (8.41)        777.1000 (8.44)
+test_xy_bounds[morecantile-(0, 0, 0)]           3,042.0000 (36.65)    108,208.0000 (212.26)    3,287.1089 (35.30)     3,250.0000 (35.29)
+test_xy_bounds[morecantile-(1, 0, 1)]           3,291.0000 (39.65)     68,166.0000 (133.71)    3,503.4695 (37.62)     3,459.0000 (37.56)
+test_xy_bounds[morecantile-(1, 1, 1)]           3,291.0000 (39.65)     72,875.0000 (142.95)    3,506.8544 (37.66)     3,500.0000 (38.01)
+test_xy_bounds[morecantile-(1, 40, 7)]          4,375.0000 (52.71)     76,375.0000 (149.82)    4,630.2893 (49.72)     4,584.0000 (49.78)
+test_xy_bounds[morecantile-(486, 332, 10)]      4,875.0000 (58.73)     85,208.0000 (167.14)    5,197.3581 (55.81)     5,125.0000 (55.65)
+test_xy_bounds[morecantile-(486, 332, 20)]      6,916.0000 (83.33)     78,458.0000 (153.90)    7,221.0669 (77.54)     7,166.0000 (77.82)
+test_xy_bounds[morecantile-(486, 332, 30)]     52,667.0000 (634.54)   404,208.0000 (792.88)   54,749.1232 (587.90)   54,333.0000 (590.00)
+------------------------------------------------------------------------------------------------------------------------------------------
+```
diff --git a/benchmarks/benchmarks.py b/benchmarks/benchmarks.py
@@ -0,0 +1,96 @@
+"""Morecantile/Mercantile/Utiles comparison benchmark
+
+The benchmark suite is adapted from jessekrubin/utiles
+https://github.com/jessekrubin/utiles/blob/ea58b9a017a2e3528f03cc20f16ef531737b863f/utiles-pyo3/bench/test_bench.py#L17-L25
+"""
+# This file is a modified version of https://github.com/jessekrubin/utiles/blob/ea58b9a017a2e3528f03cc20f16ef531737b863f/utiles-pyo3/bench/test_bench.py.
+#
+# The original license follows.
+#
+# MIT License
+#
+# Copyright (c) 2023 jessekrubin
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from typing import Callable, Tuple
+
+import mercantile
+import pytest
+import utiles
+
+import morecantile
+
+tms = morecantile.tms.get("WebMercatorQuad")
+
+TEST_TILES = (
+    (0, 0, 0),
+    (1, 0, 1),
+    (1, 1, 1),
+    (1, 40, 7),
+    (486, 332, 10),
+    # HIGH ZOOM
+    (486, 332, 20),
+    # OUTSIDE TMS Range
+    (486, 332, 30),
+)
+
+
+@pytest.mark.parametrize(
+    "tile",
+    [pytest.param(t, id=str(t)) for t in TEST_TILES],
+)
+@pytest.mark.parametrize(
+    "func",
+    [
+        pytest.param(mercantile.bounds, id="mercantile"),
+        pytest.param(tms.bounds, id="morecantile"),
+        pytest.param(utiles.bounds, id="utiles"),
+    ],
+)
+@pytest.mark.benchmark(group="bounds")
+def test_bounds(
+    tile: Tuple[int, int, int],
+    func: Callable[[Tuple[int, int, int]], Tuple[float, float]],
+    benchmark,
+) -> None:
+    """Benchmark bounds() method."""
+    _ = benchmark(func, *tile)
+
+
+@pytest.mark.parametrize(
+    "tile",
+    [pytest.param(t, id=str(t)) for t in TEST_TILES],
+)
+@pytest.mark.parametrize(
+    "func",
+    [
+        pytest.param(mercantile.xy_bounds, id="mercantile"),
+        pytest.param(tms.xy_bounds, id="morecantile"),
+        pytest.param(utiles.xy_bounds, id="utiles"),
+    ],
+)
+@pytest.mark.benchmark(group="xy_bounds")
+def test_xy_bounds(
+    tile: Tuple[int, int, int],
+    func: Callable[[Tuple[int, int, int]], Tuple[float, float]],
+    benchmark,
+) -> None:
+    """Benchmark xy_bounds() method."""
+    _ = benchmark(func, *tile)
diff --git a/benchmarks/requirements.txt b/benchmarks/requirements.txt
@@ -0,0 +1,6 @@
+pytest
+pytest-benchmark
+
+morecantile
+mercantile
+utiles
diff --git a/pyproject.toml b/pyproject.toml
@@ -36,6 +36,10 @@ test = [
     "pytest-cov",
     "rasterio>=1.2.1",
 ]
+benchmark = [
+    "pytest",
+    "pytest-benchmark",
+]
 dev = [
     "pre-commit",
     "bump-my-version",

diff --git a/tests/benchmarks.py b/tests/benchmarks.py
@@ -0,0 +1,43 @@
+"""Morecantile benchmark."""
+
+import pytest
+
+import morecantile
+from morecantile.commons import BoundingBox
+
+tms = morecantile.tms.get("WebMercatorQuad")
+
+# Test tiles from https://github.com/jessekrubin/utiles/blob/ea58b9a017a2e3528f03cc20f16ef531737b863f/utiles-pyo3/bench/test_bench.py
+TEST_TILES = (
+    (0, 0, 0),
+    (1, 0, 1),
+    (1, 1, 1),
+    (1, 40, 7),
+    (486, 332, 10),
+    # HIGH ZOOM
+    (486, 332, 20),
+    # OUTSIDE TMS Range
+    (486, 332, 30),
+)
+
+
+@pytest.mark.parametrize("tile", TEST_TILES)
+def test_bounds(tile, benchmark):
+    str_tile = "Tile(x={},y={},z={})".format(*tile)
+    benchmark.name = f"morecantile.bounds-{str_tile}"
+    benchmark.fullname = f"morecantile.bounds-{str_tile}"
+    benchmark.group = "morecantile.bounds"
+
+    r = benchmark(tms.bounds, *tile)
+    assert isinstance(r, BoundingBox)
+
+
+@pytest.mark.parametrize("tile", TEST_TILES)
+def test_xy_bounds(tile, benchmark) -> None:
+    str_tile = "Tile(x={},y={},z={})".format(*tile)
+    benchmark.name = f"morecantile.xy_bounds-{str_tile}"
+    benchmark.fullname = f"morecantile.xy_bounds-{str_tile}"
+    benchmark.group = "morecantile.xy_bounds"
+
+    r = benchmark(tms.xy_bounds, *tile)
+    assert isinstance(r, BoundingBox)