matplotlib 3.5.0b1 breaks test suite

[mwaskom]

I'm pretty sure these failures are all related to things that happen as part of tests but not normal operations and so this isn't an urgent problem, but they should get addressed before the final release drops and breaks CI.

$ pytest seaborn -q
............ssssss..................................................................................... [  8%]
...............................................................................F.FF.................... [ 17%]
.............................................................................s......................... [ 25%]
............s................................F......................................................... [ 34%]
....................................................................................................... [ 43%]
.........................sss........................................................................s.. [ 51%]
...........F..FFFFFFF..F............................................................................... [ 60%]
........................................................................................F.............. [ 68%]
.....................................ss................................................................ [ 77%]
........................................................ss.ss.........ssss...................s......... [ 86%]
....................................................................................................... [ 94%]
.......s........................................................                                        [100%]
================================================== FAILURES ===================================================
________________________________________ TestBoxPlotter.test_axes_data ________________________________________

self = <seaborn.tests.test_categorical.TestBoxPlotter object at 0x7f9dce7f2d60>

    def test_axes_data(self):

        ax = cat.boxplot(x="g", y="y", data=self.df)
>       assert len(ax.artists) == 3
E       AssertionError: assert 0 == 3
E        +  where 0 = len(<Axes.ArtistList of 0 artists>)
E        +    where <Axes.ArtistList of 0 artists> = <AxesSubplot:xlabel='g', ylabel='y'>.artists

seaborn/tests/test_categorical.py:858: AssertionError
___________________________________ TestBoxPlotter.test_draw_missing_boxes ____________________________________

self = <seaborn.tests.test_categorical.TestBoxPlotter object at 0x7f9dc93d8eb0>

    def test_draw_missing_boxes(self):

        ax = cat.boxplot(x="g", y="y", data=self.df,
                         order=["a", "b", "c", "d"])
>       assert len(ax.artists) == 3
E       AssertionError: assert 0 == 3
E        +  where 0 = len(<Axes.ArtistList of 0 artists>)
E        +    where <Axes.ArtistList of 0 artists> = <AxesSubplot:xlabel='g', ylabel='y'>.artists

seaborn/tests/test_categorical.py:887: AssertionError
______________________________________ TestBoxPlotter.test_missing_data _______________________________________

self = <seaborn.tests.test_categorical.TestBoxPlotter object at 0x7f9dceec4b50>

    def test_missing_data(self):

        x = ["a", "a", "b", "b", "c", "c", "d", "d"]
        h = ["x", "y", "x", "y", "x", "y", "x", "y"]
        y = self.rs.randn(8)
        y[-2:] = np.nan

        ax = cat.boxplot(x=x, y=y)
>       assert len(ax.artists) == 3
E       assert 0 == 3
E        +  where 0 = len(<Axes.ArtistList of 0 artists>)
E        +    where <Axes.ArtistList of 0 artists> = <AxesSubplot:>.artists

seaborn/tests/test_categorical.py:897: AssertionError
_______________________________________ TestCatPlot.test_plot_elements ________________________________________

self = <seaborn.tests.test_categorical.TestCatPlot object at 0x7f9daa710ee0>

    def test_plot_elements(self):

        g = cat.catplot(x="g", y="y", data=self.df, kind="point")
        assert len(g.ax.collections) == 1
        want_lines = self.g.unique().size + 1
        assert len(g.ax.lines) == want_lines

        g = cat.catplot(x="g", y="y", hue="h", data=self.df, kind="point")
        want_collections = self.h.unique().size
        assert len(g.ax.collections) == want_collections
        want_lines = (self.g.unique().size + 1) * self.h.unique().size
        assert len(g.ax.lines) == want_lines

        g = cat.catplot(x="g", y="y", data=self.df, kind="bar")
        want_elements = self.g.unique().size
        assert len(g.ax.patches) == want_elements
        assert len(g.ax.lines) == want_elements

        g = cat.catplot(x="g", y="y", hue="h", data=self.df, kind="bar")
        want_elements = self.g.unique().size * self.h.unique().size
        assert len(g.ax.patches) == want_elements
        assert len(g.ax.lines) == want_elements

        g = cat.catplot(x="g", data=self.df, kind="count")
        want_elements = self.g.unique().size
        assert len(g.ax.patches) == want_elements
        assert len(g.ax.lines) == 0

        g = cat.catplot(x="g", hue="h", data=self.df, kind="count")
        want_elements = self.g.unique().size * self.h.unique().size
        assert len(g.ax.patches) == want_elements
        assert len(g.ax.lines) == 0

        g = cat.catplot(x="g", y="y", data=self.df, kind="box")
        want_artists = self.g.unique().size
>       assert len(g.ax.artists) == want_artists
E       AssertionError: assert 0 == 3
E        +  where 0 = len(<Axes.ArtistList of 0 artists>)
E        +    where <Axes.ArtistList of 0 artists> = <AxesSubplot:xlabel='g', ylabel='y'>.artists
E        +      where <AxesSubplot:xlabel='g', ylabel='y'> = <seaborn.axisgrid.FacetGrid object at 0x7f9da8f82160>.ax

seaborn/tests/test_categorical.py:2769: AssertionError
______________________________________ TestKDEPlotUnivariate.test_legend ______________________________________

self = <seaborn.tests.test_distributions.TestKDEPlotUnivariate object at 0x7f9d9d1cb130>
long_df =      x         y         z  a  b  c          d                   t  s    f a_cat s_cat s_str
0   12  0.449243  6.61188...8
99  15  0.073484  1.036343  c  p  0 2005-01-01 2004-07-30 00:11:00  8  0.2     c     8     8

[100 rows x 13 columns]

    def test_legend(self, long_df):

        ax = kdeplot(data=long_df, x="x", hue="a")

        assert ax.legend_.get_title().get_text() == "a"

        legend_labels = ax.legend_.get_texts()
        order = categorical_order(long_df["a"])
        for label, level in zip(legend_labels, order):
            assert label.get_text() == level

        legend_artists = ax.legend_.findobj(mpl.lines.Line2D)[::2]
        palette = color_palette()
        for artist, color in zip(legend_artists, palette):
>           assert_colors_equal(artist.get_color(), color)

seaborn/tests/test_distributions.py:857:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

a = (0.3333333333333333, 0.6588235294117647, 0.40784313725490196, 1)
b = (0.8666666666666667, 0.5176470588235295, 0.3215686274509804), check_alpha = True

    def assert_colors_equal(a, b, check_alpha=True):

        def handle_array(x):

            if isinstance(x, np.ndarray):
                if x.ndim > 1:
                    x = np.unique(x, axis=0).squeeze()
                if x.ndim > 1:
                    raise ValueError("Color arrays must be 1 dimensional")
            return x

        a = handle_array(a)
        b = handle_array(b)

        f = to_rgba if check_alpha else to_rgb
>       assert f(a) == f(b)
E       AssertionError

seaborn/_testing.py:90: AssertionError
___________________________________ TestKDEPlotBivariate.test_fill_artists ____________________________________

self = <seaborn.tests.test_distributions.TestKDEPlotBivariate object at 0x7f9d9d6e9a00>
long_df =      x         y         z  a  b  c          d                   t  s    f a_cat s_cat s_str
0   12  0.449243  6.61188...8
99  15  0.073484  1.036343  c  p  0 2005-01-01 2004-07-30 00:11:00  8  0.2     c     8     8

[100 rows x 13 columns]

    def test_fill_artists(self, long_df):

        for fill in [True, False]:
            f, ax = plt.subplots()
            kdeplot(data=long_df, x="x", y="y", hue="c", fill=fill)
            for c in ax.collections:
                if fill:
                    assert isinstance(c, mpl.collections.PathCollection)
                else:
>                   assert isinstance(c, mpl.collections.LineCollection)
E                   AssertionError: assert False
E                    +  where False = isinstance(<matplotlib.collections.PathCollection object at 0x7f9d9d55f400>, <class 'matplotlib.collections.LineCollection'>)
E                    +    where <class 'matplotlib.collections.LineCollection'> = <module 'matplotlib.collections' from '/Users/mwaskom/miniconda3/envs/seaborn-release-test/lib/python3.9/site-packages/matplotlib/collections.py'>.LineCollection
E                    +      where <module 'matplotlib.collections' from '/Users/mwaskom/miniconda3/envs/seaborn-release-test/lib/python3.9/site-packages/matplotlib/collections.py'> = mpl.collections

seaborn/tests/test_distributions.py:908: AssertionError
____________________________________ TestKDEPlotBivariate.test_common_norm ____________________________________

self = <seaborn.tests.test_distributions.TestKDEPlotBivariate object at 0x7f9d9d6f1430>
rng = RandomState(MT19937) at 0x7F9DC56BE740

    def test_common_norm(self, rng):

        hue = np.repeat(["a", "a", "a", "b"], 40)
        x, y = rng.multivariate_normal([0, 0], [(.2, .5), (.5, 2)], len(hue)).T
        x[hue == "a"] -= 2
        x[hue == "b"] += 2

        f, (ax1, ax2) = plt.subplots(ncols=2)
        kdeplot(x=x, y=y, hue=hue, common_norm=True, ax=ax1)
        kdeplot(x=x, y=y, hue=hue, common_norm=False, ax=ax2)

>       n_seg_1 = sum([len(c.get_segments()) > 0 for c in ax1.collections])

seaborn/tests/test_distributions.py:921:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

.0 = <generator object _AxesBase.ArtistList.__iter__ at 0x7f9d9d587f20>

>   n_seg_1 = sum([len(c.get_segments()) > 0 for c in ax1.collections])
E   AttributeError: 'PathCollection' object has no attribute 'get_segments'

seaborn/tests/test_distributions.py:921: AttributeError
_____________________________________ TestKDEPlotBivariate.test_log_scale _____________________________________

self = <seaborn.tests.test_distributions.TestKDEPlotBivariate object at 0x7f9d9d5cddf0>
rng = RandomState(MT19937) at 0x7F9DC56BE640

    def test_log_scale(self, rng):

        x = rng.lognormal(0, 1, 100)
        y = rng.uniform(0, 1, 100)

        levels = .2, .5, 1

        f, ax = plt.subplots()
        kdeplot(x=x, y=y, log_scale=True, levels=levels, ax=ax)
        assert ax.get_xscale() == "log"
        assert ax.get_yscale() == "log"

        f, (ax1, ax2) = plt.subplots(ncols=2)
        kdeplot(x=x, y=y, log_scale=(10, False), levels=levels, ax=ax1)
        assert ax1.get_xscale() == "log"
        assert ax1.get_yscale() == "linear"

        p = _DistributionPlotter()
        kde = KDE()
        density, (xx, yy) = kde(np.log10(x), y)
        levels = p._quantile_to_level(density, levels)
        ax2.contour(10 ** xx, yy, density, levels=levels)

        for c1, c2 in zip(ax1.collections, ax2.collections):
>           assert_array_equal(c1.get_segments(), c2.get_segments())
E           AttributeError: 'PathCollection' object has no attribute 'get_segments'

seaborn/tests/test_distributions.py:949: AttributeError
_____________________________________ TestKDEPlotBivariate.test_bandwidth _____________________________________

self = <seaborn.tests.test_distributions.TestKDEPlotBivariate object at 0x7f9d9e4a4f10>
rng = RandomState(MT19937) at 0x7F9DC56BEB40

    def test_bandwidth(self, rng):

        n = 100
        x, y = rng.multivariate_normal([0, 0], [(.2, .5), (.5, 2)], n).T

        f, (ax1, ax2) = plt.subplots(ncols=2)

        kdeplot(x=x, y=y, ax=ax1)
        kdeplot(x=x, y=y, bw_adjust=2, ax=ax2)

        for c1, c2 in zip(ax1.collections, ax2.collections):
>           seg1, seg2 = c1.get_segments(), c2.get_segments()
E           AttributeError: 'PathCollection' object has no attribute 'get_segments'

seaborn/tests/test_distributions.py:962: AttributeError
______________________________________ TestKDEPlotBivariate.test_weights ______________________________________

self = <seaborn.tests.test_distributions.TestKDEPlotBivariate object at 0x7f9d9f60e5b0>
rng = RandomState(MT19937) at 0x7F9DC56BE840

    def test_weights(self, rng):

        import warnings
        warnings.simplefilter("error", np.VisibleDeprecationWarning)

        n = 100
        x, y = rng.multivariate_normal([1, 3], [(.2, .5), (.5, 2)], n).T
        hue = np.repeat([0, 1], n // 2)
        weights = rng.uniform(0, 1, n)

        f, (ax1, ax2) = plt.subplots(ncols=2)
        kdeplot(x=x, y=y, hue=hue, ax=ax1)
        kdeplot(x=x, y=y, hue=hue, weights=weights, ax=ax2)

        for c1, c2 in zip(ax1.collections, ax2.collections):
>           if c1.get_segments() and c2.get_segments():
E           AttributeError: 'PathCollection' object has no attribute 'get_segments'

seaborn/tests/test_distributions.py:983: AttributeError
_________________________________ TestKDEPlotBivariate.test_hue_ignores_cmap __________________________________

self = <seaborn.tests.test_distributions.TestKDEPlotBivariate object at 0x7f9d9f166760>
long_df =      x         y         z  a  b  c          d                   t  s    f a_cat s_cat s_str
0   12  0.449243  6.61188...8
99  15  0.073484  1.036343  c  p  0 2005-01-01 2004-07-30 00:11:00  8  0.2     c     8     8

[100 rows x 13 columns]

    def test_hue_ignores_cmap(self, long_df):

        with pytest.warns(UserWarning, match="cmap parameter ignored"):
            ax = kdeplot(data=long_df, x="x", y="y", hue="c", cmap="viridis")

>       assert_colors_equal(ax.collections[0].get_color(), "C0")
E       AttributeError: 'PathCollection' object has no attribute 'get_color'

seaborn/tests/test_distributions.py:993: AttributeError
________________________________ TestKDEPlotBivariate.test_contour_line_colors ________________________________

self = <seaborn.tests.test_distributions.TestKDEPlotBivariate object at 0x7f9d9f2c5e50>
long_df =      x         y         z  a  b  c          d                   t  s    f a_cat s_cat s_str
0   12  0.449243  6.61188...8
99  15  0.073484  1.036343  c  p  0 2005-01-01 2004-07-30 00:11:00  8  0.2     c     8     8

[100 rows x 13 columns]

    def test_contour_line_colors(self, long_df):

        color = (.2, .9, .8, 1)
        ax = kdeplot(data=long_df, x="x", y="y", color=color)

        for c in ax.collections:
>           assert_colors_equal(c.get_color(), color)
E           AttributeError: 'PathCollection' object has no attribute 'get_color'

seaborn/tests/test_distributions.py:1001: AttributeError
_________________________________ TestKDEPlotBivariate.test_levels_and_thresh _________________________________

self = <seaborn.tests.test_distributions.TestKDEPlotBivariate object at 0x7f9d9f821cd0>
long_df =      x         y         z  a  b  c          d                   t  s    f a_cat s_cat s_str
0   12  0.449243  6.61188...8
99  15  0.073484  1.036343  c  p  0 2005-01-01 2004-07-30 00:11:00  8  0.2     c     8     8

[100 rows x 13 columns]

    def test_levels_and_thresh(self, long_df):

        f, (ax1, ax2) = plt.subplots(ncols=2)

        n = 8
        thresh = .1
        plot_kws = dict(data=long_df, x="x", y="y")
        kdeplot(**plot_kws, levels=n, thresh=thresh, ax=ax1)
        kdeplot(**plot_kws, levels=np.linspace(thresh, 1, n), ax=ax2)

        for c1, c2 in zip(ax1.collections, ax2.collections):
>           assert_array_equal(c1.get_segments(), c2.get_segments())
E           AttributeError: 'PathCollection' object has no attribute 'get_segments'

seaborn/tests/test_distributions.py:1033: AttributeError
_____________________________________ TestDisPlot.test_bivariate_kde_norm _____________________________________

self = <seaborn.tests.test_distributions.TestDisPlot object at 0x7f9d7d930b20>
rng = RandomState(MT19937) at 0x7F9D8D67E840

    def test_bivariate_kde_norm(self, rng):

        x, y = rng.normal(0, 1, (2, 100))
        z = [0] * 80 + [1] * 20

        g = displot(x=x, y=y, col=z, kind="kde", levels=10)
>       l1 = sum(bool(c.get_segments()) for c in g.axes.flat[0].collections)

seaborn/tests/test_distributions.py:2325:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

.0 = <generator object _AxesBase.ArtistList.__iter__ at 0x7f9d7ee79510>

>   l1 = sum(bool(c.get_segments()) for c in g.axes.flat[0].collections)
E   AttributeError: 'PathCollection' object has no attribute 'get_segments'

seaborn/tests/test_distributions.py:2325: AttributeError
============================================== warnings summary ===============================================
seaborn/tests/test_axisgrid.py: 15 warnings
seaborn/tests/test_distributions.py: 44 warnings
  /Users/mwaskom/code/seaborn/seaborn/distributions.py:832: MatplotlibDeprecationWarning: Auto-removal of grids by pcolor() and pcolormesh() is deprecated since 3.5 and will be removed two minor releases later; please call grid(False) first.
    mesh = ax.pcolormesh(

seaborn/tests/test_categorical.py: 96 warnings
seaborn/tests/test_relational.py: 1 warning
  /Users/mwaskom/code/seaborn/seaborn/utils.py:115: MatplotlibDeprecationWarning: Support for passing numbers through unit converters is deprecated since 3.5 and support will be removed two minor releases later; use Axis.convert_units instead.
    scout = method(scout_x, scout_y, **kws)

seaborn/tests/test_categorical.py: 87 warnings
  /Users/mwaskom/code/seaborn/seaborn/categorical.py:281: MatplotlibDeprecationWarning: Support for passing numbers through unit converters is deprecated since 3.5 and support will be removed two minor releases later; use Axis.convert_units instead.
    ax.scatter([], [], s=60, color=mpl.colors.rgb2hex(color), label=level)

seaborn/tests/test_categorical.py: 24 warnings
  /Users/mwaskom/code/seaborn/seaborn/categorical.py:266: MatplotlibDeprecationWarning: Support for passing numbers through unit converters is deprecated since 3.5 and support will be removed two minor releases later; use Axis.convert_units instead.
    points = ax.scatter(sub_data["x"], sub_data["y"], color=color, **plot_kws)

seaborn/tests/test_categorical.py: 63 warnings
  /Users/mwaskom/code/seaborn/seaborn/categorical.py:371: MatplotlibDeprecationWarning: Support for passing numbers through unit converters is deprecated since 3.5 and support will be removed two minor releases later; use Axis.convert_units instead.
    ax.scatter([], [], s=60, color=mpl.colors.rgb2hex(color), label=level)

seaborn/tests/test_categorical.py::TestSwarmPlot::test_order[str-order3]
seaborn/tests/test_categorical.py::TestSwarmPlot::test_order[int-order7]
seaborn/tests/test_categorical.py::TestSwarmPlot::test_hue_dodged[a]
seaborn/tests/test_categorical.py::TestSwarmPlot::test_hue_dodged[a]
seaborn/tests/test_categorical.py::TestSwarmPlot::test_hue_dodged[a]
seaborn/tests/test_categorical.py::TestSwarmPlot::test_hue_dodged[a]
seaborn/tests/test_categorical.py::TestSwarmPlot::test_hue_dodged[a]
seaborn/tests/test_categorical.py::TestSwarmPlot::test_hue_dodged[a]
  /Users/mwaskom/code/seaborn/seaborn/categorical.py:319: MatplotlibDeprecationWarning: Support for passing numbers through unit converters is deprecated since 3.5 and support will be removed two minor releases later; use Axis.convert_units instead.
    points = ax.scatter(sub_data["x"], sub_data["y"], color=color, **plot_kws)

seaborn/tests/test_distributions.py::TestKDEPlotBivariate::test_colorbar
  /Users/mwaskom/code/seaborn/seaborn/distributions.py:1176: MatplotlibDeprecationWarning: Auto-removal of grids by pcolor() and pcolormesh() is deprecated since 3.5 and will be removed two minor releases later; please call grid(False) first.
    ax.figure.colorbar(cset, cbar_ax, ax, **cbar_kws)

seaborn/tests/test_distributions.py::TestHistPlotUnivariate::test_auto_linewidth[False]
seaborn/tests/test_relational.py::TestLinePlotter::test_lineplot_vs_relplot[long_semantics2]
  /Users/mwaskom/code/seaborn/seaborn/utils.py:101: MatplotlibDeprecationWarning: Support for passing numbers through unit converters is deprecated since 3.5 and support will be removed two minor releases later; use Axis.convert_units instead.
    scout, = method([], [], **kws)

seaborn/tests/test_distributions.py::TestHistPlotBivariate::test_colorbar
seaborn/tests/test_distributions.py::TestHistPlotBivariate::test_colorbar
  /Users/mwaskom/code/seaborn/seaborn/distributions.py:850: MatplotlibDeprecationWarning: Auto-removal of grids by pcolor() and pcolormesh() is deprecated since 3.5 and will be removed two minor releases later; please call grid(False) first.
    ax.figure.colorbar(mesh, cbar_ax, ax, **cbar_kws)

seaborn/tests/test_matrix.py: 77 warnings
  /Users/mwaskom/code/seaborn/seaborn/matrix.py:306: MatplotlibDeprecationWarning: Auto-removal of grids by pcolor() and pcolormesh() is deprecated since 3.5 and will be removed two minor releases later; please call grid(False) first.
    mesh = ax.pcolormesh(self.plot_data, cmap=self.cmap, **kws)

seaborn/tests/test_matrix.py: 47 warnings
  /Users/mwaskom/code/seaborn/seaborn/matrix.py:316: MatplotlibDeprecationWarning: Auto-removal of grids by pcolor() and pcolormesh() is deprecated since 3.5 and will be removed two minor releases later; please call grid(False) first.
    cb = ax.figure.colorbar(mesh, cax, ax, **self.cbar_kws)

-- Docs: https://docs.pytest.org/en/stable/warnings.html
=========================================== short test summary info ===========================================
FAILED seaborn/tests/test_categorical.py::TestBoxPlotter::test_axes_data - AssertionError: assert 0 == 3
FAILED seaborn/tests/test_categorical.py::TestBoxPlotter::test_draw_missing_boxes - AssertionError: assert 0...
FAILED seaborn/tests/test_categorical.py::TestBoxPlotter::test_missing_data - assert 0 == 3
FAILED seaborn/tests/test_categorical.py::TestCatPlot::test_plot_elements - AssertionError: assert 0 == 3
FAILED seaborn/tests/test_distributions.py::TestKDEPlotUnivariate::test_legend - AssertionError
FAILED seaborn/tests/test_distributions.py::TestKDEPlotBivariate::test_fill_artists - AssertionError: assert...
FAILED seaborn/tests/test_distributions.py::TestKDEPlotBivariate::test_common_norm - AttributeError: 'PathCo...
FAILED seaborn/tests/test_distributions.py::TestKDEPlotBivariate::test_log_scale - AttributeError: 'PathColl...
FAILED seaborn/tests/test_distributions.py::TestKDEPlotBivariate::test_bandwidth - AttributeError: 'PathColl...
FAILED seaborn/tests/test_distributions.py::TestKDEPlotBivariate::test_weights - AttributeError: 'PathCollec...
FAILED seaborn/tests/test_distributions.py::TestKDEPlotBivariate::test_hue_ignores_cmap - AttributeError: 'P...
FAILED seaborn/tests/test_distributions.py::TestKDEPlotBivariate::test_contour_line_colors - AttributeError:...
FAILED seaborn/tests/test_distributions.py::TestKDEPlotBivariate::test_levels_and_thresh - AttributeError: '...
FAILED seaborn/tests/test_distributions.py::TestDisPlot::test_bivariate_kde_norm - AttributeError: 'PathColl...
14 failed, 1159 passed, 24 skipped, 467 warnings in 459.94s (0:07:39)

[QuLogic]

Does something like this work?

diff --git a/seaborn/tests/test_categorical.py b/seaborn/tests/test_categorical.py
index d4e09b7..bb5bc51 100644
--- a/seaborn/tests/test_categorical.py
+++ b/seaborn/tests/test_categorical.py
@@ -1,5 +1,6 @@
 import itertools
 from functools import partial
+from unittest.mock import patch
 
 import numpy as np
 import pandas as pd
@@ -852,15 +853,17 @@ class TestBoxPlotter(CategoricalFixture):
         p.establish_variables("h", "y", "g", data=self.df)
         npt.assert_array_almost_equal(p.hue_offsets, [-.2, 0, .2])
 
-    def test_axes_data(self):
+    @patch('matplotlib.axes.Axes.boxplot')
+    def test_axes_data(self, boxplot):
 
-        ax = cat.boxplot(x="g", y="y", data=self.df)
-        assert len(ax.artists) == 3
+        cat.boxplot(x="g", y="y", data=self.df)
+        assert boxplot.call_count == 3
 
         plt.close("all")
+        boxplot.reset_mock()
 
-        ax = cat.boxplot(x="g", y="y", hue="h", data=self.df)
-        assert len(ax.artists) == 6
+        cat.boxplot(x="g", y="y", hue="h", data=self.df)
+        assert boxplot.call_count == 6
 
         plt.close("all")
 

I think this preserves the intent of this test, but I have not looked at all of them, and it may not work for all.

[mwaskom]

Hm I've never really thought about patching matplotlib functions and checking the call count. I actually think this would fail here because in the first case ax.boxplot is called once, with three groups, and in the second it's called twice.

I think the way forward here is to have a get_box_patches function that looks in different places depending on matplotlib version. (I never really invested the time in developing a good infrastructure for extracting matplotlib axes in the tests, alas.)

[QuLogic]

No, it does passes. The seaborn boxplot is only called once, but the Matplotlib boxplot is called multiple times, once for each group.

[QuLogic]

Ah, I just realized that some tests that zip the artists and the palette are passing (e.g., test_box_colors), but only because zip picks the shortest. They should also be failing, and using patching may not be as trivial for these ones.

[mwaskom]

Ah, seems you're right. Those functions aren't so consistent ... barplot creates multiple bars with a single call to ax.bar but boxplot behaves as you describe.

That zip issue is a recurring frustration!

[QuLogic]

So the problem with switching to ax.patches on 3.5 is that there might be additional patches listed there. For example, if there's a legend for boxplot, then each entry gets a Rectangle, which is a Patch; this occurs on, e.g., the second call in test_axes_data with hue.

I experimented a bit with catching the results with a mock patch, but it looks like you have to do a few steps to capture return values. I pulled that together into a context manager, which looks like the following:

diff --git a/seaborn/tests/test_categorical.py b/seaborn/tests/test_categorical.py
index d4e09b7..fcea099 100644
--- a/seaborn/tests/test_categorical.py
+++ b/seaborn/tests/test_categorical.py
@@ -1,5 +1,8 @@
+from contextlib import contextmanager
 import itertools
+import functools
 from functools import partial
+from unittest import mock
 
 import numpy as np
 import pandas as pd
@@ -503,6 +506,21 @@ class TestCategoricalPlotter(CategoricalFixture):
         assert p.colors == [(.25, .25, .75), (.75, .25, .25), (1, 1, 1)]
 
 
+@contextmanager
+def return_value_collector(obj, method_name):
+    func = getattr(obj, method_name)
+    collection = []
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        result = func(*args, **kwargs)
+        collection.append(result)
+        return result
+
+    with mock.patch.object(obj, method_name, side_effect=wrapper):
+        yield collection
+
+
 class TestCategoricalStatPlotter(CategoricalFixture):
 
     def test_no_bootstrappig(self):
@@ -854,53 +872,67 @@ class TestBoxPlotter(CategoricalFixture):
 
     def test_axes_data(self):
 
-        ax = cat.boxplot(x="g", y="y", data=self.df)
-        assert len(ax.artists) == 3
+        fig, ax = plt.subplots()
+        with return_value_collector(ax, 'boxplot') as bxp_artists:
+            cat.boxplot(x="g", y="y", data=self.df, ax=ax)
+            assert len(bxp_artists) == 3
 
         plt.close("all")
 
-        ax = cat.boxplot(x="g", y="y", hue="h", data=self.df)
-        assert len(ax.artists) == 6
+        fig, ax = plt.subplots()
+        with return_value_collector(ax, 'boxplot') as bxp_artists:
+            cat.boxplot(x="g", y="y", hue="h", data=self.df, ax=ax)
+        assert len(bxp_artists) == 6
 
         plt.close("all")
 
     def test_box_colors(self):
 
-        ax = cat.boxplot(x="g", y="y", data=self.df, saturation=1)
-        pal = palettes.color_palette(n_colors=3)
-        for patch, color in zip(ax.artists, pal):
-            assert patch.get_facecolor()[:3] == color
+        fig, ax = plt.subplots()
+        with return_value_collector(ax, 'boxplot') as bxp_artists:
+            cat.boxplot(x="g", y="y", data=self.df, saturation=1, ax=ax)
+
+            pal = palettes.color_palette(n_colors=3)
+            for bxp, color in itertools.zip_longest(bxp_artists, pal):
+                patch, = bxp['boxes']  # There must be 1 only.
+                assert patch.get_facecolor()[:3] == color
 
         plt.close("all")
 
-        ax = cat.boxplot(x="g", y="y", hue="h", data=self.df, saturation=1)
-        pal = palettes.color_palette(n_colors=2)
-        for patch, color in zip(ax.artists, pal * 2):
-            assert patch.get_facecolor()[:3] == color
+        fig, ax = plt.subplots()
+        with return_value_collector(ax, 'boxplot') as bxp_artists:
+            cat.boxplot(x="g", y="y", hue="h", data=self.df, saturation=1, ax=ax)
+
+            pal = palettes.color_palette(n_colors=2)
+            for bxp, color in itertools.zip_longest(bxp_artists, pal * 2):
+                patch, = bxp['boxes']  # There must be 1 only.
+                assert patch.get_facecolor()[:3] == color
 
         plt.close("all")
 

Note, this does require knowing that Matplotlb's boxplot returns a dictionary, but maybe that's okay? Also, there appears to still be a mismatch in the second call in test_box_colors, as with zip_longest they don't match right.

[mwaskom]

Alternately, could just change the tests to look for 6 * n lines instead of n patches, and then check the color of the first n patches.

Or we could exclude the patches that get added for the legend by counting patches that don't have a label.