Merge pull request #406 from Yelp/sclark_36_testify_to_pytest

Moved from testify to pytest

Author: suntzu86

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 * Changes
 
+  * Switched from `testify` to `py.test` - http://pytest.org/ (#36)
   * [cleanup] Moved bandits into their own sub directories (#375)
 
 * Bugs

Makefile

@@ -8,8 +8,7 @@ production:
 		python setup.py install
 
 test:
-		testify -v --exclude-suite=disabled moe.tests
-		testify -v --exclude-suite=disabled moe_examples.tests
+		py.test -v moe/tests moe_examples/tests
 
 style-test:
 		pip install flake8 flake8-import-order pep8-naming flake8-docstrings pyflakes

docs/contributing.rst

@@ -159,7 +159,7 @@ MOE uses ``doxygen`` (http://www.stack.nl/~dimitri/doxygen) to extract the C++ d
 Testing
 -------
 
-MOE currently uses ``testify`` (https://github.com/Yelp/Testify) to run all unit and integration tests.
+MOE currently uses ``pytest`` (http://pytest.org/) to run all unit and integration tests.
 
 Tests can be run using ``make test``. Continuous integration testing is provided by http://travis-ci.org. All builds are tested in a fresh ubuntu VM and need to be passing before being pulled into master.
 

moe/tests/bandit/bandit_interface_test.py

@@ -1,45 +1,45 @@
 # -*- coding: utf-8 -*-
 """Test bandit interface implementation."""
-import logging
+import pytest
 
-import testify as T
+import logging
 
 from moe.bandit.bandit_interface import BanditInterface
 from moe.tests.bandit.bandit_test_case import BanditTestCase
 
 
-class BanditInterfaceTest(BanditTestCase):
-
-    """Verify that different historical infos return correct results."""
-
-    @T.class_setup
-    def disable_logging(self):
-        """Disable logging (for the duration of this test case)."""
-        logging.disable(logging.CRITICAL)
+@pytest.fixture()
+def disable_logging(request):
+    """Disable logging (for the duration of this test case)."""
+    logging.disable(logging.CRITICAL)
 
-    @T.class_teardown
-    def enable_logging(self):
+    def finalize():
         """Re-enable logging (so other test cases are unaffected)."""
         logging.disable(logging.NOTSET)
+    request.addfinalizer(finalize)
 
+
+class TestBanditInterface(BanditTestCase):
+
+    """Verify that different historical infos return correct results."""
+
+    @pytest.mark.usefixtures("disable_logging")
     def test_empty_arm_invalid(self):
         """Test empty ``arms_to_allocations`` causes an ValueError."""
-        T.assert_raises(ValueError, BanditInterface.choose_arm, {})
+        with pytest.raises(ValueError):
+            BanditInterface.choose_arm({})
 
     def test_one_arm(self):
         """Check that the one-arm case always returns the given arm as the winning arm."""
         arms_to_allocations = {"arm1": 1.0}
-        T.assert_equal(BanditInterface.choose_arm(arms_to_allocations), "arm1")
+        assert BanditInterface.choose_arm(arms_to_allocations) == "arm1"
 
     def test_two_arms_one_winner(self):
         """Check that the two-arms case with one winner always returns the winning arm."""
         arms_to_allocations = {"arm1": 1.0, "arm2": 0.0}
-        T.assert_equal(BanditInterface.choose_arm(arms_to_allocations), "arm1")
+        assert BanditInterface.choose_arm(arms_to_allocations) == "arm1"
 
     def test_three_arms_two_winners(self):
         """Check that the three-arms cases with two winners return one of the two winners."""
         arms_to_allocations = {"arm1": 0.5, "arm2": 0.5, "arm3": 0.0}
-        T.assert_in(BanditInterface.choose_arm(arms_to_allocations), frozenset(["arm1", "arm2"]))
-
-if __name__ == "__main__":
-    T.run()
+        assert BanditInterface.choose_arm(arms_to_allocations) in frozenset(["arm1", "arm2"])

moe/tests/bandit/bandit_test_case.py

@@ -1,11 +1,9 @@
 # -*- coding: utf-8 -*-
 """Base test case class for bandit tests; includes different historical infos (different sampled arms)."""
-import testify as T
-
 from moe.bandit.data_containers import BernoulliArm, HistoricalData, SampleArm
 
 
-class BanditTestCase(T.TestCase):
+class BanditTestCase(object):
 
     """Base test case for the bandit library.
 
@@ -51,5 +49,5 @@ def _test_one_arm(self, bandit):
         """Check that the one-arm case always returns the given arm as the winning arm and the allocation is 1.0."""
         bandit = self.bandit_class(self.one_arm_test_case)
         arms_to_allocations = bandit.allocate_arms()
-        T.assert_dicts_equal(arms_to_allocations, {"arm1": 1.0})
-        T.assert_equal(bandit.choose_arm(arms_to_allocations), "arm1")
+        assert arms_to_allocations == {"arm1": 1.0}
+        assert bandit.choose_arm(arms_to_allocations) == "arm1"

moe/tests/bandit/bla/bla_test.py

@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-# -*- coding: utf-8 -*-
 """Test UCB1 bandit implementation.
 
 Test that the one arm case returns the given arm as the winning arm.
@@ -8,14 +7,12 @@
 """
 import numpy
 
-import testify as T
-
 from moe.bandit.bla.bla import BLA
 from moe.bandit.constant import DEFAULT_BLA_SUBTYPE
 from moe.tests.bandit.bandit_test_case import BanditTestCase
 
 
-class BLATest(BanditTestCase):
+class TestBLA(BanditTestCase):
 
     """Verify that different historical infos return correct results."""
 
@@ -32,10 +29,6 @@ def test_two_arms_one_winner(self):
         numpy.random.seed(0)
         bandit = self.bandit_class(self.two_arms_test_case, DEFAULT_BLA_SUBTYPE)
         arms_to_allocations = bandit.allocate_arms()
-        T.assert_dicts_equal(arms_to_allocations, {"arm1": 1.0, "arm2": 0.0})
-        T.assert_equal(bandit.choose_arm(arms_to_allocations), "arm1")
+        assert arms_to_allocations == {"arm1": 1.0, "arm2": 0.0}
+        assert bandit.choose_arm(arms_to_allocations) == "arm1"
         numpy.random.set_state(old_state)
-
-
-if __name__ == "__main__":
-    T.run()

moe/tests/bandit/data_containers_test.py

@@ -1,53 +1,53 @@
 # -*- coding: utf-8 -*-
 """Tests for functions in SampleArm and HistoricalData."""
-import copy
+import pytest
 
+import copy
 import logging
-
 import pprint
 
-import testify as T
-
 from moe.bandit.data_containers import HistoricalData, SampleArm
 from moe.tests.bandit.bandit_test_case import BanditTestCase
 
 
-class DataContainersTest(BanditTestCase):
+@pytest.fixture()
+def disable_logging(request):
+    """Disable logging (for the duration of this test case)."""
+    logging.disable(logging.CRITICAL)
 
-    """Tests functions in :class:`moe.bandit.data_containers.SampleArm` and :class:`moe.bandit.data_containers.HistoricalData`."""
-
-    @T.class_setup
-    def disable_logging(self):
-        """Disable logging (for the duration of this test case)."""
-        logging.disable(logging.CRITICAL)
-
-    @T.class_teardown
-    def enable_logging(self):
+    def finalize():
         """Re-enable logging (so other test cases are unaffected)."""
         logging.disable(logging.NOTSET)
+    request.addfinalizer(finalize)
+
+
+class TestDataContainers(BanditTestCase):
+
+    """Tests functions in :class:`moe.bandit.data_containers.SampleArm` and :class:`moe.bandit.data_containers.HistoricalData`."""
 
     def test_sample_arm_str(self):
         """Test SampleArm's __str__ overload operator."""
         for historical_info in self.historical_infos_to_test:
             for arm in historical_info.arms_sampled.itervalues():
-                T.assert_equals(str(arm), pprint.pformat(arm.json_payload()))
+                assert str(arm) == pprint.pformat(arm.json_payload())
 
     def test_sample_arm_add(self):
         """Test SampleArm's __add__ overload operator."""
         arm1 = SampleArm(win=2, loss=1, total=3)
         arm2 = SampleArm(win=3, loss=2, total=5)
         arm3 = arm1 + arm2
-        T.assert_equals(arm3.json_payload(), SampleArm(win=5, loss=3, total=8).json_payload())
+        assert arm3.json_payload() == SampleArm(win=5, loss=3, total=8).json_payload()
+
         # Verify that the + operator does not modify arm1 and arm2
-        T.assert_equals(arm1.json_payload(), SampleArm(win=2, loss=1, total=3).json_payload())
-        T.assert_equals(arm2.json_payload(), SampleArm(win=3, loss=2, total=5).json_payload())
+        assert arm1.json_payload() == SampleArm(win=2, loss=1, total=3).json_payload()
+        assert arm2.json_payload() == SampleArm(win=3, loss=2, total=5).json_payload()
 
         arm1 += arm2
         arm2 += arm1
-        T.assert_equals(arm1.json_payload(), SampleArm(win=5, loss=3, total=8).json_payload())
-        T.assert_equals(arm2.json_payload(), SampleArm(win=8, loss=5, total=13).json_payload())
+        assert arm1.json_payload() == SampleArm(win=5, loss=3, total=8).json_payload()
+        assert arm2.json_payload() == SampleArm(win=8, loss=5, total=13).json_payload()
         # Verify that modifying arm1 and arm2 does not change arm3
-        T.assert_equals(arm3.json_payload(), SampleArm(win=5, loss=3, total=8).json_payload())
+        assert arm3.json_payload() == SampleArm(win=5, loss=3, total=8).json_payload()
 
     def test_sample_arm_iadd(self):
         """Test SampleArm's __iadd__ overload operator.
@@ -61,29 +61,30 @@ def test_sample_arm_iadd(self):
         arm1_old_id = id(arm1)
         arm1 += arm2
         arm1_new_id = id(arm1)
-        T.assert_equals(arm1_old_id, arm1_new_id)
-        T.assert_equals(arm1.json_payload(), arm3.json_payload())
+        assert arm1_old_id == arm1_new_id
+        assert arm1.json_payload() == arm3.json_payload()
 
+    @pytest.mark.usefixtures("disable_logging")
     def test_sample_arm_add_arm_with_variance_invalid(self):
         """Test that adding arms with variance causes a ValueError. Neither of the arms can have non-None variance."""
-        arm = SampleArm(win=2, loss=1, total=500, variance=0.1)
-        T.assert_raises(ValueError, arm.__add__, SampleArm(win=2, loss=1, total=500, variance=None))
-        arm = SampleArm(win=2, loss=1, total=500, variance=None)
-        T.assert_raises(ValueError, arm.__add__, SampleArm(win=2, loss=1, total=500, variance=0.1))
+        with pytest.raises(ValueError):
+            arm = SampleArm(win=2, loss=1, total=500, variance=0.1)
+            arm.__add__(SampleArm(win=2, loss=1, total=500, variance=None))
+
+        with pytest.raises(ValueError):
+            arm = SampleArm(win=2, loss=1, total=500, variance=None)
+            arm.__add__(SampleArm(win=2, loss=1, total=500, variance=0.1))
 
     def test_historical_data_str(self):
         """Test HistoricalData's __str__ overload operator."""
         for historical_info in self.historical_infos_to_test:
-            T.assert_equals(str(historical_info), pprint.pformat(historical_info.json_payload()))
+            assert str(historical_info) == pprint.pformat(historical_info.json_payload())
 
     def test_historical_data_append_unsampled_arm(self):
         """Test that adding an unsampled arm (already exists in historical info) to HistoricalData does not change anything."""
         historical_info = self.two_unsampled_arms_test_case
         historical_info.append_sample_arms(self.one_arm_test_case.arms_sampled)
-        T.assert_dicts_equal(
-                historical_info.json_payload(),
-                self.two_unsampled_arms_test_case.json_payload()
-                )
+        assert historical_info.json_payload() == self.two_unsampled_arms_test_case.json_payload()
 
     def test_historical_data_append_arms(self):
         """Test that appending arms to HistoricalData updates historical info correctly."""
@@ -96,20 +97,13 @@ def test_historical_data_append_arms(self):
                     "arm3": SampleArm(win=0, loss=0, total=0),
                     }
                 )
-        T.assert_dicts_equal(
-                historical_info.json_payload(),
-                expected_historical_info.json_payload()
-                )
+        assert historical_info.json_payload() == expected_historical_info.json_payload()
 
+    @pytest.mark.usefixtures("disable_logging")
     def test_historical_data_append_arms_with_variance_invalid(self):
         """Test that adding arms with variance causes a ValueError."""
         historical_info = copy.deepcopy(self.three_arms_with_variance_no_unsampled_arm_test_case)
-        T.assert_raises(
-                ValueError,
-                historical_info.append_sample_arms,
-                self.three_arms_with_variance_no_unsampled_arm_test_case.arms_sampled
-                )
-
-
-if __name__ == "__main__":
-    T.run()
+        with pytest.raises(ValueError):
+                historical_info.append_sample_arms(
+                    self.three_arms_with_variance_no_unsampled_arm_test_case.arms_sampled
+                    )

moe/tests/bandit/epsilon/epsilon_first_test.py

@@ -5,13 +5,11 @@
 Test one arm with various epsilon values.
 
 """
-import testify as T
-
 from moe.bandit.epsilon.epsilon_first import EpsilonFirst
 from moe.tests.bandit.epsilon.epsilon_test_case import EpsilonTestCase
 
 
-class EpsilonFirstTest(EpsilonTestCase):
+class TestEpsilonFirst(EpsilonTestCase):
 
     """Verify that different epsilon values and historical infos return correct results."""
 
@@ -35,21 +33,21 @@ def test_two_unsampled_arms(self):
         for epsilon in self.epsilons_to_test:
             for total_samples in self.total_samples_to_test:
                 bandit = self.bandit_class(self.two_unsampled_arms_test_case, epsilon, total_samples)
-                T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 0.5, "arm2": 0.5})
+                assert bandit.allocate_arms() == {"arm1": 0.5, "arm2": 0.5}
 
     def test_two_arms_epsilon_zero(self):
         """Check that the two-arms case with zero epsilon (always exploit) always allocate arm1:1.0 and arm2:0.0 when average payoffs are arm1:1.0 and arm2:0.0."""
         epsilon = 0.0
         bandit = self.bandit_class(self.two_arms_test_case, epsilon)
         arms_to_allocations = bandit.allocate_arms()
-        T.assert_dicts_equal(arms_to_allocations, {"arm1": 1.0, "arm2": 0.0})
-        T.assert_equal(bandit.choose_arm(arms_to_allocations), "arm1")
+        assert arms_to_allocations == {"arm1": 1.0, "arm2": 0.0}
+        assert bandit.choose_arm(arms_to_allocations) == "arm1"
 
     def test_two_arms_epsilon_one(self):
         """Check that the two-arms case with one epsilon (always explore) always allocate arm1:0.5 and arm2:0.5 when average payoffs are arm1:1.0 and arm2:0.0."""
         epsilon = 1.0
         bandit = self.bandit_class(self.two_arms_test_case, epsilon)
-        T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 0.5, "arm2": 0.5})
+        assert bandit.allocate_arms() == {"arm1": 0.5, "arm2": 0.5}
 
     def test_three_arms_explore(self):
         """Check that the three-arms cases with integer and float payoffs in exploration phase return the expected arm allocations."""
@@ -58,23 +56,19 @@ def test_three_arms_explore(self):
         equal_allocation = 1.0 / 3
         for historical_info in [self.three_arms_test_case, self.three_arms_float_payoffs_test_case]:
             bandit = self.bandit_class(historical_info, epsilon, total_samples)
-            T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": equal_allocation, "arm2": equal_allocation, "arm3": equal_allocation})
+            assert bandit.allocate_arms() == {"arm1": equal_allocation, "arm2": equal_allocation, "arm3": equal_allocation}
 
     def test_three_arms_exploit(self):
         """Check that the three-arms cases with integer and float payoffs in exploitation phase return the expected arm allocations."""
         epsilon = 0.5
         total_samples = 10
         for historical_info in [self.three_arms_test_case, self.three_arms_float_payoffs_test_case]:
             bandit = self.bandit_class(historical_info, epsilon, total_samples)
-            T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 1.0, "arm2": 0.0, "arm3": 0.0})
+            assert bandit.allocate_arms() == {"arm1": 1.0, "arm2": 0.0, "arm3": 0.0}
 
     def test_three_arms_exploit_two_winners(self):
         """Check that the three-arms cases with two winners in exploitation phase return the expected arm allocations. This tests num_arms > num_winning_arms > 1."""
         epsilon = 0.5
         total_samples = 10
         bandit = self.bandit_class(self.three_arms_two_winners_test_case, epsilon, total_samples)
-        T.assert_dicts_equal(bandit.allocate_arms(), {"arm1": 0.5, "arm2": 0.5, "arm3": 0.0})
-
-
-if __name__ == "__main__":
-    T.run()
+        assert bandit.allocate_arms() == {"arm1": 0.5, "arm2": 0.5, "arm3": 0.0}