Fixing blending (#194)

* [DOC] Add circleci and sphinx documentation (#185)

* add sphinx doc

* add to using ramp workflow

* add docs

* create initial circleCI

* amend doc

* [empty] trigger circleCI

* dictate branch

* remove branch specification

* [mrg] fixing issues with tests not passing on appveyor (#186)

* Update README.md

* checking the test_notebook_testing path

* debugging convert_notebook

* checking where exactly in execute-notebook is the problem

* changed titanic starting kit not to have !command

* hashing out the Executing Shell Commands (!) in jupyter notebooks

* cleanup

* updated link for appveyor

* fixing blending

* fixing blending cli

* fixing blending

* make blending more readable

* creating training_output before using it

* flake

* assert cv

* flake

* getting rid of python 2 in travis

Author: kegl

.travis.yml

@@ -5,7 +5,6 @@ branches:
   only:
   - master
 env:
-  - PYTHON_VERSION=2.7
   - PYTHON_VERSION=3.5
   - PYTHON_VERSION=3.6
 before_install:

rampwf/__init__.py

@@ -4,6 +4,7 @@
 from . import score_types
 from . import utils
 from . import workflows
+from ._version import get_versions
 
 
 __all__ = [
@@ -15,7 +16,5 @@
     'workflows',
 ]
 
-
-from ._version import get_versions
 __version__ = get_versions()['version']
 del get_versions

rampwf/score_types/tests/test_detection.py

@@ -177,8 +177,8 @@ def test_average_precision():
 
     conf, ps, rs = precision_recall_curve_greedy(y_true, y_pred)
     assert conf.tolist() == [0.9, 0.8, 0.7, 0.6, 0.5, 0.4]
-    assert ps.tolist() == [1, 1, 2/3, 3/4, 3/5, 3/6]  # noqa
-    assert rs.tolist() == [1/4, 2/4, 2/4, 3/4, 3/4, 3/4]  # noqa
+    assert ps.tolist() == [1, 1, 2 / 3, 3 / 4, 3 / 5, 3 / 6]  # noqa
+    assert rs.tolist() == [1 / 4, 2 / 4, 2 / 4, 3 / 4, 3 / 4, 3 / 4]  # noqa
     assert ap(y_true, y_pred) == 11 / 16  # 0.5 * 1 + 0.25 * 3/4 + 0.25 * 0
 
 

rampwf/utils/combine.py

@@ -124,10 +124,11 @@ def _get_next_best_submission(predictions_list, ground_truths,
             Predictions, predictions_list, new_index_list)
         new_score = score_type.score_function(
             ground_truths, combined_predictions)
-        iltb = score_type.is_lower_the_better
-        nltb = new_score < best_score - min_improvement
-        bltn = new_score > best_score + min_improvement
-        if (iltb and nltb) or (not iltb and bltn):
+        if score_type.is_lower_the_better:
+            is_improved = new_score < best_score - min_improvement
+        else:
+            is_improved = new_score > best_score + min_improvement
+        if is_improved:
             best_predictions = combined_predictions
             best_index = i
             best_score = new_score
@@ -167,14 +168,14 @@ def blend_on_fold(predictions_list, ground_truths_valid, score_type,
         best_prediction_index = np.argmax(valid_scores)
     score = valid_scores[best_prediction_index]
     best_index_list = np.array([best_prediction_index])
-    improvement = True
-    while improvement and len(best_index_list) < max_n_ensemble:
+    is_improved = True
+    while is_improved and len(best_index_list) < max_n_ensemble:
         print('\t{}: {}'.format(best_index_list, score))
         old_best_index_list = best_index_list
         best_index_list, score = _get_next_best_submission(
             predictions_list, ground_truths_valid, score_type, best_index_list,
             min_improvement)
-        improvement = len(best_index_list) != len(old_best_index_list)
+        is_improved = len(best_index_list) != len(old_best_index_list)
     return best_index_list
     # we share a unit of 1. among the contributive submissions
     # unit_contributivity = 1. / len(best_index_list)

rampwf/utils/command_line.py

@@ -157,7 +157,7 @@ def create_ramp_blend_submissions_parser():
                         ' Specify submissions separated by a comma without'
                         ' spaces. If "ALL", all submissions in the directory'
                         ' will be blended.')
-    parser.add_argument('--save_output', dest='save_output',
+    parser.add_argument('--save-output', dest='save_output',
                         action='store_true',
                         help='Specify this flag to save predictions '
                              'after blending.')

rampwf/utils/testing.py

@@ -5,6 +5,7 @@
 
 import os
 import imp
+import shutil
 
 import numpy as np
 import pandas as pd
@@ -157,13 +158,36 @@ def assert_submission(ramp_kit_dir='.', ramp_data_dir='.',
 def blend_submissions(submissions, ramp_kit_dir='.', ramp_data_dir='.',
                       ramp_submission_dir='.', save_output=False,
                       min_improvement=0.0):
+    """Blending submissions in a ramp-kit and compute contributivities.
+
+    If save_output is True, we create three files:
+    <ramp_submission_dir>/training_output/contributivities.csv
+    <ramp_submission_dir>/training_output/bagged_scores_combined.csv
+    <ramp_submission_dir>/training_output/bagged_scores_foldwise_best.csv
+
+    Parameters
+    ----------
+    submissions : list of str
+        List of submission names (folders in <ramp_submission_dir>).
+    ramp_kit_dir : str, default='.'
+        The directory of the ramp-kit to be blended.
+    ramp_data_dir : str, default='.'
+        The directory of the data.
+    ramp_submission_dir : str, default='./submissions'
+        The directory of the submissions.
+    save_output : bool, default is False
+        Whether to store the blending results.
+    min_improvement : float, default is 0.0
+        The minimum improvement under which greedy blender is stopped.
+    """
     problem = assert_read_problem(ramp_kit_dir)
     print_title('Blending {}'.format(problem.problem_title))
     X_train, y_train, X_test, y_test = assert_data(ramp_kit_dir, ramp_data_dir)
     cv = assert_cv(ramp_kit_dir, ramp_data_dir)
     valid_is_list = [valid_is for (train_is, valid_is) in cv]
     score_types = assert_score_types(ramp_kit_dir)
-    contributivitys = np.zeros(len(submissions))
+    n_folds = len(valid_is_list)
+    contributivitys = np.zeros((len(submissions), n_folds))
 
     combined_predictions_valid_list = []
     foldwise_best_predictions_valid_list = []
@@ -198,34 +222,55 @@ def blend_submissions(submissions, ramp_kit_dir='.', ramp_data_dir='.',
         # we share a unit of 1. among the contributive submissions
         unit_contributivity = 1. / len(best_index_list)
         for i in best_index_list:
-            contributivitys[i] += unit_contributivity
+            contributivitys[i, fold_i] += unit_contributivity
 
         combined_predictions_valid_list.append(
-            problem.Predictions.combine(predictions_valid_list))
-        foldwise_best_predictions_valid_list.append(predictions_valid_list[0])
+            problem.Predictions.combine(
+                predictions_valid_list, best_index_list))
+        foldwise_best_predictions_valid_list.append(
+            predictions_valid_list[best_index_list[0]])
         combined_predictions_test_list.append(
-            problem.Predictions.combine(predictions_test_list))
-        foldwise_best_predictions_test_list.append(predictions_test_list[0])
+            problem.Predictions.combine(
+                predictions_test_list, best_index_list))
+        foldwise_best_predictions_test_list.append(
+            predictions_test_list[best_index_list[0]])
 
-    contributivitys /= len(cv)
+    contributivitys /= n_folds
     contributivitys_df = pd.DataFrame()
     contributivitys_df['submission'] = np.array(submissions)
-    contributivitys_df['contributivity'] = np.round(contributivitys, 3)
-    contributivitys_df = contributivitys_df.reset_index()
+    contributivitys_df['contributivity'] = np.zeros(len(submissions))
+    for fold_i in range(n_folds):
+        c_i = contributivitys[:, fold_i]
+        contributivitys_df['fold_{}'.format(fold_i)] = c_i
+        contributivitys_df['contributivity'] += c_i
+    percentage_factor = 100 / contributivitys_df['contributivity'].sum()
+    contributivitys_df['contributivity'] *= percentage_factor
+    rounded = contributivitys_df['contributivity'].round().astype(int)
+    contributivitys_df['contributivity'] = rounded
     contributivitys_df = contributivitys_df.sort_values(
         'contributivity', ascending=False)
     print(contributivitys_df.to_string(index=False))
 
-    training_output_path = os.path.join(ramp_kit_dir, 'training_output')
-    if not os.path.exists(training_output_path):
-        os.mkdir(training_output_path)
+    if save_output:
+        training_output_path = os.path.join(
+            ramp_submission_dir, 'training_output')
+        if not os.path.exists(training_output_path):
+            os.mkdir(training_output_path)
+        contributivitys_df.to_csv(os.path.join(
+            training_output_path, 'contributivities.csv'), index=False)
+
     # bagging the foldwise ensembles
     bag_submissions(
         problem, cv, y_train, y_test, combined_predictions_valid_list,
         combined_predictions_test_list, training_output_path,
         ramp_data_dir=ramp_data_dir, score_type_index=0,
         save_output=save_output, score_table_title='Combined bagged scores',
         score_f_name_prefix='foldwise_best')
+    if save_output:
+        shutil.move(
+            os.path.join(training_output_path, 'bagged_scores.csv'),
+            os.path.join(training_output_path, 'bagged_scores_combined.csv'))
+
     # bagging the foldwise best submissions
     bag_submissions(
         problem, cv, y_train, y_test, foldwise_best_predictions_valid_list,
@@ -234,3 +279,8 @@ def blend_submissions(submissions, ramp_kit_dir='.', ramp_data_dir='.',
         save_output=save_output,
         score_table_title='Foldwise best bagged scores',
         score_f_name_prefix='combined')
+    if save_output:
+        shutil.move(
+            os.path.join(training_output_path, 'bagged_scores.csv'),
+            os.path.join(
+                training_output_path, 'bagged_scores_foldwise_best.csv'))