313 lines
12 KiB
Python
313 lines
12 KiB
Python
|
# Author: Alexander Fabisch <afabisch@informatik.uni-bremen.de>
|
||
|
|
#
|
||
|
|
# License: BSD 3 clause
|
||
|
|
|
||
|
|
import sys
|
||
|
|
from sklearn.externals.six.moves import cStringIO as StringIO
|
||
|
|
import numpy as np
|
||
|
|
import warnings
|
||
|
|
from sklearn.base import BaseEstimator
|
||
|
|
from sklearn.utils.testing import assert_raises
|
||
|
|
from sklearn.utils.testing import assert_warns
|
||
|
|
from sklearn.utils.testing import assert_equal
|
||
|
|
from sklearn.utils.testing import assert_array_equal
|
||
|
|
from sklearn.utils.testing import assert_array_almost_equal
|
||
|
|
from sklearn.utils.testing import assert_false
|
||
|
|
from sklearn.datasets import make_classification
|
||
|
|
|
||
|
|
with warnings.catch_warnings():
|
||
|
|
warnings.simplefilter('ignore')
|
||
|
|
from sklearn.learning_curve import learning_curve, validation_curve
|
||
|
|
from sklearn.cross_validation import KFold
|
||
|
|
|
||
|
|
from sklearn.linear_model import PassiveAggressiveClassifier
|
||
|
|
|
||
|
|
|
||
|
|
class MockImprovingEstimator(BaseEstimator):
|
||
|
|
"""Dummy classifier to test the learning curve"""
|
||
|
|
def __init__(self, n_max_train_sizes):
|
||
|
|
self.n_max_train_sizes = n_max_train_sizes
|
||
|
|
self.train_sizes = 0
|
||
|
|
self.X_subset = None
|
||
|
|
|
||
|
|
def fit(self, X_subset, y_subset=None):
|
||
|
|
self.X_subset = X_subset
|
||
|
|
self.train_sizes = X_subset.shape[0]
|
||
|
|
return self
|
||
|
|
|
||
|
|
def predict(self, X):
|
||
|
|
raise NotImplementedError
|
||
|
|
|
||
|
|
def score(self, X=None, Y=None):
|
||
|
|
# training score becomes worse (2 -> 1), test error better (0 -> 1)
|
||
|
|
if self._is_training_data(X):
|
||
|
|
return 2. - float(self.train_sizes) / self.n_max_train_sizes
|
||
|
|
else:
|
||
|
|
return float(self.train_sizes) / self.n_max_train_sizes
|
||
|
|
|
||
|
|
def _is_training_data(self, X):
|
||
|
|
return X is self.X_subset
|
||
|
|
|
||
|
|
|
||
|
|
class MockIncrementalImprovingEstimator(MockImprovingEstimator):
|
||
|
|
"""Dummy classifier that provides partial_fit"""
|
||
|
|
def __init__(self, n_max_train_sizes):
|
||
|
|
super(MockIncrementalImprovingEstimator,
|
||
|
|
self).__init__(n_max_train_sizes)
|
||
|
|
self.x = None
|
||
|
|
|
||
|
|
def _is_training_data(self, X):
|
||
|
|
return self.x in X
|
||
|
|
|
||
|
|
def partial_fit(self, X, y=None, **params):
|
||
|
|
self.train_sizes += X.shape[0]
|
||
|
|
self.x = X[0]
|
||
|
|
|
||
|
|
|
||
|
|
class MockEstimatorWithParameter(BaseEstimator):
|
||
|
|
"""Dummy classifier to test the validation curve"""
|
||
|
|
def __init__(self, param=0.5):
|
||
|
|
self.X_subset = None
|
||
|
|
self.param = param
|
||
|
|
|
||
|
|
def fit(self, X_subset, y_subset):
|
||
|
|
self.X_subset = X_subset
|
||
|
|
self.train_sizes = X_subset.shape[0]
|
||
|
|
return self
|
||
|
|
|
||
|
|
def predict(self, X):
|
||
|
|
raise NotImplementedError
|
||
|
|
|
||
|
|
def score(self, X=None, y=None):
|
||
|
|
return self.param if self._is_training_data(X) else 1 - self.param
|
||
|
|
|
||
|
|
def _is_training_data(self, X):
|
||
|
|
return X is self.X_subset
|
||
|
|
|
||
|
|
|
||
|
|
class MockEstimatorFailing(BaseEstimator):
|
||
|
|
"""Dummy classifier to test error_score in learning curve"""
|
||
|
|
def fit(self, X_subset, y_subset):
|
||
|
|
raise ValueError()
|
||
|
|
|
||
|
|
def score(self, X=None, y=None):
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
class MockEstimatorWithSingleFitCallAllowed(MockEstimatorWithParameter):
|
||
|
|
"""Dummy classifier that disallows repeated calls of fit method"""
|
||
|
|
|
||
|
|
def fit(self, X_subset, y_subset):
|
||
|
|
assert_false(
|
||
|
|
hasattr(self, 'fit_called_'),
|
||
|
|
'fit is called the second time'
|
||
|
|
)
|
||
|
|
self.fit_called_ = True
|
||
|
|
return super(type(self), self).fit(X_subset, y_subset)
|
||
|
|
|
||
|
|
|
||
|
|
def test_learning_curve():
|
||
|
|
X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
|
||
|
|
n_redundant=0, n_classes=2,
|
||
|
|
n_clusters_per_class=1, random_state=0)
|
||
|
|
estimator = MockImprovingEstimator(20)
|
||
|
|
with warnings.catch_warnings(record=True) as w:
|
||
|
|
train_sizes, train_scores, test_scores = learning_curve(
|
||
|
|
estimator, X, y, cv=3, train_sizes=np.linspace(0.1, 1.0, 10))
|
||
|
|
if len(w) > 0:
|
||
|
|
raise RuntimeError("Unexpected warning: %r" % w[0].message)
|
||
|
|
assert_equal(train_scores.shape, (10, 3))
|
||
|
|
assert_equal(test_scores.shape, (10, 3))
|
||
|
|
assert_array_equal(train_sizes, np.linspace(2, 20, 10))
|
||
|
|
assert_array_almost_equal(train_scores.mean(axis=1),
|
||
|
|
np.linspace(1.9, 1.0, 10))
|
||
|
|
assert_array_almost_equal(test_scores.mean(axis=1),
|
||
|
|
np.linspace(0.1, 1.0, 10))
|
||
|
|
|
||
|
|
|
||
|
|
def test_learning_curve_unsupervised():
|
||
|
|
X, _ = make_classification(n_samples=30, n_features=1, n_informative=1,
|
||
|
|
n_redundant=0, n_classes=2,
|
||
|
|
n_clusters_per_class=1, random_state=0)
|
||
|
|
estimator = MockImprovingEstimator(20)
|
||
|
|
train_sizes, train_scores, test_scores = learning_curve(
|
||
|
|
estimator, X, y=None, cv=3, train_sizes=np.linspace(0.1, 1.0, 10))
|
||
|
|
assert_array_equal(train_sizes, np.linspace(2, 20, 10))
|
||
|
|
assert_array_almost_equal(train_scores.mean(axis=1),
|
||
|
|
np.linspace(1.9, 1.0, 10))
|
||
|
|
assert_array_almost_equal(test_scores.mean(axis=1),
|
||
|
|
np.linspace(0.1, 1.0, 10))
|
||
|
|
|
||
|
|
|
||
|
|
def test_learning_curve_verbose():
|
||
|
|
X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
|
||
|
|
n_redundant=0, n_classes=2,
|
||
|
|
n_clusters_per_class=1, random_state=0)
|
||
|
|
estimator = MockImprovingEstimator(20)
|
||
|
|
|
||
|
|
old_stdout = sys.stdout
|
||
|
|
sys.stdout = StringIO()
|
||
|
|
try:
|
||
|
|
train_sizes, train_scores, test_scores = \
|
||
|
|
learning_curve(estimator, X, y, cv=3, verbose=1)
|
||
|
|
finally:
|
||
|
|
out = sys.stdout.getvalue()
|
||
|
|
sys.stdout.close()
|
||
|
|
sys.stdout = old_stdout
|
||
|
|
|
||
|
|
assert("[learning_curve]" in out)
|
||
|
|
|
||
|
|
|
||
|
|
def test_learning_curve_error_score():
|
||
|
|
X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
|
||
|
|
n_redundant=0, n_classes=2,
|
||
|
|
n_clusters_per_class=1, random_state=0)
|
||
|
|
estimator = MockEstimatorFailing()
|
||
|
|
_, _, test_scores = learning_curve(estimator, X, y, cv=3, error_score=0)
|
||
|
|
all_zeros = not np.any(test_scores)
|
||
|
|
assert(all_zeros)
|
||
|
|
|
||
|
|
|
||
|
|
def test_learning_curve_error_score_default_raise():
|
||
|
|
X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
|
||
|
|
n_redundant=0, n_classes=2,
|
||
|
|
n_clusters_per_class=1, random_state=0)
|
||
|
|
estimator = MockEstimatorFailing()
|
||
|
|
assert_raises(ValueError, learning_curve, estimator, X, y, cv=3)
|
||
|
|
|
||
|
|
|
||
|
|
def test_learning_curve_incremental_learning_not_possible():
|
||
|
|
X, y = make_classification(n_samples=2, n_features=1, n_informative=1,
|
||
|
|
n_redundant=0, n_classes=2,
|
||
|
|
n_clusters_per_class=1, random_state=0)
|
||
|
|
# The mockup does not have partial_fit()
|
||
|
|
estimator = MockImprovingEstimator(1)
|
||
|
|
assert_raises(ValueError, learning_curve, estimator, X, y,
|
||
|
|
exploit_incremental_learning=True)
|
||
|
|
|
||
|
|
|
||
|
|
def test_learning_curve_incremental_learning():
|
||
|
|
X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
|
||
|
|
n_redundant=0, n_classes=2,
|
||
|
|
n_clusters_per_class=1, random_state=0)
|
||
|
|
estimator = MockIncrementalImprovingEstimator(20)
|
||
|
|
train_sizes, train_scores, test_scores = learning_curve(
|
||
|
|
estimator, X, y, cv=3, exploit_incremental_learning=True,
|
||
|
|
train_sizes=np.linspace(0.1, 1.0, 10))
|
||
|
|
assert_array_equal(train_sizes, np.linspace(2, 20, 10))
|
||
|
|
assert_array_almost_equal(train_scores.mean(axis=1),
|
||
|
|
np.linspace(1.9, 1.0, 10))
|
||
|
|
assert_array_almost_equal(test_scores.mean(axis=1),
|
||
|
|
np.linspace(0.1, 1.0, 10))
|
||
|
|
|
||
|
|
|
||
|
|
def test_learning_curve_incremental_learning_unsupervised():
|
||
|
|
X, _ = make_classification(n_samples=30, n_features=1, n_informative=1,
|
||
|
|
n_redundant=0, n_classes=2,
|
||
|
|
n_clusters_per_class=1, random_state=0)
|
||
|
|
estimator = MockIncrementalImprovingEstimator(20)
|
||
|
|
train_sizes, train_scores, test_scores = learning_curve(
|
||
|
|
estimator, X, y=None, cv=3, exploit_incremental_learning=True,
|
||
|
|
train_sizes=np.linspace(0.1, 1.0, 10))
|
||
|
|
assert_array_equal(train_sizes, np.linspace(2, 20, 10))
|
||
|
|
assert_array_almost_equal(train_scores.mean(axis=1),
|
||
|
|
np.linspace(1.9, 1.0, 10))
|
||
|
|
assert_array_almost_equal(test_scores.mean(axis=1),
|
||
|
|
np.linspace(0.1, 1.0, 10))
|
||
|
|
|
||
|
|
|
||
|
|
def test_learning_curve_batch_and_incremental_learning_are_equal():
|
||
|
|
X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
|
||
|
|
n_redundant=0, n_classes=2,
|
||
|
|
n_clusters_per_class=1, random_state=0)
|
||
|
|
train_sizes = np.linspace(0.2, 1.0, 5)
|
||
|
|
estimator = PassiveAggressiveClassifier(max_iter=1, tol=None,
|
||
|
|
shuffle=False)
|
||
|
|
|
||
|
|
train_sizes_inc, train_scores_inc, test_scores_inc = \
|
||
|
|
learning_curve(
|
||
|
|
estimator, X, y, train_sizes=train_sizes,
|
||
|
|
cv=3, exploit_incremental_learning=True)
|
||
|
|
train_sizes_batch, train_scores_batch, test_scores_batch = \
|
||
|
|
learning_curve(
|
||
|
|
estimator, X, y, cv=3, train_sizes=train_sizes,
|
||
|
|
exploit_incremental_learning=False)
|
||
|
|
|
||
|
|
assert_array_equal(train_sizes_inc, train_sizes_batch)
|
||
|
|
assert_array_almost_equal(train_scores_inc.mean(axis=1),
|
||
|
|
train_scores_batch.mean(axis=1))
|
||
|
|
assert_array_almost_equal(test_scores_inc.mean(axis=1),
|
||
|
|
test_scores_batch.mean(axis=1))
|
||
|
|
|
||
|
|
|
||
|
|
def test_learning_curve_n_sample_range_out_of_bounds():
|
||
|
|
X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
|
||
|
|
n_redundant=0, n_classes=2,
|
||
|
|
n_clusters_per_class=1, random_state=0)
|
||
|
|
estimator = MockImprovingEstimator(20)
|
||
|
|
assert_raises(ValueError, learning_curve, estimator, X, y, cv=3,
|
||
|
|
train_sizes=[0, 1])
|
||
|
|
assert_raises(ValueError, learning_curve, estimator, X, y, cv=3,
|
||
|
|
train_sizes=[0.0, 1.0])
|
||
|
|
assert_raises(ValueError, learning_curve, estimator, X, y, cv=3,
|
||
|
|
train_sizes=[0.1, 1.1])
|
||
|
|
assert_raises(ValueError, learning_curve, estimator, X, y, cv=3,
|
||
|
|
train_sizes=[0, 20])
|
||
|
|
assert_raises(ValueError, learning_curve, estimator, X, y, cv=3,
|
||
|
|
train_sizes=[1, 21])
|
||
|
|
|
||
|
|
|
||
|
|
def test_learning_curve_remove_duplicate_sample_sizes():
|
||
|
|
X, y = make_classification(n_samples=3, n_features=1, n_informative=1,
|
||
|
|
n_redundant=0, n_classes=2,
|
||
|
|
n_clusters_per_class=1, random_state=0)
|
||
|
|
estimator = MockImprovingEstimator(2)
|
||
|
|
train_sizes, _, _ = assert_warns(
|
||
|
|
RuntimeWarning, learning_curve, estimator, X, y, cv=3,
|
||
|
|
train_sizes=np.linspace(0.33, 1.0, 3))
|
||
|
|
assert_array_equal(train_sizes, [1, 2])
|
||
|
|
|
||
|
|
|
||
|
|
def test_learning_curve_with_boolean_indices():
|
||
|
|
X, y = make_classification(n_samples=30, n_features=1, n_informative=1,
|
||
|
|
n_redundant=0, n_classes=2,
|
||
|
|
n_clusters_per_class=1, random_state=0)
|
||
|
|
estimator = MockImprovingEstimator(20)
|
||
|
|
cv = KFold(n=30, n_folds=3)
|
||
|
|
train_sizes, train_scores, test_scores = learning_curve(
|
||
|
|
estimator, X, y, cv=cv, train_sizes=np.linspace(0.1, 1.0, 10))
|
||
|
|
assert_array_equal(train_sizes, np.linspace(2, 20, 10))
|
||
|
|
assert_array_almost_equal(train_scores.mean(axis=1),
|
||
|
|
np.linspace(1.9, 1.0, 10))
|
||
|
|
assert_array_almost_equal(test_scores.mean(axis=1),
|
||
|
|
np.linspace(0.1, 1.0, 10))
|
||
|
|
|
||
|
|
|
||
|
|
def test_validation_curve():
|
||
|
|
X, y = make_classification(n_samples=2, n_features=1, n_informative=1,
|
||
|
|
n_redundant=0, n_classes=2,
|
||
|
|
n_clusters_per_class=1, random_state=0)
|
||
|
|
param_range = np.linspace(0, 1, 10)
|
||
|
|
with warnings.catch_warnings(record=True) as w:
|
||
|
|
train_scores, test_scores = validation_curve(
|
||
|
|
MockEstimatorWithParameter(), X, y, param_name="param",
|
||
|
|
param_range=param_range, cv=2
|
||
|
|
)
|
||
|
|
if len(w) > 0:
|
||
|
|
raise RuntimeError("Unexpected warning: %r" % w[0].message)
|
||
|
|
|
||
|
|
assert_array_almost_equal(train_scores.mean(axis=1), param_range)
|
||
|
|
assert_array_almost_equal(test_scores.mean(axis=1), 1 - param_range)
|
||
|
|
|
||
|
|
|
||
|
|
def test_validation_curve_clone_estimator():
|
||
|
|
X, y = make_classification(n_samples=2, n_features=1, n_informative=1,
|
||
|
|
n_redundant=0, n_classes=2,
|
||
|
|
n_clusters_per_class=1, random_state=0)
|
||
|
|
|
||
|
|
param_range = np.linspace(1, 0, 10)
|
||
|
|
_, _ = validation_curve(
|
||
|
|
MockEstimatorWithSingleFitCallAllowed(), X, y,
|
||
|
|
param_name="param", param_range=param_range, cv=2
|
||
|
|
)
|