1575 lines
58 KiB
Python
1575 lines
58 KiB
Python
|
# -*- coding: utf-8 -*-
|
||
|
|
|
||
|
|
import numpy as np
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
from numpy.random import RandomState
|
||
|
|
from numpy import nan
|
||
|
|
from datetime import datetime
|
||
|
|
from itertools import permutations
|
||
|
|
from pandas import (Series, Categorical, CategoricalIndex,
|
||
|
|
Timestamp, DatetimeIndex, Index, IntervalIndex)
|
||
|
|
import pandas as pd
|
||
|
|
|
||
|
|
from pandas import compat
|
||
|
|
from pandas._libs import (groupby as libgroupby, algos as libalgos,
|
||
|
|
hashtable as ht)
|
||
|
|
from pandas._libs.hashtable import unique_label_indices
|
||
|
|
from pandas.compat import lrange, range
|
||
|
|
import pandas.core.algorithms as algos
|
||
|
|
import pandas.core.common as com
|
||
|
|
import pandas.util.testing as tm
|
||
|
|
import pandas.util._test_decorators as td
|
||
|
|
from pandas.core.dtypes.dtypes import CategoricalDtype as CDT
|
||
|
|
from pandas.compat.numpy import np_array_datetime64_compat
|
||
|
|
from pandas.util.testing import assert_almost_equal
|
||
|
|
|
||
|
|
|
||
|
|
class TestMatch(object):
|
||
|
|
|
||
|
|
def test_ints(self):
|
||
|
|
values = np.array([0, 2, 1])
|
||
|
|
to_match = np.array([0, 1, 2, 2, 0, 1, 3, 0])
|
||
|
|
|
||
|
|
result = algos.match(to_match, values)
|
||
|
|
expected = np.array([0, 2, 1, 1, 0, 2, -1, 0], dtype=np.int64)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = Series(algos.match(to_match, values, np.nan))
|
||
|
|
expected = Series(np.array([0, 2, 1, 1, 0, 2, np.nan, 0]))
|
||
|
|
tm.assert_series_equal(result, expected)
|
||
|
|
|
||
|
|
s = Series(np.arange(5), dtype=np.float32)
|
||
|
|
result = algos.match(s, [2, 4])
|
||
|
|
expected = np.array([-1, -1, 0, -1, 1], dtype=np.int64)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = Series(algos.match(s, [2, 4], np.nan))
|
||
|
|
expected = Series(np.array([np.nan, np.nan, 0, np.nan, 1]))
|
||
|
|
tm.assert_series_equal(result, expected)
|
||
|
|
|
||
|
|
def test_strings(self):
|
||
|
|
values = ['foo', 'bar', 'baz']
|
||
|
|
to_match = ['bar', 'foo', 'qux', 'foo', 'bar', 'baz', 'qux']
|
||
|
|
|
||
|
|
result = algos.match(to_match, values)
|
||
|
|
expected = np.array([1, 0, -1, 0, 1, 2, -1], dtype=np.int64)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = Series(algos.match(to_match, values, np.nan))
|
||
|
|
expected = Series(np.array([1, 0, np.nan, 0, 1, 2, np.nan]))
|
||
|
|
tm.assert_series_equal(result, expected)
|
||
|
|
|
||
|
|
|
||
|
|
class TestFactorize(object):
|
||
|
|
|
||
|
|
def test_basic(self):
|
||
|
|
|
||
|
|
labels, uniques = algos.factorize(['a', 'b', 'b', 'a', 'a', 'c', 'c',
|
||
|
|
'c'])
|
||
|
|
tm.assert_numpy_array_equal(
|
||
|
|
uniques, np.array(['a', 'b', 'c'], dtype=object))
|
||
|
|
|
||
|
|
labels, uniques = algos.factorize(['a', 'b', 'b', 'a',
|
||
|
|
'a', 'c', 'c', 'c'], sort=True)
|
||
|
|
exp = np.array([0, 1, 1, 0, 0, 2, 2, 2], dtype=np.intp)
|
||
|
|
tm.assert_numpy_array_equal(labels, exp)
|
||
|
|
exp = np.array(['a', 'b', 'c'], dtype=object)
|
||
|
|
tm.assert_numpy_array_equal(uniques, exp)
|
||
|
|
|
||
|
|
labels, uniques = algos.factorize(list(reversed(range(5))))
|
||
|
|
exp = np.array([0, 1, 2, 3, 4], dtype=np.intp)
|
||
|
|
tm.assert_numpy_array_equal(labels, exp)
|
||
|
|
exp = np.array([4, 3, 2, 1, 0], dtype=np.int64)
|
||
|
|
tm.assert_numpy_array_equal(uniques, exp)
|
||
|
|
|
||
|
|
labels, uniques = algos.factorize(list(reversed(range(5))), sort=True)
|
||
|
|
|
||
|
|
exp = np.array([4, 3, 2, 1, 0], dtype=np.intp)
|
||
|
|
tm.assert_numpy_array_equal(labels, exp)
|
||
|
|
exp = np.array([0, 1, 2, 3, 4], dtype=np.int64)
|
||
|
|
tm.assert_numpy_array_equal(uniques, exp)
|
||
|
|
|
||
|
|
labels, uniques = algos.factorize(list(reversed(np.arange(5.))))
|
||
|
|
exp = np.array([0, 1, 2, 3, 4], dtype=np.intp)
|
||
|
|
tm.assert_numpy_array_equal(labels, exp)
|
||
|
|
exp = np.array([4., 3., 2., 1., 0.], dtype=np.float64)
|
||
|
|
tm.assert_numpy_array_equal(uniques, exp)
|
||
|
|
|
||
|
|
labels, uniques = algos.factorize(list(reversed(np.arange(5.))),
|
||
|
|
sort=True)
|
||
|
|
exp = np.array([4, 3, 2, 1, 0], dtype=np.intp)
|
||
|
|
tm.assert_numpy_array_equal(labels, exp)
|
||
|
|
exp = np.array([0., 1., 2., 3., 4.], dtype=np.float64)
|
||
|
|
tm.assert_numpy_array_equal(uniques, exp)
|
||
|
|
|
||
|
|
def test_mixed(self):
|
||
|
|
|
||
|
|
# doc example reshaping.rst
|
||
|
|
x = Series(['A', 'A', np.nan, 'B', 3.14, np.inf])
|
||
|
|
labels, uniques = algos.factorize(x)
|
||
|
|
|
||
|
|
exp = np.array([0, 0, -1, 1, 2, 3], dtype=np.intp)
|
||
|
|
tm.assert_numpy_array_equal(labels, exp)
|
||
|
|
exp = Index(['A', 'B', 3.14, np.inf])
|
||
|
|
tm.assert_index_equal(uniques, exp)
|
||
|
|
|
||
|
|
labels, uniques = algos.factorize(x, sort=True)
|
||
|
|
exp = np.array([2, 2, -1, 3, 0, 1], dtype=np.intp)
|
||
|
|
tm.assert_numpy_array_equal(labels, exp)
|
||
|
|
exp = Index([3.14, np.inf, 'A', 'B'])
|
||
|
|
tm.assert_index_equal(uniques, exp)
|
||
|
|
|
||
|
|
def test_datelike(self):
|
||
|
|
|
||
|
|
# M8
|
||
|
|
v1 = Timestamp('20130101 09:00:00.00004')
|
||
|
|
v2 = Timestamp('20130101')
|
||
|
|
x = Series([v1, v1, v1, v2, v2, v1])
|
||
|
|
labels, uniques = algos.factorize(x)
|
||
|
|
|
||
|
|
exp = np.array([0, 0, 0, 1, 1, 0], dtype=np.intp)
|
||
|
|
tm.assert_numpy_array_equal(labels, exp)
|
||
|
|
exp = DatetimeIndex([v1, v2])
|
||
|
|
tm.assert_index_equal(uniques, exp)
|
||
|
|
|
||
|
|
labels, uniques = algos.factorize(x, sort=True)
|
||
|
|
exp = np.array([1, 1, 1, 0, 0, 1], dtype=np.intp)
|
||
|
|
tm.assert_numpy_array_equal(labels, exp)
|
||
|
|
exp = DatetimeIndex([v2, v1])
|
||
|
|
tm.assert_index_equal(uniques, exp)
|
||
|
|
|
||
|
|
# period
|
||
|
|
v1 = pd.Period('201302', freq='M')
|
||
|
|
v2 = pd.Period('201303', freq='M')
|
||
|
|
x = Series([v1, v1, v1, v2, v2, v1])
|
||
|
|
|
||
|
|
# periods are not 'sorted' as they are converted back into an index
|
||
|
|
labels, uniques = algos.factorize(x)
|
||
|
|
exp = np.array([0, 0, 0, 1, 1, 0], dtype=np.intp)
|
||
|
|
tm.assert_numpy_array_equal(labels, exp)
|
||
|
|
tm.assert_index_equal(uniques, pd.PeriodIndex([v1, v2]))
|
||
|
|
|
||
|
|
labels, uniques = algos.factorize(x, sort=True)
|
||
|
|
exp = np.array([0, 0, 0, 1, 1, 0], dtype=np.intp)
|
||
|
|
tm.assert_numpy_array_equal(labels, exp)
|
||
|
|
tm.assert_index_equal(uniques, pd.PeriodIndex([v1, v2]))
|
||
|
|
|
||
|
|
# GH 5986
|
||
|
|
v1 = pd.to_timedelta('1 day 1 min')
|
||
|
|
v2 = pd.to_timedelta('1 day')
|
||
|
|
x = Series([v1, v2, v1, v1, v2, v2, v1])
|
||
|
|
labels, uniques = algos.factorize(x)
|
||
|
|
exp = np.array([0, 1, 0, 0, 1, 1, 0], dtype=np.intp)
|
||
|
|
tm.assert_numpy_array_equal(labels, exp)
|
||
|
|
tm.assert_index_equal(uniques, pd.to_timedelta([v1, v2]))
|
||
|
|
|
||
|
|
labels, uniques = algos.factorize(x, sort=True)
|
||
|
|
exp = np.array([1, 0, 1, 1, 0, 0, 1], dtype=np.intp)
|
||
|
|
tm.assert_numpy_array_equal(labels, exp)
|
||
|
|
tm.assert_index_equal(uniques, pd.to_timedelta([v2, v1]))
|
||
|
|
|
||
|
|
def test_factorize_nan(self):
|
||
|
|
# nan should map to na_sentinel, not reverse_indexer[na_sentinel]
|
||
|
|
# rizer.factorize should not raise an exception if na_sentinel indexes
|
||
|
|
# outside of reverse_indexer
|
||
|
|
key = np.array([1, 2, 1, np.nan], dtype='O')
|
||
|
|
rizer = ht.Factorizer(len(key))
|
||
|
|
for na_sentinel in (-1, 20):
|
||
|
|
ids = rizer.factorize(key, sort=True, na_sentinel=na_sentinel)
|
||
|
|
expected = np.array([0, 1, 0, na_sentinel], dtype='int32')
|
||
|
|
assert len(set(key)) == len(set(expected))
|
||
|
|
tm.assert_numpy_array_equal(pd.isna(key),
|
||
|
|
expected == na_sentinel)
|
||
|
|
|
||
|
|
# nan still maps to na_sentinel when sort=False
|
||
|
|
key = np.array([0, np.nan, 1], dtype='O')
|
||
|
|
na_sentinel = -1
|
||
|
|
|
||
|
|
# TODO(wesm): unused?
|
||
|
|
ids = rizer.factorize(key, sort=False, na_sentinel=na_sentinel) # noqa
|
||
|
|
|
||
|
|
expected = np.array([2, -1, 0], dtype='int32')
|
||
|
|
assert len(set(key)) == len(set(expected))
|
||
|
|
tm.assert_numpy_array_equal(pd.isna(key), expected == na_sentinel)
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("data,expected_label,expected_level", [
|
||
|
|
(
|
||
|
|
[(1, 1), (1, 2), (0, 0), (1, 2), 'nonsense'],
|
||
|
|
[0, 1, 2, 1, 3],
|
||
|
|
[(1, 1), (1, 2), (0, 0), 'nonsense']
|
||
|
|
),
|
||
|
|
(
|
||
|
|
[(1, 1), (1, 2), (0, 0), (1, 2), (1, 2, 3)],
|
||
|
|
[0, 1, 2, 1, 3],
|
||
|
|
[(1, 1), (1, 2), (0, 0), (1, 2, 3)]
|
||
|
|
),
|
||
|
|
(
|
||
|
|
[(1, 1), (1, 2), (0, 0), (1, 2)],
|
||
|
|
[0, 1, 2, 1],
|
||
|
|
[(1, 1), (1, 2), (0, 0)]
|
||
|
|
)
|
||
|
|
])
|
||
|
|
def test_factorize_tuple_list(self, data, expected_label, expected_level):
|
||
|
|
# GH9454
|
||
|
|
result = pd.factorize(data)
|
||
|
|
|
||
|
|
tm.assert_numpy_array_equal(result[0],
|
||
|
|
np.array(expected_label, dtype=np.intp))
|
||
|
|
|
||
|
|
expected_level_array = com._asarray_tuplesafe(expected_level,
|
||
|
|
dtype=object)
|
||
|
|
tm.assert_numpy_array_equal(result[1], expected_level_array)
|
||
|
|
|
||
|
|
def test_complex_sorting(self):
|
||
|
|
# gh 12666 - check no segfault
|
||
|
|
# Test not valid numpy versions older than 1.11
|
||
|
|
if pd._np_version_under1p11:
|
||
|
|
pytest.skip("Test valid only for numpy 1.11+")
|
||
|
|
|
||
|
|
x17 = np.array([complex(i) for i in range(17)], dtype=object)
|
||
|
|
|
||
|
|
pytest.raises(TypeError, algos.factorize, x17[::-1], sort=True)
|
||
|
|
|
||
|
|
def test_uint64_factorize(self):
|
||
|
|
data = np.array([2**63, 1, 2**63], dtype=np.uint64)
|
||
|
|
exp_labels = np.array([0, 1, 0], dtype=np.intp)
|
||
|
|
exp_uniques = np.array([2**63, 1], dtype=np.uint64)
|
||
|
|
|
||
|
|
labels, uniques = algos.factorize(data)
|
||
|
|
tm.assert_numpy_array_equal(labels, exp_labels)
|
||
|
|
tm.assert_numpy_array_equal(uniques, exp_uniques)
|
||
|
|
|
||
|
|
data = np.array([2**63, -1, 2**63], dtype=object)
|
||
|
|
exp_labels = np.array([0, 1, 0], dtype=np.intp)
|
||
|
|
exp_uniques = np.array([2**63, -1], dtype=object)
|
||
|
|
|
||
|
|
labels, uniques = algos.factorize(data)
|
||
|
|
tm.assert_numpy_array_equal(labels, exp_labels)
|
||
|
|
tm.assert_numpy_array_equal(uniques, exp_uniques)
|
||
|
|
|
||
|
|
def test_deprecate_order(self):
|
||
|
|
# gh 19727 - check warning is raised for deprecated keyword, order.
|
||
|
|
# Test not valid once order keyword is removed.
|
||
|
|
data = np.array([2**63, 1, 2**63], dtype=np.uint64)
|
||
|
|
with tm.assert_produces_warning(expected_warning=FutureWarning):
|
||
|
|
algos.factorize(data, order=True)
|
||
|
|
with tm.assert_produces_warning(False):
|
||
|
|
algos.factorize(data)
|
||
|
|
|
||
|
|
@pytest.mark.parametrize('data', [
|
||
|
|
np.array([0, 1, 0], dtype='u8'),
|
||
|
|
np.array([-2**63, 1, -2**63], dtype='i8'),
|
||
|
|
np.array(['__nan__', 'foo', '__nan__'], dtype='object'),
|
||
|
|
])
|
||
|
|
def test_parametrized_factorize_na_value_default(self, data):
|
||
|
|
# arrays that include the NA default for that type, but isn't used.
|
||
|
|
l, u = algos.factorize(data)
|
||
|
|
expected_uniques = data[[0, 1]]
|
||
|
|
expected_labels = np.array([0, 1, 0], dtype=np.intp)
|
||
|
|
tm.assert_numpy_array_equal(l, expected_labels)
|
||
|
|
tm.assert_numpy_array_equal(u, expected_uniques)
|
||
|
|
|
||
|
|
@pytest.mark.parametrize('data, na_value', [
|
||
|
|
(np.array([0, 1, 0, 2], dtype='u8'), 0),
|
||
|
|
(np.array([1, 0, 1, 2], dtype='u8'), 1),
|
||
|
|
(np.array([-2**63, 1, -2**63, 0], dtype='i8'), -2**63),
|
||
|
|
(np.array([1, -2**63, 1, 0], dtype='i8'), 1),
|
||
|
|
(np.array(['a', '', 'a', 'b'], dtype=object), 'a'),
|
||
|
|
(np.array([(), ('a', 1), (), ('a', 2)], dtype=object), ()),
|
||
|
|
(np.array([('a', 1), (), ('a', 1), ('a', 2)], dtype=object),
|
||
|
|
('a', 1)),
|
||
|
|
])
|
||
|
|
def test_parametrized_factorize_na_value(self, data, na_value):
|
||
|
|
l, u = algos._factorize_array(data, na_value=na_value)
|
||
|
|
expected_uniques = data[[1, 3]]
|
||
|
|
expected_labels = np.array([-1, 0, -1, 1], dtype=np.intp)
|
||
|
|
tm.assert_numpy_array_equal(l, expected_labels)
|
||
|
|
tm.assert_numpy_array_equal(u, expected_uniques)
|
||
|
|
|
||
|
|
|
||
|
|
class TestUnique(object):
|
||
|
|
|
||
|
|
def test_ints(self):
|
||
|
|
arr = np.random.randint(0, 100, size=50)
|
||
|
|
|
||
|
|
result = algos.unique(arr)
|
||
|
|
assert isinstance(result, np.ndarray)
|
||
|
|
|
||
|
|
def test_objects(self):
|
||
|
|
arr = np.random.randint(0, 100, size=50).astype('O')
|
||
|
|
|
||
|
|
result = algos.unique(arr)
|
||
|
|
assert isinstance(result, np.ndarray)
|
||
|
|
|
||
|
|
def test_object_refcount_bug(self):
|
||
|
|
lst = ['A', 'B', 'C', 'D', 'E']
|
||
|
|
for i in range(1000):
|
||
|
|
len(algos.unique(lst))
|
||
|
|
|
||
|
|
def test_on_index_object(self):
|
||
|
|
|
||
|
|
mindex = pd.MultiIndex.from_arrays([np.arange(5).repeat(5), np.tile(
|
||
|
|
np.arange(5), 5)])
|
||
|
|
expected = mindex.values
|
||
|
|
expected.sort()
|
||
|
|
|
||
|
|
mindex = mindex.repeat(2)
|
||
|
|
|
||
|
|
result = pd.unique(mindex)
|
||
|
|
result.sort()
|
||
|
|
|
||
|
|
tm.assert_almost_equal(result, expected)
|
||
|
|
|
||
|
|
def test_datetime64_dtype_array_returned(self):
|
||
|
|
# GH 9431
|
||
|
|
expected = np_array_datetime64_compat(
|
||
|
|
['2015-01-03T00:00:00.000000000+0000',
|
||
|
|
'2015-01-01T00:00:00.000000000+0000'],
|
||
|
|
dtype='M8[ns]')
|
||
|
|
|
||
|
|
dt_index = pd.to_datetime(['2015-01-03T00:00:00.000000000+0000',
|
||
|
|
'2015-01-01T00:00:00.000000000+0000',
|
||
|
|
'2015-01-01T00:00:00.000000000+0000'])
|
||
|
|
result = algos.unique(dt_index)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
assert result.dtype == expected.dtype
|
||
|
|
|
||
|
|
s = Series(dt_index)
|
||
|
|
result = algos.unique(s)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
assert result.dtype == expected.dtype
|
||
|
|
|
||
|
|
arr = s.values
|
||
|
|
result = algos.unique(arr)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
assert result.dtype == expected.dtype
|
||
|
|
|
||
|
|
def test_timedelta64_dtype_array_returned(self):
|
||
|
|
# GH 9431
|
||
|
|
expected = np.array([31200, 45678, 10000], dtype='m8[ns]')
|
||
|
|
|
||
|
|
td_index = pd.to_timedelta([31200, 45678, 31200, 10000, 45678])
|
||
|
|
result = algos.unique(td_index)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
assert result.dtype == expected.dtype
|
||
|
|
|
||
|
|
s = Series(td_index)
|
||
|
|
result = algos.unique(s)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
assert result.dtype == expected.dtype
|
||
|
|
|
||
|
|
arr = s.values
|
||
|
|
result = algos.unique(arr)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
assert result.dtype == expected.dtype
|
||
|
|
|
||
|
|
def test_uint64_overflow(self):
|
||
|
|
s = Series([1, 2, 2**63, 2**63], dtype=np.uint64)
|
||
|
|
exp = np.array([1, 2, 2**63], dtype=np.uint64)
|
||
|
|
tm.assert_numpy_array_equal(algos.unique(s), exp)
|
||
|
|
|
||
|
|
def test_nan_in_object_array(self):
|
||
|
|
l = ['a', np.nan, 'c', 'c']
|
||
|
|
result = pd.unique(l)
|
||
|
|
expected = np.array(['a', np.nan, 'c'], dtype=object)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
def test_categorical(self):
|
||
|
|
|
||
|
|
# we are expecting to return in the order
|
||
|
|
# of appearance
|
||
|
|
expected = Categorical(list('bac'), categories=list('bac'))
|
||
|
|
|
||
|
|
# we are expecting to return in the order
|
||
|
|
# of the categories
|
||
|
|
expected_o = Categorical(
|
||
|
|
list('bac'), categories=list('abc'), ordered=True)
|
||
|
|
|
||
|
|
# GH 15939
|
||
|
|
c = Categorical(list('baabc'))
|
||
|
|
result = c.unique()
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
result = algos.unique(c)
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
c = Categorical(list('baabc'), ordered=True)
|
||
|
|
result = c.unique()
|
||
|
|
tm.assert_categorical_equal(result, expected_o)
|
||
|
|
|
||
|
|
result = algos.unique(c)
|
||
|
|
tm.assert_categorical_equal(result, expected_o)
|
||
|
|
|
||
|
|
# Series of categorical dtype
|
||
|
|
s = Series(Categorical(list('baabc')), name='foo')
|
||
|
|
result = s.unique()
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
result = pd.unique(s)
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
# CI -> return CI
|
||
|
|
ci = CategoricalIndex(Categorical(list('baabc'),
|
||
|
|
categories=list('bac')))
|
||
|
|
expected = CategoricalIndex(expected)
|
||
|
|
result = ci.unique()
|
||
|
|
tm.assert_index_equal(result, expected)
|
||
|
|
|
||
|
|
result = pd.unique(ci)
|
||
|
|
tm.assert_index_equal(result, expected)
|
||
|
|
|
||
|
|
def test_datetime64tz_aware(self):
|
||
|
|
# GH 15939
|
||
|
|
|
||
|
|
result = Series(
|
||
|
|
Index([Timestamp('20160101', tz='US/Eastern'),
|
||
|
|
Timestamp('20160101', tz='US/Eastern')])).unique()
|
||
|
|
expected = np.array([Timestamp('2016-01-01 00:00:00-0500',
|
||
|
|
tz='US/Eastern')], dtype=object)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = Index([Timestamp('20160101', tz='US/Eastern'),
|
||
|
|
Timestamp('20160101', tz='US/Eastern')]).unique()
|
||
|
|
expected = DatetimeIndex(['2016-01-01 00:00:00'],
|
||
|
|
dtype='datetime64[ns, US/Eastern]', freq=None)
|
||
|
|
tm.assert_index_equal(result, expected)
|
||
|
|
|
||
|
|
result = pd.unique(
|
||
|
|
Series(Index([Timestamp('20160101', tz='US/Eastern'),
|
||
|
|
Timestamp('20160101', tz='US/Eastern')])))
|
||
|
|
expected = np.array([Timestamp('2016-01-01 00:00:00-0500',
|
||
|
|
tz='US/Eastern')], dtype=object)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = pd.unique(Index([Timestamp('20160101', tz='US/Eastern'),
|
||
|
|
Timestamp('20160101', tz='US/Eastern')]))
|
||
|
|
expected = DatetimeIndex(['2016-01-01 00:00:00'],
|
||
|
|
dtype='datetime64[ns, US/Eastern]', freq=None)
|
||
|
|
tm.assert_index_equal(result, expected)
|
||
|
|
|
||
|
|
def test_order_of_appearance(self):
|
||
|
|
# 9346
|
||
|
|
# light testing of guarantee of order of appearance
|
||
|
|
# these also are the doc-examples
|
||
|
|
result = pd.unique(Series([2, 1, 3, 3]))
|
||
|
|
tm.assert_numpy_array_equal(result,
|
||
|
|
np.array([2, 1, 3], dtype='int64'))
|
||
|
|
|
||
|
|
result = pd.unique(Series([2] + [1] * 5))
|
||
|
|
tm.assert_numpy_array_equal(result,
|
||
|
|
np.array([2, 1], dtype='int64'))
|
||
|
|
|
||
|
|
result = pd.unique(Series([Timestamp('20160101'),
|
||
|
|
Timestamp('20160101')]))
|
||
|
|
expected = np.array(['2016-01-01T00:00:00.000000000'],
|
||
|
|
dtype='datetime64[ns]')
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = pd.unique(Index(
|
||
|
|
[Timestamp('20160101', tz='US/Eastern'),
|
||
|
|
Timestamp('20160101', tz='US/Eastern')]))
|
||
|
|
expected = DatetimeIndex(['2016-01-01 00:00:00'],
|
||
|
|
dtype='datetime64[ns, US/Eastern]',
|
||
|
|
freq=None)
|
||
|
|
tm.assert_index_equal(result, expected)
|
||
|
|
|
||
|
|
result = pd.unique(list('aabc'))
|
||
|
|
expected = np.array(['a', 'b', 'c'], dtype=object)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = pd.unique(Series(Categorical(list('aabc'))))
|
||
|
|
expected = Categorical(list('abc'))
|
||
|
|
tm.assert_categorical_equal(result, expected)
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("arg ,expected", [
|
||
|
|
(('1', '1', '2'), np.array(['1', '2'], dtype=object)),
|
||
|
|
(('foo',), np.array(['foo'], dtype=object))
|
||
|
|
])
|
||
|
|
def test_tuple_with_strings(self, arg, expected):
|
||
|
|
# see GH 17108
|
||
|
|
result = pd.unique(arg)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
def test_obj_none_preservation(self):
|
||
|
|
# GH 20866
|
||
|
|
arr = np.array(['foo', None], dtype=object)
|
||
|
|
result = pd.unique(arr)
|
||
|
|
expected = np.array(['foo', None], dtype=object)
|
||
|
|
|
||
|
|
tm.assert_numpy_array_equal(result, expected, strict_nan=True)
|
||
|
|
|
||
|
|
|
||
|
|
class TestIsin(object):
|
||
|
|
|
||
|
|
def test_invalid(self):
|
||
|
|
|
||
|
|
pytest.raises(TypeError, lambda: algos.isin(1, 1))
|
||
|
|
pytest.raises(TypeError, lambda: algos.isin(1, [1]))
|
||
|
|
pytest.raises(TypeError, lambda: algos.isin([1], 1))
|
||
|
|
|
||
|
|
def test_basic(self):
|
||
|
|
|
||
|
|
result = algos.isin([1, 2], [1])
|
||
|
|
expected = np.array([True, False])
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = algos.isin(np.array([1, 2]), [1])
|
||
|
|
expected = np.array([True, False])
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = algos.isin(Series([1, 2]), [1])
|
||
|
|
expected = np.array([True, False])
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = algos.isin(Series([1, 2]), Series([1]))
|
||
|
|
expected = np.array([True, False])
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = algos.isin(Series([1, 2]), set([1]))
|
||
|
|
expected = np.array([True, False])
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = algos.isin(['a', 'b'], ['a'])
|
||
|
|
expected = np.array([True, False])
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = algos.isin(Series(['a', 'b']), Series(['a']))
|
||
|
|
expected = np.array([True, False])
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = algos.isin(Series(['a', 'b']), set(['a']))
|
||
|
|
expected = np.array([True, False])
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = algos.isin(['a', 'b'], [1])
|
||
|
|
expected = np.array([False, False])
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
def test_i8(self):
|
||
|
|
|
||
|
|
arr = pd.date_range('20130101', periods=3).values
|
||
|
|
result = algos.isin(arr, [arr[0]])
|
||
|
|
expected = np.array([True, False, False])
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = algos.isin(arr, arr[0:2])
|
||
|
|
expected = np.array([True, True, False])
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = algos.isin(arr, set(arr[0:2]))
|
||
|
|
expected = np.array([True, True, False])
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
arr = pd.timedelta_range('1 day', periods=3).values
|
||
|
|
result = algos.isin(arr, [arr[0]])
|
||
|
|
expected = np.array([True, False, False])
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = algos.isin(arr, arr[0:2])
|
||
|
|
expected = np.array([True, True, False])
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = algos.isin(arr, set(arr[0:2]))
|
||
|
|
expected = np.array([True, True, False])
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
def test_large(self):
|
||
|
|
|
||
|
|
s = pd.date_range('20000101', periods=2000000, freq='s').values
|
||
|
|
result = algos.isin(s, s[0:2])
|
||
|
|
expected = np.zeros(len(s), dtype=bool)
|
||
|
|
expected[0] = True
|
||
|
|
expected[1] = True
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
def test_categorical_from_codes(self):
|
||
|
|
# GH 16639
|
||
|
|
vals = np.array([0, 1, 2, 0])
|
||
|
|
cats = ['a', 'b', 'c']
|
||
|
|
Sd = Series(Categorical(1).from_codes(vals, cats))
|
||
|
|
St = Series(Categorical(1).from_codes(np.array([0, 1]), cats))
|
||
|
|
expected = np.array([True, True, False, True])
|
||
|
|
result = algos.isin(Sd, St)
|
||
|
|
tm.assert_numpy_array_equal(expected, result)
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("empty", [[], Series(), np.array([])])
|
||
|
|
def test_empty(self, empty):
|
||
|
|
# see gh-16991
|
||
|
|
vals = Index(["a", "b"])
|
||
|
|
expected = np.array([False, False])
|
||
|
|
|
||
|
|
result = algos.isin(vals, empty)
|
||
|
|
tm.assert_numpy_array_equal(expected, result)
|
||
|
|
|
||
|
|
|
||
|
|
class TestValueCounts(object):
|
||
|
|
|
||
|
|
def test_value_counts(self):
|
||
|
|
np.random.seed(1234)
|
||
|
|
from pandas.core.reshape.tile import cut
|
||
|
|
|
||
|
|
arr = np.random.randn(4)
|
||
|
|
factor = cut(arr, 4)
|
||
|
|
|
||
|
|
# assert isinstance(factor, n)
|
||
|
|
result = algos.value_counts(factor)
|
||
|
|
breaks = [-1.194, -0.535, 0.121, 0.777, 1.433]
|
||
|
|
index = IntervalIndex.from_breaks(breaks).astype(CDT(ordered=True))
|
||
|
|
expected = Series([1, 1, 1, 1], index=index)
|
||
|
|
tm.assert_series_equal(result.sort_index(), expected.sort_index())
|
||
|
|
|
||
|
|
def test_value_counts_bins(self):
|
||
|
|
s = [1, 2, 3, 4]
|
||
|
|
result = algos.value_counts(s, bins=1)
|
||
|
|
expected = Series([4],
|
||
|
|
index=IntervalIndex.from_tuples([(0.996, 4.0)]))
|
||
|
|
tm.assert_series_equal(result, expected)
|
||
|
|
|
||
|
|
result = algos.value_counts(s, bins=2, sort=False)
|
||
|
|
expected = Series([2, 2],
|
||
|
|
index=IntervalIndex.from_tuples([(0.996, 2.5),
|
||
|
|
(2.5, 4.0)]))
|
||
|
|
tm.assert_series_equal(result, expected)
|
||
|
|
|
||
|
|
def test_value_counts_dtypes(self):
|
||
|
|
result = algos.value_counts([1, 1.])
|
||
|
|
assert len(result) == 1
|
||
|
|
|
||
|
|
result = algos.value_counts([1, 1.], bins=1)
|
||
|
|
assert len(result) == 1
|
||
|
|
|
||
|
|
result = algos.value_counts(Series([1, 1., '1'])) # object
|
||
|
|
assert len(result) == 2
|
||
|
|
|
||
|
|
pytest.raises(TypeError, lambda s: algos.value_counts(s, bins=1),
|
||
|
|
['1', 1])
|
||
|
|
|
||
|
|
def test_value_counts_nat(self):
|
||
|
|
td = Series([np.timedelta64(10000), pd.NaT], dtype='timedelta64[ns]')
|
||
|
|
dt = pd.to_datetime(['NaT', '2014-01-01'])
|
||
|
|
|
||
|
|
for s in [td, dt]:
|
||
|
|
vc = algos.value_counts(s)
|
||
|
|
vc_with_na = algos.value_counts(s, dropna=False)
|
||
|
|
assert len(vc) == 1
|
||
|
|
assert len(vc_with_na) == 2
|
||
|
|
|
||
|
|
exp_dt = Series({Timestamp('2014-01-01 00:00:00'): 1})
|
||
|
|
tm.assert_series_equal(algos.value_counts(dt), exp_dt)
|
||
|
|
# TODO same for (timedelta)
|
||
|
|
|
||
|
|
def test_value_counts_datetime_outofbounds(self):
|
||
|
|
# GH 13663
|
||
|
|
s = Series([datetime(3000, 1, 1), datetime(5000, 1, 1),
|
||
|
|
datetime(5000, 1, 1), datetime(6000, 1, 1),
|
||
|
|
datetime(3000, 1, 1), datetime(3000, 1, 1)])
|
||
|
|
res = s.value_counts()
|
||
|
|
|
||
|
|
exp_index = Index([datetime(3000, 1, 1), datetime(5000, 1, 1),
|
||
|
|
datetime(6000, 1, 1)], dtype=object)
|
||
|
|
exp = Series([3, 2, 1], index=exp_index)
|
||
|
|
tm.assert_series_equal(res, exp)
|
||
|
|
|
||
|
|
# GH 12424
|
||
|
|
res = pd.to_datetime(Series(['2362-01-01', np.nan]),
|
||
|
|
errors='ignore')
|
||
|
|
exp = Series(['2362-01-01', np.nan], dtype=object)
|
||
|
|
tm.assert_series_equal(res, exp)
|
||
|
|
|
||
|
|
def test_categorical(self):
|
||
|
|
s = Series(Categorical(list('aaabbc')))
|
||
|
|
result = s.value_counts()
|
||
|
|
expected = Series([3, 2, 1], index=CategoricalIndex(['a', 'b', 'c']))
|
||
|
|
|
||
|
|
tm.assert_series_equal(result, expected, check_index_type=True)
|
||
|
|
|
||
|
|
# preserve order?
|
||
|
|
s = s.cat.as_ordered()
|
||
|
|
result = s.value_counts()
|
||
|
|
expected.index = expected.index.as_ordered()
|
||
|
|
tm.assert_series_equal(result, expected, check_index_type=True)
|
||
|
|
|
||
|
|
def test_categorical_nans(self):
|
||
|
|
s = Series(Categorical(list('aaaaabbbcc'))) # 4,3,2,1 (nan)
|
||
|
|
s.iloc[1] = np.nan
|
||
|
|
result = s.value_counts()
|
||
|
|
expected = Series([4, 3, 2], index=CategoricalIndex(
|
||
|
|
['a', 'b', 'c'], categories=['a', 'b', 'c']))
|
||
|
|
tm.assert_series_equal(result, expected, check_index_type=True)
|
||
|
|
result = s.value_counts(dropna=False)
|
||
|
|
expected = Series([
|
||
|
|
4, 3, 2, 1
|
||
|
|
], index=CategoricalIndex(['a', 'b', 'c', np.nan]))
|
||
|
|
tm.assert_series_equal(result, expected, check_index_type=True)
|
||
|
|
|
||
|
|
# out of order
|
||
|
|
s = Series(Categorical(
|
||
|
|
list('aaaaabbbcc'), ordered=True, categories=['b', 'a', 'c']))
|
||
|
|
s.iloc[1] = np.nan
|
||
|
|
result = s.value_counts()
|
||
|
|
expected = Series([4, 3, 2], index=CategoricalIndex(
|
||
|
|
['a', 'b', 'c'], categories=['b', 'a', 'c'], ordered=True))
|
||
|
|
tm.assert_series_equal(result, expected, check_index_type=True)
|
||
|
|
|
||
|
|
result = s.value_counts(dropna=False)
|
||
|
|
expected = Series([4, 3, 2, 1], index=CategoricalIndex(
|
||
|
|
['a', 'b', 'c', np.nan], categories=['b', 'a', 'c'], ordered=True))
|
||
|
|
tm.assert_series_equal(result, expected, check_index_type=True)
|
||
|
|
|
||
|
|
def test_categorical_zeroes(self):
|
||
|
|
# keep the `d` category with 0
|
||
|
|
s = Series(Categorical(
|
||
|
|
list('bbbaac'), categories=list('abcd'), ordered=True))
|
||
|
|
result = s.value_counts()
|
||
|
|
expected = Series([3, 2, 1, 0], index=Categorical(
|
||
|
|
['b', 'a', 'c', 'd'], categories=list('abcd'), ordered=True))
|
||
|
|
tm.assert_series_equal(result, expected, check_index_type=True)
|
||
|
|
|
||
|
|
def test_dropna(self):
|
||
|
|
# https://github.com/pandas-dev/pandas/issues/9443#issuecomment-73719328
|
||
|
|
|
||
|
|
tm.assert_series_equal(
|
||
|
|
Series([True, True, False]).value_counts(dropna=True),
|
||
|
|
Series([2, 1], index=[True, False]))
|
||
|
|
tm.assert_series_equal(
|
||
|
|
Series([True, True, False]).value_counts(dropna=False),
|
||
|
|
Series([2, 1], index=[True, False]))
|
||
|
|
|
||
|
|
tm.assert_series_equal(
|
||
|
|
Series([True, True, False, None]).value_counts(dropna=True),
|
||
|
|
Series([2, 1], index=[True, False]))
|
||
|
|
tm.assert_series_equal(
|
||
|
|
Series([True, True, False, None]).value_counts(dropna=False),
|
||
|
|
Series([2, 1, 1], index=[True, False, np.nan]))
|
||
|
|
tm.assert_series_equal(
|
||
|
|
Series([10.3, 5., 5.]).value_counts(dropna=True),
|
||
|
|
Series([2, 1], index=[5., 10.3]))
|
||
|
|
tm.assert_series_equal(
|
||
|
|
Series([10.3, 5., 5.]).value_counts(dropna=False),
|
||
|
|
Series([2, 1], index=[5., 10.3]))
|
||
|
|
|
||
|
|
tm.assert_series_equal(
|
||
|
|
Series([10.3, 5., 5., None]).value_counts(dropna=True),
|
||
|
|
Series([2, 1], index=[5., 10.3]))
|
||
|
|
|
||
|
|
# 32-bit linux has a different ordering
|
||
|
|
if not compat.is_platform_32bit():
|
||
|
|
result = Series([10.3, 5., 5., None]).value_counts(dropna=False)
|
||
|
|
expected = Series([2, 1, 1], index=[5., 10.3, np.nan])
|
||
|
|
tm.assert_series_equal(result, expected)
|
||
|
|
|
||
|
|
def test_value_counts_normalized(self):
|
||
|
|
# GH12558
|
||
|
|
s = Series([1, 2, np.nan, np.nan, np.nan])
|
||
|
|
dtypes = (np.float64, np.object, 'M8[ns]')
|
||
|
|
for t in dtypes:
|
||
|
|
s_typed = s.astype(t)
|
||
|
|
result = s_typed.value_counts(normalize=True, dropna=False)
|
||
|
|
expected = Series([0.6, 0.2, 0.2],
|
||
|
|
index=Series([np.nan, 2.0, 1.0], dtype=t))
|
||
|
|
tm.assert_series_equal(result, expected)
|
||
|
|
|
||
|
|
result = s_typed.value_counts(normalize=True, dropna=True)
|
||
|
|
expected = Series([0.5, 0.5],
|
||
|
|
index=Series([2.0, 1.0], dtype=t))
|
||
|
|
tm.assert_series_equal(result, expected)
|
||
|
|
|
||
|
|
def test_value_counts_uint64(self):
|
||
|
|
arr = np.array([2**63], dtype=np.uint64)
|
||
|
|
expected = Series([1], index=[2**63])
|
||
|
|
result = algos.value_counts(arr)
|
||
|
|
|
||
|
|
tm.assert_series_equal(result, expected)
|
||
|
|
|
||
|
|
arr = np.array([-1, 2**63], dtype=object)
|
||
|
|
expected = Series([1, 1], index=[-1, 2**63])
|
||
|
|
result = algos.value_counts(arr)
|
||
|
|
|
||
|
|
# 32-bit linux has a different ordering
|
||
|
|
if not compat.is_platform_32bit():
|
||
|
|
tm.assert_series_equal(result, expected)
|
||
|
|
|
||
|
|
|
||
|
|
class TestDuplicated(object):
|
||
|
|
|
||
|
|
def test_duplicated_with_nas(self):
|
||
|
|
keys = np.array([0, 1, np.nan, 0, 2, np.nan], dtype=object)
|
||
|
|
|
||
|
|
result = algos.duplicated(keys)
|
||
|
|
expected = np.array([False, False, False, True, False, True])
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = algos.duplicated(keys, keep='first')
|
||
|
|
expected = np.array([False, False, False, True, False, True])
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = algos.duplicated(keys, keep='last')
|
||
|
|
expected = np.array([True, False, True, False, False, False])
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = algos.duplicated(keys, keep=False)
|
||
|
|
expected = np.array([True, False, True, True, False, True])
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
keys = np.empty(8, dtype=object)
|
||
|
|
for i, t in enumerate(zip([0, 0, np.nan, np.nan] * 2,
|
||
|
|
[0, np.nan, 0, np.nan] * 2)):
|
||
|
|
keys[i] = t
|
||
|
|
|
||
|
|
result = algos.duplicated(keys)
|
||
|
|
falses = [False] * 4
|
||
|
|
trues = [True] * 4
|
||
|
|
expected = np.array(falses + trues)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = algos.duplicated(keys, keep='last')
|
||
|
|
expected = np.array(trues + falses)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = algos.duplicated(keys, keep=False)
|
||
|
|
expected = np.array(trues + trues)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
@pytest.mark.parametrize('case', [
|
||
|
|
np.array([1, 2, 1, 5, 3,
|
||
|
|
2, 4, 1, 5, 6]),
|
||
|
|
np.array([1.1, 2.2, 1.1, np.nan, 3.3,
|
||
|
|
2.2, 4.4, 1.1, np.nan, 6.6]),
|
||
|
|
pytest.param(np.array([1 + 1j, 2 + 2j, 1 + 1j, 5 + 5j, 3 + 3j,
|
||
|
|
2 + 2j, 4 + 4j, 1 + 1j, 5 + 5j, 6 + 6j]),
|
||
|
|
marks=pytest.mark.xfail(reason="Complex bug. GH 16399")
|
||
|
|
),
|
||
|
|
np.array(['a', 'b', 'a', 'e', 'c',
|
||
|
|
'b', 'd', 'a', 'e', 'f'], dtype=object),
|
||
|
|
np.array([1, 2**63, 1, 3**5, 10, 2**63, 39, 1, 3**5, 7],
|
||
|
|
dtype=np.uint64),
|
||
|
|
])
|
||
|
|
def test_numeric_object_likes(self, case):
|
||
|
|
exp_first = np.array([False, False, True, False, False,
|
||
|
|
True, False, True, True, False])
|
||
|
|
exp_last = np.array([True, True, True, True, False,
|
||
|
|
False, False, False, False, False])
|
||
|
|
exp_false = exp_first | exp_last
|
||
|
|
|
||
|
|
res_first = algos.duplicated(case, keep='first')
|
||
|
|
tm.assert_numpy_array_equal(res_first, exp_first)
|
||
|
|
|
||
|
|
res_last = algos.duplicated(case, keep='last')
|
||
|
|
tm.assert_numpy_array_equal(res_last, exp_last)
|
||
|
|
|
||
|
|
res_false = algos.duplicated(case, keep=False)
|
||
|
|
tm.assert_numpy_array_equal(res_false, exp_false)
|
||
|
|
|
||
|
|
# index
|
||
|
|
for idx in [Index(case), Index(case, dtype='category')]:
|
||
|
|
res_first = idx.duplicated(keep='first')
|
||
|
|
tm.assert_numpy_array_equal(res_first, exp_first)
|
||
|
|
|
||
|
|
res_last = idx.duplicated(keep='last')
|
||
|
|
tm.assert_numpy_array_equal(res_last, exp_last)
|
||
|
|
|
||
|
|
res_false = idx.duplicated(keep=False)
|
||
|
|
tm.assert_numpy_array_equal(res_false, exp_false)
|
||
|
|
|
||
|
|
# series
|
||
|
|
for s in [Series(case), Series(case, dtype='category')]:
|
||
|
|
res_first = s.duplicated(keep='first')
|
||
|
|
tm.assert_series_equal(res_first, Series(exp_first))
|
||
|
|
|
||
|
|
res_last = s.duplicated(keep='last')
|
||
|
|
tm.assert_series_equal(res_last, Series(exp_last))
|
||
|
|
|
||
|
|
res_false = s.duplicated(keep=False)
|
||
|
|
tm.assert_series_equal(res_false, Series(exp_false))
|
||
|
|
|
||
|
|
def test_datetime_likes(self):
|
||
|
|
|
||
|
|
dt = ['2011-01-01', '2011-01-02', '2011-01-01', 'NaT', '2011-01-03',
|
||
|
|
'2011-01-02', '2011-01-04', '2011-01-01', 'NaT', '2011-01-06']
|
||
|
|
td = ['1 days', '2 days', '1 days', 'NaT', '3 days',
|
||
|
|
'2 days', '4 days', '1 days', 'NaT', '6 days']
|
||
|
|
|
||
|
|
cases = [np.array([Timestamp(d) for d in dt]),
|
||
|
|
np.array([Timestamp(d, tz='US/Eastern') for d in dt]),
|
||
|
|
np.array([pd.Period(d, freq='D') for d in dt]),
|
||
|
|
np.array([np.datetime64(d) for d in dt]),
|
||
|
|
np.array([pd.Timedelta(d) for d in td])]
|
||
|
|
|
||
|
|
exp_first = np.array([False, False, True, False, False,
|
||
|
|
True, False, True, True, False])
|
||
|
|
exp_last = np.array([True, True, True, True, False,
|
||
|
|
False, False, False, False, False])
|
||
|
|
exp_false = exp_first | exp_last
|
||
|
|
|
||
|
|
for case in cases:
|
||
|
|
res_first = algos.duplicated(case, keep='first')
|
||
|
|
tm.assert_numpy_array_equal(res_first, exp_first)
|
||
|
|
|
||
|
|
res_last = algos.duplicated(case, keep='last')
|
||
|
|
tm.assert_numpy_array_equal(res_last, exp_last)
|
||
|
|
|
||
|
|
res_false = algos.duplicated(case, keep=False)
|
||
|
|
tm.assert_numpy_array_equal(res_false, exp_false)
|
||
|
|
|
||
|
|
# index
|
||
|
|
for idx in [Index(case), Index(case, dtype='category'),
|
||
|
|
Index(case, dtype=object)]:
|
||
|
|
res_first = idx.duplicated(keep='first')
|
||
|
|
tm.assert_numpy_array_equal(res_first, exp_first)
|
||
|
|
|
||
|
|
res_last = idx.duplicated(keep='last')
|
||
|
|
tm.assert_numpy_array_equal(res_last, exp_last)
|
||
|
|
|
||
|
|
res_false = idx.duplicated(keep=False)
|
||
|
|
tm.assert_numpy_array_equal(res_false, exp_false)
|
||
|
|
|
||
|
|
# series
|
||
|
|
for s in [Series(case), Series(case, dtype='category'),
|
||
|
|
Series(case, dtype=object)]:
|
||
|
|
res_first = s.duplicated(keep='first')
|
||
|
|
tm.assert_series_equal(res_first, Series(exp_first))
|
||
|
|
|
||
|
|
res_last = s.duplicated(keep='last')
|
||
|
|
tm.assert_series_equal(res_last, Series(exp_last))
|
||
|
|
|
||
|
|
res_false = s.duplicated(keep=False)
|
||
|
|
tm.assert_series_equal(res_false, Series(exp_false))
|
||
|
|
|
||
|
|
def test_unique_index(self):
|
||
|
|
cases = [Index([1, 2, 3]), pd.RangeIndex(0, 3)]
|
||
|
|
for case in cases:
|
||
|
|
assert case.is_unique
|
||
|
|
tm.assert_numpy_array_equal(case.duplicated(),
|
||
|
|
np.array([False, False, False]))
|
||
|
|
|
||
|
|
@pytest.mark.parametrize('arr, unique', [
|
||
|
|
([(0, 0), (0, 1), (1, 0), (1, 1), (0, 0), (0, 1), (1, 0), (1, 1)],
|
||
|
|
[(0, 0), (0, 1), (1, 0), (1, 1)]),
|
||
|
|
([('b', 'c'), ('a', 'b'), ('a', 'b'), ('b', 'c')],
|
||
|
|
[('b', 'c'), ('a', 'b')]),
|
||
|
|
([('a', 1), ('b', 2), ('a', 3), ('a', 1)],
|
||
|
|
[('a', 1), ('b', 2), ('a', 3)]),
|
||
|
|
])
|
||
|
|
def test_unique_tuples(self, arr, unique):
|
||
|
|
# https://github.com/pandas-dev/pandas/issues/16519
|
||
|
|
expected = np.empty(len(unique), dtype=object)
|
||
|
|
expected[:] = unique
|
||
|
|
|
||
|
|
result = pd.unique(arr)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
|
||
|
|
class GroupVarTestMixin(object):
|
||
|
|
|
||
|
|
def test_group_var_generic_1d(self):
|
||
|
|
prng = RandomState(1234)
|
||
|
|
|
||
|
|
out = (np.nan * np.ones((5, 1))).astype(self.dtype)
|
||
|
|
counts = np.zeros(5, dtype='int64')
|
||
|
|
values = 10 * prng.rand(15, 1).astype(self.dtype)
|
||
|
|
labels = np.tile(np.arange(5), (3, )).astype('int64')
|
||
|
|
|
||
|
|
expected_out = (np.squeeze(values)
|
||
|
|
.reshape((5, 3), order='F')
|
||
|
|
.std(axis=1, ddof=1) ** 2)[:, np.newaxis]
|
||
|
|
expected_counts = counts + 3
|
||
|
|
|
||
|
|
self.algo(out, counts, values, labels)
|
||
|
|
assert np.allclose(out, expected_out, self.rtol)
|
||
|
|
tm.assert_numpy_array_equal(counts, expected_counts)
|
||
|
|
|
||
|
|
def test_group_var_generic_1d_flat_labels(self):
|
||
|
|
prng = RandomState(1234)
|
||
|
|
|
||
|
|
out = (np.nan * np.ones((1, 1))).astype(self.dtype)
|
||
|
|
counts = np.zeros(1, dtype='int64')
|
||
|
|
values = 10 * prng.rand(5, 1).astype(self.dtype)
|
||
|
|
labels = np.zeros(5, dtype='int64')
|
||
|
|
|
||
|
|
expected_out = np.array([[values.std(ddof=1) ** 2]])
|
||
|
|
expected_counts = counts + 5
|
||
|
|
|
||
|
|
self.algo(out, counts, values, labels)
|
||
|
|
|
||
|
|
assert np.allclose(out, expected_out, self.rtol)
|
||
|
|
tm.assert_numpy_array_equal(counts, expected_counts)
|
||
|
|
|
||
|
|
def test_group_var_generic_2d_all_finite(self):
|
||
|
|
prng = RandomState(1234)
|
||
|
|
|
||
|
|
out = (np.nan * np.ones((5, 2))).astype(self.dtype)
|
||
|
|
counts = np.zeros(5, dtype='int64')
|
||
|
|
values = 10 * prng.rand(10, 2).astype(self.dtype)
|
||
|
|
labels = np.tile(np.arange(5), (2, )).astype('int64')
|
||
|
|
|
||
|
|
expected_out = np.std(values.reshape(2, 5, 2), ddof=1, axis=0) ** 2
|
||
|
|
expected_counts = counts + 2
|
||
|
|
|
||
|
|
self.algo(out, counts, values, labels)
|
||
|
|
assert np.allclose(out, expected_out, self.rtol)
|
||
|
|
tm.assert_numpy_array_equal(counts, expected_counts)
|
||
|
|
|
||
|
|
def test_group_var_generic_2d_some_nan(self):
|
||
|
|
prng = RandomState(1234)
|
||
|
|
|
||
|
|
out = (np.nan * np.ones((5, 2))).astype(self.dtype)
|
||
|
|
counts = np.zeros(5, dtype='int64')
|
||
|
|
values = 10 * prng.rand(10, 2).astype(self.dtype)
|
||
|
|
values[:, 1] = np.nan
|
||
|
|
labels = np.tile(np.arange(5), (2, )).astype('int64')
|
||
|
|
|
||
|
|
expected_out = np.vstack([values[:, 0]
|
||
|
|
.reshape(5, 2, order='F')
|
||
|
|
.std(ddof=1, axis=1) ** 2,
|
||
|
|
np.nan * np.ones(5)]).T.astype(self.dtype)
|
||
|
|
expected_counts = counts + 2
|
||
|
|
|
||
|
|
self.algo(out, counts, values, labels)
|
||
|
|
tm.assert_almost_equal(out, expected_out, check_less_precise=6)
|
||
|
|
tm.assert_numpy_array_equal(counts, expected_counts)
|
||
|
|
|
||
|
|
def test_group_var_constant(self):
|
||
|
|
# Regression test from GH 10448.
|
||
|
|
|
||
|
|
out = np.array([[np.nan]], dtype=self.dtype)
|
||
|
|
counts = np.array([0], dtype='int64')
|
||
|
|
values = 0.832845131556193 * np.ones((3, 1), dtype=self.dtype)
|
||
|
|
labels = np.zeros(3, dtype='int64')
|
||
|
|
|
||
|
|
self.algo(out, counts, values, labels)
|
||
|
|
|
||
|
|
assert counts[0] == 3
|
||
|
|
assert out[0, 0] >= 0
|
||
|
|
tm.assert_almost_equal(out[0, 0], 0.0)
|
||
|
|
|
||
|
|
|
||
|
|
class TestGroupVarFloat64(GroupVarTestMixin):
|
||
|
|
__test__ = True
|
||
|
|
|
||
|
|
algo = libgroupby.group_var_float64
|
||
|
|
dtype = np.float64
|
||
|
|
rtol = 1e-5
|
||
|
|
|
||
|
|
def test_group_var_large_inputs(self):
|
||
|
|
|
||
|
|
prng = RandomState(1234)
|
||
|
|
|
||
|
|
out = np.array([[np.nan]], dtype=self.dtype)
|
||
|
|
counts = np.array([0], dtype='int64')
|
||
|
|
values = (prng.rand(10 ** 6) + 10 ** 12).astype(self.dtype)
|
||
|
|
values.shape = (10 ** 6, 1)
|
||
|
|
labels = np.zeros(10 ** 6, dtype='int64')
|
||
|
|
|
||
|
|
self.algo(out, counts, values, labels)
|
||
|
|
|
||
|
|
assert counts[0] == 10 ** 6
|
||
|
|
tm.assert_almost_equal(out[0, 0], 1.0 / 12, check_less_precise=True)
|
||
|
|
|
||
|
|
|
||
|
|
class TestGroupVarFloat32(GroupVarTestMixin):
|
||
|
|
__test__ = True
|
||
|
|
|
||
|
|
algo = libgroupby.group_var_float32
|
||
|
|
dtype = np.float32
|
||
|
|
rtol = 1e-2
|
||
|
|
|
||
|
|
|
||
|
|
class TestHashTable(object):
|
||
|
|
|
||
|
|
def test_lookup_nan(self):
|
||
|
|
xs = np.array([2.718, 3.14, np.nan, -7, 5, 2, 3])
|
||
|
|
m = ht.Float64HashTable()
|
||
|
|
m.map_locations(xs)
|
||
|
|
tm.assert_numpy_array_equal(m.lookup(xs), np.arange(len(xs),
|
||
|
|
dtype=np.int64))
|
||
|
|
|
||
|
|
def test_lookup_overflow(self):
|
||
|
|
xs = np.array([1, 2, 2**63], dtype=np.uint64)
|
||
|
|
m = ht.UInt64HashTable()
|
||
|
|
m.map_locations(xs)
|
||
|
|
tm.assert_numpy_array_equal(m.lookup(xs), np.arange(len(xs),
|
||
|
|
dtype=np.int64))
|
||
|
|
|
||
|
|
def test_get_unique(self):
|
||
|
|
s = Series([1, 2, 2**63, 2**63], dtype=np.uint64)
|
||
|
|
exp = np.array([1, 2, 2**63], dtype=np.uint64)
|
||
|
|
tm.assert_numpy_array_equal(s.unique(), exp)
|
||
|
|
|
||
|
|
def test_vector_resize(self):
|
||
|
|
# Test for memory errors after internal vector
|
||
|
|
# reallocations (pull request #7157)
|
||
|
|
|
||
|
|
def _test_vector_resize(htable, uniques, dtype, nvals, safely_resizes):
|
||
|
|
vals = np.array(np.random.randn(1000), dtype=dtype)
|
||
|
|
# get_labels may append to uniques
|
||
|
|
htable.get_labels(vals[:nvals], uniques, 0, -1)
|
||
|
|
# to_array() set an external_view_exists flag on uniques.
|
||
|
|
tmp = uniques.to_array()
|
||
|
|
oldshape = tmp.shape
|
||
|
|
# subsequent get_labels() calls can no longer append to it
|
||
|
|
# (for all but StringHashTables + ObjectVector)
|
||
|
|
if safely_resizes:
|
||
|
|
htable.get_labels(vals, uniques, 0, -1)
|
||
|
|
else:
|
||
|
|
with pytest.raises(ValueError) as excinfo:
|
||
|
|
htable.get_labels(vals, uniques, 0, -1)
|
||
|
|
assert str(excinfo.value).startswith('external reference')
|
||
|
|
uniques.to_array() # should not raise here
|
||
|
|
assert tmp.shape == oldshape
|
||
|
|
|
||
|
|
test_cases = [
|
||
|
|
(ht.PyObjectHashTable, ht.ObjectVector, 'object', False),
|
||
|
|
(ht.StringHashTable, ht.ObjectVector, 'object', True),
|
||
|
|
(ht.Float64HashTable, ht.Float64Vector, 'float64', False),
|
||
|
|
(ht.Int64HashTable, ht.Int64Vector, 'int64', False),
|
||
|
|
(ht.UInt64HashTable, ht.UInt64Vector, 'uint64', False)]
|
||
|
|
|
||
|
|
for (tbl, vect, dtype, safely_resizes) in test_cases:
|
||
|
|
# resizing to empty is a special case
|
||
|
|
_test_vector_resize(tbl(), vect(), dtype, 0, safely_resizes)
|
||
|
|
_test_vector_resize(tbl(), vect(), dtype, 10, safely_resizes)
|
||
|
|
|
||
|
|
|
||
|
|
def test_quantile():
|
||
|
|
s = Series(np.random.randn(100))
|
||
|
|
|
||
|
|
result = algos.quantile(s, [0, .25, .5, .75, 1.])
|
||
|
|
expected = algos.quantile(s.values, [0, .25, .5, .75, 1.])
|
||
|
|
tm.assert_almost_equal(result, expected)
|
||
|
|
|
||
|
|
|
||
|
|
def test_unique_label_indices():
|
||
|
|
|
||
|
|
a = np.random.randint(1, 1 << 10, 1 << 15).astype('i8')
|
||
|
|
|
||
|
|
left = unique_label_indices(a)
|
||
|
|
right = np.unique(a, return_index=True)[1]
|
||
|
|
|
||
|
|
tm.assert_numpy_array_equal(left, right,
|
||
|
|
check_dtype=False)
|
||
|
|
|
||
|
|
a[np.random.choice(len(a), 10)] = -1
|
||
|
|
left = unique_label_indices(a)
|
||
|
|
right = np.unique(a, return_index=True)[1][1:]
|
||
|
|
tm.assert_numpy_array_equal(left, right,
|
||
|
|
check_dtype=False)
|
||
|
|
|
||
|
|
|
||
|
|
class TestRank(object):
|
||
|
|
|
||
|
|
@td.skip_if_no_scipy
|
||
|
|
def test_scipy_compat(self):
|
||
|
|
from scipy.stats import rankdata
|
||
|
|
|
||
|
|
def _check(arr):
|
||
|
|
mask = ~np.isfinite(arr)
|
||
|
|
arr = arr.copy()
|
||
|
|
result = libalgos.rank_1d_float64(arr)
|
||
|
|
arr[mask] = np.inf
|
||
|
|
exp = rankdata(arr)
|
||
|
|
exp[mask] = nan
|
||
|
|
assert_almost_equal(result, exp)
|
||
|
|
|
||
|
|
_check(np.array([nan, nan, 5., 5., 5., nan, 1, 2, 3, nan]))
|
||
|
|
_check(np.array([4., nan, 5., 5., 5., nan, 1, 2, 4., nan]))
|
||
|
|
|
||
|
|
def test_basic(self):
|
||
|
|
exp = np.array([1, 2], dtype=np.float64)
|
||
|
|
|
||
|
|
for dtype in np.typecodes['AllInteger']:
|
||
|
|
s = Series([1, 100], dtype=dtype)
|
||
|
|
tm.assert_numpy_array_equal(algos.rank(s), exp)
|
||
|
|
|
||
|
|
def test_uint64_overflow(self):
|
||
|
|
exp = np.array([1, 2], dtype=np.float64)
|
||
|
|
|
||
|
|
for dtype in [np.float64, np.uint64]:
|
||
|
|
s = Series([1, 2**63], dtype=dtype)
|
||
|
|
tm.assert_numpy_array_equal(algos.rank(s), exp)
|
||
|
|
|
||
|
|
def test_too_many_ndims(self):
|
||
|
|
arr = np.array([[[1, 2, 3], [4, 5, 6], [7, 8, 9]]])
|
||
|
|
msg = "Array with ndim > 2 are not supported"
|
||
|
|
|
||
|
|
with tm.assert_raises_regex(TypeError, msg):
|
||
|
|
algos.rank(arr)
|
||
|
|
|
||
|
|
|
||
|
|
def test_pad_backfill_object_segfault():
|
||
|
|
|
||
|
|
old = np.array([], dtype='O')
|
||
|
|
new = np.array([datetime(2010, 12, 31)], dtype='O')
|
||
|
|
|
||
|
|
result = libalgos.pad_object(old, new)
|
||
|
|
expected = np.array([-1], dtype=np.int64)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = libalgos.pad_object(new, old)
|
||
|
|
expected = np.array([], dtype=np.int64)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = libalgos.backfill_object(old, new)
|
||
|
|
expected = np.array([-1], dtype=np.int64)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
result = libalgos.backfill_object(new, old)
|
||
|
|
expected = np.array([], dtype=np.int64)
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
|
||
|
|
def test_arrmap():
|
||
|
|
values = np.array(['foo', 'foo', 'bar', 'bar', 'baz', 'qux'], dtype='O')
|
||
|
|
result = libalgos.arrmap_object(values, lambda x: x in ['foo', 'bar'])
|
||
|
|
assert (result.dtype == np.bool_)
|
||
|
|
|
||
|
|
|
||
|
|
class TestTseriesUtil(object):
|
||
|
|
|
||
|
|
def test_combineFunc(self):
|
||
|
|
pass
|
||
|
|
|
||
|
|
def test_reindex(self):
|
||
|
|
pass
|
||
|
|
|
||
|
|
def test_isna(self):
|
||
|
|
pass
|
||
|
|
|
||
|
|
def test_groupby(self):
|
||
|
|
pass
|
||
|
|
|
||
|
|
def test_groupby_withnull(self):
|
||
|
|
pass
|
||
|
|
|
||
|
|
def test_backfill(self):
|
||
|
|
old = Index([1, 5, 10])
|
||
|
|
new = Index(lrange(12))
|
||
|
|
|
||
|
|
filler = libalgos.backfill_int64(old.values, new.values)
|
||
|
|
|
||
|
|
expect_filler = np.array([0, 0, 1, 1, 1, 1,
|
||
|
|
2, 2, 2, 2, 2, -1], dtype=np.int64)
|
||
|
|
tm.assert_numpy_array_equal(filler, expect_filler)
|
||
|
|
|
||
|
|
# corner case
|
||
|
|
old = Index([1, 4])
|
||
|
|
new = Index(lrange(5, 10))
|
||
|
|
filler = libalgos.backfill_int64(old.values, new.values)
|
||
|
|
|
||
|
|
expect_filler = np.array([-1, -1, -1, -1, -1], dtype=np.int64)
|
||
|
|
tm.assert_numpy_array_equal(filler, expect_filler)
|
||
|
|
|
||
|
|
def test_pad(self):
|
||
|
|
old = Index([1, 5, 10])
|
||
|
|
new = Index(lrange(12))
|
||
|
|
|
||
|
|
filler = libalgos.pad_int64(old.values, new.values)
|
||
|
|
|
||
|
|
expect_filler = np.array([-1, 0, 0, 0, 0, 1,
|
||
|
|
1, 1, 1, 1, 2, 2], dtype=np.int64)
|
||
|
|
tm.assert_numpy_array_equal(filler, expect_filler)
|
||
|
|
|
||
|
|
# corner case
|
||
|
|
old = Index([5, 10])
|
||
|
|
new = Index(lrange(5))
|
||
|
|
filler = libalgos.pad_int64(old.values, new.values)
|
||
|
|
expect_filler = np.array([-1, -1, -1, -1, -1], dtype=np.int64)
|
||
|
|
tm.assert_numpy_array_equal(filler, expect_filler)
|
||
|
|
|
||
|
|
|
||
|
|
def test_is_lexsorted():
|
||
|
|
failure = [
|
||
|
|
np.array([3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
|
||
|
|
3, 3,
|
||
|
|
3, 3,
|
||
|
|
3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2,
|
||
|
|
2, 2, 2, 2, 2, 2, 2,
|
||
|
|
2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
|
||
|
|
1, 1, 1, 1, 1, 1, 1,
|
||
|
|
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
|
||
|
|
1, 1, 1, 1, 1, 1, 1,
|
||
|
|
1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
|
||
|
|
0, 0, 0, 0, 0, 0, 0,
|
||
|
|
0, 0, 0, 0, 0, 0, 0, 0, 0], dtype='int64'),
|
||
|
|
np.array([30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16,
|
||
|
|
15, 14,
|
||
|
|
13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 30, 29, 28,
|
||
|
|
27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13,
|
||
|
|
12, 11,
|
||
|
|
10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 30, 29, 28, 27, 26, 25,
|
||
|
|
24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10,
|
||
|
|
9, 8,
|
||
|
|
7, 6, 5, 4, 3, 2, 1, 0, 30, 29, 28, 27, 26, 25, 24, 23, 22,
|
||
|
|
21, 20, 19, 18, 17, 16, 15, 14, 13, 12, 11, 10, 9, 8, 7,
|
||
|
|
6, 5,
|
||
|
|
4, 3, 2, 1, 0], dtype='int64')]
|
||
|
|
|
||
|
|
assert (not libalgos.is_lexsorted(failure))
|
||
|
|
|
||
|
|
|
||
|
|
def test_groupsort_indexer():
|
||
|
|
a = np.random.randint(0, 1000, 100).astype(np.int64)
|
||
|
|
b = np.random.randint(0, 1000, 100).astype(np.int64)
|
||
|
|
|
||
|
|
result = libalgos.groupsort_indexer(a, 1000)[0]
|
||
|
|
|
||
|
|
# need to use a stable sort
|
||
|
|
# np.argsort returns int, groupsort_indexer
|
||
|
|
# always returns int64
|
||
|
|
expected = np.argsort(a, kind='mergesort')
|
||
|
|
expected = expected.astype(np.int64)
|
||
|
|
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
# compare with lexsort
|
||
|
|
# np.lexsort returns int, groupsort_indexer
|
||
|
|
# always returns int64
|
||
|
|
key = a * 1000 + b
|
||
|
|
result = libalgos.groupsort_indexer(key, 1000000)[0]
|
||
|
|
expected = np.lexsort((b, a))
|
||
|
|
expected = expected.astype(np.int64)
|
||
|
|
|
||
|
|
tm.assert_numpy_array_equal(result, expected)
|
||
|
|
|
||
|
|
|
||
|
|
def test_infinity_sort():
|
||
|
|
# GH 13445
|
||
|
|
# numpy's argsort can be unhappy if something is less than
|
||
|
|
# itself. Instead, let's give our infinities a self-consistent
|
||
|
|
# ordering, but outside the float extended real line.
|
||
|
|
|
||
|
|
Inf = libalgos.Infinity()
|
||
|
|
NegInf = libalgos.NegInfinity()
|
||
|
|
|
||
|
|
ref_nums = [NegInf, float("-inf"), -1e100, 0, 1e100, float("inf"), Inf]
|
||
|
|
|
||
|
|
assert all(Inf >= x for x in ref_nums)
|
||
|
|
assert all(Inf > x or x is Inf for x in ref_nums)
|
||
|
|
assert Inf >= Inf and Inf == Inf
|
||
|
|
assert not Inf < Inf and not Inf > Inf
|
||
|
|
assert libalgos.Infinity() == libalgos.Infinity()
|
||
|
|
assert not libalgos.Infinity() != libalgos.Infinity()
|
||
|
|
|
||
|
|
assert all(NegInf <= x for x in ref_nums)
|
||
|
|
assert all(NegInf < x or x is NegInf for x in ref_nums)
|
||
|
|
assert NegInf <= NegInf and NegInf == NegInf
|
||
|
|
assert not NegInf < NegInf and not NegInf > NegInf
|
||
|
|
assert libalgos.NegInfinity() == libalgos.NegInfinity()
|
||
|
|
assert not libalgos.NegInfinity() != libalgos.NegInfinity()
|
||
|
|
|
||
|
|
for perm in permutations(ref_nums):
|
||
|
|
assert sorted(perm) == ref_nums
|
||
|
|
|
||
|
|
# smoke tests
|
||
|
|
np.array([libalgos.Infinity()] * 32).argsort()
|
||
|
|
np.array([libalgos.NegInfinity()] * 32).argsort()
|
||
|
|
|
||
|
|
|
||
|
|
def test_infinity_against_nan():
|
||
|
|
Inf = libalgos.Infinity()
|
||
|
|
NegInf = libalgos.NegInfinity()
|
||
|
|
|
||
|
|
assert not Inf > np.nan
|
||
|
|
assert not Inf >= np.nan
|
||
|
|
assert not Inf < np.nan
|
||
|
|
assert not Inf <= np.nan
|
||
|
|
assert not Inf == np.nan
|
||
|
|
assert Inf != np.nan
|
||
|
|
|
||
|
|
assert not NegInf > np.nan
|
||
|
|
assert not NegInf >= np.nan
|
||
|
|
assert not NegInf < np.nan
|
||
|
|
assert not NegInf <= np.nan
|
||
|
|
assert not NegInf == np.nan
|
||
|
|
assert NegInf != np.nan
|
||
|
|
|
||
|
|
|
||
|
|
def test_ensure_platform_int():
|
||
|
|
arr = np.arange(100, dtype=np.intp)
|
||
|
|
|
||
|
|
result = libalgos.ensure_platform_int(arr)
|
||
|
|
assert (result is arr)
|
||
|
|
|
||
|
|
|
||
|
|
def test_int64_add_overflow():
|
||
|
|
# see gh-14068
|
||
|
|
msg = "Overflow in int64 addition"
|
||
|
|
m = np.iinfo(np.int64).max
|
||
|
|
n = np.iinfo(np.int64).min
|
||
|
|
|
||
|
|
with tm.assert_raises_regex(OverflowError, msg):
|
||
|
|
algos.checked_add_with_arr(np.array([m, m]), m)
|
||
|
|
with tm.assert_raises_regex(OverflowError, msg):
|
||
|
|
algos.checked_add_with_arr(np.array([m, m]), np.array([m, m]))
|
||
|
|
with tm.assert_raises_regex(OverflowError, msg):
|
||
|
|
algos.checked_add_with_arr(np.array([n, n]), n)
|
||
|
|
with tm.assert_raises_regex(OverflowError, msg):
|
||
|
|
algos.checked_add_with_arr(np.array([n, n]), np.array([n, n]))
|
||
|
|
with tm.assert_raises_regex(OverflowError, msg):
|
||
|
|
algos.checked_add_with_arr(np.array([m, n]), np.array([n, n]))
|
||
|
|
with tm.assert_raises_regex(OverflowError, msg):
|
||
|
|
algos.checked_add_with_arr(np.array([m, m]), np.array([m, m]),
|
||
|
|
arr_mask=np.array([False, True]))
|
||
|
|
with tm.assert_raises_regex(OverflowError, msg):
|
||
|
|
algos.checked_add_with_arr(np.array([m, m]), np.array([m, m]),
|
||
|
|
b_mask=np.array([False, True]))
|
||
|
|
with tm.assert_raises_regex(OverflowError, msg):
|
||
|
|
algos.checked_add_with_arr(np.array([m, m]), np.array([m, m]),
|
||
|
|
arr_mask=np.array([False, True]),
|
||
|
|
b_mask=np.array([False, True]))
|
||
|
|
with tm.assert_raises_regex(OverflowError, msg):
|
||
|
|
with tm.assert_produces_warning(RuntimeWarning):
|
||
|
|
algos.checked_add_with_arr(np.array([m, m]),
|
||
|
|
np.array([np.nan, m]))
|
||
|
|
|
||
|
|
# Check that the nan boolean arrays override whether or not
|
||
|
|
# the addition overflows. We don't check the result but just
|
||
|
|
# the fact that an OverflowError is not raised.
|
||
|
|
with pytest.raises(AssertionError):
|
||
|
|
with tm.assert_raises_regex(OverflowError, msg):
|
||
|
|
algos.checked_add_with_arr(np.array([m, m]), np.array([m, m]),
|
||
|
|
arr_mask=np.array([True, True]))
|
||
|
|
with pytest.raises(AssertionError):
|
||
|
|
with tm.assert_raises_regex(OverflowError, msg):
|
||
|
|
algos.checked_add_with_arr(np.array([m, m]), np.array([m, m]),
|
||
|
|
b_mask=np.array([True, True]))
|
||
|
|
with pytest.raises(AssertionError):
|
||
|
|
with tm.assert_raises_regex(OverflowError, msg):
|
||
|
|
algos.checked_add_with_arr(np.array([m, m]), np.array([m, m]),
|
||
|
|
arr_mask=np.array([True, False]),
|
||
|
|
b_mask=np.array([False, True]))
|
||
|
|
|
||
|
|
|
||
|
|
class TestMode(object):
|
||
|
|
|
||
|
|
def test_no_mode(self):
|
||
|
|
exp = Series([], dtype=np.float64)
|
||
|
|
tm.assert_series_equal(algos.mode([]), exp)
|
||
|
|
|
||
|
|
def test_mode_single(self):
|
||
|
|
# GH 15714
|
||
|
|
exp_single = [1]
|
||
|
|
data_single = [1]
|
||
|
|
|
||
|
|
exp_multi = [1]
|
||
|
|
data_multi = [1, 1]
|
||
|
|
|
||
|
|
for dt in np.typecodes['AllInteger'] + np.typecodes['Float']:
|
||
|
|
s = Series(data_single, dtype=dt)
|
||
|
|
exp = Series(exp_single, dtype=dt)
|
||
|
|
tm.assert_series_equal(algos.mode(s), exp)
|
||
|
|
|
||
|
|
s = Series(data_multi, dtype=dt)
|
||
|
|
exp = Series(exp_multi, dtype=dt)
|
||
|
|
tm.assert_series_equal(algos.mode(s), exp)
|
||
|
|
|
||
|
|
exp = Series([1], dtype=np.int)
|
||
|
|
tm.assert_series_equal(algos.mode([1]), exp)
|
||
|
|
|
||
|
|
exp = Series(['a', 'b', 'c'], dtype=np.object)
|
||
|
|
tm.assert_series_equal(algos.mode(['a', 'b', 'c']), exp)
|
||
|
|
|
||
|
|
def test_number_mode(self):
|
||
|
|
exp_single = [1]
|
||
|
|
data_single = [1] * 5 + [2] * 3
|
||
|
|
|
||
|
|
exp_multi = [1, 3]
|
||
|
|
data_multi = [1] * 5 + [2] * 3 + [3] * 5
|
||
|
|
|
||
|
|
for dt in np.typecodes['AllInteger'] + np.typecodes['Float']:
|
||
|
|
s = Series(data_single, dtype=dt)
|
||
|
|
exp = Series(exp_single, dtype=dt)
|
||
|
|
tm.assert_series_equal(algos.mode(s), exp)
|
||
|
|
|
||
|
|
s = Series(data_multi, dtype=dt)
|
||
|
|
exp = Series(exp_multi, dtype=dt)
|
||
|
|
tm.assert_series_equal(algos.mode(s), exp)
|
||
|
|
|
||
|
|
def test_strobj_mode(self):
|
||
|
|
exp = ['b']
|
||
|
|
data = ['a'] * 2 + ['b'] * 3
|
||
|
|
|
||
|
|
s = Series(data, dtype='c')
|
||
|
|
exp = Series(exp, dtype='c')
|
||
|
|
tm.assert_series_equal(algos.mode(s), exp)
|
||
|
|
|
||
|
|
exp = ['bar']
|
||
|
|
data = ['foo'] * 2 + ['bar'] * 3
|
||
|
|
|
||
|
|
for dt in [str, object]:
|
||
|
|
s = Series(data, dtype=dt)
|
||
|
|
exp = Series(exp, dtype=dt)
|
||
|
|
tm.assert_series_equal(algos.mode(s), exp)
|
||
|
|
|
||
|
|
def test_datelike_mode(self):
|
||
|
|
exp = Series(['1900-05-03', '2011-01-03',
|
||
|
|
'2013-01-02'], dtype="M8[ns]")
|
||
|
|
s = Series(['2011-01-03', '2013-01-02',
|
||
|
|
'1900-05-03'], dtype='M8[ns]')
|
||
|
|
tm.assert_series_equal(algos.mode(s), exp)
|
||
|
|
|
||
|
|
exp = Series(['2011-01-03', '2013-01-02'], dtype='M8[ns]')
|
||
|
|
s = Series(['2011-01-03', '2013-01-02', '1900-05-03',
|
||
|
|
'2011-01-03', '2013-01-02'], dtype='M8[ns]')
|
||
|
|
tm.assert_series_equal(algos.mode(s), exp)
|
||
|
|
|
||
|
|
def test_timedelta_mode(self):
|
||
|
|
exp = Series(['-1 days', '0 days', '1 days'],
|
||
|
|
dtype='timedelta64[ns]')
|
||
|
|
s = Series(['1 days', '-1 days', '0 days'],
|
||
|
|
dtype='timedelta64[ns]')
|
||
|
|
tm.assert_series_equal(algos.mode(s), exp)
|
||
|
|
|
||
|
|
exp = Series(['2 min', '1 day'], dtype='timedelta64[ns]')
|
||
|
|
s = Series(['1 day', '1 day', '-1 day', '-1 day 2 min',
|
||
|
|
'2 min', '2 min'], dtype='timedelta64[ns]')
|
||
|
|
tm.assert_series_equal(algos.mode(s), exp)
|
||
|
|
|
||
|
|
def test_mixed_dtype(self):
|
||
|
|
exp = Series(['foo'])
|
||
|
|
s = Series([1, 'foo', 'foo'])
|
||
|
|
tm.assert_series_equal(algos.mode(s), exp)
|
||
|
|
|
||
|
|
def test_uint64_overflow(self):
|
||
|
|
exp = Series([2**63], dtype=np.uint64)
|
||
|
|
s = Series([1, 2**63, 2**63], dtype=np.uint64)
|
||
|
|
tm.assert_series_equal(algos.mode(s), exp)
|
||
|
|
|
||
|
|
exp = Series([1, 2**63], dtype=np.uint64)
|
||
|
|
s = Series([1, 2**63], dtype=np.uint64)
|
||
|
|
tm.assert_series_equal(algos.mode(s), exp)
|
||
|
|
|
||
|
|
def test_categorical(self):
|
||
|
|
c = Categorical([1, 2])
|
||
|
|
exp = c
|
||
|
|
tm.assert_categorical_equal(algos.mode(c), exp)
|
||
|
|
tm.assert_categorical_equal(c.mode(), exp)
|
||
|
|
|
||
|
|
c = Categorical([1, 'a', 'a'])
|
||
|
|
exp = Categorical(['a'], categories=[1, 'a'])
|
||
|
|
tm.assert_categorical_equal(algos.mode(c), exp)
|
||
|
|
tm.assert_categorical_equal(c.mode(), exp)
|
||
|
|
|
||
|
|
c = Categorical([1, 1, 2, 3, 3])
|
||
|
|
exp = Categorical([1, 3], categories=[1, 2, 3])
|
||
|
|
tm.assert_categorical_equal(algos.mode(c), exp)
|
||
|
|
tm.assert_categorical_equal(c.mode(), exp)
|
||
|
|
|
||
|
|
def test_index(self):
|
||
|
|
idx = Index([1, 2, 3])
|
||
|
|
exp = Series([1, 2, 3], dtype=np.int64)
|
||
|
|
tm.assert_series_equal(algos.mode(idx), exp)
|
||
|
|
|
||
|
|
idx = Index([1, 'a', 'a'])
|
||
|
|
exp = Series(['a'], dtype=object)
|
||
|
|
tm.assert_series_equal(algos.mode(idx), exp)
|
||
|
|
|
||
|
|
idx = Index([1, 1, 2, 3, 3])
|
||
|
|
exp = Series([1, 3], dtype=np.int64)
|
||
|
|
tm.assert_series_equal(algos.mode(idx), exp)
|
||
|
|
|
||
|
|
exp = Series(['2 min', '1 day'], dtype='timedelta64[ns]')
|
||
|
|
idx = Index(['1 day', '1 day', '-1 day', '-1 day 2 min',
|
||
|
|
'2 min', '2 min'], dtype='timedelta64[ns]')
|
||
|
|
tm.assert_series_equal(algos.mode(idx), exp)
|