226 lines
8 KiB
Python
226 lines
8 KiB
Python
|
# -*- coding: utf-8 -*-
|
||
|
|
|
||
|
|
"""
|
||
|
|
Tests that skipped rows are properly handled during
|
||
|
|
parsing for all of the parsers defined in parsers.py
|
||
|
|
"""
|
||
|
|
|
||
|
|
from datetime import datetime
|
||
|
|
|
||
|
|
import numpy as np
|
||
|
|
|
||
|
|
import pandas.util.testing as tm
|
||
|
|
|
||
|
|
from pandas import DataFrame
|
||
|
|
from pandas.errors import EmptyDataError
|
||
|
|
from pandas.compat import StringIO, range, lrange
|
||
|
|
|
||
|
|
|
||
|
|
class SkipRowsTests(object):
|
||
|
|
|
||
|
|
def test_skiprows_bug(self):
|
||
|
|
# see gh-505
|
||
|
|
text = """#foo,a,b,c
|
||
|
|
#foo,a,b,c
|
||
|
|
#foo,a,b,c
|
||
|
|
#foo,a,b,c
|
||
|
|
#foo,a,b,c
|
||
|
|
#foo,a,b,c
|
||
|
|
1/1/2000,1.,2.,3.
|
||
|
|
1/2/2000,4,5,6
|
||
|
|
1/3/2000,7,8,9
|
||
|
|
"""
|
||
|
|
data = self.read_csv(StringIO(text), skiprows=lrange(6), header=None,
|
||
|
|
index_col=0, parse_dates=True)
|
||
|
|
|
||
|
|
data2 = self.read_csv(StringIO(text), skiprows=6, header=None,
|
||
|
|
index_col=0, parse_dates=True)
|
||
|
|
|
||
|
|
expected = DataFrame(np.arange(1., 10.).reshape((3, 3)),
|
||
|
|
columns=[1, 2, 3],
|
||
|
|
index=[datetime(2000, 1, 1), datetime(2000, 1, 2),
|
||
|
|
datetime(2000, 1, 3)])
|
||
|
|
expected.index.name = 0
|
||
|
|
tm.assert_frame_equal(data, expected)
|
||
|
|
tm.assert_frame_equal(data, data2)
|
||
|
|
|
||
|
|
def test_deep_skiprows(self):
|
||
|
|
# see gh-4382
|
||
|
|
text = "a,b,c\n" + \
|
||
|
|
"\n".join([",".join([str(i), str(i + 1), str(i + 2)])
|
||
|
|
for i in range(10)])
|
||
|
|
condensed_text = "a,b,c\n" + \
|
||
|
|
"\n".join([",".join([str(i), str(i + 1), str(i + 2)])
|
||
|
|
for i in [0, 1, 2, 3, 4, 6, 8, 9]])
|
||
|
|
data = self.read_csv(StringIO(text), skiprows=[6, 8])
|
||
|
|
condensed_data = self.read_csv(StringIO(condensed_text))
|
||
|
|
tm.assert_frame_equal(data, condensed_data)
|
||
|
|
|
||
|
|
def test_skiprows_blank(self):
|
||
|
|
# see gh-9832
|
||
|
|
text = """#foo,a,b,c
|
||
|
|
#foo,a,b,c
|
||
|
|
|
||
|
|
#foo,a,b,c
|
||
|
|
#foo,a,b,c
|
||
|
|
|
||
|
|
1/1/2000,1.,2.,3.
|
||
|
|
1/2/2000,4,5,6
|
||
|
|
1/3/2000,7,8,9
|
||
|
|
"""
|
||
|
|
data = self.read_csv(StringIO(text), skiprows=6, header=None,
|
||
|
|
index_col=0, parse_dates=True)
|
||
|
|
|
||
|
|
expected = DataFrame(np.arange(1., 10.).reshape((3, 3)),
|
||
|
|
columns=[1, 2, 3],
|
||
|
|
index=[datetime(2000, 1, 1), datetime(2000, 1, 2),
|
||
|
|
datetime(2000, 1, 3)])
|
||
|
|
expected.index.name = 0
|
||
|
|
tm.assert_frame_equal(data, expected)
|
||
|
|
|
||
|
|
def test_skiprow_with_newline(self):
|
||
|
|
# see gh-12775 and gh-10911
|
||
|
|
data = """id,text,num_lines
|
||
|
|
1,"line 11
|
||
|
|
line 12",2
|
||
|
|
2,"line 21
|
||
|
|
line 22",2
|
||
|
|
3,"line 31",1"""
|
||
|
|
expected = [[2, 'line 21\nline 22', 2],
|
||
|
|
[3, 'line 31', 1]]
|
||
|
|
expected = DataFrame(expected, columns=[
|
||
|
|
'id', 'text', 'num_lines'])
|
||
|
|
df = self.read_csv(StringIO(data), skiprows=[1])
|
||
|
|
tm.assert_frame_equal(df, expected)
|
||
|
|
|
||
|
|
data = ('a,b,c\n~a\n b~,~e\n d~,'
|
||
|
|
'~f\n f~\n1,2,~12\n 13\n 14~')
|
||
|
|
expected = [['a\n b', 'e\n d', 'f\n f']]
|
||
|
|
expected = DataFrame(expected, columns=[
|
||
|
|
'a', 'b', 'c'])
|
||
|
|
df = self.read_csv(StringIO(data),
|
||
|
|
quotechar="~",
|
||
|
|
skiprows=[2])
|
||
|
|
tm.assert_frame_equal(df, expected)
|
||
|
|
|
||
|
|
data = ('Text,url\n~example\n '
|
||
|
|
'sentence\n one~,url1\n~'
|
||
|
|
'example\n sentence\n two~,url2\n~'
|
||
|
|
'example\n sentence\n three~,url3')
|
||
|
|
expected = [['example\n sentence\n two', 'url2']]
|
||
|
|
expected = DataFrame(expected, columns=[
|
||
|
|
'Text', 'url'])
|
||
|
|
df = self.read_csv(StringIO(data),
|
||
|
|
quotechar="~",
|
||
|
|
skiprows=[1, 3])
|
||
|
|
tm.assert_frame_equal(df, expected)
|
||
|
|
|
||
|
|
def test_skiprow_with_quote(self):
|
||
|
|
# see gh-12775 and gh-10911
|
||
|
|
data = """id,text,num_lines
|
||
|
|
1,"line '11' line 12",2
|
||
|
|
2,"line '21' line 22",2
|
||
|
|
3,"line '31' line 32",1"""
|
||
|
|
expected = [[2, "line '21' line 22", 2],
|
||
|
|
[3, "line '31' line 32", 1]]
|
||
|
|
expected = DataFrame(expected, columns=[
|
||
|
|
'id', 'text', 'num_lines'])
|
||
|
|
df = self.read_csv(StringIO(data), skiprows=[1])
|
||
|
|
tm.assert_frame_equal(df, expected)
|
||
|
|
|
||
|
|
def test_skiprow_with_newline_and_quote(self):
|
||
|
|
# see gh-12775 and gh-10911
|
||
|
|
data = """id,text,num_lines
|
||
|
|
1,"line \n'11' line 12",2
|
||
|
|
2,"line \n'21' line 22",2
|
||
|
|
3,"line \n'31' line 32",1"""
|
||
|
|
expected = [[2, "line \n'21' line 22", 2],
|
||
|
|
[3, "line \n'31' line 32", 1]]
|
||
|
|
expected = DataFrame(expected, columns=[
|
||
|
|
'id', 'text', 'num_lines'])
|
||
|
|
df = self.read_csv(StringIO(data), skiprows=[1])
|
||
|
|
tm.assert_frame_equal(df, expected)
|
||
|
|
|
||
|
|
data = """id,text,num_lines
|
||
|
|
1,"line '11\n' line 12",2
|
||
|
|
2,"line '21\n' line 22",2
|
||
|
|
3,"line '31\n' line 32",1"""
|
||
|
|
expected = [[2, "line '21\n' line 22", 2],
|
||
|
|
[3, "line '31\n' line 32", 1]]
|
||
|
|
expected = DataFrame(expected, columns=[
|
||
|
|
'id', 'text', 'num_lines'])
|
||
|
|
df = self.read_csv(StringIO(data), skiprows=[1])
|
||
|
|
tm.assert_frame_equal(df, expected)
|
||
|
|
|
||
|
|
data = """id,text,num_lines
|
||
|
|
1,"line '11\n' \r\tline 12",2
|
||
|
|
2,"line '21\n' \r\tline 22",2
|
||
|
|
3,"line '31\n' \r\tline 32",1"""
|
||
|
|
expected = [[2, "line '21\n' \r\tline 22", 2],
|
||
|
|
[3, "line '31\n' \r\tline 32", 1]]
|
||
|
|
expected = DataFrame(expected, columns=[
|
||
|
|
'id', 'text', 'num_lines'])
|
||
|
|
df = self.read_csv(StringIO(data), skiprows=[1])
|
||
|
|
tm.assert_frame_equal(df, expected)
|
||
|
|
|
||
|
|
def test_skiprows_lineterminator(self):
|
||
|
|
# see gh-9079
|
||
|
|
data = '\n'.join(['SMOSMANIA ThetaProbe-ML2X ',
|
||
|
|
'2007/01/01 01:00 0.2140 U M ',
|
||
|
|
'2007/01/01 02:00 0.2141 M O ',
|
||
|
|
'2007/01/01 04:00 0.2142 D M '])
|
||
|
|
expected = DataFrame([['2007/01/01', '01:00', 0.2140, 'U', 'M'],
|
||
|
|
['2007/01/01', '02:00', 0.2141, 'M', 'O'],
|
||
|
|
['2007/01/01', '04:00', 0.2142, 'D', 'M']],
|
||
|
|
columns=['date', 'time', 'var', 'flag',
|
||
|
|
'oflag'])
|
||
|
|
|
||
|
|
# test with default line terminators "LF" and "CRLF"
|
||
|
|
df = self.read_csv(StringIO(data), skiprows=1, delim_whitespace=True,
|
||
|
|
names=['date', 'time', 'var', 'flag', 'oflag'])
|
||
|
|
tm.assert_frame_equal(df, expected)
|
||
|
|
|
||
|
|
df = self.read_csv(StringIO(data.replace('\n', '\r\n')),
|
||
|
|
skiprows=1, delim_whitespace=True,
|
||
|
|
names=['date', 'time', 'var', 'flag', 'oflag'])
|
||
|
|
tm.assert_frame_equal(df, expected)
|
||
|
|
|
||
|
|
# "CR" is not respected with the Python parser yet
|
||
|
|
if self.engine == 'c':
|
||
|
|
df = self.read_csv(StringIO(data.replace('\n', '\r')),
|
||
|
|
skiprows=1, delim_whitespace=True,
|
||
|
|
names=['date', 'time', 'var', 'flag', 'oflag'])
|
||
|
|
tm.assert_frame_equal(df, expected)
|
||
|
|
|
||
|
|
def test_skiprows_infield_quote(self):
|
||
|
|
# see gh-14459
|
||
|
|
data = 'a"\nb"\na\n1'
|
||
|
|
expected = DataFrame({'a': [1]})
|
||
|
|
|
||
|
|
df = self.read_csv(StringIO(data), skiprows=2)
|
||
|
|
tm.assert_frame_equal(df, expected)
|
||
|
|
|
||
|
|
def test_skiprows_callable(self):
|
||
|
|
data = 'a\n1\n2\n3\n4\n5'
|
||
|
|
|
||
|
|
skiprows = lambda x: x % 2 == 0
|
||
|
|
expected = DataFrame({'1': [3, 5]})
|
||
|
|
df = self.read_csv(StringIO(data), skiprows=skiprows)
|
||
|
|
tm.assert_frame_equal(df, expected)
|
||
|
|
|
||
|
|
expected = DataFrame({'foo': [3, 5]})
|
||
|
|
df = self.read_csv(StringIO(data), skiprows=skiprows,
|
||
|
|
header=0, names=['foo'])
|
||
|
|
tm.assert_frame_equal(df, expected)
|
||
|
|
|
||
|
|
skiprows = lambda x: True
|
||
|
|
msg = "No columns to parse from file"
|
||
|
|
with tm.assert_raises_regex(EmptyDataError, msg):
|
||
|
|
self.read_csv(StringIO(data), skiprows=skiprows)
|
||
|
|
|
||
|
|
# This is a bad callable and should raise.
|
||
|
|
msg = "by zero"
|
||
|
|
skiprows = lambda x: 1 / 0
|
||
|
|
with tm.assert_raises_regex(ZeroDivisionError, msg):
|
||
|
|
self.read_csv(StringIO(data), skiprows=skiprows)
|