pandas加速读取数据记录csv大文件处理

def readf(file):
    t0 = time.time()
    data=pd.read_csv(file,low_memory=False,encoding='gbk'
                     #,nrows=100
                     )
    t1 = time.time()
    print('耗时%0.3f秒钟'%(t1-t0))
    return data
d1=readf(file1)
t0 = time.time()
d1.to_pickle('d1.pkl')
t1 = time.time()
print('耗时%0.3f秒钟'%(t1-t0))
t0 = time.time()
d2=pd.read_pickle('d1.pkl')
t1 = time.time()
print('耗时%0.3f秒钟'%(t1-t0))

print(d1.shape)
print(d2.shape)

测试读取40多万数据的速度：

耗时12.768秒钟
耗时1.561秒钟
耗时1.751秒钟
(468372, 77)
(468372, 77)