python数据处理（三）之处理pdf文件

代码以及资料 https://github.com/jackiekazil/data-wrangling

1.前言

尽可能地寻找可以替代pdf格式的数据

2.解析pdf的编程方法

安装slate

pip install slate
pip install pdfminer

2.1 利用slate库打开并读取PDF

import slate  #导入slate

pdf = 'EN-FINAL Table 9.pdf' # pdf文件名

with open(pdf) as f: # 打开pdf文件
    doc = slate.PDF(f) # 读取pdf文件

for page in doc[:2]: # 遍历文档doc的前两页并输出
    print(page)

2.2 将PDF转换成文本

转换文本代码 pdf2txt.py

pdf_txt = 'en-final-table9.txt'
openfile = open(pdf_txt, 'r')
country_line = total_line = False
previous_line = ''
countries = [] # 创建空的国家列表
totals = [] # 创建空的总数列表

double_lined_countries = [
    'Bolivia (Plurinational \n',
    'Democratic People\xe2\x80\x99s \n',
    'Democratic Republic \n',
    'Lao People\xe2\x80\x99s Democratic \n',
    'Micronesia (Federated \n',
    'Saint Vincent and \n',
    'The former Yugoslav \n',
    'United Republic \n',
    'Venezuela (Bolivarian \n',
]

def turn_on_off(line, status, start, prev_line, end='\n'):
    """
    This function checks to see if a line starts/ends with a certain
    value. If the line starts/ends with that value, the status is
    set to on/off (True/False).
    """
    if line.startswith(start):
        status = True
    elif status:
        if line == end and prev_line != 'and areas':
            status = False
    return status

def clean(line):
    """
    Cleans line breaks, spaces, and special characters from our line.
    """
    line = line.strip('\n').strip()
    line = line.replace('\xe2\x80\x93', '-')
    line = line.replace('\xe2\x80\x99', '\'')
    return line

for line in openfile:
    if country_line:  #如果包含国家则将国家添加到国家列表中
        if previous_line in double_lined_countries: # 名字占两行的国家
            line = ' '.join([clean(previous_line), clean(line)])
        countries.append(clean(line))

    elif total_line: #采集总数
        if len(line.replace('\n', '').strip()) > 0:
            totals.append(clean(line))

    country_line = turn_on_off(line, country_line, 'and areas', previous_line)
    total_line = turn_on_off(line, total_line, 'total', previous_line)

    previous_line = line

import pprint #导入pprint库
data = dict(zip(countries, totals)) #见国家和总数合并到一起，转换成字典
pprint.pprint(data)

3 利用pdfminer解析PDF

3.1 表格中的名字被分为两条记录

a.合并两行  b.手动添加国家名

4 学习解决问题的方法

4.1 使用表格提取 pdftables

from pdftables import get_tables
import pprint

headers = ['Country', 'Child Labor 2005-2012 (%) total',
           'Child Labor 2005-2012 (%) male',
           'Child Labor 2005-2012 (%) female',
           'Child Marriage 2005-2012 (%) married by 15',
           'Child Marriage 2005-2012 (%) married by 18',
           'Birth registration 2005-2012 (%)',
           'Female Genital mutilation 2002-2012 (prevalence), women',
           'Female Genital mutilation 2002-2012 (prevalence), girls',
           'Female Genital mutilation 2002-2012 (support)',
           'Justification of wife beating 2005-2012 (%) male',
           'Justification of wife beating 2005-2012 (%) female',
           'Violent discipline 2005-2012 (%) total',
           'Violent discipline 2005-2012 (%) male',
           'Violent discipline 2005-2012 (%) female']

all_tables = get_tables(open('EN-FINAL Table 9.pdf', 'rb'))

first_name = False
final_data = []

for table in all_tables:
    for row in table[5:]:
        if row[0] == '' or row[0][0].isdigit(): # 如果是空行或为数据 不是国名
            continue
        elif row[2] == '': # 缺失 可能是国名的前半部分
            first_name = row[0]
            continue
        if first_name: # 如果这一行有first_name, 那么在该行内将国名合并
            row[0] = u'{} {}'.format(first_name, row[0])
            first_name = False # 保证下一次迭代正常运行

        final_data.append(dict(zip(headers, row)))

        if row[0] == 'Zimbabwe':
                break

pprint.pprint(final_data)

4.2 手动清洗数据

4.3 使用Tabula