Singer 修改tap-s3-csv 支持minio 连接

singer 团队官方处了一个tap-s3-csv 的tap，对于没有使用aws 的人来说并不是很方便了，所以简单修改了
下源码，可以支持通用的s3 csv 文件的处理，同时发布到了官方pip 仓库中，方便大家使用。
以下是简单代码修改部分的说明，以及如何发布pip包

修改说明

主要是关于连接s3 的部分，因为tap-s3-csv 使用的是boto3 我们需要修改的就是关于boto3 连接s3 的部署
添加上aws_access_key_id,aws_secret_access_key,endpoint_url
关于s3 自定义连接的说明，格式如下：

 

  s3_client = boto3.session.Session().client(

        service_name='s3',

        aws_access_key_id=aws_access_key_id,

        aws_secret_access_key=aws_secret_access_key,

        endpoint_url=endpoint_url,

    )

几个需要修改的部分

• s3.py
  get_input_files_for_table 部分，主要是传递参数的
  修改如下：

 

def get_input_files_for_table(config, table_spec, modified_since=None):

    bucket = config['bucket']

    aws_access_key_id = config['aws_access_key_id']

    aws_secret_access_key =config['aws_secret_access_key']

    endpoint_url =config['endpoint_url']

    to_return = []

​

    pattern = table_spec['search_pattern']

    try:

        matcher = re.compile(pattern)

    except re.error as e:

        raise ValueError(

            ("search_pattern for table `{}` is not a valid regular "

             "expression. See "

             "https://docs.python.org/3.5/library/re.html#regular-expression-syntax").format(table_spec['table_name']),

            pattern) from e

​

    LOGGER.info(

        'Checking bucket "%s" for keys matching "%s"', bucket, pattern)

​

    matched_files_count = 0

    unmatched_files_count = 0

    max_files_before_log = 30000

    for s3_object in list_files_in_bucket(bucket,aws_access_key_id,aws_secret_access_key,endpoint_url, table_spec.get('search_prefix')):

        key = s3_object['Key']

        last_modified = s3_object['LastModified']

        LOGGER.info(key)

        LOGGER.info(last_modified)

        if s3_object['Size'] == 0:

            LOGGER.info('Skipping matched file "%s" as it is empty', key)

            unmatched_files_count += 1

            continue

​

        if matcher.search(key):

            matched_files_count += 1

            if modified_since is None or modified_since < last_modified:

                LOGGER.info('Will download key "%s" as it was last modified %s',

                            key,

                            last_modified)

                yield {'key': key, 'last_modified': last_modified}

        else:

            unmatched_files_count += 1

​

        if (unmatched_files_count + matched_files_count) % max_files_before_log == 0:

            # Are we skipping greater than 50% of the files?

            if 0.5 < (unmatched_files_count / (matched_files_count + unmatched_files_count)):

                LOGGER.warn(("Found %s matching files and %s non-matching files. "

                             "You should consider adding a `search_prefix` to the config "

                             "or removing non-matching files from the bucket."),

                            matched_files_count, unmatched_files_count)

            else:

                LOGGER.info("Found %s matching files and %s non-matching files",

                            matched_files_count, unmatched_files_count)

​

    if 0 == matched_files_count:

        raise Exception("No files found matching pattern {}".format(pattern))

list_files_in_bucket 修改核心部分，关于连接s3 的
修改如下：

 

 @retry_pattern()

def list_files_in_bucket(bucket,aws_access_key_id,aws_secret_access_key,endpoint_url, search_prefix=None):

    s3_client = boto3.session.Session().client(

        service_name='s3',

        aws_access_key_id=aws_access_key_id,

        aws_secret_access_key=aws_secret_access_key,

        endpoint_url=endpoint_url,

    )

    s3_object_count = 0

​

    max_results = 1000

    args = {

        'Bucket': bucket,

        'MaxKeys': max_results,

    }

​

    if search_prefix is not None:

        args['Prefix'] = search_prefix

​

    paginator = s3_client.get_paginator('list_objects_v2')

    pages = 0

    for page in paginator.paginate(**args):

        pages += 1

        LOGGER.debug("On page %s", pages)

        s3_object_count += len(page['Contents'])

        yield from page['Contents']

​

    if 0 < s3_object_count:

        LOGGER.info("Found %s files.", s3_object_count)

    else:

        LOGGER.warning('Found no files for bucket "%s" that match prefix "%s"', bucket, search_prefix)

get_file_handle 部分，主要是关于获取s3 对象内容的

@retry_pattern()

def get_file_handle(config, s3_path):

    bucket = config['bucket']

    aws_access_key_id = config['aws_access_key_id']

    aws_secret_access_key =config['aws_secret_access_key']

    endpoint_url =config['endpoint_url']

    s3_client = boto3.resource(

       service_name="s3",

       aws_access_key_id=aws_access_key_id,

       aws_secret_access_key=aws_secret_access_key,

       endpoint_url=endpoint_url)

    s3_bucket = s3_client.Bucket(bucket)

    s3_object = s3_bucket.Object(s3_path)

    return s3_object.get()['Body']

• init.py
  关于tap 命令处理的部分，比如模式发现，执行同步,以及参数检验的
  参数校验修改，修改为我们配置参数需要的

REQUIRED_CONFIG_KEYS = ["start_date", "bucket", "aws_access_key_id", "aws_secret_access_key", "endpoint_url"]

main 函数：

@singer.utils.handle_top_exception(LOGGER)

def main():

    args = singer.utils.parse_args(REQUIRED_CONFIG_KEYS)

    config = args.config

    bucket = config['bucket']

    aws_access_key_id = config['aws_access_key_id']

    aws_secret_access_key =config['aws_secret_access_key']

    endpoint_url =config['endpoint_url']

    config['tables'] = validate_table_config(config)

    try:

        for page in s3.list_files_in_bucket(bucket,aws_access_key_id,aws_secret_access_key,endpoint_url):

            break

        LOGGER.warning("I have direct access to the bucket without assuming the configured role.")

    except:

        LOGGER.error("can't connect to s3 storage")

    if args.discover:

        do_discover(args.config)

    elif args.properties:

        do_sync(config, args.properties, args.state)

• pip 包约定处理
  为了不和官方冲突，重新别名
  setup.py:

 

#!/usr/bin/env python

​

from setuptools import setup

​

setup(name='tap-minio-csv',

      version='1.2.2',

      description='Singer.io tap for extracting CSV files from minio',

      author='rongfengliang',

      url='https://github.com/rongfengliang/tap-minio-csv',

      classifiers=['Programming Language :: Python :: 3 :: Only'],

      py_modules=['tap_minio_csv'],

      install_requires=[

          'backoff==1.3.2',

          'boto3==1.9.57',

          'singer-encodings==0.0.3',

          'singer-python==5.1.5',

          'voluptuous==0.10.5'

      ],

      extras_require={

          'dev': [

              'ipdb==0.11'

          ]

      },

      entry_points='''

          [console_scripts]

          tap-minio-csv=tap_minio_csv:main

      ''',

      packages=['tap_minio_csv'])

• 项目包名称

发布pip 包

• 安装工具

python3 -m pip install --user --upgrade setuptools wheel twine

• 生成文件

python3 setup.py sdist bdist_wheel

• 上传
  需要先注册账户,执行以下命令，按照提示输入账户信息即可

 

twine upload dist/*

• pip 包

说明

以上是一个简单的说明，详细代码可以参考https://github.com/rongfengliang/tap-minio-csv

参考资料

https://boto3.amazonaws.com/v1/documentation/api/latest/reference/core/session.html
https://github.com/singer-io/tap-s3-csv
https://github.com/rongfengliang/tap-minio-csv