python-gitlab 一个简单demo

背景

• 需要收集git仓库信息到数据库供前端展示
  • 包括：仓库信息、仓库所有者、成员列表、提交信息、活跃情况等
• 需要定时启动、灵活触发

实现简介

• 使用gitlab v4 restful 接口
• 使用python-gitlab 依赖库（pyapi-gitlab 不支持gitlab 10.0+ 版本，故弃用）
• 使用flask 提供接口，远程灵活触发
• 使用requests 完成数据汇报与接口访问（因频率不高，性能优化相关暂不考虑）
• 使用datetime/dateutil 完成gitlab 的时间信息向mysql datatime转化
• 外层使用docker进行封装与部署

api使用

• projects

class GitProjectEntry(CCInterfaceHandler):
    def __init__(self, _project):
        super(GitProjectEntry, self).__init__()
        self.logger = logger
        self.project = _project
        self.attribute = self.project.attributes
        self.logger.debug("parse project name : {PROJECT_NAME} with id : {PROJECT_ID}.".format(
            PROJECT_NAME = self._self_attribute_get("name"),
            PROJECT_ID = self.project.get_id()
        ))

        # id, name, description, owner, web_url, last_activity_at, created_at, num_commits, member_list
        self.KEY_ID = "id"
        self.KEY_NAME = "name"
        self.KEY_DESCRIPTION = "description"

        self.KEY_OWNER = "owner"
        self.KEY_OWNER_WEB_URL = "owner_web_url"

        self.KEY_LAST_ACTIVITY_AT = "last_activity_at"
        self.KEY_CREATED_AT = "created_at"
        self.KEY_ACTIVE_STATUS = "status"

        self.KEY_NAMESPACE_PATH = "namespace_path"
        self.KEY_NAMESPACE_WEB_URL = "namespace_web_url"

        self.KEY_MEMBER_USERNAME_LIST = "member_username_list"

        self.KEY_WEB_URL = "web_url"
        self.KEY_HTTP_URL_TO_REPO = "http_url_to_repo"
        self.logger.debug(str(self._self_init()))

    def _date_formatter(self, _git_date_stamp_str):
        try:
            return date_parser.parse(_git_date_stamp_str).strftime(database_datefmt)
        except Exception,e:
            self.logger.warn("date stamp convert failed. with {INNER_GIT_STAMP}".format(INNER_GIT_STAMP = _git_date_stamp_str))

    def _git_status_active_judge(self, _git_last_active_at_dt):
        # status 1 活跃，0不活跃
        KEY_STATUS_ACTIVE = 1
        KEY_STATUS_INACTIVE = 0

        today=datetime.datetime.today()
        month_time_delta=datetime.timedelta(days = 30, hours=1)
        activity_threshold = today - month_time_delta
        activity_status = KEY_STATUS_ACTIVE if(
            date_parser.parse(_git_last_active_at_dt) > activity_threshold) else KEY_STATUS_INACTIVE
        return activity_status

    def _self_init(self):

        self.member_list = []
        self.guess_owner = None

        self.last_activity_at = self._date_formatter(self._self_attribute_get("last_activity_at"))

        self.data = {

            self.KEY_ID  : self.project.get_id(), # self.project.id
            self.KEY_NAME : self._self_attribute_get("name"), # self.project.name
            self.KEY_DESCRIPTION : self._self_attribute_get("description"),

            self.KEY_OWNER : self._self_attribute_get("owner", {}).get("username"),
            self.KEY_OWNER_WEB_URL : self._self_attribute_get("owner", {}).get("web_url"),

            self.KEY_LAST_ACTIVITY_AT : self.last_activity_at,
            self.KEY_CREATED_AT : self._date_formatter(self._self_attribute_get("created_at")),
            self.KEY_ACTIVE_STATUS : self._git_status_active_judge(self.last_activity_at),

            self.KEY_NAMESPACE_PATH : self._self_attribute_get("namespace", {}).get("path"),
            self.KEY_NAMESPACE_WEB_URL : self._self_attribute_get("namespace", {}).get("web_url"),

            self.KEY_MEMBER_USERNAME_LIST : self._self_member_username_list(),

            self.KEY_WEB_URL : self._self_attribute_get("web_url"),
            self.KEY_HTTP_URL_TO_REPO : self._self_attribute_get("description")

        }

        if(self.data.get(self.KEY_OWNER) == None):
            self.data[self.KEY_OWNER] = self._self_member_top_access_user_name()

        if(self.data.get(self.KEY_OWNER_WEB_URL) == None):
            self.data[self.KEY_OWNER_WEB_URL] = self._self_member_top_access_user_name()

        self.git_project_member_report(self.project.get_id(),self.member_list)
        if('200' != str(self.git_project_report(self.data))):
            self.logger.warn("reported git_project failed! with project_id : {PR_ID} and project_name : {PR_NAME} .\n"
                             "while data is {PR_DATA}".format(PR_ID = str(self.data.get(self.KEY_ID, "NULL_ID")),
                                                              PR_NAME = str(self.data.get(self.KEY_NAME, "NULL_NAME")),
                                                              PR_DATA = str(self.data)
                                                              )
                             )
        else:
            self.logger.info("reported git_project succeed! with project_id : {PR_ID} and project_name : {PR_NAME} .\n"
                             "while data is {PR_DATA}".format(PR_ID = str(self.data.get(self.KEY_ID, "NULL_ID")),
                                                              PR_NAME = str(self.data.get(self.KEY_NAME, "NULL_NAME")),
                                                              PR_DATA = str(self.data)
                                                              )
                             )
        return self.data

    def _self_member_list(self):
        if(self.member_list != None and len(self.member_list) > 0):
            return self.member_list
        self.member_list =  self.project.members.list(all=True)
        if(len(self.member_list)<1):
            self.member_list = self.project.members.all(all=True)
        return self.member_list

    def _self_member_top_access_user(self):
        if(not self.guess_owner):
            self.member_list = self._self_member_list()
            self.sorted_member_list = sorted(self.member_list,key=lambda x : x.attributes.get("access_level",0),reverse=True)
            self.guess_owner = self.sorted_member_list[0] if(len(self.sorted_member_list) > 0) else None
        return self.guess_owner

    def _self_member_top_access_user_weburl(self):
        self.guess_owner = self._self_member_top_access_user()
        self.top_access_username = self.guess_owner.attributes.get("web_url", None) if(self.guess_owner) else None
        return self.top_access_username

    def _self_member_top_access_user_name(self):
        self.guess_owner = self._self_member_top_access_user()
        self.top_access_username = self.guess_owner.attributes.get("username", None) if(self.guess_owner) else None
        return self.top_access_username

    def _self_member_username_list(self):
        '''
        gitlab document:
            Examples
            List the project members:

            members = project.members.list()
            List the project members recursively (including inherited members through ancestor groups):

            members = project.members.all(all=True)
            Search project members matching a query string:

            members = project.members.list(query='bar')
            Get a single project member:

            member = project.members.get(user_id)
            Add a project member:

            member = project.members.create({'user_id': user.id, 'access_level':
                                             gitlab.DEVELOPER_ACCESS})
            Modify a project member (change the access level):

            member.access_level = gitlab.MAINTAINER_ACCESS
            member.save()
            Remove a member from the project team:

            project.members.delete(user.id)
            # or
            member.delete()
            Share/unshare the project with a group:

            project.share(group.id, gitlab.DEVELOPER_ACCESS)
            project.unshare(group.id)
        :return:
        '''
        self.member_username_list = [
            member.attributes.get("username") for member in self._self_member_list()
        ]
        return self.member_username_list

    def _self_attribute_get(self, _key, _default_value = ""):
        return self.attribute.get(_key, _default_value)

    def to_json(self):
        self.json_string = json.dumps(self.data)
        return self.json_string

• members

    def _self_member_username_list(self):
        '''
        gitlab document:
            Examples
            List the project members:

            members = project.members.list()
            List the project members recursively (including inherited members through ancestor groups):

            members = project.members.all(all=True)
            Search project members matching a query string:

            members = project.members.list(query='bar')
            Get a single project member:

            member = project.members.get(user_id)
            Add a project member:

            member = project.members.create({'user_id': user.id, 'access_level':
                                             gitlab.DEVELOPER_ACCESS})
            Modify a project member (change the access level):

            member.access_level = gitlab.MAINTAINER_ACCESS
            member.save()
            Remove a member from the project team:

            project.members.delete(user.id)
            # or
            member.delete()
            Share/unshare the project with a group:

            project.share(group.id, gitlab.DEVELOPER_ACCESS)
            project.unshare(group.id)
        :return:
        '''
        self.member_username_list = [
            member.attributes.get("username") for member in self._self_member_list()
        ]
        return self.member_username_list

• users

In [260]: user.attributes
Out[260]:
    {u'access_level': 50,
     u'avatar_url': None,
     u'expires_at': None,
     u'id': 224,
     u'name': u'mosaic',
     'project_id': 18915,
     u'state': u'active',
     u'username': u'mosaic',
     u'web_url': u'http://git.intra.mosaic.com/mosaic'}

完整代码

• 新增http连接池

# !/usr/bin/python
# -*- coding: UTF-8 -*-
__author__ = 'enzhao'
# Created by enzhao on 2020/5/25.

# -*- coding: utf-8 -*-
import gitlab
from gitlab.v4 import objects as git_const
import json
import logging
from logging.handlers import RotatingFileHandler
# fh = RotatingFileHandler(filename, maxBytes=1024, backupCount=5)

import datetime
from dateutil import parser as date_parser
import time
from functools import wraps

from flask import Flask
import requests

from gitlab.v4.objects import Project

import sys
reload(sys)
sys.setdefaultencoding("utf-8")

app = Flask(__name__)

RESPONSE_SUCCESS_CODE = 200
RESPONSE_FAILED_CODE = 500
RESPONSE_SUCCESS_MESSAGE = "success"
RESPONSE_RESULT_KEY = "result"
RESPONSE_CODE_KEY = "code"
RESPONSE_MESSAGE_KEY = "message"
RESPONSE_PARAMS_KEY = "params"
RESPONSE_DICT_TEMPLATE = {
    RESPONSE_RESULT_KEY  : {},
    RESPONSE_CODE_KEY    : RESPONSE_SUCCESS_CODE,
    RESPONSE_MESSAGE_KEY : RESPONSE_SUCCESS_MESSAGE,
    RESPONSE_PARAMS_KEY  : {}
}

logger_file_suffix = time.strftime('%Y-%m-%d', time.localtime( float( time.time() )))
logger_file_name = "./waic-gitlab-info-retrive.log"
logger_file_name = "".join(["./develop-python-gitlab/host-py-source/waic-gitlab-info-retrive-", logger_file_suffix, ".log"])
# logger_file_name = "./waic-gitlab-info-retrive.log"

logger_time_fmt = '%Y/%m/%d %H:%M:%S %A'
datefmt='%Y-%m-%d %H:%M:%S@%A'
database_datefmt = "%Y-%m-%d %H:%M:%S"
git_datefmt = "%Y-%m-%d %H:%M:%S"

CC_HOST = "http://10.73.13.163"
# CC_HOST = "http://controlcenter.ds.sina.com.cn"

def get_local_ip():
    local_ip = ''
    try:
        import socket
        socket_objs = [socket.socket(socket.AF_INET, socket.SOCK_DGRAM)]
        ip_from_ip_port = [(s.connect(('8.8.8.8', 53)), s.getsockname()[0], s.close()) for s in socket_objs][0][1]
        ip_from_host_name = [ip for ip in socket.gethostbyname_ex(socket.gethostname())[2] if not ip.startswith('127.')][:1]
        local_ip = [l for l in (ip_from_ip_port, ip_from_host_name) if l][0]
    except (Exception),e:
        print('get_local_ip found exception : %s' % e)
    return local_ip if('' != local_ip and None != local_ip) else socket.gethostbyname(socket.gethostname())

def result_decorator(_func_obj):
    @wraps(_func_obj)
    def wrapTheFunction(*args, **kwargs):
        receive_time = time.time()
        # RESPONSE_DICT = RESPONSE_DICT_TEMPLATE.copy()
        RESPONSE_DICT = {
            RESPONSE_RESULT_KEY  : {},
            RESPONSE_CODE_KEY    : RESPONSE_SUCCESS_CODE,
            RESPONSE_MESSAGE_KEY : RESPONSE_SUCCESS_MESSAGE,
            RESPONSE_PARAMS_KEY  : {}
        }
        RESPONSE_DICT[RESPONSE_RESULT_KEY]["server_ip"] = get_local_ip()
        RESPONSE_DICT[RESPONSE_RESULT_KEY]["time"] = time.strftime(datefmt,time.localtime())
        result_dict = {}
        try:
            result_dict = _func_obj(*args, **kwargs)
        except Exception,e:
            finish_time = time.time()
            RESPONSE_DICT[RESPONSE_RESULT_KEY]["duration"] = str(finish_time - receive_time) + "ms"
            RESPONSE_DICT[RESPONSE_PARAMS_KEY].update(**kwargs)
            RESPONSE_DICT[RESPONSE_CODE_KEY] = RESPONSE_FAILED_CODE
            RESPONSE_DICT[RESPONSE_MESSAGE_KEY] = e.message
            return json.dumps(RESPONSE_DICT)
        RESPONSE_DICT[RESPONSE_RESULT_KEY].update(result_dict)
        RESPONSE_DICT[RESPONSE_PARAMS_KEY].update(**kwargs)
        finish_time = time.time()
        RESPONSE_DICT[RESPONSE_RESULT_KEY]["duration"] = str(finish_time - receive_time) + "ms"
        RESPONSE_DICT[RESPONSE_CODE_KEY] = RESPONSE_SUCCESS_CODE
        RESPONSE_DICT[RESPONSE_MESSAGE_KEY] = RESPONSE_SUCCESS_MESSAGE
        return json.dumps(RESPONSE_DICT)
    return wrapTheFunction

def wei_logger(_cls_name = "logger_helper",
               _log_level = logging.INFO):# !/usr/bin/python
    _format_str = '[%(levelname)s] : %(asctime)s %(filename)s[line:%(lineno)d]  %(message)s'
    logging.basicConfig(level=logging.DEBUG,
                        format=_format_str,
                        filename=logger_file_name,
                        datefmt=logger_time_fmt)

    console = logging.StreamHandler()
    console.setLevel(logging.INFO)
    console_formatter = logging.Formatter(_format_str)
    console.setFormatter(console_formatter)

    rotating_file_handler = RotatingFileHandler(logger_file_name, maxBytes=1024 * 1024 * 50, backupCount=5)
    rotating_file_handler.setLevel(logging.DEBUG)
    rotating_file_handler.setFormatter(console_formatter)

    logger = logging.getLogger(_cls_name)
    logger.addHandler(console)
    logger.addHandler(rotating_file_handler)
    logger.debug('%s logger init success!!' % _cls_name)
    return logger

logger = wei_logger("GitManageEntry")

def get_http_session(pool_connections=1, pool_maxsize=10, max_retries=3):
    '''
    http连接池
    pool_connections 要缓存的 urllib3 连接池的数量。
    pool_maxsize 要保存在池中的最大连接数。
    max_retries 每个连接的最大重试次数
    '''
    session =requests.session()
    # 创建适配器
    adapter = requests.adapters.HTTPAdapter(pool_connections=pool_connections,
                                            pool_maxsize=pool_maxsize, max_retries=max_retries)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session

session = get_http_session()

class CCInterfaceHandler(object):
    def __init__(self):
        self.cc_host = CC_HOST
        self.url_sep = "/"
        self.git_project_query_url = "waic/v1/data/git_project"
        self.git_project_add_url = "waic/v1/data/git_project/add"
        self.git_project_update_url = "waic/v1/data/git_project/update"
        self.git_project_member_query_url = "waic/v1/data/git_project_member"
        self.git_project_member_add_url = "waic/v1/data/git_project_member/add"
        self.git_project_member_update_url = "waic/v1/data/git_project_member/update"
        self.interface_git_project_query = self.url_sep.join([self.cc_host, self.git_project_query_url ])
        self.interface_git_project_add = self.url_sep.join([self.cc_host, self.git_project_add_url ])
        self.interface_git_project_update = self.url_sep.join([self.cc_host, self.git_project_update_url ])
        self.interface_git_project_member_query = self.url_sep.join([self.cc_host, self.git_project_member_query_url])
        self.interface_git_project_member_add = self.url_sep.join([self.cc_host, self.git_project_member_add_url ])
        self.interface_git_project_member_update = self.url_sep.join([self.cc_host, self.git_project_member_update_url ])
        self.response_format_param = {"responseFormat" : "json"}
        self.role_mapping = {
            git_const.ACCESS_GUEST : "guest",
            git_const.ACCESS_REPORTER : "reporter",
            git_const.ACCESS_DEVELOPER : "developer",
            git_const.ACCESS_MASTER : "master",
            git_const.ACCESS_OWNER : "owner",
        }

    def get_value(self, _key_path = "cluster.json.weibox_path", _dict = {}):
        if(_dict == None):
            return ""
        tmp_obj = _dict.get(_key_path)
        if(tmp_obj == None):
            point_idx = _key_path.find(".")
            if(point_idx < 0):
                return ""
            tail_dict = _dict.get(_key_path[0:point_idx])
            if(isinstance(tail_dict,list)):
                tail_dict = tail_dict[0] if len(tail_dict)>0 else None
            tmp_obj = self.get_value(_key_path[point_idx+1:], tail_dict)
        return tmp_obj

    def _git_project_report_interface(self, _id):
        self.cc_project_count = 0
        try:
            self.id_param = self.response_format_param.copy()
            self.id_param.update({"id" : _id})
            r = session.get(self.interface_git_project_query, params = self.id_param)
            self.cc_project_count = self.get_value("result.totalCount",r.json())
        except Exception,e:
            logger.warn(str(e) + e.message)
        return self.interface_git_project_update if(self.cc_project_count > 0) else self.interface_git_project_add

    def git_project_report(self, _project_data = {}):
        self.id = _project_data.get("id")
        try:
            self.interface_git_project_report = self._git_project_report_interface(_id=self.id)
            self.report_param = self.response_format_param.copy()
            self.report_param.update(_project_data)
            r = session.post(self.interface_git_project_report, self.report_param)
            return self.get_value("code",r.json())
        except Exception,e:
            logger.warn(str(e) + e.message)

    def _git_project_member_query(self, _id):
        try:
            self.id_param = self.response_format_param.copy()
            self.id_param.update({"id" : _id})
            r = session.get(self.interface_git_project_member_query,  params = self.id_param)
            self.cc_project_member = self.get_value("result.data",r.json())
            self.cc_project_member_username_set = set([
                user_dict.get("user_id", "") for user_dict in self.cc_project_member
            ])
            return self.cc_project_member_username_set
        except Exception,e:
            logger.warn(str(e) + e.message)

    def _git_project_member_report_interface(self, _project_id, _user_id):
        self.cc_project_member_count = 0
        try:
            self.id_param = self.response_format_param.copy()
            self.id_param.update({"project_id" : _project_id, "user_id" : _user_id})
            r = session.get(self.interface_git_project_member_query, params = self.id_param)
            self.cc_project_count = self.get_value("result.totalCount",r.json())
        except Exception,e:
            logger.warn(str(e) + e.message)
        return self.interface_git_project_member_update if(self.cc_project_count > 0) else self.interface_git_project_member_add

    def git_project_member_report(self, _project_id, _project_member_data = []):
        self.id = _project_id
        try:
            for project_user in _project_member_data:
                cc_project_user_role = self.role_mapping.get(project_user.attributes.get("access_level",git_const.ACCESS_DEVELOPER))
                cc_project_user_id = project_user.attributes.get("username")
                cc_project_member_id = "".join([str(_project_id), "0", str(project_user.attributes.get("id"))])
                self.interface_git_project_member_report = self._git_project_member_report_interface(
                    _project_id=_project_id, _user_id=cc_project_user_id)
                self.id_param = self.response_format_param.copy()
                self.id_param.update({
                    "id" : cc_project_member_id,
                    "project_id" : _project_id,
                    "user_id" : cc_project_user_id,
                    "role" : cc_project_user_role
                })
                r = session.post(self.interface_git_project_member_report, self.id_param)

        except Exception,e:
            logger.warn(str(e) + e.message)

class GitProjectEntry(CCInterfaceHandler):
    def __init__(self, _project):
        super(GitProjectEntry, self).__init__()
        self.logger = logger
        self.project = _project
        self.attribute = self.project.attributes
        self.logger.debug("parse project name : {PROJECT_NAME} with id : {PROJECT_ID}.".format(
            PROJECT_NAME = self._self_attribute_get("name"),
            PROJECT_ID = self.project.get_id()
        ))

        # id, name, description, owner, web_url, last_activity_at, created_at, num_commits, member_list
        self.KEY_ID = "id"
        self.KEY_NAME = "name"
        self.KEY_DESCRIPTION = "description"

        self.KEY_OWNER = "owner"
        self.KEY_OWNER_WEB_URL = "owner_web_url"

        self.KEY_LAST_ACTIVITY_AT = "last_activity_at"
        self.KEY_CREATED_AT = "created_at"
        self.KEY_ACTIVE_STATUS = "status"

        self.KEY_NAMESPACE_PATH = "namespace_path"
        self.KEY_NAMESPACE_WEB_URL = "namespace_web_url"

        self.KEY_MEMBER_USERNAME_LIST = "member_username_list"

        self.KEY_WEB_URL = "web_url"
        self.KEY_HTTP_URL_TO_REPO = "http_url_to_repo"
        self.logger.debug(str(self._self_init()))

    def _date_formatter(self, _git_date_stamp_str):
        try:
            return date_parser.parse(_git_date_stamp_str).strftime(database_datefmt)
        except Exception,e:
            self.logger.warn("date stamp convert failed. with {INNER_GIT_STAMP}".format(INNER_GIT_STAMP = _git_date_stamp_str))

    def _git_status_active_judge(self, _git_last_active_at_dt):
        # status 1 活跃，0不活跃
        KEY_STATUS_ACTIVE = 1
        KEY_STATUS_INACTIVE = 0

        today=datetime.datetime.today()
        month_time_delta=datetime.timedelta(days = 30, hours=1)
        activity_threshold = today - month_time_delta
        activity_status = KEY_STATUS_ACTIVE if(
            date_parser.parse(_git_last_active_at_dt) > activity_threshold) else KEY_STATUS_INACTIVE
        return activity_status

    def _self_init(self):

        self.member_list = []
        self.guess_owner = None

        self.last_activity_at = self._date_formatter(self._self_attribute_get("last_activity_at"))

        self.data = {

            self.KEY_ID  : self.project.get_id(), # self.project.id
            self.KEY_NAME : self._self_attribute_get("name"), # self.project.name
            self.KEY_DESCRIPTION : self._self_attribute_get("description"),

            self.KEY_OWNER : self._self_attribute_get("owner", {}).get("username"),
            self.KEY_OWNER_WEB_URL : self._self_attribute_get("owner", {}).get("web_url"),

            self.KEY_LAST_ACTIVITY_AT : self.last_activity_at,
            self.KEY_CREATED_AT : self._date_formatter(self._self_attribute_get("created_at")),
            self.KEY_ACTIVE_STATUS : self._git_status_active_judge(self.last_activity_at),

            self.KEY_NAMESPACE_PATH : self._self_attribute_get("namespace", {}).get("path"),
            self.KEY_NAMESPACE_WEB_URL : self._self_attribute_get("namespace", {}).get("web_url"),

            self.KEY_MEMBER_USERNAME_LIST : self._self_member_username_list(),

            self.KEY_WEB_URL : self._self_attribute_get("web_url"),
            self.KEY_HTTP_URL_TO_REPO : self._self_attribute_get("description")

        }

        if(self.data.get(self.KEY_OWNER) == None):
            self.data[self.KEY_OWNER] = self._self_member_top_access_user_name()

        if(self.data.get(self.KEY_OWNER_WEB_URL) == None):
            self.data[self.KEY_OWNER_WEB_URL] = self._self_member_top_access_user_name()

        self.git_project_member_report(self.project.get_id(),self.member_list)
        if('200' != str(self.git_project_report(self.data))):
            self.logger.warn("reported git_project failed! with project_id : {PR_ID} and project_name : {PR_NAME} .\n"
                             "while data is {PR_DATA}".format(PR_ID = str(self.data.get(self.KEY_ID, "NULL_ID")),
                                                              PR_NAME = str(self.data.get(self.KEY_NAME, "NULL_NAME")),
                                                              PR_DATA = str(self.data)
                                                              )
                             )
        else:
            self.logger.info("reported git_project succeed! with project_id : {PR_ID} and project_name : {PR_NAME} .\n"
                             "while data is {PR_DATA}".format(PR_ID = str(self.data.get(self.KEY_ID, "NULL_ID")),
                                                              PR_NAME = str(self.data.get(self.KEY_NAME, "NULL_NAME")),
                                                              PR_DATA = str(self.data)
                                                              )
                             )
        return self.data

    def _self_member_list(self):
        if(self.member_list != None and len(self.member_list) > 0):
            return self.member_list
        self.member_list =  self.project.members.list(all=True)
        if(len(self.member_list)<1):
            self.member_list = self.project.members.all(all=True)
        return self.member_list

    def _self_member_top_access_user(self):
        if(not self.guess_owner):
            self.member_list = self._self_member_list()
            self.sorted_member_list = sorted(self.member_list,key=lambda x : x.attributes.get("access_level",0),reverse=True)
            self.guess_owner = self.sorted_member_list[0] if(len(self.sorted_member_list) > 0) else None
        return self.guess_owner

    def _self_member_top_access_user_weburl(self):
        self.guess_owner = self._self_member_top_access_user()
        self.top_access_username = self.guess_owner.attributes.get("web_url", None) if(self.guess_owner) else None
        return self.top_access_username

    def _self_member_top_access_user_name(self):
        self.guess_owner = self._self_member_top_access_user()
        self.top_access_username = self.guess_owner.attributes.get("username", None) if(self.guess_owner) else None
        return self.top_access_username

    def _self_member_username_list(self):
        '''
        gitlab document:
            Examples
            List the project members:

            members = project.members.list()
            List the project members recursively (including inherited members through ancestor groups):

            members = project.members.all(all=True)
            Search project members matching a query string:

            members = project.members.list(query='bar')
            Get a single project member:

            member = project.members.get(user_id)
            Add a project member:

            member = project.members.create({'user_id': user.id, 'access_level':
                                             gitlab.DEVELOPER_ACCESS})
            Modify a project member (change the access level):

            member.access_level = gitlab.MAINTAINER_ACCESS
            member.save()
            Remove a member from the project team:

            project.members.delete(user.id)
            # or
            member.delete()
            Share/unshare the project with a group:

            project.share(group.id, gitlab.DEVELOPER_ACCESS)
            project.unshare(group.id)
        :return:
        '''
        self.member_username_list = [
            member.attributes.get("username") for member in self._self_member_list()
        ]
        return self.member_username_list

    def _self_attribute_get(self, _key, _default_value = ""):
        return self.attribute.get(_key, _default_value)

    def to_json(self):
        self.json_string = json.dumps(self.data)
        return self.json_string

class GitManageEntry(object):
    def __init__(self, _private_token, _fetch_all = False, _exclude_project = True, _visibility = git_const.VISIBILITY_PRIVATE):
        '''
        gitlab document:
            The API provides several filtering parameters for the listing methods:

            archived: if True only archived projects will be returned
            visibility: returns only projects with the specified visibility (can be public, internal or private)
            search: returns project matching the given pattern
            Results can also be sorted using the following parameters:

            order_by: sort using the given argument. Valid values are id, name, path, created_at, updated_at and last_activity_at. The default is to sort by created_at
            sort: sort order (asc or desc)
            # List all projects (default 20)
            projects = gl.projects.list(all=True)
            # Archived projects
            projects = gl.projects.list(archived=1)
            # Limit to projects with a defined visibility
            projects = gl.projects.list(visibility='public')

            # List owned projects
            projects = gl.projects.list(owned=True)

            # List starred projects
            projects = gl.projects.list(starred=True)

            # Search projects
            projects = gl.projects.list(search='keyword')
        :param _private_token:
        :param _fetch_all:
        :param _exclude_project:
        :param _visibility:
        '''
        self.url = "http://git.intra.weibo.com/"
        self.private_token = _private_token
        self.fetch_all = _fetch_all
        self.exclude_project = _exclude_project
        self.logger = logger
        self.visibility = _visibility
        self.git = gitlab.Gitlab(self.url, self.private_token)
        self.git.auth()
        self.username = self.git.user.username
        self.user_id = self.git.user.id
        self.logger_info = "parse private token for username : {USER_NAME} with id : {USER_ID}.".format(
            USER_NAME = self.username,
            USER_ID = self.user_id
        )
        print (self.logger_info)
        self.logger.info(self.logger_info)
        self.user_last_sign_in_at = self.git.user.last_sign_in_at
        self.user_last_activity_on = self.git.user.last_activity_on

        self.logger.info("start retrieve git projects")
        self.projects = self.git.projects.list(all=True, visibility = self.visibility) \
            if(self.fetch_all) else \
            self.git.projects.list(visibility = self.visibility)
        self.logger.info("retrieve git projects done!")

        self.gpe = [GitProjectEntry(_project=project) for project in self.projects]
        self.gpe_data = [_gpe.data for _gpe in self.gpe]
        self.data = {
            "url" : self.url ,
            "private_token" : self.private_token ,
            "username" : self.username ,
            "user_id" : self.user_id ,
            "user_last_sign_in_at" : self.user_last_sign_in_at ,
            "user_last_activity_on" : self.user_last_activity_on,
            "count_of_projects" : len(self.gpe)
        }

        if(not self.exclude_project):
            self.data.update({"projects" : self.gpe_data})
        self.logger.debug(str(self.data))

    def to_json(self):
        self.json_string = json.dumps(self.data)
        return self.json_string

@app.route('/waic/git-retrieve/user-auth/<private_token>',methods = ['POST','GET'])
@result_decorator
def git_user_auth(private_token):
    gme = GitManageEntry(_private_token=private_token)
    return gme.data
    # return gme.to_json()

@app.route('/waic/git-retrieve/preview/<private_token>',methods = ['POST','GET'])
@result_decorator
def git_retrieve_preview(private_token):
    gme = GitManageEntry(_private_token=private_token, _exclude_project=False)
    return gme.data
    # return gme.to_json()

@app.route('/waic/git-retrieve/all/<private_token>',methods = ['POST','GET'])
@result_decorator
def git_retrieve_all_projects(private_token):
    gme = GitManageEntry(_private_token=private_token, _fetch_all=True,  _exclude_project=False)
    return gme.data
    # return gme.to_json()

@app.route('/waic/git-retrieve/all-projects/<private_token>',methods = ['POST','GET'])
@result_decorator
def git_retrieve_all_internal_projects(private_token):
    gme = GitManageEntry(_private_token=private_token, _visibility=git_const.VISIBILITY_INTERNAL, _fetch_all=True,  _exclude_project=False)
    return gme.data
    # return gme.to_json()

if __name__ == '__main__':
    app.run(host=get_local_ip())
    # job_state("12345678")
    token = "anNN8srBMewKzvokyxaL"
    # token = "yk-1v6J_txopP7x7xtif"
    # gme = GitManageEntry(_private_token=token, _fetch_all=True, _exclude_project=False)
    gme = GitManageEntry(_private_token=token, _exclude_project=False)

code_stage = '''
# -*- coding: utf-8 -*-
import gitlab
from gitlab.v4.objects import Project
url = "http://git.intra.weibo.com/"
token = "fv29Z9seyCs7xRhYSUt8"
token = "anNN8srBMewKzvokyxaL"
gl = gitlab.Gitlab(url, token, per_page=50)

projects = gl.projects.list(all=True)

class GitManageEntry(Project):
    def toJson(self):

        "url" : self.url ,
        "private_token" : self.private_token ,
        "git" : self.git ,
        "username" : self.username ,
        "user_id" : self.user_id ,
        "user_last_sign_in_at" : self.user_last_sign_in_at ,
        "user_last_activity_on" : self.user_last_activity_on ,

# 登录
gl = gitlab.Gitlab(url, token)

# ---------------------------------------------------------------- #
# 获取第一页project
projects = gl.projects.list()
# 获取所有的project
projects = gl.projects.list(all=True)
# ---------------------------------------------------------------- #

# ---------------------------------------------------------------- #
# 获取所有project的name,id
for p in gl.projects.list(all=True, as_list=False):
    print(p.name, p.id)
# ---------------------------------------------------------------- #

# ---------------------------------------------------------------- #
# 获取第一页project的name,id
for p in gl.projects.list(page=1):
    print(p.name, p.id)
# ---------------------------------------------------------------- #

# ---------------------------------------------------------------- #
# 通过指定id 获取 project 对象
project = gl.projects.get(501)
# ---------------------------------------------------------------- #

# ---------------------------------------------------------------- #
# 查找项目
projects = gl.projects.list(search='keyword')
# ---------------------------------------------------------------- #

# ---------------------------------------------------------------- #
# 创建一个项目
project = gl.projects.create({'name':'project1'})
# ---------------------------------------------------------------- #

# ---------------------------------------------------------------- #
# 获取公开的项目
projects = gl.projects.list(visibility='public')  # public, internal or private
# ---------------------------------------------------------------- #

#  获取 project 对象是以下操作的基础

# ---------------------------------------------------------------- #
# 通过指定project对象获取该项目的所有分支
branches = project.branches.list()
print(branches)
# ---------------------------------------------------------------- #

# ---------------------------------------------------------------- #
# 获取指定分支的属性
branch = project.branches.get('master')
print(branch)
# ---------------------------------------------------------------- #

# ---------------------------------------------------------------- #
# 创建分支
branch = project.branches.create({'branch_name': 'feature1',
                                  'ref': 'master'})
# ---------------------------------------------------------------- #

# ---------------------------------------------------------------- #
# 删除分支
project.branches.delete('feature1')
# ---------------------------------------------------------------- #

# ---------------------------------------------------------------- #
# 分支保护/取消保护
branch.protect()
branch.unprotect()
# ---------------------------------------------------------------- #

# ---------------------------------------------------------------- #
# 获取指定项目的所有tags
tags = project.tags.list()

# 获取某个指定tag 的信息
tags = project.tags.list('1.0')

# 创建一个tag
tag = project.tags.create({'tag_name':'1.0', 'ref':'master'})

# 设置tags 说明:
tag.set_release_description('awesome v1.0 release')

# 删除tags
project.tags.delete('1.0')
# or
tag.delete()

# ---------------------------------------------------------------- #
# 获取所有commit info
commits = project.commits.list()
for c in commits:
    print(c.author_name, c.message, c.title)
# ---------------------------------------------------------------- #

# ---------------------------------------------------------------- #
# 获取指定commit的info
commit = project.commits.get('e3d5a71b')
# ---------------------------------------------------------------- #

# ---------------------------------------------------------------- #
# 获取指定项目的所有merge request
mrs = project.mergerequests.list()
print(mrs)
# ---------------------------------------------------------------- #

# ---------------------------------------------------------------- #
# 获取 指定mr info
mr = project.mergesession.get(mr_id)
# ---------------------------------------------------------------- #

# ---------------------------------------------------------------- #
#  创建一个merge request
mr = project.mergerequests.create({'source_branch':'cool_feature',
                                   'target_branch':'master',
                                   'title':'merge cool feature', })
# ---------------------------------------------------------------- #

# ---------------------------------------------------------------- #
# 更新一个merge request 的描述
mr.description = 'New description'
mr.save()
# ---------------------------------------------------------------- #

# ---------------------------------------------------------------- #
# 开关一个merge request  (close or reopen):
mr.state_event = 'close'  # or 'reopen'
mr.save()
# ---------------------------------------------------------------- #

# ---------------------------------------------------------------- #
# Delete a MR:
project.mergerequests.delete(mr_id)
# or
mr.delete()
# ---------------------------------------------------------------- #

# ---------------------------------------------------------------- #
# Accept a MR:
mr.merge()
# ---------------------------------------------------------------- #

# ---------------------------------------------------------------- #
# 指定条件过滤 所有的merge request
# state: state of the MR. It can be one of all, merged, opened or closed
# order_by: sort by created_at or updated_at
# sort: sort order (asc or desc)
mrs = project.mergerequests.list(state='merged', sort='asc')  # all, merged, opened or closed
# ---------------------------------------------------------------- #

# ---------------------------------------------------------------- #
# 创建一个commit
data = {
    'branch_name': 'master',  # v3
    'commit_message': 'blah blah blah',
    'actions': [
        {
            'action': 'create',
            'file_path': 'blah',
            'content': 'blah'
        }
    ]
}
commit = project.commits.create(data)
# ---------------------------------------------------------------- #

# ---------------------------------------------------------------- #
# Compare two branches, tags or commits:
result = project.repository_compare('develop', 'feature-20180104')
print(result)
# get the commits

for commit in result['commits']:
    print(commit)
#
# get the diffs
for file_diff in result['diffs']:
    print(file_diff)
# ---------------------------------------------------------------- #

# ---------------------------------------------------------------- #
# get the commits
for commit in result['commits']:
    print(commit)
#
# get the diffs
for file_diff in result['diffs']:
    print(file_diff)
# ---------------------------------------------------------------- #

'''