使用terraform 进行gitlab 代码仓库批量迁移

 

gitlab 的代码是在文件目录中，这个对于批量迁移很简单，只需要copy 文件夹（但是对于不同gitlab server 可能需要重新设置目录权限）

几个问题

• 大批量仓库tf resource问题

直接使用默认的不是很好，需要配置的比较多，解决方法，可以通过直接查询数据库数据，使用模版引擎生成tf 文件，可选的开发语言很多，但是使用nodejs
开发可能会比较方便的

• 仓库代码拷贝问题

推荐通过rsync 等工具，进行数据同步，因为git repo 存在软连接的问题，一般的scp 不能附带信息

• tf 模版编写

因为可能存在多层级group，所以需要查询原有gitlab pg 数据库进行分析处理，这个分析下sql 就可以了（通过自关联查询，多级处理）

• 用户权限处理

这个稍有复杂，tf gitlab provider 提供了用户权限操作的，但是对于不用的gitlab server 系统层面只识别的是用户id，但是这个id 是系统生成的
这个可以通过email 等能确认是唯一的进行关联处理，通过查询，在模版生成的时候指定用户id

一个参考实现

• 模版处理

  通过nodejs,为了方便管理，对于仓库信息通过rest 接口暴露，使用dotenv 管理环境变量

yarn add node-fetch mustache dotenv
app.js： 模版生成代码
const fetch = require('node-fetch');
require("dotenv").config()
const Mustache = require("mustache");
const fs = require("fs")
fetch(process.env.gitlab_api)
    .then(res => res.json())
    .then(body => {
        fs.readFile("template/temp.mu", {
            encoding: "utf8"
        }, (err,result) => {
            if (err) {
                console.log("some wrong")
            } else {
                let output = Mustache.render(result, body);
                console.log(output)
            }
        })
});
模版：
provider "gitlab" {
    base_url = "http://gitlabserver/api/v4/"
    token = "${var.gitlab_token}"
}
{{#data.groups_parent}}
resource "gitlab_group" "{{name}}" {
    name = "{{name}}"
    path = "{{path}}"
    description = "{{description}}"
}
{{/data.groups_parent}}

{{#data.subgroups}}
resource "gitlab_group" "{{name}}" {

    name = "{{name}}"
    path = "{{path}}"
    description = "{{description}}"
    parent_id = "${gitlab_group.{{parent}}.id}"
}
{{/data.subgroups}}
{{#data.projects}}
resource "gitlab_project" "{{name}}" {
    name = "{{name}}"
    namespace_id = "${gitlab_group.{{group}}.id}"
}
{{/data.projects}}
仓库代码restapi格式：
{
   "data": {
    "projects": [
        {
          "name": "firstrong",
          "path":"firstrong",
          "description":"firstrong",
          "group" : "firstrong"
        },
        {
           "name": "secondrong",
           "path":"secondrong",
           "description":"secondrong",
           "group": "secondrong"
        },
        {
            "name": "thirdrong",
            "path":"thirdrong",
            "description":"thirdrong",
            "group": "thirdrong"
        }
    ],
    "groups_parent":[
        {
          "name":"firstrong",
          "path":"firstrong",
          "description":"firstrong"
        },
        {
            "name":"secondrong",
            "path":"secondrong",
             "description":"secondrong"
        },
        {
            "name":"thirdrong",
            "path":"thirdrong",
            "description":"thirdrong"
        }
    ],
    "subgroups":[
        {
          "name":"demoapp",
          "path":"demoapp",
          "parent":"firstrong"
        },
        {
            "name":"demoapp2",
            "path":"demoapp2",
            "parent":"secondrong"
        },
        {
            "name":"demoapp3",
            "path":"demoapp3",
            "parent":"thirdrong"
        }
    ]
   }
}

• 生成的tf 内容（参考）

provider "gitlab" {
    base_url = "http://gitlabserver/api/v4/"
    token = "${var.gitlab_token}"
}
resource "gitlab_group" "firstrong" {
    name = "firstrong"
    path = "firstrong"
    description = "firstrong"
}
resource "gitlab_group" "secondrong" {
    name = "secondrong"
    path = "secondrong"
    description = "secondrong"
}
resource "gitlab_group" "thirdrong" {
    name = "thirdrong"
    path = "thirdrong"
    description = "thirdrong"
}

resource "gitlab_group" "demoapp" {

    name = "demoapp"
    path = "demoapp"
    description = ""
    parent_id = "${gitlab_group.firstrong.id}"
}
resource "gitlab_group" "demoapp2" {

    name = "demoapp2"
    path = "demoapp2"
    description = ""
    parent_id = "${gitlab_group.secondrong.id}"
}
resource "gitlab_group" "demoapp3" {

    name = "demoapp3"
    path = "demoapp3"
    description = ""
    parent_id = "${gitlab_group.thirdrong.id}"
}
resource "gitlab_project" "firstrong" {
    name = "firstrong"
    namespace_id = "${gitlab_group.firstrong.id}"
}
resource "gitlab_project" "secondrong" {
    name = "secondrong"
    namespace_id = "${gitlab_group.secondrong.id}"
}
resource "gitlab_project" "thirdrong" {
    name = "thirdrong"
    namespace_id = "${gitlab_group.thirdrong.id}"
}

• gitlab group层级查询(主要是判断使用子group 的情况)

  这是一个简单四级的处理，实际情况在处理下

select a.name as parent,b.name,b.path,b.description,c.name as thirdname,c.path as thirdpath ,c.description as thirddescription, d.name as fouthname, d.path as fouthpaht from namespaces a join namespaces b on a.id=b.parent_id join namespaces c on b.id=c.parent_id join namespaces d on c.id=d.parent_id where b.parent_id is not null and c.parent_id is not null;

rest api 生成方式

上边有一个简单的rest 内容格式，设计的生成我们可以直接使用sqler（简单方便，写几个简单的sql 就可以搞定了）

• sqler docker 运行配置

version: "3"
services:
  sqler:
    image: dalongrong/sqler:2.0
    volumes:
    - "./config/config-example.hcl:/app/config.example.hcl"
    environment:
    - "DSN=postgresql://username:password@server:port/dbname"
    ports:
    - "3678:3678"
    - "8025:8025"

• 配置文件(参考，很简单，写了几个，按照实际补充下，这个可能实际运行有问题)

group {
    exec = <<SQL
    SELECT disinct(name) from projects where namespace_id !=nil
    SQL
}

group_project {
    exec = <<SQL
    SELECT disinct(name) from projects where namespace_id !=nil
    SQL
}

gitlab_project {
    aggregate = ["group", "group_project"]
}

说明

以上只是一个简单的说明，以及简单实践，实际迁移可能没有这么简单，或有各类的问题，还是需要部分人员的介入处理，因为毕竟每个
团队对于git 的使用情况是不一样的，需要实际问题实际考虑，变通下,实际上对于git 的同步可以使用tf 操作，使用remote_exec
provisioners

参考资料

https://www.terraform.io/docs/providers/gitlab/index.html