cube.js 通过presto-gateway 进行连接

cube.js 对于presto 的支持是通过presto-client 刚好简单修改了一个可以支持presto-gateway 连接的
以下是一个简单的集成，以及关于集成中原有的一些修改

环境准备

• docker-compose 文件

version: "3"

services:

  proxy:

     image: dalongrong/presto-gateway:1.6.1

     ports:

     - "8888:8888"

     - "8082:8082"

     - "8083:8083"

     build: ./

     volumes:

     - "./config.yml.template:/config.yml.template"

  presto1:

     image: starburstdata/presto

     ports:

     - "8080:8080"

  presto2:

     image: starburstdata/presto

     ports:

     - "8081:8080"

• presto-gateway 配置

requestRouter:

  port: 8888

  name: prestoRouter

  cacheDir: /var/log/prestoproxy/cache

  historySize: 1000

backends:

  - localPort: 8082

    name: presto1

    proxyTo: http://presto1:8080

    routingGroup: adhoc

  - localPort: 8083

    name: presto2

    proxyTo: http://presto2:8080

    routingGroup: scheduled

server:

  applicationConnectors:

    - type: http

      port: 8090

  adminConnectors:

    - type: http

      port: 8091

notifier:

  smtpHost: localhost

  smtpPort: 587

  sender: presto-gw-monitor-noreply@lyft.com

  recipients:

    - prestodev@yourorg.com

modules:

  - com.lyft.data.gateway.module.ProxyBackendProviderModule

  - com.lyft.data.gateway.module.GatewayProviderModule

  - com.lyft.data.gateway.module.NotifierModule

managedApps:

  - com.lyft.data.gateway.GatewayManagedApp

  - com.lyft.data.gateway.ActiveClusterMonitor

# Logging settings.

logging:

  # The default level of all loggers. Can be OFF, ERROR, WARN, INFO, DEBUG, TRACE, or ALL.

  level: INFO

  # Logger-specific levels.

  loggers:

    com.lyft: DEBUG

  appenders:

    - type: console

    - type: file

      currentLogFilename: /var/log/prestoproxy/prestoproxy-java.log

      archivedLogFilenamePattern: /var/log/prestoproxy/prestoproxy-java-%d{yyyy-MM-dd}-%i.log.gz

      archivedFileCount: 7

      timeZone: UTC

      maxFileSize: 100MB

cube.js presto 项目

• 初始化

cubejs  create -d presto mypresto

• 修改生成的项目
  主要是关于presto-client 的替换以及环境变量的添加，还有就是部分代码的简单修改
• package.json 改进

 

yarn add @dalongrong/presto-client

• 添加环境变量
  .env

 

CUBEJS_DB_HOST=localhost

CUBEJS_DB_USER=dalong

CUBEJS_DB_PORT=8888

CUBEJS_DB_CATALOG=memory

CUBEJS_DB_TYPE=prestodb

CUBEJS_ROUTING_GROUP=scheduled

CUBEJS_DB_SCHEMA=default

CUBEJS_API_SECRET=71a4454929e8b429158759a29f4a7ddfcc957505efed7e5a93e02a859d7bd9f7ed7e49daac2243024a94cdaa3e88faa4cd3946dae7aea562f947bb1ed525125b

• 修改关于presto 连接
  文件node_modules/@cubejs-backend/prestodb-driver/driver/PrestoDriver.js

const presto = require('@dalongrong/presto-client');

const {

  map, zipObj, prop, concat

} = require('ramda');

const BaseDriver = require('@cubejs-backend/query-orchestrator/driver/BaseDriver');

const SqlString = require('sqlstring');

​

class PrestoDriver extends BaseDriver {

  constructor(config) {

    super();

​

    this.config = {

      host: process.env.CUBEJS_DB_HOST,

      port: process.env.CUBEJS_DB_PORT,

      catalog: process.env.CUBEJS_DB_CATALOG,

      schema: process.env.CUBEJS_DB_SCHEMA,

      user: process.env.CUBEJS_DB_USER,

      routingGroup: process.env.CUBEJS_ROUTING_GROUP,

      basic_auth: process.env.CUBEJS_DB_PASS ? {

        user: process.env.CUBEJS_DB_USER,

        password: process.env.CUBEJS_DB_PASS

      } : undefined,

      ...config

    };

​

    this.catalog = this.config.catalog;

    this.client = new presto.Client(this.config);

  }

​

  testConnection() {

    const query = SqlString.format(`show catalogs like ?`, [`%${this.catalog}%`]);

​

    return this.queryPromised(query)

      .then(catalogs => {

        if (catalogs.length === 0) {

          throw new Error(`Catalog not found '${this.catalog}'`);

        }

      });

  }

​

  query(query, values) {

    const queryWithParams = SqlString.format(query, values);

    return this.queryPromised(queryWithParams);

  }

​

  queryPromised(query) {

    return new Promise((resolve, reject) => {

      let fullData = [];

​

      this.client.execute({

        query,

        schema: this.config.schema || 'default',

        routingGroup: this.config.routingGroup,

        data: (error, data, columns) => {

          const normalData = this.normalizeResultOverColumns(data, columns);

          fullData = concat(normalData, fullData);

        },

        success: () => {

          resolve(fullData);

        },

        error: error => {

          reject(error.message || JSON.stringify(error));

        }

      });

    });

  }

​

  normalizeResultOverColumns(data, columns) {

    const columnNames = map(prop('name'), columns || []);

    const arrayToObject = zipObj(columnNames);

    return map(arrayToObject, data || []);

  }

}

​

module.exports = PrestoDriver;

启动&&效果

• 启动

yarn  dev

• 效果

• 生成schema
• 集成效果

• presto-gatesway 效果

说明

以上是一个简单的集成，通过presto-gateway 可以方便我们的管理，是一个很不错的工具

参考资料

https://github.com/cube-js/cube.js
https://github.com/rongfengliang/presto-client-node
https://github.com/lyft/presto-gateway