From aad22e1cb377955ef37b5c50c52e202964d27c11 Mon Sep 17 00:00:00 2001 From: kokokuo Date: Thu, 6 Jul 2023 14:37:05 +0800 Subject: [PATCH 1/9] feat(extension-driver-clickhouse): create clickhouse driver extension - Create the "extension-driver-clickhouse" package - Create "ClickHouseDataSource". - update package.json to install "@clickhouse/client" --- .../.eslintrc.json | 18 ++ .../jest.config.ts | 14 ++ .../extension-driver-clickhouse/package.json | 32 ++++ .../extension-driver-clickhouse/project.json | 85 +++++++++ .../extension-driver-clickhouse/src/index.ts | 3 + .../src/lib/clickhouseDataSource.ts | 168 ++++++++++++++++++ .../src/lib/sqlBuilder.ts | 40 +++++ .../src/lib/typeMapper.ts | 63 +++++++ .../extension-driver-clickhouse/tsconfig.json | 22 +++ .../tsconfig.lib.json | 10 ++ .../tsconfig.spec.json | 15 ++ .../extension-driver-clickhouse/yarn.lock | 15 ++ tsconfig.base.json | 3 + workspace.json | 1 + 14 files changed, 489 insertions(+) create mode 100644 packages/extension-driver-clickhouse/.eslintrc.json create mode 100644 packages/extension-driver-clickhouse/jest.config.ts create mode 100644 packages/extension-driver-clickhouse/package.json create mode 100644 packages/extension-driver-clickhouse/project.json create mode 100644 packages/extension-driver-clickhouse/src/index.ts create mode 100644 packages/extension-driver-clickhouse/src/lib/clickhouseDataSource.ts create mode 100644 packages/extension-driver-clickhouse/src/lib/sqlBuilder.ts create mode 100644 packages/extension-driver-clickhouse/src/lib/typeMapper.ts create mode 100644 packages/extension-driver-clickhouse/tsconfig.json create mode 100644 packages/extension-driver-clickhouse/tsconfig.lib.json create mode 100644 packages/extension-driver-clickhouse/tsconfig.spec.json create mode 100644 packages/extension-driver-clickhouse/yarn.lock diff --git a/packages/extension-driver-clickhouse/.eslintrc.json b/packages/extension-driver-clickhouse/.eslintrc.json new file mode 100644 index 00000000..9d9c0db5 --- /dev/null +++ b/packages/extension-driver-clickhouse/.eslintrc.json @@ -0,0 +1,18 @@ +{ + "extends": ["../../.eslintrc.json"], + "ignorePatterns": ["!**/*"], + "overrides": [ + { + "files": ["*.ts", "*.tsx", "*.js", "*.jsx"], + "rules": {} + }, + { + "files": ["*.ts", "*.tsx"], + "rules": {} + }, + { + "files": ["*.js", "*.jsx"], + "rules": {} + } + ] +} diff --git a/packages/extension-driver-clickhouse/jest.config.ts b/packages/extension-driver-clickhouse/jest.config.ts new file mode 100644 index 00000000..97fe1657 --- /dev/null +++ b/packages/extension-driver-clickhouse/jest.config.ts @@ -0,0 +1,14 @@ +module.exports = { + displayName: 'extension-driver-clickhouse', + preset: '../../jest.preset.ts', + globals: { + 'ts-jest': { + tsconfig: '/tsconfig.spec.json', + }, + }, + transform: { + '^.+\\.[tj]s$': 'ts-jest', + }, + moduleFileExtensions: ['ts', 'js', 'html'], + coverageDirectory: '../../coverage/packages/extension-driver-clickhouse', +}; diff --git a/packages/extension-driver-clickhouse/package.json b/packages/extension-driver-clickhouse/package.json new file mode 100644 index 00000000..ff3f7c0c --- /dev/null +++ b/packages/extension-driver-clickhouse/package.json @@ -0,0 +1,32 @@ +{ + "name": "@vulcan-sql/extension-driver-clickhouse", + "description": "Clickhouse driver for VulcanSQL", + "version": "0.5.3", + "type": "commonjs", + "publishConfig": { + "access": "public" + }, + "keywords": [ + "vulcan", + "vulcan-sql", + "data", + "sql", + "database", + "data-warehouse", + "data-lake", + "api-builder", + "snowflake", + "snow" + ], + "repository": { + "type": "git", + "url": "https://github.com/Canner/vulcan.git" + }, + "license": "MIT", + "dependencies": { + "@clickhouse/client": "^0.1.1" + }, + "peerDependencies": { + "@vulcan-sql/core": "~0.5.3-0" + } +} diff --git a/packages/extension-driver-clickhouse/project.json b/packages/extension-driver-clickhouse/project.json new file mode 100644 index 00000000..3837061c --- /dev/null +++ b/packages/extension-driver-clickhouse/project.json @@ -0,0 +1,85 @@ +{ + "root": "packages/extension-driver-clickhouse", + "sourceRoot": "packages/extension-driver-clickhouse/src", + "targets": { + "build": { + "executor": "@nrwl/workspace:run-commands", + "options": { + "command": "yarn ts-node ./tools/scripts/replaceAlias.ts extension-driver-clickhouse" + }, + "dependsOn": [ + { + "projects": "self", + "target": "tsc" + }, + { + "projects": "self", + "target": "install-dependencies" + } + ] + }, + "tsc": { + "executor": "@nrwl/js:tsc", + "outputs": ["{options.outputPath}"], + "options": { + "outputPath": "dist/packages/extension-driver-clickhouse", + "main": "packages/extension-driver-clickhouse/src/index.ts", + "tsConfig": "packages/extension-driver-clickhouse/tsconfig.lib.json", + "assets": ["packages/extension-driver-clickhouse/*.md"], + "buildableProjectDepsInPackageJsonType": "dependencies" + }, + "dependsOn": [ + { + "projects": "dependencies", + "target": "build" + }, + { + "projects": "self", + "target": "install-dependencies" + } + ] + }, + "lint": { + "executor": "@nrwl/linter:eslint", + "outputs": ["{options.outputFile}"], + "options": { + "lintFilePatterns": ["packages/extension-driver-clickhouse/**/*.ts"] + } + }, + "test": { + "executor": "@nrwl/jest:jest", + "outputs": ["coverage/packages/extension-driver-clickhouse"], + "options": { + "jestConfig": "packages/extension-driver-clickhouse/jest.config.ts", + "passWithNoTests": true + }, + "dependsOn": [ + { + "projects": "self", + "target": "install-dependencies" + } + ] + }, + "publish": { + "executor": "@nrwl/workspace:run-commands", + "options": { + "command": "node ../../../tools/scripts/publish.mjs {args.tag} {args.version}", + "cwd": "dist/packages/extension-driver-clickhouse" + }, + "dependsOn": [ + { + "projects": "self", + "target": "build" + } + ] + }, + "install-dependencies": { + "executor": "@nrwl/workspace:run-commands", + "options": { + "command": "yarn", + "cwd": "packages/extension-driver-clickhouse" + } + } + }, + "tags": [] +} diff --git a/packages/extension-driver-clickhouse/src/index.ts b/packages/extension-driver-clickhouse/src/index.ts new file mode 100644 index 00000000..93a24cfb --- /dev/null +++ b/packages/extension-driver-clickhouse/src/index.ts @@ -0,0 +1,3 @@ +export * from './lib/clickhouseDataSource'; +import { ClickHouseDataSource } from './lib/clickhouseDataSource'; +export default [ClickHouseDataSource]; diff --git a/packages/extension-driver-clickhouse/src/lib/clickhouseDataSource.ts b/packages/extension-driver-clickhouse/src/lib/clickhouseDataSource.ts new file mode 100644 index 00000000..a43d4ece --- /dev/null +++ b/packages/extension-driver-clickhouse/src/lib/clickhouseDataSource.ts @@ -0,0 +1,168 @@ +import { + BindParameters, + DataResult, + DataSource, + ExecuteOptions, + ExportOptions, + InternalError, + RequestParameter, + VulcanExtensionId, +} from '@vulcan-sql/core'; +import { Stream } from 'stream'; +import * as fs from 'fs'; +import { + createClient, + ClickHouseClientConfigOptions, + ClickHouseClient, + ResultSet, + Row, +} from '@clickhouse/client'; +import { buildSQL } from './sqlBuilder'; +import { mapFromClickHouseType, mapToClickHouseType } from './typeMapper'; +import { omit } from 'lodash'; + +export type ClickHouseOptions = ClickHouseClientConfigOptions; +export type ClickHouseTLSOptions = { + ca_cert?: string; + cert?: string; + key?: string; +}; + +@VulcanExtensionId('clickhouse') +export class ClickHouseDataSource extends DataSource { + private logger = this.getLogger(); + private clientMapping = new Map< + string, + { client: ClickHouseClient; options?: ClickHouseOptions } + >(); + public override async onActivate() { + const profiles = this.getProfiles().values(); + for (const profile of profiles) { + this.logger.debug( + `Initializing profile: ${profile.name} using clickhouse driver` + ); + + // Omit and get the tls from connection, because clickhouse client support tls for bytes data, + const tls = omit(profile.connection, ['tls']) as ClickHouseTLSOptions; + const options: ClickHouseOptions = { + ...{ application: 'VulcanSQL' }, + ...profile.connection!, + }; + // Set TLS options is existed + if (tls.ca_cert && tls.cert && tls.key) { + options.tls = { + ca_cert: fs.readFileSync(tls.ca_cert), + cert: fs.readFileSync(tls.cert), + key: fs.readFileSync(tls.key), + }; + } else if (tls.ca_cert) { + options.tls = { ca_cert: fs.readFileSync(tls.ca_cert) }; + } + + const client = createClient(options); + this.clientMapping.set(profile.name, { + client, + options: profile.connection, + }); + + await client.query({ query: 'SELECT 1;' }); + this.logger.debug(`Profile ${profile.name} initialized`); + } + } + + public async execute({ + statement: sql, + bindParams, + profileName, + operations, + }: ExecuteOptions): Promise { + this.checkProfileExist(profileName); + const { client } = this.clientMapping.get(profileName)!; + + // convert to clickhouse support type of query params. + const params = this.convertToQueryParams(bindParams); + try { + const builtSQL = buildSQL(sql, operations); + const result = await client.query({ + query: builtSQL, + // get result with column name and type, refer: https://clickhouse.com/docs/en/integrations/language-clients/nodejs#supported-data-formats + format: 'JSONCompactEachRowWithNamesAndTypes', + query_params: params, + }); + return await this.getResultFromRestfulSet(result); + } catch (e: any) { + this.logger.debug( + `Errors occurred, release connection from ${profileName}` + ); + throw e; + } + } + + public async prepare({ parameterIndex, value }: RequestParameter) { + // ClickHouse use {name:type} be a placeholder, so if we only use number string as name e.g: {1:Unit8} + // it will face issue when converting to the query params => {1: value1}, because the key is value not string type, so here add prefix "p" to avoid this issue. + return `{p${parameterIndex}:${mapToClickHouseType(value)}`; + } + + private async getResultFromRestfulSet(result: ResultSet) { + const dataRowStream = new Stream.Readable({ + objectMode: true, + read: () => null, + // automatically destroy() the stream when it emits 'finish' or errors. Node > 10.16 + autoDestroy: true, + }); + // data is all rows according to https://clickhouse.com/docs/en/integrations/language-clients/nodejs#resultset-and-row-abstractions + // first row is column names, second row is column types, the rest is data + let names: string[] = []; + let types: string[] = []; + const rawStream = result.stream(); + rawStream.on('data', (rows: Row[]) => { + const [namesRow, typesRow, ...dataRows] = rows; + names = JSON.parse(namesRow.text); + types = JSON.parse(typesRow.text); + dataRows.forEach((row) => dataRowStream.push(row.text)); + }); + await new Promise((resolve) => { + rawStream.on('end', () => { + dataRowStream.push(null); + resolve(null); + }); + }); + return { + getColumns: () => { + return names.map((name, idx) => ({ + name: name || '', + // Convert ClickHouse type to FieldDataType supported by VulcanSQL for generating the response schema in the specification + // please see https://github.com/Canner/vulcan-sql/pull/78#issuecomment-1621532674 + type: mapFromClickHouseType(types[idx] || ''), + })); + }, + getData: () => dataRowStream, + }; + } + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + public override async export(options: ExportOptions): Promise { + throw new InternalError( + 'ClickHouse not yet support exporting data to parquet file for caching datasets feature' + ); + } + + private checkProfileExist(profileName: string) { + if (!this.clientMapping.has(profileName)) { + throw new InternalError(`Profile instance ${profileName} not found`); + } + } + + private convertToQueryParams(bind: BindParameters) { + // find the param name from named placeholder from prepare method + const pattern = /{(\w+):\w+}/; + const params: Record = {}; + bind.forEach((value, key) => { + // get the param name from key. e.g: "{p1:Unit8}" => "p1" + const paramName = key.match(pattern)![1]; + params[paramName] = value; + }); + return params; + } +} diff --git a/packages/extension-driver-clickhouse/src/lib/sqlBuilder.ts b/packages/extension-driver-clickhouse/src/lib/sqlBuilder.ts new file mode 100644 index 00000000..b5b94e95 --- /dev/null +++ b/packages/extension-driver-clickhouse/src/lib/sqlBuilder.ts @@ -0,0 +1,40 @@ +import { Parameterized, SQLClauseOperation } from '@vulcan-sql/core'; +import { isNull, isUndefined } from 'lodash'; + +const isNullOrUndefine = (value: any) => isUndefined(value) || isNull(value); + +export const removeEndingSemiColon = (sql: string) => { + return sql.replace(/;([ \n]+)?$/, ''); +}; + +export const addLimit = (sql: string, limit?: string | null) => { + if (isNullOrUndefine(limit)) return sql; + return [sql, `LIMIT`, limit].join(' '); +}; + +export const addOffset = (sql: string, offset?: string | null) => { + if (isNullOrUndefine(offset)) return sql; + return [sql, `OFFSET`, offset].join(' '); +}; + +// Check if there is no operations +export const isNoOP = ( + operations: Partial> +): boolean => { + if (!isNullOrUndefine(operations.limit)) return false; + if (!isNullOrUndefine(operations.offset)) return false; + return true; +}; + +export const buildSQL = ( + sql: string, + operations: Partial> +): string => { + if (isNoOP(operations)) return sql; + let builtSQL = ''; + builtSQL += `SELECT * FROM (${removeEndingSemiColon(sql)})`; + builtSQL = addLimit(builtSQL, operations.limit); + builtSQL = addOffset(builtSQL, operations.offset); + builtSQL += ';'; + return builtSQL; +}; diff --git a/packages/extension-driver-clickhouse/src/lib/typeMapper.ts b/packages/extension-driver-clickhouse/src/lib/typeMapper.ts new file mode 100644 index 00000000..45b9e98c --- /dev/null +++ b/packages/extension-driver-clickhouse/src/lib/typeMapper.ts @@ -0,0 +1,63 @@ +const typeMapping = new Map(); + +const register = (clickHouseType: string, type: string) => { + typeMapping.set(clickHouseType, type); +}; + +// Reference +// https://clickhouse.com/docs/en/integrations/language-clients/nodejs#resultset-and-row-abstractions +// Currently, FieldDataType only support number, string, boolean, Date for generating response schema in the specification. +register('Int', 'number'); +register('UInt', 'number'); +register('UInt8', 'number'); +register('LowCardinality', 'string'); +register('UInt16', 'number'); +register('UInt32', 'number'); +register('UInt64', 'string'); +register('UInt128', 'string'); +register('UInt256', 'string'); +register('Int8', 'number'); +register('Int16', 'number'); +register('Int32', 'number'); +register('Int64', 'string'); +register('Int128', 'string'); +register('Int256', 'string'); +register('Float32', 'number'); +register('Float64', 'number'); +register('Decimal', 'number'); +register('Boolean', 'boolean'); +register('String', 'string'); +register('FixedString', 'string'); +register('UUID', 'string'); +register('Date32', 'string'); +register('Date64', 'string'); +register('DateTime32', 'string'); +register('DateTime64', 'string'); +register('IPv4', 'string'); +register('IPv6', 'string'); + +export const mapFromClickHouseType = (clickHouseType: string) => { + if (typeMapping.has(clickHouseType)) return typeMapping.get(clickHouseType)!; + return 'string'; +}; + +/** + * Convert the js type to the corresponding ClickHouse type for generating named placeholder of parameterize query. + * Only support to convert number to Int or Float, boolean to Boolean, string to String, other types will convert to String. + * If exist complex type e.g: object, Array, null, undefined, Date, Record.. etc, just convert to string type by ClickHouse function in SQL. + * ClickHouse support converting string to other types function. + * Please see Each section of the https://clickhouse.com/docs/en/sql-reference/functions and https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions + * @param value + * @returns 'FLoat', 'Int', 'Boolean', 'String' + */ + +export const mapToClickHouseType = (value: any) => { + if (typeof value === 'number') { + // infer the float or int according to exist remainder or not + if (value % 1 !== 0) return 'Float'; + return 'Int'; + } + if (typeof value === 'boolean') return 'Boolean'; + if (typeof value === 'string') return 'String'; + return 'String'; +}; diff --git a/packages/extension-driver-clickhouse/tsconfig.json b/packages/extension-driver-clickhouse/tsconfig.json new file mode 100644 index 00000000..f5b85657 --- /dev/null +++ b/packages/extension-driver-clickhouse/tsconfig.json @@ -0,0 +1,22 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "module": "commonjs", + "forceConsistentCasingInFileNames": true, + "strict": true, + "noImplicitOverride": true, + "noPropertyAccessFromIndexSignature": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true + }, + "files": [], + "include": [], + "references": [ + { + "path": "./tsconfig.lib.json" + }, + { + "path": "./tsconfig.spec.json" + } + ] +} diff --git a/packages/extension-driver-clickhouse/tsconfig.lib.json b/packages/extension-driver-clickhouse/tsconfig.lib.json new file mode 100644 index 00000000..1925baa1 --- /dev/null +++ b/packages/extension-driver-clickhouse/tsconfig.lib.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../dist/out-tsc", + "declaration": true, + "types": [] + }, + "include": ["**/*.ts", "../../types/*.d.ts"], + "exclude": ["jest.config.ts", "**/*.spec.ts", "**/*.test.ts"] +} diff --git a/packages/extension-driver-clickhouse/tsconfig.spec.json b/packages/extension-driver-clickhouse/tsconfig.spec.json new file mode 100644 index 00000000..eb72f635 --- /dev/null +++ b/packages/extension-driver-clickhouse/tsconfig.spec.json @@ -0,0 +1,15 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../dist/out-tsc", + "module": "commonjs", + "types": ["jest", "node"] + }, + "include": [ + "jest.config.ts", + "**/*.test.ts", + "**/*.spec.ts", + "**/*.d.ts", + "../../types/*.d.ts" + ] +} diff --git a/packages/extension-driver-clickhouse/yarn.lock b/packages/extension-driver-clickhouse/yarn.lock new file mode 100644 index 00000000..7dcbb8ed --- /dev/null +++ b/packages/extension-driver-clickhouse/yarn.lock @@ -0,0 +1,15 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@clickhouse/client@^0.1.1": + version "0.1.1" + resolved "https://registry.yarnpkg.com/@clickhouse/client/-/client-0.1.1.tgz#1a848438baf5deefadf7dcee40ad441c8666c398" + integrity sha512-oeALCAjNFEXHPxMHJgj0QERiLM2ZknOOavvHB1mxmztZLhTuj86HaQEfh9q8x7LgKnv3jep7lb/fhFgD71WTjA== + dependencies: + uuid "^9.0.0" + +uuid@^9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.0.tgz#592f550650024a38ceb0c562f2f6aa435761efb5" + integrity sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg== diff --git a/tsconfig.base.json b/tsconfig.base.json index 1c55f315..a4797347 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -83,6 +83,9 @@ "@vulcan-sql/extension-driver-canner": [ "packages/extension-driver-canner/src/index.ts" ], + "@vulcan-sql/extension-driver-clickhouse": [ + "packages/extension-driver-clickhouse/src/index.ts" + ], "@vulcan-sql/extension-driver-duckdb": [ "packages/extension-driver-duckdb/src/index.ts" ], diff --git a/workspace.json b/workspace.json index 46b9579e..4f5b83dd 100644 --- a/workspace.json +++ b/workspace.json @@ -11,6 +11,7 @@ "extension-debug-tools": "packages/extension-debug-tools", "extension-driver-bq": "packages/extension-driver-bq", "extension-driver-canner": "packages/extension-driver-canner", + "extension-driver-clickhouse": "packages/extension-driver-clickhouse", "extension-driver-duckdb": "packages/extension-driver-duckdb", "extension-driver-pg": "packages/extension-driver-pg", "extension-driver-snowflake": "packages/extension-driver-snowflake", From 466b8995c5edf7ee61efbc658c8eb74b49ca104f Mon Sep 17 00:00:00 2001 From: kokokuo Date: Thu, 6 Jul 2023 14:37:36 +0800 Subject: [PATCH 2/9] chore(extension-driver-clickhouse): update README --- .../extension-driver-clickhouse/README.md | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 packages/extension-driver-clickhouse/README.md diff --git a/packages/extension-driver-clickhouse/README.md b/packages/extension-driver-clickhouse/README.md new file mode 100644 index 00000000..e25cee10 --- /dev/null +++ b/packages/extension-driver-clickhouse/README.md @@ -0,0 +1,105 @@ +# extension-driver-clickhouse + +[ClickHouse](https://clickhouse.com/) driver for VulcanSQL. + +## Install + +1. Install package + + ```sql + npm i @vulcan-sql/extension-driver-clickhouse + ``` + +2. Update `vulcan.yaml`, enable the extension. + + ```yaml + extensions: + clickhouse: '@vulcan-sql/extension-driver-clickhouse' + ``` + +3. Create a new profile in `profiles.yaml` or in your profiles' paths. Please check [arguments of ClickHouse Client](https://clickhouse.com/docs/en/integrations/language-clients/nodejs) for further information. + + ```yaml + - name: ch # profile name + type: clickhouse + connection: + # Optional: ClickHouse instance URL. default is http://localhost:8123. + host: 'www.example.com:8123' + # Optional: The request timeout in milliseconds. Default value: 30000 + request_timeout: 60000 + # Optional: Maximum number of sockets to allow per host. Default value: Infinity. + max_open_connections: 10 + # Optional: Data applications operating with large datasets over the wire can benefit from enabling compression. Currently, only GZIP is supported using zlib. Please see https://clickhouse.com/docs/en/integrations/language-clients/nodejs#compression + compression: + # Optional: "response: true" instructs ClickHouse server to respond with compressed response body. Default: true + response: true + # Optional: "request: true" enabled compression on the client request body. Default value: false + request: false + # Optional: The name of the user on whose behalf requests are made. Default value: 'default' + username: 'user' + # The user password. Default: ''. + password: 'pass' + # Optional: The name of the application using the Node.js client. Default value: VulcanSQL + application: 'VulcanSQL' + # Optional: Database name to use. Default value: 'default' + database: 'hello-clickhouse' + # Optional: ClickHouse settings to apply to all requests, below is a sample. + # For all settings, please see the https://clickhouse.com/docs/en/operations/settings, the definition see https://github.com/ClickHouse/clickhouse-js/blob/0.1.1/src/settings.ts + clickhouse_settings: + # Optional: Allow Nullable types as primary keys. (default: false) + allow_nullable_key: true + # Optional: configure TLS certificates, please see https://clickhouse.com/docs/en/integrations/language-clients/nodejs#tls-certificates + tls: + # Optional The CA Cert file path + ca_cert: 'ca-cert-file-path' + # Optional: The Cert file path + cert: 'cert-file-path' + # Optional: The key file path + key: 'key-path' + # Optional: ClickHouse Session ID to send with every request + session_id: '' + # Optional: HTTP Keep-Alive related settings, please see https://clickhouse.com/docs/en/integrations/language-clients/nodejs#keep-alive + keep_alive: + # Optional: Enable or disable HTTP Keep-Alive mechanism. Default: true + enabled: true + # Optional: How long to keep a particular open socket alive on the client side (in milliseconds). + # Should be less than the server setting (see `keep_alive_timeout` in server's `config.xml`). + # Currently, has no effect if is unset or false. Default value: 2500 (based on the default ClickHouse server setting, which is 3000) + socket_ttl: 2500 + # Optional: If the client detects a potentially expired socket based on the this socket will be immediately destroyed before sending the request ,and this request will be retried with a new socket up to 3 times. Default: false (no retries) + retry_on_expired_socket: false + ``` + +At the above, it not contains `log` option, because the `logs` need to define a Logger class and assign to it, so it could not set by `profiles.yaml`. + +## Note + +The ClickHouse support parameterize query to prevent SQL Injection by prepared statement. The named placeholder define by `{name:type}`, please see [Query with Parameters](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#queries-with-parameters). + +However, The VulcanSQL API support API query parameter is JSON format, so it not support [variety types like ClickHouse](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#supported-clickhouse-data-types), The VulcanSQL will only support to convert below types: + +- `boolean` to `Boolean` ClickHouse type +- `number` to `Int` or `Float` ClickHouse type +- `string` to `String` ClickHouse type + +Therefore, if you would like to query the data is a special type from ClickHouse, e.g: `Array(Unit8)`, `Record`, `Date`, `DateTime` ...etc, you could use the ClickHouse [Regular Function](https://clickhouse.com/docs/en/sql-reference/functions) or [Type Conversion Function](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions) to do it. + +Example: + +```sql +-- If the val from API query parameter is '1990-11-01', and the born_date columns type is Date32 type +-- What is the toDate function, please see https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions#todate +SELECT * FROM users WHERE born_date = toDate({val:String}); +``` + +## ⚠️ Caution + +ClickHouse driver currently not yet support for caching datasets feature. + +If you use the ClickHouse driver and setup the cache options in API Schema yaml, it will throw error. + +## Testing + +```bash +nx test extension-driver-clickhouse +``` From 135dfb58afbf0b140f3aa4047240af1088e93e33 Mon Sep 17 00:00:00 2001 From: kokokuo Date: Thu, 6 Jul 2023 14:38:01 +0800 Subject: [PATCH 3/9] feat(docs): add ClickHouse connector to docs package --- packages/doc/docs/connect/clickhouse.mdx | 82 +++++++++++++++++ packages/doc/docs/connect/overview.mdx | 22 +++-- packages/doc/docs/connectors.mdx | 1 + packages/doc/docs/connectors/clickhouse.mdx | 87 +++++++++++++++++++ packages/doc/sidebars.js | 23 +++-- .../extension-driver-clickhouse/README.md | 6 +- .../extension-driver-clickhouse/package.json | 4 +- .../src/lib/clickhouseDataSource.ts | 2 + .../src/lib/typeMapper.ts | 2 +- 9 files changed, 206 insertions(+), 23 deletions(-) create mode 100644 packages/doc/docs/connect/clickhouse.mdx create mode 100644 packages/doc/docs/connectors/clickhouse.mdx diff --git a/packages/doc/docs/connect/clickhouse.mdx b/packages/doc/docs/connect/clickhouse.mdx new file mode 100644 index 00000000..0c62ac97 --- /dev/null +++ b/packages/doc/docs/connect/clickhouse.mdx @@ -0,0 +1,82 @@ +# ClickHouse + +## Installation + +1. Install package + + **If you are developing with binary, the package is already bundled in the binary. You can skip this step.** + + ```bash + npm i @vulcan-sql/extension-driver-clickhouse + ``` + +2. Update `vulcan.yaml`, and enable the extension. + + ```yaml + extensions: + ... + // highlight-next-line + ch: '@vulcan-sql/extension-driver-clickhouse' # Add this line + ``` + +3. Create a new profile in `profiles.yaml` or in your profile files. For example: + + ```yaml + - name: ch # profile name + type: clickhouse + connection: + host: www.example.com:8123 + request_timeout: 60000 + compression: + request: true + max_open_connections: 10 + username: user + password: pass + database: hello-clickhouse + allow: '*' + ``` + +## Configuration + +Please check [arguments of ClickHouse Client](https://clickhouse.com/docs/en/integrations/language-clients/nodejs) for further information. + +| Name | Required | Default | Description | +| -------------------- | -------- | --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| host | N | http://localhost:8123 | ClickHouse instance URL. | +| request_timeout | N | 30000 | The request timeout in milliseconds. | +| max_open_connections | N | Infinity | Maximum number of sockets to allow per host. | +| compression | N | | Data applications operating with large datasets over the wire can benefit from enabling compression. Currently, only GZIP is supported using zlib.Please see [Compression docs](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#compression). | +| username | N | default | The name of the user on whose behalf requests are made. | +| password | N | | The user password. | +| application | N | VulcanSQL | The name of the application using the Node.js client. | +| database | N | default | Database name to use. | +| clickhouse_settings | N | | ClickHouse settings to apply to all requests. please see the [Advance Settings](https://clickhouse.com/docs/en/operations/settings), and [Definition](https://github.com/ClickHouse/clickhouse-js/blob/0.1.1/src/settings.ts) | +| tls | N | | Configure TLS certificates. Please see [TLS docs](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#tls-certificates). | +| session_id | N | | ClickHouse Session ID to send with every request. | +| keep_alive | N | | HTTP Keep-Alive related settings. Please see [Keep Alive docs](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#keep-alive) | + +At the above, it not contains `log` option, because the `logs` need to define a Logger class and assign to it, so it could not set by `profiles.yaml`. + +## Note + +The ClickHouse support parameterized query to prevent SQL Injection by prepared statement. The named placeholder define by `{name:type}`, please see [Query with Parameters](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#queries-with-parameters). + +However, The VulcanSQL API support API query parameter is JSON format, so it not support [variety types like ClickHouse](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#supported-clickhouse-data-types), The VulcanSQL will only support to convert below types: + +- `boolean` to `Boolean` ClickHouse type +- `number` to `Int` or `Float` ClickHouse type +- `string` to `String` ClickHouse type + +Therefore, if you would like to query the data is a special type from ClickHouse, e.g: `Array(Unit8)`, `Record`, `Date`, `DateTime` ...etc, you could use the ClickHouse [Regular Function](https://clickhouse.com/docs/en/sql-reference/functions) or [Type Conversion Function](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions) to do it. + +Example: + +```sql +-- If the val from API query parameter is '1990-11-01', and the born_date columns type is Date32 type +-- What is the toDate function, please see https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions#todate +SELECT * FROM users WHERE born_date = toDate({val:String}); +``` + +## ⚠️ Caution + +ClickHouse driver currently not yet support for caching datasets feature. If you use the ClickHouse driver with caching dataset feature, it will be failed. diff --git a/packages/doc/docs/connect/overview.mdx b/packages/doc/docs/connect/overview.mdx index 2fb85b0d..832f9eb1 100644 --- a/packages/doc/docs/connect/overview.mdx +++ b/packages/doc/docs/connect/overview.mdx @@ -5,20 +5,24 @@ pagination_next: connect/bigquery # Overview We support the following data warehouses to connect with, you can choose multiple connectors in a single project, please check the connectors’ document for the installation guide. -* [PostgreSQL](./postgresql) -* [DuckDB](./duckdb) -* [Snowflake](./snowflake) -* [BigQuery](./bigquery) + +- [PostgreSQL](./postgresql) +- [DuckDB](./duckdb) +- [Snowflake](./snowflake) +- [BigQuery](./bigquery) +- [ClickHouse](./clickhouse) ## How to use + Setting up a data warehouse connector is easy, you can follow the steps below to set up a connector. 1. Once you install the connector package, update the `extensions` section in `vulcan.yaml` to include the connector package. For example: - ```yaml + + ```yaml extensions: - ... - // highlight-next-line - pg: '@vulcan-sql/extension-driver-pg' # Add this line + ... + // highlight-next-line + pg: '@vulcan-sql/extension-driver-pg' # Add this line ``` 2. Create a new profile in `profiles.yaml` or in your profile files. @@ -42,4 +46,4 @@ Setting up a data warehouse connector is easy, you can follow the steps below to - pg # profile name ``` - Then, you can query the data warehouse in your APIs. +Then, you can query the data warehouse in your APIs. diff --git a/packages/doc/docs/connectors.mdx b/packages/doc/docs/connectors.mdx index 6ccb2e9b..d6dac1e4 100644 --- a/packages/doc/docs/connectors.mdx +++ b/packages/doc/docs/connectors.mdx @@ -8,6 +8,7 @@ We support the following data warehouses to connect with, you can choose multipl | [DuckDB](./connectors/duckdb) | ✅ Yes | ✅ Yes | ❌ No | | [Snowflake](./connectors/snowflake) | ✅ Yes | ✅ Yes | ❌ No | | [BigQuery](./connectors/bigquery) | ✅ Yes | ✅ Yes | ❌ No | +| [ClickHouse](./connectors/clickhouse) | ✅ Yes | ✅ Yes | ❌ No | \* Fetching rows only when we need them, it has better performance with large query results. diff --git a/packages/doc/docs/connectors/clickhouse.mdx b/packages/doc/docs/connectors/clickhouse.mdx new file mode 100644 index 00000000..5043bce2 --- /dev/null +++ b/packages/doc/docs/connectors/clickhouse.mdx @@ -0,0 +1,87 @@ +# ClickHouse + +Connect with your ClickHouse servers via the official [Node.js Driver](https://clickhouse.com/docs/en/integrations/language-clients/nodejs). + +## Installation + +1. Install package + + ```bash + npm i @vulcan-sql/extension-driver-clickhouse + ``` + + :::info + If you run VulcanSQL with Docker, you should use the command `vulcan-install @vulcan-sql/extension-driver-clickhouse` instead. + + ::: + +2. Update `vulcan.yaml`, and enable the extension. + + ```yaml + extensions: + ... + // highlight-next-line + ch: '@vulcan-sql/extension-driver-clickhouse' # Add this line + ``` + +3. Create a new profile in `profiles.yaml` or in your profile files. For example: + + ```yaml + - name: ch # profile name + type: clickhouse + connection: + host: www.example.com:8123 + request_timeout: 60000 + compression: + request: true + max_open_connections: 10 + username: user + password: pass + database: hello-clickhouse + allow: '*' + ``` + +## Configuration + +Please check [arguments of ClickHouse Client](https://clickhouse.com/docs/en/integrations/language-clients/nodejs) for further information. + +| Name | Required | Default | Description | +| -------------------- | -------- | --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| host | N | http://localhost:8123 | ClickHouse instance URL. | +| request_timeout | N | 30000 | The request timeout in milliseconds. | +| max_open_connections | N | Infinity | Maximum number of sockets to allow per host. | +| compression | N | | Data applications operating with large datasets over the wire can benefit from enabling compression. Currently, only GZIP is supported using zlib.Please see [Compression docs](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#compression). | +| username | N | default | The name of the user on whose behalf requests are made. | +| password | N | | The user password. | +| application | N | VulcanSQL | The name of the application using the Node.js client. | +| database | N | default | Database name to use. | +| clickhouse_settings | N | | ClickHouse settings to apply to all requests. please see the [Advance Settings](https://clickhouse.com/docs/en/operations/settings), and [Definition](https://github.com/ClickHouse/clickhouse-js/blob/0.1.1/src/settings.ts) | +| tls | N | | Configure TLS certificates. Please see [TLS docs](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#tls-certificates). | +| session_id | N | | ClickHouse Session ID to send with every request. | +| keep_alive | N | | HTTP Keep-Alive related settings. Please see [Keep Alive docs](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#keep-alive) | + +At the above, it not contains `log` option, because the `logs` need to define a Logger class and assign to it, so it could not set by `profiles.yaml`. + +## Note + +The ClickHouse support parameterized query to prevent SQL Injection by prepared statement. The named placeholder define by `{name:type}`, please see [Query with Parameters](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#queries-with-parameters). + +However, The VulcanSQL API support API query parameter is JSON format, so it not support [variety types like ClickHouse](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#supported-clickhouse-data-types), The VulcanSQL will only support to convert below types: + +- `boolean` to `Boolean` ClickHouse type +- `number` to `Int` or `Float` ClickHouse type +- `string` to `String` ClickHouse type + +Therefore, if you would like to query the data is a special type from ClickHouse, e.g: `Array(Unit8)`, `Record`, `Date`, `DateTime` ...etc, you could use the ClickHouse [Regular Function](https://clickhouse.com/docs/en/sql-reference/functions) or [Type Conversion Function](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions) to do it. + +Example: + +```sql +-- If the val from API query parameter is '1990-11-01', and the born_date columns type is Date32 type +-- What is the toDate function, please see https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions#todate +SELECT * FROM users WHERE born_date = toDate({val:String}); +``` + +## ⚠️ Caution + +ClickHouse driver currently not yet support for caching datasets feature. If you use the ClickHouse driver with caching dataset feature, it will be failed. diff --git a/packages/doc/sidebars.js b/packages/doc/sidebars.js index c52031cb..85ae9a90 100644 --- a/packages/doc/sidebars.js +++ b/packages/doc/sidebars.js @@ -21,7 +21,8 @@ const sidebars = { tutorialSidebar: [ { type: 'html', - value: ' Getting Started', + value: + ' Getting Started', className: 'sidebar-title', }, { @@ -38,7 +39,8 @@ const sidebars = { }, { type: 'html', - value: ' Building Data API', + value: + ' Building Data API', className: 'sidebar-title', }, { @@ -69,7 +71,11 @@ const sidebars = { { type: 'doc', id: 'connect/duckdb', - } + }, + { + type: 'doc', + id: 'connect/clickhouse', + }, ], }, { @@ -97,7 +103,7 @@ const sidebars = { type: 'doc', id: 'develop/advance', }, - ] + ], }, { type: 'doc', @@ -144,7 +150,8 @@ const sidebars = { }, { type: 'html', - value: ' API Catalog & Documentation', + value: + ' API Catalog & Documentation', className: 'sidebar-title', }, { @@ -164,7 +171,8 @@ const sidebars = { }, { type: 'html', - value: ' API Configuration', + value: + ' API Configuration', className: 'sidebar-title', }, { @@ -319,7 +327,8 @@ const sidebars = { // }, { type: 'html', - value: ' Deployment and Maintenance', + value: + ' Deployment and Maintenance', className: 'sidebar-title', }, 'deployment', diff --git a/packages/extension-driver-clickhouse/README.md b/packages/extension-driver-clickhouse/README.md index e25cee10..5464719b 100644 --- a/packages/extension-driver-clickhouse/README.md +++ b/packages/extension-driver-clickhouse/README.md @@ -74,7 +74,7 @@ At the above, it not contains `log` option, because the `logs` need to define a ## Note -The ClickHouse support parameterize query to prevent SQL Injection by prepared statement. The named placeholder define by `{name:type}`, please see [Query with Parameters](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#queries-with-parameters). +The ClickHouse support parameterized query to prevent SQL Injection by prepared statement. The named placeholder define by `{name:type}`, please see [Query with Parameters](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#queries-with-parameters). However, The VulcanSQL API support API query parameter is JSON format, so it not support [variety types like ClickHouse](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#supported-clickhouse-data-types), The VulcanSQL will only support to convert below types: @@ -94,9 +94,7 @@ SELECT * FROM users WHERE born_date = toDate({val:String}); ## ⚠️ Caution -ClickHouse driver currently not yet support for caching datasets feature. - -If you use the ClickHouse driver and setup the cache options in API Schema yaml, it will throw error. +ClickHouse driver currently not yet support for caching datasets feature. If you use the ClickHouse driver with caching dataset feature, it will be failed. ## Testing diff --git a/packages/extension-driver-clickhouse/package.json b/packages/extension-driver-clickhouse/package.json index ff3f7c0c..c3772892 100644 --- a/packages/extension-driver-clickhouse/package.json +++ b/packages/extension-driver-clickhouse/package.json @@ -15,8 +15,8 @@ "data-warehouse", "data-lake", "api-builder", - "snowflake", - "snow" + "clickhouse", + "ch" ], "repository": { "type": "git", diff --git a/packages/extension-driver-clickhouse/src/lib/clickhouseDataSource.ts b/packages/extension-driver-clickhouse/src/lib/clickhouseDataSource.ts index a43d4ece..95d07710 100644 --- a/packages/extension-driver-clickhouse/src/lib/clickhouseDataSource.ts +++ b/packages/extension-driver-clickhouse/src/lib/clickhouseDataSource.ts @@ -120,6 +120,8 @@ export class ClickHouseDataSource extends DataSource { const [namesRow, typesRow, ...dataRows] = rows; names = JSON.parse(namesRow.text); types = JSON.parse(typesRow.text); + // ClickHouse stream only called once and return all data row in one chuck, so we need to push each row to the stream by loop. + // Please see https://clickhouse.com/docs/en/integrations/language-clients/nodejs#resultset-and-row-abstractions dataRows.forEach((row) => dataRowStream.push(row.text)); }); await new Promise((resolve) => { diff --git a/packages/extension-driver-clickhouse/src/lib/typeMapper.ts b/packages/extension-driver-clickhouse/src/lib/typeMapper.ts index 45b9e98c..19e62206 100644 --- a/packages/extension-driver-clickhouse/src/lib/typeMapper.ts +++ b/packages/extension-driver-clickhouse/src/lib/typeMapper.ts @@ -42,7 +42,7 @@ export const mapFromClickHouseType = (clickHouseType: string) => { }; /** - * Convert the js type to the corresponding ClickHouse type for generating named placeholder of parameterize query. + * Convert the JS type (source is JSON format by API query parameter) to the corresponding ClickHouse type for generating named placeholder of parameterized query. * Only support to convert number to Int or Float, boolean to Boolean, string to String, other types will convert to String. * If exist complex type e.g: object, Array, null, undefined, Date, Record.. etc, just convert to string type by ClickHouse function in SQL. * ClickHouse support converting string to other types function. From 385b908f6157707aca9809531d329ad3f91af14a Mon Sep 17 00:00:00 2001 From: kokokuo Date: Fri, 7 Jul 2023 11:59:25 +0800 Subject: [PATCH 4/9] feat(extension-driver-clickhouse): add test cases and refactor querying method - add clickhouse server for running test cases by docker. - refactor clickhouse data source by querying data with column name and evaluating column with type by "describe" method. - set "testEnvironment" to "node" in jest environment to make clickhouse client could work when running test cases. - fix typeMapper for converting "Bool" to "boolean" --- .../jest.config.ts | 3 + .../extension-driver-clickhouse/src/index.ts | 4 +- ...eDataSource.ts => clickHouseDataSource.ts} | 59 ++-- .../src/lib/typeMapper.ts | 2 +- .../test/clickHouseDataSource.spec.ts | 316 ++++++++++++++++++ .../test/clickHouseServer.ts | 131 ++++++++ .../test/errorHandler.spec.ts | 86 +++++ .../test/sqlBuilder.spec.ts | 73 ++++ 8 files changed, 646 insertions(+), 28 deletions(-) rename packages/extension-driver-clickhouse/src/lib/{clickhouseDataSource.ts => clickHouseDataSource.ts} (69%) create mode 100644 packages/extension-driver-clickhouse/test/clickHouseDataSource.spec.ts create mode 100644 packages/extension-driver-clickhouse/test/clickHouseServer.ts create mode 100644 packages/extension-driver-clickhouse/test/errorHandler.spec.ts create mode 100644 packages/extension-driver-clickhouse/test/sqlBuilder.spec.ts diff --git a/packages/extension-driver-clickhouse/jest.config.ts b/packages/extension-driver-clickhouse/jest.config.ts index 97fe1657..4b230079 100644 --- a/packages/extension-driver-clickhouse/jest.config.ts +++ b/packages/extension-driver-clickhouse/jest.config.ts @@ -1,6 +1,9 @@ module.exports = { displayName: 'extension-driver-clickhouse', preset: '../../jest.preset.ts', + // Use node environment to avoid facing "TypeError: The "listener" argument must be of type function. Received an instance of Object" error + // when using clickhouse client executing query in the jest environment. + testEnvironment: 'node', globals: { 'ts-jest': { tsconfig: '/tsconfig.spec.json', diff --git a/packages/extension-driver-clickhouse/src/index.ts b/packages/extension-driver-clickhouse/src/index.ts index 93a24cfb..97ff6499 100644 --- a/packages/extension-driver-clickhouse/src/index.ts +++ b/packages/extension-driver-clickhouse/src/index.ts @@ -1,3 +1,3 @@ -export * from './lib/clickhouseDataSource'; -import { ClickHouseDataSource } from './lib/clickhouseDataSource'; +export * from './lib/clickHouseDataSource'; +import { ClickHouseDataSource } from './lib/clickHouseDataSource'; export default [ClickHouseDataSource]; diff --git a/packages/extension-driver-clickhouse/src/lib/clickhouseDataSource.ts b/packages/extension-driver-clickhouse/src/lib/clickHouseDataSource.ts similarity index 69% rename from packages/extension-driver-clickhouse/src/lib/clickhouseDataSource.ts rename to packages/extension-driver-clickhouse/src/lib/clickHouseDataSource.ts index 95d07710..dee17d65 100644 --- a/packages/extension-driver-clickhouse/src/lib/clickhouseDataSource.ts +++ b/packages/extension-driver-clickhouse/src/lib/clickHouseDataSource.ts @@ -1,5 +1,6 @@ import { BindParameters, + DataColumn, DataResult, DataSource, ExecuteOptions, @@ -17,7 +18,7 @@ import { ResultSet, Row, } from '@clickhouse/client'; -import { buildSQL } from './sqlBuilder'; +import { buildSQL, removeEndingSemiColon } from './sqlBuilder'; import { mapFromClickHouseType, mapToClickHouseType } from './typeMapper'; import { omit } from 'lodash'; @@ -62,10 +63,10 @@ export class ClickHouseDataSource extends DataSource { const client = createClient(options); this.clientMapping.set(profile.name, { client, - options: profile.connection, + options, }); - await client.query({ query: 'SELECT 1;' }); + await client.query({ query: 'SELECT 1' }); this.logger.debug(`Profile ${profile.name} initialized`); } } @@ -82,14 +83,22 @@ export class ClickHouseDataSource extends DataSource { // convert to clickhouse support type of query params. const params = this.convertToQueryParams(bindParams); try { + // Use JSONEachFormat to get data result with column name , refer: https://clickhouse.com/docs/en/integrations/language-clients/nodejs#supported-data-formats const builtSQL = buildSQL(sql, operations); - const result = await client.query({ + const data = await client.query({ query: builtSQL, - // get result with column name and type, refer: https://clickhouse.com/docs/en/integrations/language-clients/nodejs#supported-data-formats - format: 'JSONCompactEachRowWithNamesAndTypes', + format: 'JSONEachRow', query_params: params, }); - return await this.getResultFromRestfulSet(result); + // Get the query metadata e.g: column name, type by DESCRIBE TABLE method. + // DESCRIBE TABLE will return the type after evaluating the query, without execution, and will return only one entry per column, see: https://www.tinybird.co/blog-posts/tips-11-how-to-get-the-types-returned-by-a-query + const metadata = await client.query({ + // remove semicolon at the end, because could not exist semicolon in sub-query when using DESCRIBE TABLE + query: `DESCRIBE TABLE (${removeEndingSemiColon(builtSQL)})`, + format: 'JSONEachRow', + query_params: params, + }); + return await this.getResultFromRestfulSet(metadata, data); } catch (e: any) { this.logger.debug( `Errors occurred, release connection from ${profileName}` @@ -101,28 +110,29 @@ export class ClickHouseDataSource extends DataSource { public async prepare({ parameterIndex, value }: RequestParameter) { // ClickHouse use {name:type} be a placeholder, so if we only use number string as name e.g: {1:Unit8} // it will face issue when converting to the query params => {1: value1}, because the key is value not string type, so here add prefix "p" to avoid this issue. - return `{p${parameterIndex}:${mapToClickHouseType(value)}`; + return `{p${parameterIndex}:${mapToClickHouseType(value)}}`; + } + + public async destroy() { + for (const { client } of this.clientMapping.values()) { + await client.close(); + } } - private async getResultFromRestfulSet(result: ResultSet) { + private async getResultFromRestfulSet(metadata: ResultSet, data: ResultSet) { const dataRowStream = new Stream.Readable({ objectMode: true, read: () => null, // automatically destroy() the stream when it emits 'finish' or errors. Node > 10.16 autoDestroy: true, }); - // data is all rows according to https://clickhouse.com/docs/en/integrations/language-clients/nodejs#resultset-and-row-abstractions - // first row is column names, second row is column types, the rest is data - let names: string[] = []; - let types: string[] = []; - const rawStream = result.stream(); + // Get the metadata and only need column name and type + const columns = (await metadata.json()) as Array; + const rawStream = data.stream(); + // ClickHouse stream only called once and return all data row in one chuck, so we need to push each row to the stream by loop. + // Please see https://clickhouse.com/docs/en/integrations/language-clients/nodejs#resultset-and-row-abstractions rawStream.on('data', (rows: Row[]) => { - const [namesRow, typesRow, ...dataRows] = rows; - names = JSON.parse(namesRow.text); - types = JSON.parse(typesRow.text); - // ClickHouse stream only called once and return all data row in one chuck, so we need to push each row to the stream by loop. - // Please see https://clickhouse.com/docs/en/integrations/language-clients/nodejs#resultset-and-row-abstractions - dataRows.forEach((row) => dataRowStream.push(row.text)); + rows.forEach((row) => dataRowStream.push(JSON.parse(row.text))); }); await new Promise((resolve) => { rawStream.on('end', () => { @@ -132,11 +142,10 @@ export class ClickHouseDataSource extends DataSource { }); return { getColumns: () => { - return names.map((name, idx) => ({ - name: name || '', - // Convert ClickHouse type to FieldDataType supported by VulcanSQL for generating the response schema in the specification - // please see https://github.com/Canner/vulcan-sql/pull/78#issuecomment-1621532674 - type: mapFromClickHouseType(types[idx] || ''), + return columns.map((column) => ({ + name: column.name || '', + // Convert ClickHouse type to FieldDataType supported by VulcanSQL for generating the response schema in the specification, see: https://github.com/Canner/vulcan-sql/pull/78#issuecomment-1621532674 + type: mapFromClickHouseType(column.type || ''), })); }, getData: () => dataRowStream, diff --git a/packages/extension-driver-clickhouse/src/lib/typeMapper.ts b/packages/extension-driver-clickhouse/src/lib/typeMapper.ts index 19e62206..eb47ca01 100644 --- a/packages/extension-driver-clickhouse/src/lib/typeMapper.ts +++ b/packages/extension-driver-clickhouse/src/lib/typeMapper.ts @@ -25,7 +25,7 @@ register('Int256', 'string'); register('Float32', 'number'); register('Float64', 'number'); register('Decimal', 'number'); -register('Boolean', 'boolean'); +register('Bool', 'boolean'); register('String', 'string'); register('FixedString', 'string'); register('UUID', 'string'); diff --git a/packages/extension-driver-clickhouse/test/clickHouseDataSource.spec.ts b/packages/extension-driver-clickhouse/test/clickHouseDataSource.spec.ts new file mode 100644 index 00000000..4bdbca03 --- /dev/null +++ b/packages/extension-driver-clickhouse/test/clickHouseDataSource.spec.ts @@ -0,0 +1,316 @@ +import { ClickHouseServer } from './clickHouseServer'; +import { ClickHouseDataSource, ClickHouseOptions } from '../src'; +import { streamToArray } from '@vulcan-sql/core'; +import { Writable } from 'stream'; + +const clickHouse = new ClickHouseServer(); +let dataSource: ClickHouseDataSource; + +beforeAll(async () => { + await clickHouse.prepare(); +}, 5 * 60 * 1000); // it might take some time to pull images. + +afterAll(async () => { + await clickHouse.destroy(); +}, 30000); + +it('Data source should be activate without any error when all profiles are valid', async () => { + // Arrange + dataSource = new ClickHouseDataSource({}, '', [ + clickHouse.getProfile('profile1'), + ]); + // Act, Assert + await expect(dataSource.activate()).resolves.not.toThrow(); +}, 10000); + +it('Data source should throw error when activating if any profile is invalid', async () => { + // Arrange + dataSource = new ClickHouseDataSource({}, '', [ + clickHouse.getProfile('profile1'), + { + name: 'wrong-password', + type: 'clickhouse', + connection: { + host: `http://${clickHouse.host}:${clickHouse.port}`, + user: clickHouse.user, + password: clickHouse.password + 'wrong', + database: clickHouse.database, + } as ClickHouseOptions, + allow: '*', + }, + ]); + // Act, Assert + await expect(dataSource.activate()).rejects.toThrow(); +}, 10000); + +it.each([[193], [2]])( + 'Data source should return correct rows with limit %s', + async (limit: number) => { + // Arrange + dataSource = new ClickHouseDataSource({}, '', [ + clickHouse.getProfile('profile1'), + ]); + await dataSource.activate(); + // Act + const { getData } = await dataSource.execute({ + statement: `SELECT * FROM products LIMIT ${limit.toString()}`, + bindParams: new Map(), + profileName: 'profile1', + operations: {} as any, + }); + const rows = await streamToArray(getData()); + // Assert + expect(rows.length).toBe(limit); + }, + 30000 +); + +it('Data source should return empty data with no row', async () => { + // Arrange + dataSource = new ClickHouseDataSource({}, '', [ + clickHouse.getProfile('profile1'), + ]); + await dataSource.activate(); + // Act + const { getData } = await dataSource.execute({ + statement: 'SELECT * FROM products LIMIT 0', + bindParams: new Map(), + profileName: 'profile1', + operations: {} as any, + }); + const rows = await streamToArray(getData()); + // Assert + expect(rows.length).toBe(0); +}, 30000); + +it('Data source should return correct data with rows', async () => { + // Arrange + dataSource = new ClickHouseDataSource({}, '', [ + clickHouse.getProfile('profile1'), + ]); + await dataSource.activate(); + // Act + const { getData } = await dataSource.execute({ + statement: 'SELECT * FROM products ORDER BY serial ASC LIMIT 2', + bindParams: new Map(), + profileName: 'profile1', + operations: {} as any, + }); + const rows = await streamToArray(getData()); + // Assert + expect(rows.length).toEqual(2); + expect(rows[0]).toEqual({ + serial: 1, + product_id: '6fd1080f-c186-42ba-b32c-10211a8689a2', + name: 'juice', + price: 30, + enabled: true, + }); + expect(rows[1]).toEqual({ + serial: 2, + product_id: '17a677ca-8f50-49c5-82d7-bce85031ee09', + name: 'egg', + price: 50, + enabled: false, + }); +}, 30000); + +it('Data source should work with prepare statements', async () => { + // Arrange + dataSource = new ClickHouseDataSource({}, '', [ + clickHouse.getProfile('profile1'), + ]); + await dataSource.activate(); + // Act + const bindParams = new Map(); + const var1Name = await dataSource.prepare({ + parameterIndex: 1, + value: '123', + profileName: 'profile1', + }); + bindParams.set(var1Name, '123'); + + const var2Name = await dataSource.prepare({ + parameterIndex: 2, + value: '456', + profileName: 'profile1', + }); + bindParams.set(var2Name, '456'); + + const { getData } = await dataSource.execute({ + statement: `SELECT ${var1Name} AS v1, ${var2Name} AS v2;`, + bindParams, + profileName: 'profile1', + operations: {} as any, + }); + const rows = await streamToArray(getData()); + // Assert + expect(rows[0].v1).toBe('123'); + expect(rows[0].v2).toBe('456'); +}, 30000); + +it('Data source should query correct result with prepare statements', async () => { + // Arrange + dataSource = new ClickHouseDataSource({}, '', [ + clickHouse.getProfile('profile1'), + ]); + await dataSource.activate(); + // Act + const bindParams = new Map(); + const param1 = await dataSource.prepare({ + parameterIndex: 1, + value: 'juice', + profileName: 'profile1', + }); + bindParams.set(param1, 'juice'); + + const param2 = await dataSource.prepare({ + parameterIndex: 2, + value: 30, + profileName: 'profile1', + }); + bindParams.set(param2, 30); + + const { getData } = await dataSource.execute({ + statement: `SELECT * FROM products WHERE name = ${param1} AND price = ${param2} ORDER BY serial ASC;`, + bindParams, + profileName: 'profile1', + operations: {} as any, + }); + const rows = await streamToArray(getData()); + // Assert + expect(rows.length).toBe(1); + expect(rows[0]).toEqual({ + serial: 1, + product_id: '6fd1080f-c186-42ba-b32c-10211a8689a2', + name: 'juice', + price: 30, + enabled: true, + }); +}, 30000); + +it('Data source should return correct column types', async () => { + // Arrange + dataSource = new ClickHouseDataSource({}, '', [ + clickHouse.getProfile('profile1'), + ]); + await dataSource.activate(); + // Act + const { getColumns, getData } = await dataSource.execute({ + statement: 'select * from products limit 0', + bindParams: new Map(), + profileName: 'profile1', + operations: {} as any, + }); + const column = getColumns(); + // We need to destroy the data stream or the driver waits for us + const data = getData(); + data.destroy(); + + // Assert + expect(column[0]).toEqual({ name: 'serial', type: 'number' }); + expect(column[1]).toEqual({ name: 'product_id', type: 'string' }); + expect(column[2]).toEqual({ name: 'name', type: 'string' }); + expect(column[3]).toEqual({ name: 'price', type: 'number' }); + expect(column[4]).toEqual({ name: 'enabled', type: 'boolean' }); +}, 30000); + +it('Data source should release connection when readable stream is destroyed', async () => { + // Arrange + dataSource = new ClickHouseDataSource({}, '', [ + clickHouse.getProfile('profile1'), + ]); + await dataSource.activate(); + // Act + const { getData } = await dataSource.execute({ + statement: 'select * from products limit 100', + bindParams: new Map(), + profileName: 'profile1', + operations: {} as any, + }); + const readStream = getData(); + const rows: any[] = []; + let resolve: any; + const waitForStream = () => new Promise((res) => (resolve = res)); + const writeStream = new Writable({ + write(chunk, _, cb) { + rows.push(chunk); + // After read 5 records, destroy the upstream + if (rows.length === 5) { + readStream.destroy(); + resolve(); + } else cb(); + }, + objectMode: true, + }); + readStream.pipe(writeStream); + await waitForStream(); + // Assert + expect(rows.length).toBe(5); + // afterEach hook will timeout if any leak occurred. +}, 30000); + +it('Data source should release the connection when finished no matter success or not', async () => { + // Arrange + dataSource = new ClickHouseDataSource({}, '', [ + { + name: 'profile1', + type: 'clickhouse', + connection: { + host: `http://${clickHouse.host}:${clickHouse.port}`, + username: clickHouse.user, + password: clickHouse.password, + database: clickHouse.database, + // Limit the max connection size to 1, we'll get blocked with any leak. + max_open_connections: 1, + } as ClickHouseOptions, + allow: '*', + }, + ]); + await dataSource.activate(); + + // Act + // send parallel queries to test pool leak + const result = await Promise.all( + [ + async () => { + const { getData } = await dataSource.execute({ + statement: 'select * from products limit 1', + bindParams: new Map(), + profileName: 'profile1', + operations: {} as any, + }); + return await streamToArray(getData()); + }, + async () => { + try { + const { getData } = await dataSource.execute({ + statement: 'wrong sql', + bindParams: new Map(), + profileName: 'profile1', + operations: {} as any, + }); + await streamToArray(getData()); + return [{}]; // fake data + } catch { + // ignore error + return []; + } + }, + async () => { + const { getData } = await dataSource.execute({ + statement: 'select * from products limit 1', + bindParams: new Map(), + profileName: 'profile1', + operations: {} as any, + }); + return await streamToArray(getData()); + }, + ].map((task) => task()) + ); + + // Assert + expect(result[0].length).toBe(1); + expect(result[1].length).toBe(0); + expect(result[2].length).toBe(1); +}, 30000); diff --git a/packages/extension-driver-clickhouse/test/clickHouseServer.ts b/packages/extension-driver-clickhouse/test/clickHouseServer.ts new file mode 100644 index 00000000..ae2e7c08 --- /dev/null +++ b/packages/extension-driver-clickhouse/test/clickHouseServer.ts @@ -0,0 +1,131 @@ +/* istanbul ignore file */ +import * as Docker from 'dockerode'; +import faker from '@faker-js/faker'; +import { createClient } from '@clickhouse/client'; +import * as BPromise from 'bluebird'; +import { ClickHouseOptions } from '../src/lib/clickHouseDataSource'; + +const docker = new Docker(); + +/** + * ClickHouse Server in docker + * table: users (id Int32, name String, enabled Boolean) + * rows: 200 rows. + */ +export class ClickHouseServer { + public readonly password = '123'; + // https://hub.docker.com/r/clickhouse/clickhouse-server/ + public readonly image = 'clickhouse/clickhouse-server:23'; + public readonly port = faker.datatype.number({ min: 20000, max: 30000 }); + public readonly host = 'localhost'; + public readonly user = 'user'; + public readonly database = 'db'; + private container?: Docker.Container; + + public async prepare() { + const pullStream = await docker.pull(this.image); + // https://github.com/apocas/dockerode/issues/647 + await new Promise((res) => docker.modem.followProgress(pullStream, res)); + this.container = await docker.createContainer({ + name: `vulcan-clickhouse-test-${faker.random.word()}`, + Image: this.image, + ExposedPorts: { + '8123/tcp': {}, + }, + Env: [ + `CLICKHOUSE_DB=${this.database}`, + `CLICKHOUSE_USER=${this.user}`, + `CLICKHOUSE_PASSWORD=${this.password}`, + ], + HostConfig: { + PortBindings: { + '8123/tcp': [{ HostPort: `${this.port}` }], + }, + // Set unlimited options according to https://hub.docker.com/r/clickhouse/clickhouse-server/ + // Docs: https://docs.docker.com/engine/api/v1.37/#tag/Container/operation/ContainerCreate + Ulimits: [{ Name: 'nofile', Soft: 262144, Hard: 262144 }], + }, + }); + await this.container.start({}); + await this.waitClickHouseReady(); + // Init data + const client = createClient({ + host: `http://${this.host}:${this.port}`, + username: this.user, + password: this.password, + database: this.database, + }); + + await client.exec({ + query: `CREATE TABLE IF NOT EXISTS products + ( + serial UInt32, + product_id UUID, + name String, + price UInt32, + enabled Boolean + ) + ENGINE = MergeTree + PRIMARY KEY (product_id)`, + }); + + await client.exec({ + query: `INSERT INTO products (*) VALUES + (1, '6fd1080f-c186-42ba-b32c-10211a8689a2', 'juice', 30, true), + (2, '17a677ca-8f50-49c5-82d7-bce85031ee09', 'egg', 50, false), + (3, '9fb20bb4-75ea-46bf-ab00-4898a28283fc', 'milk', 20, true)`, + }); + for (let i = 4; i <= 200; i++) { + await client.exec({ + query: + 'INSERT INTO products (*) VALUES ({serial: Int}, {product_id: UUID}, {name:String}, {price:Float}, {enabled:Boolean})', + query_params: { + serial: i, + product_id: faker.datatype.uuid(), + name: faker.commerce.product(), + price: faker.commerce.price(100, 300, 0), + enabled: faker.datatype.boolean(), + }, + }); + } + } + + public async destroy() { + await this.container?.remove({ force: true }); + } + + public getProfile(name: string) { + return { + name, + type: 'clickhouse', + connection: { + host: `http://${this.host}:${this.port}`, + username: this.user, + password: this.password, + database: this.database, + } as ClickHouseOptions, + allow: '*', + }; + } + + private async waitClickHouseReady() { + const waitConnection = await this.container?.exec({ + Cmd: [ + '/bin/bash', + '-c', + 'until grep " Application: Ready for connections" /var/log/clickhouse-server/clickhouse-server.log; do sleep 5; echo "not ready"; done', + ], + }); + if (!waitConnection) return; + await waitConnection.start({}); + let wait = 20; + while (wait--) { + const { Running, ExitCode } = await waitConnection.inspect(); + if (!Running && ExitCode === 0) return; + else if (!Running && ExitCode && ExitCode > 0) + throw new Error(`ClickHouse wait commend return exit code ${ExitCode}`); + await BPromise.delay(1000); + } + throw new Error(`ClickHouse timeout`); + } +} diff --git a/packages/extension-driver-clickhouse/test/errorHandler.spec.ts b/packages/extension-driver-clickhouse/test/errorHandler.spec.ts new file mode 100644 index 00000000..a9d4c9be --- /dev/null +++ b/packages/extension-driver-clickhouse/test/errorHandler.spec.ts @@ -0,0 +1,86 @@ +import { ClickHouseServer } from './clickHouseServer'; +import { ClickHouseDataSource } from '../src'; +import { streamToArray } from '@vulcan-sql/core'; + +let clickHouse: ClickHouseServer; +let dataSource: ClickHouseDataSource; + +afterEach(async () => { + try { + await clickHouse.destroy(); + } catch { + // ignore + } +}, 30000); + +it( + 'Data source should throw an error when executing queries on disconnected sources', + async () => { + // Arrange + clickHouse = new ClickHouseServer(); + await clickHouse.prepare(); + dataSource = new ClickHouseDataSource({}, '', [ + clickHouse.getProfile('profile1'), + ]); + await dataSource.activate(); + const { getData } = await dataSource.execute({ + statement: 'SELECT 1', + bindParams: new Map(), + profileName: 'profile1', + operations: {} as any, + }); + // We must consume the stream or the connection won't be released. + await streamToArray(getData()); + // Close the pg server + await clickHouse.destroy(); + + // Act, Assert + await expect( + dataSource.execute({ + statement: 'SELECT 1', + bindParams: new Map(), + profileName: 'profile1', + operations: {} as any, + }) + ).rejects.toThrow(); + }, + 5 * 60 * 1000 +); + +it( + 'Data source should reconnect when the source was disconnected and reconnected', + async () => { + // Arrange + clickHouse = new ClickHouseServer(); + await clickHouse.prepare(); + dataSource = new ClickHouseDataSource({}, '', [ + clickHouse.getProfile('profile1'), + ]); + await dataSource.activate(); + const { getData } = await dataSource.execute({ + statement: 'SELECT 1', + bindParams: new Map(), + profileName: 'profile1', + operations: {} as any, + }); + // We must consume the stream or the connection won't be released. + await streamToArray(getData()); + // Close the pg server + await clickHouse.destroy(); + // Start the pg server again + await clickHouse.prepare(); + + // Act + const { getData: getSecondData } = await dataSource.execute({ + statement: 'SELECT 1 as a', + bindParams: new Map(), + profileName: 'profile1', + operations: {} as any, + }); + const data = await streamToArray(getSecondData()); + + // Assert + expect(data).toEqual([{ a: 1 }]); + }, + 5 * 60 * 1000 +); diff --git a/packages/extension-driver-clickhouse/test/sqlBuilder.spec.ts b/packages/extension-driver-clickhouse/test/sqlBuilder.spec.ts new file mode 100644 index 00000000..5d0424d0 --- /dev/null +++ b/packages/extension-driver-clickhouse/test/sqlBuilder.spec.ts @@ -0,0 +1,73 @@ +import * as builder from '../src/lib/sqlBuilder'; + +describe('SQL builders components test', () => { + it('removeEndingSemiColon', async () => { + // Arrange + const statement = `SELECT * FROM users; \n `; + // Act + const result = builder.removeEndingSemiColon(statement); + // Arrange + expect(result).toBe('SELECT * FROM users'); + }); + + it('addLimit - string value', async () => { + // Arrange + const statement = `SELECT * FROM users`; + // Act + const result = builder.addLimit(statement, '$1'); + // Arrange + expect(result).toBe('SELECT * FROM users LIMIT $1'); + }); + + it('addLimit - null value', async () => { + // Arrange + const statement = `SELECT * FROM users`; + // Act + const result = builder.addLimit(statement, null); + // Arrange + expect(result).toBe('SELECT * FROM users'); + }); + + it('addOffset - string value', async () => { + // Arrange + const statement = `SELECT * FROM users`; + // Act + const result = builder.addOffset(statement, '$1'); + // Arrange + expect(result).toBe('SELECT * FROM users OFFSET $1'); + }); + + it('addOffset - null value', async () => { + // Arrange + const statement = `SELECT * FROM users`; + // Act + const result = builder.addOffset(statement, null); + // Arrange + expect(result).toBe('SELECT * FROM users'); + }); + + it('isNoOP - empty operation', async () => { + // Act + const result = builder.isNoOP({}); + // Arrange + expect(result).toBe(true); + }); + + it('isNoOP - some operations', async () => { + // Act + const results = [{ limit: '$1' }, { offset: '$1' }].map(builder.isNoOP); + // Arrange + expect(results.every((result) => result === false)).toBeTruthy(); + }); +}); + +it('BuildSQL function should build sql with operations', async () => { + // Arrange + const statement = `SELECT * FROM users;`; + // Act + const result = builder.buildSQL(statement, { limit: '$1', offset: '$2' }); + // Arrange + expect(result).toBe( + 'SELECT * FROM (SELECT * FROM users) LIMIT $1 OFFSET $2;' + ); +}); From 3e43ae83229333c8c312f241c488b85ae4373149 Mon Sep 17 00:00:00 2001 From: kokokuo Date: Fri, 7 Jul 2023 12:23:29 +0800 Subject: [PATCH 5/9] chore(docs, extension-driver-clickhouse): optimizing and correcting clickhouse documentation content and syntax. --- packages/doc/docs/connect/clickhouse.mdx | 60 +++++++-------- packages/doc/docs/connectors/clickhouse.mdx | 64 +++++++++------- .../extension-driver-clickhouse/README.md | 75 ++++++++++--------- 3 files changed, 105 insertions(+), 94 deletions(-) diff --git a/packages/doc/docs/connect/clickhouse.mdx b/packages/doc/docs/connect/clickhouse.mdx index 0c62ac97..0bd7e67b 100644 --- a/packages/doc/docs/connect/clickhouse.mdx +++ b/packages/doc/docs/connect/clickhouse.mdx @@ -2,7 +2,7 @@ ## Installation -1. Install package +1. Install the package: **If you are developing with binary, the package is already bundled in the binary. You can skip this step.** @@ -10,7 +10,7 @@ npm i @vulcan-sql/extension-driver-clickhouse ``` -2. Update `vulcan.yaml`, and enable the extension. +2. Update your `vulcan.yaml` file to enable the extension: ```yaml extensions: @@ -19,7 +19,7 @@ ch: '@vulcan-sql/extension-driver-clickhouse' # Add this line ``` -3. Create a new profile in `profiles.yaml` or in your profile files. For example: +3. Create a new profile in your `profiles.yaml` file or in the designated profile paths. For example: ```yaml - name: ch # profile name @@ -38,45 +38,45 @@ ## Configuration -Please check [arguments of ClickHouse Client](https://clickhouse.com/docs/en/integrations/language-clients/nodejs) for further information. - -| Name | Required | Default | Description | -| -------------------- | -------- | --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| host | N | http://localhost:8123 | ClickHouse instance URL. | -| request_timeout | N | 30000 | The request timeout in milliseconds. | -| max_open_connections | N | Infinity | Maximum number of sockets to allow per host. | -| compression | N | | Data applications operating with large datasets over the wire can benefit from enabling compression. Currently, only GZIP is supported using zlib.Please see [Compression docs](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#compression). | -| username | N | default | The name of the user on whose behalf requests are made. | -| password | N | | The user password. | -| application | N | VulcanSQL | The name of the application using the Node.js client. | -| database | N | default | Database name to use. | -| clickhouse_settings | N | | ClickHouse settings to apply to all requests. please see the [Advance Settings](https://clickhouse.com/docs/en/operations/settings), and [Definition](https://github.com/ClickHouse/clickhouse-js/blob/0.1.1/src/settings.ts) | -| tls | N | | Configure TLS certificates. Please see [TLS docs](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#tls-certificates). | -| session_id | N | | ClickHouse Session ID to send with every request. | -| keep_alive | N | | HTTP Keep-Alive related settings. Please see [Keep Alive docs](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#keep-alive) | - -At the above, it not contains `log` option, because the `logs` need to define a Logger class and assign to it, so it could not set by `profiles.yaml`. +For more information, please refer to the [ClickHouse Client documentation](https://clickhouse.com/docs/en/integrations/language-clients/nodejs) to learn about the available arguments for the ClickHouse Client. + +| Name | Required | Default | Description | +| -------------------- | -------- | --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| host | N | http://localhost:8123 | ClickHouse instance URL. | +| request_timeout | N | 30000 | Request timeout in milliseconds. | +| max_open_connections | N | Infinity | Maximum number of sockets to allow per host. | +| compression | N | | Compression settings for data transfer. Currently, only GZIP compression using zlib is supported. See [Compression docs](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#compression) for details. | +| username | N | default | The name of the user on whose behalf requests are made. | +| password | N | | The user's password. | +| application | N | VulcanSQL | The name of the application using the Node.js client. | +| database | N | default | Database name to use. | +| clickhouse_settings | N | | ClickHouse settings to apply to all requests. For all available settings, see [Advance Settings](https://clickhouse.com/docs/en/operations/settings), and For the definition, see [Definition](https://github.com/ClickHouse/clickhouse-js/blob/0.1.1/src/settings.ts) | +| tls | N | | Configure TLS certificates. See [TLS docs](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#tls-certificates). | +| session_id | N | | ClickHouse Session ID to send with every request. | +| keep_alive | N | | HTTP Keep-Alive related settings. See [Keep Alive docs](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#keep-alive) | + +The `log` option is not included above because it requires defining a Logger class and assigning it. Therefore, it cannot be set through `profiles.yaml`. ## Note -The ClickHouse support parameterized query to prevent SQL Injection by prepared statement. The named placeholder define by `{name:type}`, please see [Query with Parameters](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#queries-with-parameters). +ClickHouse supports parameterized queries to prevent SQL injection using prepared statements. Named placeholders are defined using the `{name:type}` syntax. For more information, refer to the [Query with Parameters](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#queries-with-parameters) section in the ClickHouse documentation. -However, The VulcanSQL API support API query parameter is JSON format, so it not support [variety types like ClickHouse](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#supported-clickhouse-data-types), The VulcanSQL will only support to convert below types: +However, the VulcanSQL API supports JSON format for API query parameters and does not support the full range of types available in ClickHouse. VulcanSQL only supports the conversion of the following types: -- `boolean` to `Boolean` ClickHouse type -- `number` to `Int` or `Float` ClickHouse type -- `string` to `String` ClickHouse type +- `boolean` to ClickHouse type `Boolean` +- `number` to ClickHouse types `Int` or `Float` +- `string` to ClickHouse type `String` -Therefore, if you would like to query the data is a special type from ClickHouse, e.g: `Array(Unit8)`, `Record`, `Date`, `DateTime` ...etc, you could use the ClickHouse [Regular Function](https://clickhouse.com/docs/en/sql-reference/functions) or [Type Conversion Function](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions) to do it. +Therefore, if you need to query data with special types in ClickHouse, such as `Array(Unit8)`, `Record`, `Date`, `DateTime`, and so on, you can use ClickHouse [Regular Functions](https://clickhouse.com/docs/en/sql-reference/functions) or [Type Conversion Functions](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions) to handle them. Example: ```sql --- If the val from API query parameter is '1990-11-01', and the born_date columns type is Date32 type --- What is the toDate function, please see https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions#todate +-- If the `val` from the API query parameter is '1990-11-01' and the `born_date` column is of type `Date32`, +-- you can use the toDate function to convert the value. See https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions#todate SELECT * FROM users WHERE born_date = toDate({val:String}); ``` ## ⚠️ Caution -ClickHouse driver currently not yet support for caching datasets feature. If you use the ClickHouse driver with caching dataset feature, it will be failed. +The ClickHouse driver currently does not support caching datasets. If you use the ClickHouse driver with caching dataset features, it will result in failure. diff --git a/packages/doc/docs/connectors/clickhouse.mdx b/packages/doc/docs/connectors/clickhouse.mdx index 5043bce2..81b92d47 100644 --- a/packages/doc/docs/connectors/clickhouse.mdx +++ b/packages/doc/docs/connectors/clickhouse.mdx @@ -1,6 +1,6 @@ # ClickHouse -Connect with your ClickHouse servers via the official [Node.js Driver](https://clickhouse.com/docs/en/integrations/language-clients/nodejs). +Connect to your ClickHouse servers using the official [Node.js Driver](https://clickhouse.com/docs/en/integrations/language-clients/nodejs). ## Installation @@ -43,45 +43,53 @@ Connect with your ClickHouse servers via the official [Node.js Driver](https://c ## Configuration -Please check [arguments of ClickHouse Client](https://clickhouse.com/docs/en/integrations/language-clients/nodejs) for further information. - -| Name | Required | Default | Description | -| -------------------- | -------- | --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| host | N | http://localhost:8123 | ClickHouse instance URL. | -| request_timeout | N | 30000 | The request timeout in milliseconds. | -| max_open_connections | N | Infinity | Maximum number of sockets to allow per host. | -| compression | N | | Data applications operating with large datasets over the wire can benefit from enabling compression. Currently, only GZIP is supported using zlib.Please see [Compression docs](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#compression). | -| username | N | default | The name of the user on whose behalf requests are made. | -| password | N | | The user password. | -| application | N | VulcanSQL | The name of the application using the Node.js client. | -| database | N | default | Database name to use. | -| clickhouse_settings | N | | ClickHouse settings to apply to all requests. please see the [Advance Settings](https://clickhouse.com/docs/en/operations/settings), and [Definition](https://github.com/ClickHouse/clickhouse-js/blob/0.1.1/src/settings.ts) | -| tls | N | | Configure TLS certificates. Please see [TLS docs](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#tls-certificates). | -| session_id | N | | ClickHouse Session ID to send with every request. | -| keep_alive | N | | HTTP Keep-Alive related settings. Please see [Keep Alive docs](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#keep-alive) | - -At the above, it not contains `log` option, because the `logs` need to define a Logger class and assign to it, so it could not set by `profiles.yaml`. +For more information, please refer to the [ClickHouse Client documentation](https://clickhouse.com/docs/en/integrations/language-clients/nodejs) to learn about the available arguments for the ClickHouse Client. + +| Name | Required | Default | Description | +| -------------------- | -------- | --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| host | N | http://localhost:8123 | ClickHouse instance URL. | +| request_timeout | N | 30000 | Request timeout in milliseconds. | +| max_open_connections | N | Infinity | Maximum number of sockets to allow per host. | +| compression | N | | Compression settings for data transfer. Currently, only GZIP compression using zlib is supported. See [Compression docs](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#compression) for details. | +| username | N | default | The name of the user on whose behalf requests are made. | +| password | N | | The user's password. | +| application | N | VulcanSQL | The name of the application using the Node.js client. | +| database | N | default | Database name to use. | +| clickhouse_settings | N | | ClickHouse settings to apply to all requests. For all available settings, see [Advance Settings](https://clickhouse.com/docs/en/operations/settings), and For the definition, see [Definition](https://github.com/ClickHouse/clickhouse-js/blob/0.1.1/src/settings.ts) | +| tls | N | | Configure TLS certificates. See [TLS docs](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#tls-certificates). | +| session_id | N | | ClickHouse Session ID to send with every request. | +| keep_alive | N | | HTTP Keep-Alive related settings. See [Keep Alive docs](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#keep-alive) | + +The `log` option is not included above because it requires defining a Logger class and assigning it. Therefore, it cannot be set through `profiles.yaml`. ## Note -The ClickHouse support parameterized query to prevent SQL Injection by prepared statement. The named placeholder define by `{name:type}`, please see [Query with Parameters](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#queries-with-parameters). +ClickHouse supports parameterized queries to prevent SQL injection using prepared statements. Named placeholders are defined using the `{name:type}` syntax. For more information, refer to the [Query with Parameters](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#queries-with-parameters) section in the ClickHouse documentation. -However, The VulcanSQL API support API query parameter is JSON format, so it not support [variety types like ClickHouse](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#supported-clickhouse-data-types), The VulcanSQL will only support to convert below types: +However, the VulcanSQL API supports JSON format for API query parameters and does not support the full range of types available in ClickHouse. VulcanSQL only supports the conversion of the following types: -- `boolean` to `Boolean` ClickHouse type -- `number` to `Int` or `Float` ClickHouse type -- `string` to `String` ClickHouse type +- `boolean` to ClickHouse type `Boolean` +- `number` to ClickHouse types `Int` or `Float` +- `string` to ClickHouse type `String` -Therefore, if you would like to query the data is a special type from ClickHouse, e.g: `Array(Unit8)`, `Record`, `Date`, `DateTime` ...etc, you could use the ClickHouse [Regular Function](https://clickhouse.com/docs/en/sql-reference/functions) or [Type Conversion Function](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions) to do it. +Therefore, if you need to query data with special types in ClickHouse, such as `Array(Unit8)`, `Record`, `Date`, `DateTime`, and so on, you can use ClickHouse [Regular Functions](https://clickhouse.com/docs/en/sql-reference/functions) or [Type Conversion Functions](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions) to handle them. Example: ```sql --- If the val from API query parameter is '1990-11-01', and the born_date columns type is Date32 type --- What is the toDate function, please see https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions#todate +-- If the `val` from the API query parameter is '1990-11-01' and the `born_date` column is of type `Date32`, +-- you can use the toDate function to convert the value. See https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions#todate SELECT * FROM users WHERE born_date = toDate({val:String}); ``` ## ⚠️ Caution -ClickHouse driver currently not yet support for caching datasets feature. If you use the ClickHouse driver with caching dataset feature, it will be failed. +The ClickHouse driver currently does not support caching datasets. If you use the ClickHouse driver with caching dataset features, it will result in failure. + +## Testing + +To run tests for the `extension-driver-clickhouse` module, use the following command: + +```bash +nx test extension-driver-clickhouse +``` diff --git a/packages/extension-driver-clickhouse/README.md b/packages/extension-driver-clickhouse/README.md index 5464719b..8c682625 100644 --- a/packages/extension-driver-clickhouse/README.md +++ b/packages/extension-driver-clickhouse/README.md @@ -1,103 +1,106 @@ # extension-driver-clickhouse -[ClickHouse](https://clickhouse.com/) driver for VulcanSQL. +This is the ClickHouse driver for VulcanSQL, provided by [Canner](https://canner.io/). -## Install +## Installation -1. Install package +1. Install the package: - ```sql + ```bash npm i @vulcan-sql/extension-driver-clickhouse ``` -2. Update `vulcan.yaml`, enable the extension. +2. Update your `vulcan.yaml` file to enable the extension: ```yaml extensions: clickhouse: '@vulcan-sql/extension-driver-clickhouse' ``` -3. Create a new profile in `profiles.yaml` or in your profiles' paths. Please check [arguments of ClickHouse Client](https://clickhouse.com/docs/en/integrations/language-clients/nodejs) for further information. +3. Create a new profile in your `profiles.yaml` file or in the designated profile paths. For more information, please refer to the [ClickHouse Client documentation](https://clickhouse.com/docs/en/integrations/language-clients/nodejs) for the available arguments. ```yaml - - name: ch # profile name + - name: ch # Profile name type: clickhouse connection: - # Optional: ClickHouse instance URL. default is http://localhost:8123. + # Optional: ClickHouse instance URL. Default is http://localhost:8123. host: 'www.example.com:8123' - # Optional: The request timeout in milliseconds. Default value: 30000 + # Optional: Request timeout in milliseconds. Default value: 30000 request_timeout: 60000 # Optional: Maximum number of sockets to allow per host. Default value: Infinity. max_open_connections: 10 - # Optional: Data applications operating with large datasets over the wire can benefit from enabling compression. Currently, only GZIP is supported using zlib. Please see https://clickhouse.com/docs/en/integrations/language-clients/nodejs#compression + # Optional: Compression settings for data transfer. Currently, only GZIP compression using zlib is supported. See https://clickhouse.com/docs/en/integrations/language-clients/nodejs#compression for details. compression: - # Optional: "response: true" instructs ClickHouse server to respond with compressed response body. Default: true + # Optional: "response: true" instructs ClickHouse server to respond with a compressed response body. Default: true response: true - # Optional: "request: true" enabled compression on the client request body. Default value: false + # Optional: "request: true" enables compression on the client request body. Default value: false request: false - # Optional: The name of the user on whose behalf requests are made. Default value: 'default' + # Optional: The name of the user on whose behalf requests are made. Default value for username: 'default' username: 'user' - # The user password. Default: ''. + # The user's password. Default: ''. password: 'pass' # Optional: The name of the application using the Node.js client. Default value: VulcanSQL application: 'VulcanSQL' # Optional: Database name to use. Default value: 'default' database: 'hello-clickhouse' - # Optional: ClickHouse settings to apply to all requests, below is a sample. - # For all settings, please see the https://clickhouse.com/docs/en/operations/settings, the definition see https://github.com/ClickHouse/clickhouse-js/blob/0.1.1/src/settings.ts + # Optional: ClickHouse settings to apply to all requests. The following is a sample configuration. + # For all available settings, see https://clickhouse.com/docs/en/operations/settings. + # For the definition, see https://github.com/ClickHouse/clickhouse-js/blob/0.1.1/src/settings.ts clickhouse_settings: - # Optional: Allow Nullable types as primary keys. (default: false) + # Optional: Allow Nullable types as primary keys. Default: false allow_nullable_key: true - # Optional: configure TLS certificates, please see https://clickhouse.com/docs/en/integrations/language-clients/nodejs#tls-certificates + # Optional: Configure TLS certificates. See https://clickhouse.com/docs/en/integrations/language-clients/nodejs#tls-certificates tls: - # Optional The CA Cert file path + # Optional: The path to the CA Cert file ca_cert: 'ca-cert-file-path' - # Optional: The Cert file path + # Optional: The path to the Cert file cert: 'cert-file-path' - # Optional: The key file path + # Optional: The path to the key file key: 'key-path' # Optional: ClickHouse Session ID to send with every request session_id: '' - # Optional: HTTP Keep-Alive related settings, please see https://clickhouse.com/docs/en/integrations/language-clients/nodejs#keep-alive + # Optional: HTTP Keep-Alive related settings. See https://clickhouse.com/docs/en/integrations/language-clients/nodejs#keep-alive keep_alive: # Optional: Enable or disable HTTP Keep-Alive mechanism. Default: true enabled: true - # Optional: How long to keep a particular open socket alive on the client side (in milliseconds). - # Should be less than the server setting (see `keep_alive_timeout` in server's `config.xml`). - # Currently, has no effect if is unset or false. Default value: 2500 (based on the default ClickHouse server setting, which is 3000) + # Optional: How long to keep a particular open socket alive on the client side (in milliseconds). + # Should be less than the server setting (see `keep_alive_timeout` in the server's `config.xml`). + # Currently, has no effect if unset or false. Default value: 2500 (based on the default ClickHouse server setting, which is 3000) socket_ttl: 2500 - # Optional: If the client detects a potentially expired socket based on the this socket will be immediately destroyed before sending the request ,and this request will be retried with a new socket up to 3 times. Default: false (no retries) + # Optional: If the client detects a potentially expired socket based on this value, the socket will be immediately destroyed before sending the request, and the request will be retried with a new socket up to 3 times. Default: false (no retries) retry_on_expired_socket: false ``` -At the above, it not contains `log` option, because the `logs` need to define a Logger class and assign to it, so it could not set by `profiles.yaml`. +The `log` option is not included above because it requires defining a Logger class and assigning it. Therefore, it cannot be set through `profiles.yaml`. ## Note -The ClickHouse support parameterized query to prevent SQL Injection by prepared statement. The named placeholder define by `{name:type}`, please see [Query with Parameters](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#queries-with-parameters). +ClickHouse supports parameterized queries to prevent SQL injection using prepared statements. Named placeholders are defined using the `{name:type}` syntax. For more information, refer to the [Query with Parameters](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#queries-with-parameters) section in the ClickHouse documentation. -However, The VulcanSQL API support API query parameter is JSON format, so it not support [variety types like ClickHouse](https://clickhouse.com/docs/en/integrations/language-clients/nodejs#supported-clickhouse-data-types), The VulcanSQL will only support to convert below types: +However, the VulcanSQL API supports JSON format for API query parameters and does not support the full range of types available in ClickHouse. VulcanSQL only supports the conversion of the following types: -- `boolean` to `Boolean` ClickHouse type -- `number` to `Int` or `Float` ClickHouse type -- `string` to `String` ClickHouse type +- `boolean` to ClickHouse type `Boolean` +- `number` to ClickHouse types `Int` or `Float` +- `string` to ClickHouse type `String` -Therefore, if you would like to query the data is a special type from ClickHouse, e.g: `Array(Unit8)`, `Record`, `Date`, `DateTime` ...etc, you could use the ClickHouse [Regular Function](https://clickhouse.com/docs/en/sql-reference/functions) or [Type Conversion Function](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions) to do it. +Therefore, if you need to query data with special types in ClickHouse, such as `Array(Unit8)`, `Record`, `Date`, `DateTime`, and so on, you can use ClickHouse [Regular Functions](https://clickhouse.com/docs/en/sql-reference/functions) or [Type Conversion Functions](https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions) to handle them. Example: ```sql --- If the val from API query parameter is '1990-11-01', and the born_date columns type is Date32 type --- What is the toDate function, please see https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions#todate +-- If the `val` from the API query parameter is '1990-11-01' and the `born_date` column is of type `Date32`, +-- you can use the toDate function to convert the value. See https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions#todate SELECT * FROM users WHERE born_date = toDate({val:String}); ``` ## ⚠️ Caution -ClickHouse driver currently not yet support for caching datasets feature. If you use the ClickHouse driver with caching dataset feature, it will be failed. +The ClickHouse driver currently does not support caching datasets. If you use the ClickHouse driver with caching dataset features, it will result in failure. ## Testing +To run tests for the `extension-driver-clickhouse` module, use the following command: + ```bash nx test extension-driver-clickhouse ``` From 3ccd30126c11712e30ca10bc091abc86b019a551 Mon Sep 17 00:00:00 2001 From: kokokuo Date: Fri, 7 Jul 2023 14:21:09 +0800 Subject: [PATCH 6/9] chore(labs, extension-driver-clickhouse): update Makefile to add extension-driver-clickhouse --- labs/playground1/Makefile | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/labs/playground1/Makefile b/labs/playground1/Makefile index f1cc0eb4..5ad9b499 100644 --- a/labs/playground1/Makefile +++ b/labs/playground1/Makefile @@ -6,7 +6,7 @@ start: build test-data/moma.db ../../node_modules @vulcan start # build the required packages -build: pkg-core pkg-build pkg-serve pkg-catalog-server pkg-cli pkg-extension-driver-duckdb pkg-extension-authenticator-canner +build: pkg-core pkg-build pkg-serve pkg-catalog-server pkg-cli pkg-extension-driver-duckdb pkg-extension-authenticator-canner pkg-extension-driver-clickhouse # build for core pakge @@ -56,6 +56,15 @@ pkg-extension-authenticator-canner: ../../node_modules rm -rf ./labs/playground1/node_modules/@vulcan-sql/extension-authenticator-canner; \ cp -R ./dist/packages/extension-authenticator-canner ./labs/playground1/node_modules/@vulcan-sql +pkg-extension-driver-clickhouse: ../../node_modules + @cd ../..; \ + yarn nx build extension-driver-clickhouse; \ + mkdir -p ./labs/playground1/node_modules/@vulcan-sql; \ + rm -rf ./labs/playground1/node_modules/@vulcan-sql/extension-driver-clickhouse; \ + cp -R ./dist/packages/extension-driver-clickhouse ./labs/playground1/node_modules/@vulcan-sql; \ + cp -R ./packages/extension-driver-clickhouse/node_modules/@clickhouse ./labs/playground1/node_modules + + # build and install for cli pakge pkg-cli: ../../node_modules @cd ../..; \ From 9f3e4ce18269b81a3c79e095d7f6679a707f5759 Mon Sep 17 00:00:00 2001 From: kokokuo Date: Fri, 7 Jul 2023 14:22:21 +0800 Subject: [PATCH 7/9] fix(extension-driver-clickhouse): fix the typo and make the clickhouse server docker name may not conflict --- packages/extension-driver-clickhouse/test/clickHouseServer.ts | 2 +- packages/extension-driver-clickhouse/test/errorHandler.spec.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/extension-driver-clickhouse/test/clickHouseServer.ts b/packages/extension-driver-clickhouse/test/clickHouseServer.ts index ae2e7c08..110989fb 100644 --- a/packages/extension-driver-clickhouse/test/clickHouseServer.ts +++ b/packages/extension-driver-clickhouse/test/clickHouseServer.ts @@ -27,7 +27,7 @@ export class ClickHouseServer { // https://github.com/apocas/dockerode/issues/647 await new Promise((res) => docker.modem.followProgress(pullStream, res)); this.container = await docker.createContainer({ - name: `vulcan-clickhouse-test-${faker.random.word()}`, + name: `vulcan-clickhouse-test-${faker.git.shortSha()}`, Image: this.image, ExposedPorts: { '8123/tcp': {}, diff --git a/packages/extension-driver-clickhouse/test/errorHandler.spec.ts b/packages/extension-driver-clickhouse/test/errorHandler.spec.ts index a9d4c9be..b8813211 100644 --- a/packages/extension-driver-clickhouse/test/errorHandler.spec.ts +++ b/packages/extension-driver-clickhouse/test/errorHandler.spec.ts @@ -31,7 +31,7 @@ it( }); // We must consume the stream or the connection won't be released. await streamToArray(getData()); - // Close the pg server + // Close the clickhouse server await clickHouse.destroy(); // Act, Assert From c7b26e4deb62911b79a21473056bcaa6b98859a8 Mon Sep 17 00:00:00 2001 From: kokokuo Date: Fri, 7 Jul 2023 15:27:56 +0800 Subject: [PATCH 8/9] chore(extension-driver-clickhouse): normalize to convert to Bool clickhouse type - normalize to convert to Bool clickhouse type. - change to use number to prevent docker name duplicated. - update README. --- packages/extension-driver-clickhouse/README.md | 2 +- .../src/lib/typeMapper.ts | 12 ++++++++---- .../test/clickHouseServer.ts | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/extension-driver-clickhouse/README.md b/packages/extension-driver-clickhouse/README.md index 8c682625..6b2e965f 100644 --- a/packages/extension-driver-clickhouse/README.md +++ b/packages/extension-driver-clickhouse/README.md @@ -79,7 +79,7 @@ ClickHouse supports parameterized queries to prevent SQL injection using prepare However, the VulcanSQL API supports JSON format for API query parameters and does not support the full range of types available in ClickHouse. VulcanSQL only supports the conversion of the following types: -- `boolean` to ClickHouse type `Boolean` +- `boolean` to ClickHouse type `Bool` - `number` to ClickHouse types `Int` or `Float` - `string` to ClickHouse type `String` diff --git a/packages/extension-driver-clickhouse/src/lib/typeMapper.ts b/packages/extension-driver-clickhouse/src/lib/typeMapper.ts index eb47ca01..3918c237 100644 --- a/packages/extension-driver-clickhouse/src/lib/typeMapper.ts +++ b/packages/extension-driver-clickhouse/src/lib/typeMapper.ts @@ -5,7 +5,7 @@ const register = (clickHouseType: string, type: string) => { }; // Reference -// https://clickhouse.com/docs/en/integrations/language-clients/nodejs#resultset-and-row-abstractions +// https://clickhouse.com/docs/en/native-protocol/columns#integers // Currently, FieldDataType only support number, string, boolean, Date for generating response schema in the specification. register('Int', 'number'); register('UInt', 'number'); @@ -25,6 +25,8 @@ register('Int256', 'string'); register('Float32', 'number'); register('Float64', 'number'); register('Decimal', 'number'); +// When define column type or query result with parameterized query, The Bool or Boolean type both supported. +// But the column type of query result only return Bool, so we only support Bool type for safety. register('Bool', 'boolean'); register('String', 'string'); register('FixedString', 'string'); @@ -43,12 +45,12 @@ export const mapFromClickHouseType = (clickHouseType: string) => { /** * Convert the JS type (source is JSON format by API query parameter) to the corresponding ClickHouse type for generating named placeholder of parameterized query. - * Only support to convert number to Int or Float, boolean to Boolean, string to String, other types will convert to String. + * Only support to convert number to Int or Float, boolean to Bool, string to String, other types will convert to String. * If exist complex type e.g: object, Array, null, undefined, Date, Record.. etc, just convert to string type by ClickHouse function in SQL. * ClickHouse support converting string to other types function. * Please see Each section of the https://clickhouse.com/docs/en/sql-reference/functions and https://clickhouse.com/docs/en/sql-reference/functions/type-conversion-functions * @param value - * @returns 'FLoat', 'Int', 'Boolean', 'String' + * @returns 'FLoat', 'Int', 'Bool', 'String' */ export const mapToClickHouseType = (value: any) => { @@ -57,7 +59,9 @@ export const mapToClickHouseType = (value: any) => { if (value % 1 !== 0) return 'Float'; return 'Int'; } - if (typeof value === 'boolean') return 'Boolean'; + // When define column type or query result with parameterized query, The Bool or Boolean type both supported. + // But the column type of query result only return Bool, so we only support Bool type for safety. + if (typeof value === 'boolean') return 'Bool'; if (typeof value === 'string') return 'String'; return 'String'; }; diff --git a/packages/extension-driver-clickhouse/test/clickHouseServer.ts b/packages/extension-driver-clickhouse/test/clickHouseServer.ts index 110989fb..1653b44c 100644 --- a/packages/extension-driver-clickhouse/test/clickHouseServer.ts +++ b/packages/extension-driver-clickhouse/test/clickHouseServer.ts @@ -27,7 +27,7 @@ export class ClickHouseServer { // https://github.com/apocas/dockerode/issues/647 await new Promise((res) => docker.modem.followProgress(pullStream, res)); this.container = await docker.createContainer({ - name: `vulcan-clickhouse-test-${faker.git.shortSha()}`, + name: `vulcan-clickhouse-test-${faker.datatype.number({ min: 1 })}`, Image: this.image, ExposedPorts: { '8123/tcp': {}, From 59d4c39f030f0f97a08414f9b78ca1e6a0a22bf4 Mon Sep 17 00:00:00 2001 From: kokokuo Date: Fri, 7 Jul 2023 15:28:12 +0800 Subject: [PATCH 9/9] chore(docs): update clickhouse document --- packages/doc/docs/connect/clickhouse.mdx | 2 +- packages/doc/docs/connectors/clickhouse.mdx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/doc/docs/connect/clickhouse.mdx b/packages/doc/docs/connect/clickhouse.mdx index 0bd7e67b..3e78114d 100644 --- a/packages/doc/docs/connect/clickhouse.mdx +++ b/packages/doc/docs/connect/clickhouse.mdx @@ -63,7 +63,7 @@ ClickHouse supports parameterized queries to prevent SQL injection using prepare However, the VulcanSQL API supports JSON format for API query parameters and does not support the full range of types available in ClickHouse. VulcanSQL only supports the conversion of the following types: -- `boolean` to ClickHouse type `Boolean` +- `boolean` to ClickHouse type `Bool` - `number` to ClickHouse types `Int` or `Float` - `string` to ClickHouse type `String` diff --git a/packages/doc/docs/connectors/clickhouse.mdx b/packages/doc/docs/connectors/clickhouse.mdx index 81b92d47..c9933c04 100644 --- a/packages/doc/docs/connectors/clickhouse.mdx +++ b/packages/doc/docs/connectors/clickhouse.mdx @@ -68,7 +68,7 @@ ClickHouse supports parameterized queries to prevent SQL injection using prepare However, the VulcanSQL API supports JSON format for API query parameters and does not support the full range of types available in ClickHouse. VulcanSQL only supports the conversion of the following types: -- `boolean` to ClickHouse type `Boolean` +- `boolean` to ClickHouse type `Bool` - `number` to ClickHouse types `Int` or `Float` - `string` to ClickHouse type `String`