diff --git a/README.md b/README.md index 06d272f5bdec41..5a9faef4e7c257 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,6 @@ Views [Apache Parquet](https://parquet.apache.org/) files as JSON. -Requires [parquet-tools](https://mvnrepository.com/artifact/org.apache.parquet/parquet-tools). - ## Features When opening a Parquet file, a JSON presentation of the file will open automatically: @@ -18,8 +16,10 @@ After closing the JSON view, it is possible to reopen it by clicking on the link ## Requirements -The extension requires [parquet-tools](https://github.com/apache/parquet-mr/tree/master/parquet-tools-deprecated). -It should be in your PATH, or a path can be set in settings. +The extension used to require [parquet-tools](https://mvnrepository.com/artifact/org.apache.parquet/parquet-tools). +Now the extension uses the [parquets](https://github.com/dvirtz/parquets) TypeScript library to do parse the files. + +If you still want to use `parquet-tools`, you should set `parquet-viewer.useParquetTools` to `true` and `paruqet-tools` should be in your `PATH`, or pointed by the `parquet-viewer.parquetToolsPath` setting. ![settings](images/settings.png) @@ -29,10 +29,11 @@ The following setting options are available: |name|default|description| |----|-------|-----------| -|`parquet-viewer.parquetToolsPath`|`parquet-tools`| the name of the parquet-tools executable or a path to the parquet-tools jar| -|`parquet-viewer.logPanel`|`false`|whether to write diagnostic logs to an output panel| +|`parquet-viewer.parquetToolsPath`|`parquet-tools`|The name of the parquet-tools executable or a path to the parquet-tools jar| +|`parquet-viewer.logPanel`|`false`|Whether to write diagnostic logs to an output panel| |`parquet-viewer.logFolder`|empty|Write diagnostic logs under the given directory| |`parquet-viewer.logLevel`|info|Diagnostic log level. Choose between: `off`, `fatal`, `error`, `warn`, `info`, `debug` or `trace`| +|`parquet-viewer.useParquetTools`|`false`|Use the legacy `parquet-tools` application for reading the files| ### What's new diff --git a/package-lock.json b/package-lock.json index 78554cc5c379a4..5368e025a1665a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -350,6 +350,18 @@ "kuler": "^2.0.0" } }, + "@dvirtz/parquets": { + "version": "0.11.2-develop.1", + "resolved": "https://registry.npmjs.org/@dvirtz/parquets/-/parquets-0.11.2-develop.1.tgz", + "integrity": "sha512-8pkZI6txBVxxtqXm8TxO6ENuSHM40zTwfk8IxoArPdINTBsYLwJzQy2vTjYHvdKnjSt2eEf5Vv+FYyLQqvAwBA==", + "requires": { + "bson": "^4.0.2", + "int53": "^1.0.0", + "node-int64": "^0.4.0", + "thrift": "^0.12.0", + "varint": "^5.0.0" + } + }, "@eslint/eslintrc": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.0.tgz", @@ -445,11 +457,37 @@ "dev": true }, "@types/node": { - "version": "14.14.6", - "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.6.tgz", - "integrity": "sha512-6QlRuqsQ/Ox/aJEQWBEJG7A9+u7oSYl3mem/K8IzxXG/kAGbV1YPD9Bg9Zw3vyxC/YP+zONKwy8hGkSt1jxFMw==", + "version": "12.20.11", + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.20.11.tgz", + "integrity": "sha512-gema+apZ6qLQK7k7F0dGkGCWQYsL0qqKORWOQO6tq46q+x+1C0vbOiOqOwRVlh4RAdbQwV/j/ryr3u5NOG1fPQ==", + "dev": true + }, + "@types/node-int64": { + "version": "0.4.29", + "resolved": "https://registry.npmjs.org/@types/node-int64/-/node-int64-0.4.29.tgz", + "integrity": "sha512-rHXvenLTj/CcsmNAebaBOhxQ2MqEGl3yXZZcZ21XYR+gzGTTcpOy2N4IxpvTCz48loyQNatHvfn6GhIbbZ1R3Q==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, + "@types/q": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/@types/q/-/q-1.5.4.tgz", + "integrity": "sha512-1HcDas8SEj4z1Wc696tH56G8OlRaH/sqZOynNNB+HF0WOeXPaxTtbYzJY2oEfiUxjSKjhCKr+MvR7dCHcEelug==", "dev": true }, + "@types/thrift": { + "version": "0.10.10", + "resolved": "https://registry.npmjs.org/@types/thrift/-/thrift-0.10.10.tgz", + "integrity": "sha512-sEhRbmmQgiTn+JdophEPiuTFkc9QRl7R/Kxs/le5GXZAc2K1/VCHauW734nCwpu1cFI8jL1niyrqk7E4tGL92w==", + "dev": true, + "requires": { + "@types/node": "*", + "@types/node-int64": "*", + "@types/q": "*" + } + }, "@types/vscode": { "version": "1.50.0", "resolved": "https://registry.npmjs.org/@types/vscode/-/vscode-1.50.0.tgz", @@ -734,6 +772,11 @@ "resolved": "https://registry.npmjs.org/async/-/async-3.2.0.tgz", "integrity": "sha512-TR2mEZFVOj2pLStYxLht7TyfuRzaydfpxr3k9RpHIzMgw7A64dzsdqCxH1WJyQdoe8T10nDXd9wnEigmiuHIZw==" }, + "async-limiter": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/async-limiter/-/async-limiter-1.0.1.tgz", + "integrity": "sha512-csOlWGAcRFJaI6m+F2WKdnMKr4HhdhFVBk0H/QbJFMCr+uO2kwohwXQPxw/9OCxp05r5ghVBFSyioixx3gfkNQ==" + }, "at-least-node": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz", @@ -758,6 +801,11 @@ "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "dev": true }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, "binary-extensions": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.1.0.tgz", @@ -795,6 +843,23 @@ "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", "dev": true }, + "bson": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/bson/-/bson-4.3.0.tgz", + "integrity": "sha512-LkKKeFJx5D6RRCRvLE+fDs40M2ZQNuk7W7tFXmKd7OOcQQ+BHdzCgRdL4XEGjc1UEGtiYuMvIVk91Bv8qsI50A==", + "requires": { + "buffer": "^5.6.0" + } + }, + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", @@ -1872,6 +1937,11 @@ "safer-buffer": ">= 2.1.2 < 3.0.0" } }, + "ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + }, "ignore": { "version": "5.1.8", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.8.tgz", @@ -1929,6 +1999,11 @@ "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", "dev": true }, + "int53": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/int53/-/int53-1.0.0.tgz", + "integrity": "sha512-u8BMiMa05OPBgd32CKTead0CVTsFVgwFk23nNXo1teKPF6Sxcu0lXxEzP//zTcaKzXbGgPDXGmj/woyv+I4C5w==" + }, "is-arrayish": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", @@ -2273,6 +2348,11 @@ "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", "dev": true }, + "node-int64": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", + "integrity": "sha1-h6kGXNs1XTGC2PlM4RGIuCXGijs=" + }, "normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -2472,6 +2552,11 @@ "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", "dev": true }, + "q": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/q/-/q-1.5.1.tgz", + "integrity": "sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=" + }, "queue-microtask": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.2.tgz", @@ -2895,6 +2980,16 @@ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, + "thrift": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/thrift/-/thrift-0.12.0.tgz", + "integrity": "sha512-qE9PZi4XSbSQLz/sNxj6+ZiiFQYgbM4GmlO3CS/EVJBjCVfd46Zw0aiVIqOvVn74M7XUGyjOs2chAOwK4d4/hQ==", + "requires": { + "node-int64": "^0.4.0", + "q": "^1.5.0", + "ws": "^5.0.0" + } + }, "tmp": { "version": "0.0.29", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.29.tgz", @@ -3038,6 +3133,11 @@ "integrity": "sha512-gTpR5XQNKFwOd4clxfnhaqvfqMpqEwr4tOtCyz4MtYZX2JYhfr1JvBFKdS+7K/9rfpZR3VLX+YWBbKoxCgS43Q==", "dev": true }, + "varint": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/varint/-/varint-5.0.2.tgz", + "integrity": "sha512-lKxKYG6H03yCZUpAGOPOsMcGxd1RHCu1iKvEHYDPmTyq2HueGhD73ssNBqqQWfvYs04G9iUFRvmAVLW20Jw6ow==" + }, "vsce": { "version": "1.81.1", "resolved": "https://registry.npmjs.org/vsce/-/vsce-1.81.1.tgz", @@ -3358,6 +3458,14 @@ "typedarray-to-buffer": "^3.1.5" } }, + "ws": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/ws/-/ws-5.2.2.tgz", + "integrity": "sha512-jaHFD6PFv6UgoIVda6qZllptQsMlDEJkTQcybzzXDYM1XO9Y8em691FGMPmM46WGyLU4z9KMgQN+qrux/nhlHA==", + "requires": { + "async-limiter": "~1.0.0" + } + }, "xdg-basedir": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-4.0.0.tgz", @@ -3365,9 +3473,9 @@ "dev": true }, "y18n": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.1.tgz", - "integrity": "sha512-wNcy4NvjMYL8gogWWYAO7ZFWFfHcbdbE57tZO8e4cbpj8tfUcwrwqSl3ad8HxpYWCdXcJUCeKKZS62Av1affwQ==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.3.tgz", + "integrity": "sha512-JKhqTOwSrqNA1NY5lSztJ1GrBiUodLMmIZuLiDaMRJ+itFd+ABVE8XBjOvIWL+rSqNDC74LCSFmlb/U4UZ4hJQ==", "dev": true }, "yallist": { diff --git a/package.json b/package.json index 0cb7aee5a7c6a1..42670255b664e5 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,10 @@ "trace" ], "default": "info" + }, + "parquet-viewer.useParquetTools": { + "description": "use legacy parquet-tools application for reading parquet files", + "type": "boolean" } } } @@ -97,7 +101,8 @@ "@types/chai-as-promised": "^7.1.3", "@types/glob": "^7.1.3", "@types/mocha": "^8.0.3", - "@types/node": "^14.14.6", + "@types/node": "^12.20.11", + "@types/thrift": "^0.10.10", "@types/vscode": "^1.46.0", "@typescript-eslint/eslint-plugin": "^4.16.1", "@typescript-eslint/parser": "^4.16.1", @@ -113,6 +118,7 @@ }, "dependencies": { "@async-generators/to-array": "^0.1.0", + "@dvirtz/parquets": "^0.11.2-develop.1", "@vscode-logging/logger": "^1.2.2", "@vscode-logging/wrapper": "^1.0.0" } diff --git a/src/extension.ts b/src/extension.ts index 1b6a9910c6feff..b45254787a06ec 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -3,7 +3,6 @@ // Import the module and reference it with the alias vscode in your code below import * as vscode from 'vscode'; import { ParquetEditorProvider } from './parquet-editor-provider'; -import { ParquetToolsRunner } from './parquet-tools-runner'; import { getLogger, initLogger } from './logger'; // this method is called when your extension is activated @@ -15,12 +14,5 @@ export async function activate(context: vscode.ExtensionContext): Promise // restart logger on configuration change vscode.workspace.onDidChangeConfiguration(() => initLogger(context)); - const parquetTools = await ParquetToolsRunner.spawnParquetTools(['-h']); - parquetTools.on('error', async (err: string) => { - const message = `parquet-tools not in PATH ('${err}')`; - getLogger().error(message); - await vscode.window.showErrorMessage(message); - }); - context.subscriptions.push(ParquetEditorProvider.register(context)); } diff --git a/src/parquet-editor-provider.ts b/src/parquet-editor-provider.ts index 6e0a362b6e85c9..1d8b81f4211f7b 100644 --- a/src/parquet-editor-provider.ts +++ b/src/parquet-editor-provider.ts @@ -2,10 +2,12 @@ import * as vscode from "vscode"; import * as path from 'path'; import { getNonce } from './util'; import { Disposable } from "./dispose"; -import { ParquetTextDocumentContentProvider } from "./parquet-document-provider"; -import { ParquetToolsRunner } from "./parquet-tools-runner"; +import { ParquetTextDocumentContentProvider } from './parquet-document-provider'; +import { ParquetToolsBackend } from './parquet-tools-backend'; +import { ParquetsBackend } from './parquets-backend'; import toArray from '@async-generators/to-array'; import { getLogger } from './logger'; +import { useParquetTools } from "./settings"; class DummyDocument extends Disposable implements vscode.CustomDocument { uri: vscode.Uri; @@ -24,29 +26,36 @@ class DummyDocument extends Disposable implements vscode.CustomDocument { ); } + private async * toJson(parquetPath: string, token?: vscode.CancellationToken): AsyncGenerator { + if (useParquetTools()) { + yield* ParquetToolsBackend.toJson(parquetPath, token); + } + + yield* ParquetsBackend.toJson(parquetPath, token); + } + public async show() { getLogger().info(`showing ${this.path}.as.json`); if (ParquetTextDocumentContentProvider.has(this.path)) { return await this.open(); } - return await vscode.window.withProgress({ - location: vscode.ProgressLocation.Notification, - title: `opening ${path.basename(this.path)}`, - cancellable: true - }, - async (progress, token) => { - try { - const json = await toArray(ParquetToolsRunner.toJson(this.path, token)); + try { + return await vscode.window.withProgress({ + location: vscode.ProgressLocation.Notification, + title: `opening ${path.basename(this.path)}`, + cancellable: true + }, + async (progress, token) => { + const json = await toArray(this.toJson(this.path, token)); if (!token.isCancellationRequested) { ParquetTextDocumentContentProvider.add(this.path, json.join('')); await this.open(); } - } catch (err) { - getLogger().error(err.message); - await vscode.window.showErrorMessage(err.message); - } - }); + }); + } catch (err) { + await vscode.window.showErrorMessage(err); + } } dispose(): void { @@ -86,11 +95,19 @@ export class ParquetEditorProvider implements vscode.CustomReadonlyEditorProvide webviewPanel.webview.html = this.getHtmlForWebview(webviewPanel.webview, document); - webviewPanel.webview.onDidReceiveMessage(_ => document.show()); + webviewPanel.webview.onDidReceiveMessage(e => this.onMessage(document, e)); await document.show(); } + private async onMessage(document: DummyDocument, message: string) { + switch (message) { + case 'clicked': + await document.show(); + break; + } + } + private getHtmlForWebview(webview: vscode.Webview, document: DummyDocument): string { // Use a nonce to whitelist which scripts can be run const nonce = getNonce(); @@ -109,7 +126,7 @@ export class ParquetEditorProvider implements vscode.CustomReadonlyEditorProvide //# sourceURL=to-json.js const vscode = acquireVsCodeApi(); document.getElementById('here').addEventListener('click', _ => { - vscode.postMessage(); + vscode.postMessage('clicked'); }); diff --git a/src/parquet-tools-runner.ts b/src/parquet-tools-backend.ts similarity index 93% rename from src/parquet-tools-runner.ts rename to src/parquet-tools-backend.ts index f0c6ed21b51579..15beea6806ac02 100644 --- a/src/parquet-tools-runner.ts +++ b/src/parquet-tools-backend.ts @@ -5,7 +5,7 @@ import * as assert from 'assert'; import { getLogger } from './logger'; import { parquetTools as getParquetTools } from './settings'; -export class ParquetToolsRunner { +export class ParquetToolsBackend { public static async spawnParquetTools(params: string[]): Promise { let parquetTools = getParquetTools(); if (!parquetTools) { @@ -33,7 +33,7 @@ export class ParquetToolsRunner { return; } - const parquetTools = await ParquetToolsRunner.spawnParquetTools(['cat', '-j', parquetPath]); + const parquetTools = await ParquetToolsBackend.spawnParquetTools(['cat', '-j', parquetPath]); token?.onCancellationRequested(_ => { getLogger().info(cancelledMessage); diff --git a/src/parquets-backend.ts b/src/parquets-backend.ts new file mode 100644 index 00000000000000..77db6c402362fd --- /dev/null +++ b/src/parquets-backend.ts @@ -0,0 +1,32 @@ +import * as vscode from 'vscode'; +import { getLogger } from './logger'; +import { ParquetReader } from '@dvirtz/parquets'; +import * as os from 'os'; + +export class ParquetsBackend { + public static async * toJson(parquetPath: string, token?: vscode.CancellationToken): AsyncGenerator { + const cancelledMessage = `parsing ${parquetPath} was cancelled by user`; + if (token?.isCancellationRequested) { + getLogger().info(cancelledMessage); + return; + } + + getLogger().info(`opening ${parquetPath}`) + try { + const reader = await ParquetReader.openFile(parquetPath); + const cursor = reader.getCursor(); + + // read all records from the file and print them + let record = null; + while (!token?.isCancellationRequested && (record = await cursor.next())) { + yield `${JSON.stringify(record)}${os.EOL}`; + } + + await reader.close(); + } catch (error) { + const message = `error while reading ${parquetPath}: ${error}`; + getLogger().error(message); + throw message; + } + } +} diff --git a/src/settings.ts b/src/settings.ts index 63dccec84fb000..f9ccac9d7a4c85 100644 --- a/src/settings.ts +++ b/src/settings.ts @@ -36,3 +36,7 @@ export function logLevel() : LogLevel { export async function setLogLevel(logLevel : LogLevel | undefined): Promise { await settings().update('logLevel', logLevel); } + +export function useParquetTools(): boolean { + return settings().get('useParquetTools', false); +} diff --git a/test/suite/logger.test.ts b/test/suite/logger.test.ts index 7cd592b3c67b1a..f684b3c04008cc 100644 --- a/test/suite/logger.test.ts +++ b/test/suite/logger.test.ts @@ -1,12 +1,12 @@ import { logLevel, logFolder, setLogFolder } from "../../src/settings"; -import { ParquetToolsRunner } from "../../src/parquet-tools-runner"; import { getUri } from "./utils"; import { promises } from "fs"; import { expect } from "chai"; import path = require("path"); import toArray from '@async-generators/to-array'; +import { ParquetsBackend } from "../../src/parquets-backend"; -suite('logger tests', function () { +suite('Logger tests', function () { const folder = path.resolve(__dirname); const logPath = path.join(folder, 'parquet-viewer.log'); @@ -24,10 +24,10 @@ suite('logger tests', function () { expect(logFolder()).to.equal(folder); const parquet = await getUri('small.parquet'); - const contents = await toArray(ParquetToolsRunner.toJson(parquet.fsPath)); + const contents = await toArray(ParquetsBackend.toJson(parquet.fsPath)); const logContents = await promises.readFile(logPath, 'utf-8'); expect(contents).to.have.lengthOf(2, `contents are ${contents}`); - expect(logContents).to.match(/\{\s+"label": "parquet-viewer",\s+"level": "debug",\s+"message": "spawning java -jar .*parquet-tools-1.12.0-SNAPSHOT.jar cat -j .*small.parquet",\s+"time": "\S+"\s+\}/); + expect(logContents).to.match(/\{\s+"label": "parquet-viewer",\s+"level": "info",\s+"message": "opening .*small.parquet",\s+"time": "\S+"\s+\}/); }); }); diff --git a/test/suite/parquet-tools-runner.test.ts b/test/suite/parquet-tools-backend.test.ts similarity index 61% rename from test/suite/parquet-tools-runner.test.ts rename to test/suite/parquet-tools-backend.test.ts index 3b0430cb287b0f..22110f41c4200d 100644 --- a/test/suite/parquet-tools-runner.test.ts +++ b/test/suite/parquet-tools-backend.test.ts @@ -1,17 +1,17 @@ -import { getUri, fileRead } from './utils'; import { expect, use } from 'chai'; import * as chaiAsPromised from 'chai-as-promised'; -import { ParquetToolsRunner } from "../../src/parquet-tools-runner"; +import { ParquetToolsBackend } from "../../src/parquet-tools-backend"; +import { fileRead, getUri } from './utils'; import toArray from '@async-generators/to-array'; use(chaiAsPromised); -suite("ParquetToolsRunner tests", () => { +suite("ParquetToolsBackend tests", () => { test('Converts Parquet to JSON', async function () { for (const name of ["small", "large"]) { const parquet = await getUri(`${name}.parquet`); - const json = (await toArray(ParquetToolsRunner.toJson(parquet.fsPath))).join(''); + const json = (await toArray(ParquetToolsBackend.toJson(parquet.fsPath))).join(''); const expected = await fileRead(`${name}.json`); if (name === "small") { @@ -23,15 +23,15 @@ suite("ParquetToolsRunner tests", () => { }); test("Error on not existing file", async function () { - await expect(toArray(ParquetToolsRunner.toJson("no-such-file"))).be.rejectedWith( + await expect(toArray(ParquetToolsBackend.toJson("no-such-file"))).be.rejectedWith( /parquet-tools exited with code 1:\n.*java.io.FileNotFoundException: File no-such-file does not exist/s ); }); test("-h works", async function () { - const parquetTools = await ParquetToolsRunner.spawnParquetTools(['-h']); - const exitCode = await new Promise( (resolve) => { - parquetTools.on('close', resolve); + const parquetTools = await ParquetToolsBackend.spawnParquetTools(['-h']); + const exitCode = await new Promise((resolve) => { + parquetTools.on('close', resolve); }); expect(exitCode).to.equal(0); }); diff --git a/test/suite/parquets-backend.test.ts b/test/suite/parquets-backend.test.ts new file mode 100644 index 00000000000000..fee95cd71605db --- /dev/null +++ b/test/suite/parquets-backend.test.ts @@ -0,0 +1,30 @@ + +import { getUri, fileRead } from './utils'; +import { expect, use } from 'chai'; +import * as chaiAsPromised from 'chai-as-promised'; +import toArray from '@async-generators/to-array'; +import { ParquetsBackend } from '../../src/parquets-backend'; + +use(chaiAsPromised); + +suite("ParquetsBackend tests", () => { + test('Converts Parquet to JSON', async function () { + for (const name of ["small", "large"]) { + const parquet = await getUri(`${name}.parquet`); + const json = (await toArray(ParquetsBackend.toJson(parquet.fsPath))).join(''); + const expected = await fileRead(`${name}.json`); + + if (name === "small") { + expect(json).to.equal(expected); + } else if (json !== expected) { + expect.fail("large JSON differ"); + } + } + }); + + test("Error on not existing file", async function () { + await expect(toArray(ParquetsBackend.toJson("no-such-file"))).be.rejectedWith( + /error while reading no-such-file: Error: ENOENT: no such file or directory, stat '.*no-such-file'/ + ); + }); +}); diff --git a/tsconfig.json b/tsconfig.json index bd844f9a4c7970..18c3852f53f000 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,7 +4,8 @@ "target": "es6", "outDir": "out", "lib": [ - "es6" + "es6", + "DOM" ], "sourceMap": true, "resolveJsonModule": true, @@ -12,6 +13,10 @@ "strict": true, /* enable all strict type-checking options */ /* Additional Checks */ "noUnusedLocals": true, /* Report errors on unused locals. */ + "typeRoots": [ + "src/@types", + "node_modules/@types" + ], }, "exclude": [ "node_modules",