-
Notifications
You must be signed in to change notification settings - Fork 22
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add partial update to pipelines (#1453)
## Changes I've changed the "normal" pipeline run action to use `run-all` icon, and the partial update run uses the `run` instead. Currently we don't pre-load pipeline updates, so if a user hasn't executed a pipeline from the extension, the partial update action will not show a list of tables to select from (and users will have to type the table names manually). After the pipeline is executed we show the tables selection in the partial update action. Pre-loading pipeline updates is for later. New action: <img width="676" alt="Screenshot 2024-11-18 at 15 31 07" src="https://github.com/user-attachments/assets/31022043-9820-497a-a5bf-8d2b71fed24a"> The case when we don't know about past events: <img width="896" alt="Screenshot 2024-11-15 at 11 24 52" src="https://github.com/user-attachments/assets/6bdf0f49-399b-4657-a2d1-8207d4e8f78c"> <img width="896" alt="Screenshot 2024-11-15 at 11 25 01" src="https://github.com/user-attachments/assets/3044ddf2-1059-4775-aa00-9a28165c1e77"> The case when we offer a selection of table names based on past runs: <img width="896" alt="Screenshot 2024-11-15 at 11 26 28" src="https://github.com/user-attachments/assets/e4f834c3-918a-4f0a-92ab-182305fe450c"> <img width="896" alt="Screenshot 2024-11-15 at 11 25 13" src="https://github.com/user-attachments/assets/48c5a21f-7423-4585-99f9-62a682101ece"> ## Tests Partially covered by unit tests
- Loading branch information
Showing
9 changed files
with
439 additions
and
24 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
119 changes: 119 additions & 0 deletions
119
packages/databricks-vscode/src/bundle/BundlePipelinesManager.test.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,119 @@ | ||
| import {BundlePipelinesManager} from "./BundlePipelinesManager"; | ||
| import {BundleRunStatusManager} from "./run/BundleRunStatusManager"; | ||
| import {ConfigModel} from "../configuration/models/ConfigModel"; | ||
| import {mock, instance, when} from "ts-mockito"; | ||
| import assert from "assert"; | ||
| import {EventEmitter} from "vscode"; | ||
| import {install, InstalledClock} from "@sinonjs/fake-timers"; | ||
|
|
||
| describe(__filename, () => { | ||
| let runStatusManager: BundleRunStatusManager; | ||
| let configModel: ConfigModel; | ||
| let manager: BundlePipelinesManager; | ||
| let eventEmitter: EventEmitter<void>; | ||
| let clock: InstalledClock; | ||
|
|
||
| beforeEach(() => { | ||
| clock = install(); | ||
| eventEmitter = new EventEmitter(); | ||
| runStatusManager = mock<BundleRunStatusManager>(); | ||
| configModel = mock<ConfigModel>(); | ||
| when(runStatusManager.onDidChange).thenReturn(eventEmitter.event); | ||
| when(configModel.onDidChangeKey("remoteStateConfig")).thenReturn( | ||
| new EventEmitter<void>().event | ||
| ); | ||
| when(configModel.onDidChangeTarget).thenReturn( | ||
| new EventEmitter<void>().event | ||
| ); | ||
| manager = new BundlePipelinesManager( | ||
| instance(runStatusManager), | ||
| instance(configModel) | ||
| ); | ||
| }); | ||
|
|
||
| afterEach(() => { | ||
| clock.uninstall(); | ||
| }); | ||
|
|
||
| it("should update pipeline datasets from run events", async () => { | ||
| let datasets; | ||
| const remoteState = {resources: {pipelines: {pipeline1: {}}}}; | ||
| when(configModel.get("remoteStateConfig")).thenResolve(remoteState); | ||
| const runStatuses = new Map(); | ||
| when(runStatusManager.runStatuses).thenReturn(runStatuses); | ||
|
|
||
| /* eslint-disable @typescript-eslint/naming-convention */ | ||
| const firstRun = { | ||
| data: { | ||
| update: {creation_time: 10}, | ||
| }, | ||
| events: [ | ||
| {origin: {dataset_name: "table1"}}, | ||
| {origin: {not_a_dataset_name: "table1.5"}}, | ||
| {origin: {dataset_name: "table2"}}, | ||
| ], | ||
| }; | ||
| /* eslint-enable @typescript-eslint/naming-convention */ | ||
| runStatuses.set("pipelines.pipeline1", firstRun); | ||
|
|
||
| eventEmitter.fire(); | ||
| await clock.runToLastAsync(); | ||
|
|
||
| datasets = manager.getDatasets("pipeline1"); | ||
| assert.strictEqual(datasets.size, 2); | ||
| assert(datasets.has("table1")); | ||
| assert(datasets.has("table2")); | ||
|
|
||
| /* eslint-disable @typescript-eslint/naming-convention */ | ||
| const secondPartialRun = { | ||
| data: { | ||
| update: { | ||
| creation_time: 100, | ||
| refresh_selection: ["table3", "table4"], | ||
| }, | ||
| }, | ||
| events: [ | ||
| {origin: {dataset_name: "table3"}}, | ||
| {origin: {not_a_dataset_name: "table3.5"}}, | ||
| {origin: {dataset_name: "table4"}}, | ||
| ], | ||
| }; | ||
| /* eslint-enable @typescript-eslint/naming-convention */ | ||
|
|
||
| runStatuses.set("pipelines.pipeline1", secondPartialRun); | ||
| eventEmitter.fire(); | ||
| await clock.runToLastAsync(); | ||
|
|
||
| datasets = manager.getDatasets("pipeline1"); | ||
| assert.strictEqual(datasets.size, 4); | ||
| assert(datasets.has("table1")); | ||
| assert(datasets.has("table2")); | ||
| assert(datasets.has("table3")); | ||
| assert(datasets.has("table4")); | ||
|
|
||
| /* eslint-disable @typescript-eslint/naming-convention */ | ||
| const finalFullRefreshRun = { | ||
| data: { | ||
| update: { | ||
| creation_time: 200, | ||
| refresh_selection: [], | ||
| }, | ||
| }, | ||
| events: [ | ||
| {origin: {dataset_name: "table_new"}}, | ||
| {origin: {not_a_dataset_name: "not a table"}}, | ||
| {origin: {dataset_name: "table_final"}}, | ||
| ], | ||
| }; | ||
| /* eslint-enable @typescript-eslint/naming-convention */ | ||
| runStatuses.set("pipelines.pipeline1", finalFullRefreshRun); | ||
| eventEmitter.fire(); | ||
| await clock.runToLastAsync(); | ||
|
|
||
| // Only the datasets from the final full-refresh run should be left | ||
| datasets = manager.getDatasets("pipeline1"); | ||
| assert.strictEqual(datasets.size, 2); | ||
| assert(datasets.has("table_new")); | ||
| assert(datasets.has("table_final")); | ||
| }); | ||
| }); |
Oops, something went wrong.