From d20c4ba897c312933cb1e39dd0b53a6dcf9c20df Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Fri, 23 Jun 2023 07:45:36 +1000 Subject: [PATCH 1/6] Changes to storage format --- .../jupyter/connection/serverUriStorage.ts | 6 +-- src/platform/common/cache.ts | 3 +- .../userServerUriProvider.unit.test.ts | 9 +++- .../userServerUrlProvider.ts | 50 +++++++++++++++++-- 4 files changed, 58 insertions(+), 10 deletions(-) diff --git a/src/kernels/jupyter/connection/serverUriStorage.ts b/src/kernels/jupyter/connection/serverUriStorage.ts index 248a36f8138..8747edb21ee 100644 --- a/src/kernels/jupyter/connection/serverUriStorage.ts +++ b/src/kernels/jupyter/connection/serverUriStorage.ts @@ -280,7 +280,7 @@ class OldStorage { } public async getAll(): Promise { if (this.lastSavedList) { - return this.lastSavedList; + return this.lastSavedList.then((items) => items.sort((a, b) => b.time - a.time)); } const promise = async () => { // List is in the global memento, URIs are in encrypted storage @@ -305,7 +305,7 @@ class OldStorage { return result.filter((item) => !!item) as IJupyterServerUriEntry[]; }; this.lastSavedList = promise(); - return this.lastSavedList; + return this.lastSavedList.then((items) => items.sort((a, b) => b.time - a.time)); } public async getAllRaw(): Promise { // List is in the global memento, URIs are in encrypted storage @@ -534,7 +534,7 @@ class NewStorage { .catch(noop)); } public async getAll(): Promise { - return this.getAllImpl(true); + return this.getAllImpl(true).then((items) => items.sort((a, b) => b.time - a.time)); } public async clear(): Promise { const all = await this.getAllImpl(false); diff --git a/src/platform/common/cache.ts b/src/platform/common/cache.ts index 5871dc53d70..5f5f98e69af 100644 --- a/src/platform/common/cache.ts +++ b/src/platform/common/cache.ts @@ -36,7 +36,8 @@ export class OldCacheCleaner implements IExtensionSyncActivationService { 'JUPYTER_REMOTE_KERNELSPECS_V3', 'JUPYTER_LOCAL_KERNELSPECS_V4', 'LOCAL_KERNEL_SPECS_CACHE_KEY_V_2022_10', - 'LOCAL_KERNEL_PYTHON_AND_RELATED_SPECS_CACHE_KEY_V_2022_10' + 'LOCAL_KERNEL_PYTHON_AND_RELATED_SPECS_CACHE_KEY_V_2022_10', + 'user-jupyter-server-uri-list-v2' ] .filter((key) => this.globalState.get(key, undefined) !== undefined) .map((key) => this.globalState.update(key, undefined).then(noop, noop)) diff --git a/src/standalone/userJupyterServer/userServerUriProvider.unit.test.ts b/src/standalone/userJupyterServer/userServerUriProvider.unit.test.ts index d0aebb43bc8..7ad98a17d45 100644 --- a/src/standalone/userJupyterServer/userServerUriProvider.unit.test.ts +++ b/src/standalone/userJupyterServer/userServerUriProvider.unit.test.ts @@ -43,7 +43,7 @@ import { IJupyterPasswordConnectInfo, JupyterPasswordConnect } from './jupyterPa suite('User Uri Provider', () => { ['Old Password Manager', 'New Password Manager'].forEach((passwordManager) => { ['Old Storage Format', 'New Storage Format'].forEach((storageFormat) => { - suite(storageFormat, () => { + suite(`${passwordManager} - ${storageFormat}`, () => { const isNewPasswordManager = passwordManager === 'New Password Manager'; const isNewStorageFormat = storageFormat === 'New Storage Format'; let provider: UserJupyterServerUrlProvider; @@ -159,6 +159,13 @@ suite('User Uri Provider', () => { return Promise.resolve(); }); + when( + encryptedStorage.store( + Settings.JupyterServerRemoteLaunchService, + 'user-jupyter-server-uri-list-v2', + anything() + ) + ).thenResolve(); when( encryptedStorage.retrieve( Settings.JupyterServerRemoteLaunchService, diff --git a/src/standalone/userJupyterServer/userServerUrlProvider.ts b/src/standalone/userJupyterServer/userServerUrlProvider.ts index aa6e0b4fd54..6fac346cae4 100644 --- a/src/standalone/userJupyterServer/userServerUrlProvider.ts +++ b/src/standalone/userJupyterServer/userServerUrlProvider.ts @@ -52,7 +52,7 @@ import { validateSelectJupyterURI } from '../../kernels/jupyter/connection/serve import { Deferred, createDeferred } from '../../platform/common/utils/async'; export const UserJupyterServerUriListKey = 'user-jupyter-server-uri-list'; -export const UserJupyterServerUriListKeyV2 = 'user-jupyter-server-uri-list-v2'; +export const UserJupyterServerUriListKeyV2 = 'user-jupyter-server-uri-list-version2'; export const UserJupyterServerUriListMementoKey = '_builtin.jupyterServerUrlProvider.uriList'; const GlobalStateUserAllowsInsecureConnections = 'DataScienceAllowInsecureConnections'; @@ -696,6 +696,40 @@ export class OldStorage { } } +type StorageItem = { + handle: string; + uri: string; + displayName: string; +}; +function serverToStorageFormat( + servers: { + handle: string; + uri: string; + serverInfo: IJupyterServerUri; + }[] +): StorageItem[] { + return servers.map((s) => ({ handle: s.handle, uri: s.uri, displayName: s.serverInfo.displayName })); +} +function storageFormatToServers(items: StorageItem[]) { + const servers: { + handle: string; + uri: string; + serverInfo: IJupyterServerUri; + }[] = []; + items.forEach((s) => { + const server = parseUri(s.uri); + if (!server) { + return; + } + servers.push({ + handle: s.handle, + uri: s.uri, + serverInfo: server + }); + }); + return servers; +} + export class NewStorage { private readonly _migrationDone: Deferred; private updatePromise = Promise.resolve(); @@ -721,7 +755,13 @@ export class NewStorage { // Already migrated once before. return this._migrationDone.resolve(); } - + this.encryptedStorage + .store( + Settings.JupyterServerRemoteLaunchService, + 'user-jupyter-server-uri-list-v2', // Removed as this storage data is not in the best format. + undefined + ) + .catch(noop); await this.encryptedStorage.store( Settings.JupyterServerRemoteLaunchService, UserJupyterServerUriListKeyV2, @@ -738,7 +778,7 @@ export class NewStorage { return []; } try { - return JSON.parse(data); + return storageFormatToServers(JSON.parse(data)); } catch { return []; } @@ -751,7 +791,7 @@ export class NewStorage { await this.encryptedStorage.store( Settings.JupyterServerRemoteLaunchService, UserJupyterServerUriListKeyV2, - JSON.stringify(servers) + JSON.stringify(serverToStorageFormat(servers)) ); }) .catch(noop)); @@ -763,7 +803,7 @@ export class NewStorage { return this.encryptedStorage.store( Settings.JupyterServerRemoteLaunchService, UserJupyterServerUriListKeyV2, - JSON.stringify(servers) + JSON.stringify(serverToStorageFormat(servers)) ); }) .catch(noop)); From 2c46f331469915c66c8f40eb29abc6755c6359fe Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Fri, 23 Jun 2023 09:10:57 +1000 Subject: [PATCH 2/6] fixes --- src/kernels/jupyter/connection/serverUriStorage.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/kernels/jupyter/connection/serverUriStorage.ts b/src/kernels/jupyter/connection/serverUriStorage.ts index 8747edb21ee..edaece447a3 100644 --- a/src/kernels/jupyter/connection/serverUriStorage.ts +++ b/src/kernels/jupyter/connection/serverUriStorage.ts @@ -111,6 +111,7 @@ export class JupyterServerUriStorage extends Disposables implements IJupyterServ } public async get(id: string): Promise { this.hookupStorageEvents(); + await this.newStorage.migrateMRU(); const savedList = await this.getAll(); return savedList.find((item) => item.serverId === id); } @@ -119,6 +120,7 @@ export class JupyterServerUriStorage extends Disposables implements IJupyterServ options?: { time: number; displayName: string } ): Promise { this.hookupStorageEvents(); + await this.newStorage.migrateMRU(); traceInfoIfCI(`setUri: ${jupyterHandle.id}.${jupyterHandle.handle}`); const uri = generateUriFromRemoteProvider(jupyterHandle.id, jupyterHandle.handle); const serverId = await computeServerId(uri); @@ -142,10 +144,12 @@ export class JupyterServerUriStorage extends Disposables implements IJupyterServ } public async update(serverId: string) { this.hookupStorageEvents(); + await this.newStorage.migrateMRU(); await Promise.all([this.newStorage.update(serverId), this.oldStorage.update(serverId)]); } public async remove(serverId: string) { this.hookupStorageEvents(); + await this.newStorage.migrateMRU(); await Promise.all([this.newStorage.remove(serverId), this.oldStorage.remove(serverId)]); } } @@ -577,6 +581,9 @@ class NewStorage { return entries; } private async getAllRaw(): Promise { + if (!(await this.fs.exists(this.storageFile))) { + return []; + } const json = await this.fs.readFile(this.storageFile); return JSON.parse(json) as StorageMRUItem[]; } From f5b2bee41d63db07f5bae5a15984af907725b686 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Fri, 23 Jun 2023 09:16:25 +1000 Subject: [PATCH 3/6] fixes --- src/kernels/jupyter/connection/serverUriStorage.unit.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/kernels/jupyter/connection/serverUriStorage.unit.test.ts b/src/kernels/jupyter/connection/serverUriStorage.unit.test.ts index ad7f3af35be..64ef9b74ceb 100644 --- a/src/kernels/jupyter/connection/serverUriStorage.unit.test.ts +++ b/src/kernels/jupyter/connection/serverUriStorage.unit.test.ts @@ -84,6 +84,7 @@ suite('Server Uri Storage', async () => { const itemsInNewStorage: StorageMRUItem[] = []; when(fs.writeFile(anything(), anything())).thenCall((_, data) => { itemsInNewStorage.push(...JSON.parse(data.toString())); + when(fs.exists(anything())).thenResolve(true); return Promise.resolve(); }); when(fs.readFile(anything())).thenCall(() => JSON.stringify(itemsInNewStorage)); From d1371a555627067802ac1782921559b6d18fd187 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Fri, 23 Jun 2023 09:16:50 +1000 Subject: [PATCH 4/6] fixes --- src/kernels/jupyter/connection/serverUriStorage.unit.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kernels/jupyter/connection/serverUriStorage.unit.test.ts b/src/kernels/jupyter/connection/serverUriStorage.unit.test.ts index 64ef9b74ceb..4f107d308ad 100644 --- a/src/kernels/jupyter/connection/serverUriStorage.unit.test.ts +++ b/src/kernels/jupyter/connection/serverUriStorage.unit.test.ts @@ -114,7 +114,7 @@ suite('Server Uri Storage', async () => { .sort((a, b) => (a.displayName || '').localeCompare(b.displayName || '')) ); }); - test('Clear when we have some old data', async () => { + test.only('Clear when we have some old data', async () => { generateDummyData(2); when(fs.exists(anything())).thenResolve(false); when(fs.exists(uriEquals(globalStorageUri))).thenResolve(true); From 60bbd866f96064862a3dd85f4730e1644c3bd56d Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Fri, 23 Jun 2023 09:17:23 +1000 Subject: [PATCH 5/6] fixes --- src/kernels/jupyter/connection/serverUriStorage.unit.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/kernels/jupyter/connection/serverUriStorage.unit.test.ts b/src/kernels/jupyter/connection/serverUriStorage.unit.test.ts index 4f107d308ad..61690725ecf 100644 --- a/src/kernels/jupyter/connection/serverUriStorage.unit.test.ts +++ b/src/kernels/jupyter/connection/serverUriStorage.unit.test.ts @@ -114,7 +114,7 @@ suite('Server Uri Storage', async () => { .sort((a, b) => (a.displayName || '').localeCompare(b.displayName || '')) ); }); - test.only('Clear when we have some old data', async () => { + test('Clear when we have some old data', async () => { generateDummyData(2); when(fs.exists(anything())).thenResolve(false); when(fs.exists(uriEquals(globalStorageUri))).thenResolve(true); @@ -123,6 +123,7 @@ suite('Server Uri Storage', async () => { const itemsInNewStorage: StorageMRUItem[] = []; when(fs.writeFile(anything(), anything())).thenCall((_, data) => { itemsInNewStorage.push(...JSON.parse(data.toString())); + when(fs.exists(anything())).thenResolve(true); return Promise.resolve(); }); when(fs.readFile(anything())).thenCall(() => JSON.stringify(itemsInNewStorage)); From e98d0757437928c6e35c54fdf5b84bd49b215505 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Fri, 23 Jun 2023 09:23:12 +1000 Subject: [PATCH 6/6] fixes --- src/standalone/userJupyterServer/userServerUrlProvider.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/standalone/userJupyterServer/userServerUrlProvider.ts b/src/standalone/userJupyterServer/userServerUrlProvider.ts index 6fac346cae4..0ba0fcaff9f 100644 --- a/src/standalone/userJupyterServer/userServerUrlProvider.ts +++ b/src/standalone/userJupyterServer/userServerUrlProvider.ts @@ -699,7 +699,6 @@ export class OldStorage { type StorageItem = { handle: string; uri: string; - displayName: string; }; function serverToStorageFormat( servers: { @@ -708,7 +707,7 @@ function serverToStorageFormat( serverInfo: IJupyterServerUri; }[] ): StorageItem[] { - return servers.map((s) => ({ handle: s.handle, uri: s.uri, displayName: s.serverInfo.displayName })); + return servers.map((s) => ({ handle: s.handle, uri: s.uri })); } function storageFormatToServers(items: StorageItem[]) { const servers: {