Divide the package manager into separate components (#2188)

* Feature tests running with refactored package manager

* Refactoring packages-1

* Test for the downloader running using the mock server

* Using network settings

* Changing the package path

* Dividing packages.ts into separate components

* use filter async

* Use tmpfile interface

* Check for the event stream

* Using FilePathResolver

* Remove using

* Testing for normal and fallback case working

* Resolve the paths

* Remove failing case

* package installer test-1

* Add package installer test

* Create tmp asset

* Package Downloader test refactored

* Rename files

* resolve compile issues

* Clean up installer

* Clean up the tests

* Rename packages

* Package installer test

* PR feedback

* Package Filter test

* Remove yauzl.d.ts

* Filter test

* Added test for invalid zip file

* method for getting the request options

* remove imports

* please resolve the path

* remove  latest in settings

* Package Manager test executing

* Use free port in package manager test

* Package Manager (using a https server running)

* using http mock server

* Downloader test running using free port

package.json

@@ -70,16 +70,18 @@
     "https-proxy-agent": "^2.1.1",
     "jsonc-parser": "^1.0.0",
     "mkdirp": "^0.5.1",
+    "node-filter-async": "0.0.4",
     "open": "*",
     "request-light": "^0.2.0",
     "rxjs": "^5.5.6",
     "semver": "*",
     "tmp": "0.0.33",
     "vscode-debugprotocol": "^1.6.1",
     "vscode-extension-telemetry": "0.0.15",
-    "yauzl": "^2.5.0"
+    "yauzl": "^2.9.1"
   },
   "devDependencies": {
+    "@types/archiver": "^2.1.1",
     "@types/chai": "^4.1.2",
     "@types/chai-arrays": "1.0.2",
     "@types/chai-as-promised": "^7.1.0",
@@ -95,6 +97,8 @@
     "@types/node": "^9.4.7",
     "@types/semver": "^5.5.0",
     "@types/tmp": "0.0.33",
+    "@types/yauzl": "^2.9.0",
+    "archiver": "^2.1.1",
     "async-child-process": "^1.1.1",
     "async-file": "^2.0.2",
     "async-shelljs": "^0.1.2",
@@ -107,6 +111,7 @@
     "copyfiles": "^2.0.0",
     "cross-env": "^5.1.4",
     "del": "3.0.0",
+    "get-port": "^3.2.0",
     "glob-promise": "^3.4.0",
     "gulp": "3.9.1",
     "gulp-mocha": "^5.0.0",
@@ -118,6 +123,7 @@
     "minimist": "^1.2.0",
     "mocha": "^5.0.4",
     "mocha-typescript": "^1.1.12",
+    "mock-http-server": "^0.2.0",
     "npm-run-all": "^4.1.2",
     "nyc": "^11.6.0",
     "plist": "^3.0.1",

src/CSharpExtDownloader.ts

@@ -4,68 +4,60 @@
  *--------------------------------------------------------------------------------------------*/
 
 import * as util from './common';
-import { GetNetworkConfiguration, defaultPackageManagerFactory, IPackageManagerFactory } from './downloader.helper';
-import { PackageManager } from './packages';
 import { PlatformInformation } from './platform';
 import { PackageInstallation, LogPlatformInfo, InstallationSuccess, InstallationFailure } from './omnisharp/loggingEvents';
 import { EventStream } from './EventStream';
-import { vscode } from './vscodeAdapter';
+import { DownloadAndInstallPackages } from './packageManager/PackageManager';
+import { Package } from './packageManager/Package';
+import { NetworkSettingsProvider } from './NetworkSettings';
+import { ResolveFilePaths } from './packageManager/PackageFilePathResolver';
 
 /*
  * Class used to download the runtime dependencies of the C# Extension
  */
 export class CSharpExtDownloader {
-    private proxy: string;
-    private strictSSL: boolean;
-    private packageManager: PackageManager;
 
     public constructor(
-        vscode: vscode,
+        private networkSettingsProvider: NetworkSettingsProvider,
         private eventStream: EventStream,
         private packageJSON: any,
-        private platformInfo: PlatformInformation,
-        packageManagerFactory: IPackageManagerFactory = defaultPackageManagerFactory) {
-
-        let networkConfiguration = GetNetworkConfiguration(vscode);
-        this.proxy = networkConfiguration.Proxy;
-        this.strictSSL = networkConfiguration.StrictSSL;
-        this.packageManager = packageManagerFactory(this.platformInfo, this.packageJSON);
+        private platformInfo: PlatformInformation) {
     }
 
     public async installRuntimeDependencies(): Promise<boolean> {
         this.eventStream.post(new PackageInstallation("C# dependencies"));
         let installationStage = 'touchBeginFile';
-        let success = false;
 
         try {
             await util.touchInstallFile(util.InstallFileType.Begin);
-
             // Display platform information and RID
             this.eventStream.post(new LogPlatformInfo(this.platformInfo));
-
-            installationStage = 'downloadPackages';
-            await this.packageManager.DownloadPackages(this.eventStream, this.proxy, this.strictSSL);
-
-            installationStage = 'installPackages';
-            await this.packageManager.InstallPackages(this.eventStream);
-
+            let runTimeDependencies = GetRunTimeDependenciesPackages(this.packageJSON);
+            runTimeDependencies.forEach(pkg => ResolveFilePaths(pkg));
+            installationStage = 'downloadAndInstallPackages';
+            await DownloadAndInstallPackages(runTimeDependencies, this.networkSettingsProvider, this.platformInfo, this.eventStream);
             installationStage = 'touchLockFile';
             await util.touchInstallFile(util.InstallFileType.Lock);
-
-            success = true;
             this.eventStream.post(new InstallationSuccess());
+            return true;
         }
         catch (error) {
             this.eventStream.post(new InstallationFailure(installationStage, error));
+            return false;
         }
         finally {
-            // We do this step at the end so that we clean up the begin file in the case that we hit above catch block
-            // Attach a an empty catch to this so that errors here do not propogate
             try {
                 util.deleteInstallFile(util.InstallFileType.Begin);
             }
             catch (error) { }
-            return success;
         }
     }
 }
+
+export function GetRunTimeDependenciesPackages(packageJSON: any): Package[] {
+    if (packageJSON.runtimeDependencies) {
+        return JSON.parse(JSON.stringify(<Package[]>packageJSON.runtimeDependencies));
+    }
+
+    throw new Error("No runtime dependencies found");
+}

src/CreateTmpAsset.ts

@@ -0,0 +1,59 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+import * as tmp from 'tmp';
+import { rimraf } from 'async-file';
+import { NestedError } from './NestedError';
+
+export async function CreateTmpFile(): Promise<TmpAsset> {
+    const tmpFile = await new Promise<tmp.SynchrounousResult>((resolve, reject) => {
+        tmp.file({ prefix: 'package-' }, (err, path, fd, cleanupCallback) => {
+            if (err) {
+                return reject(new NestedError('Error from tmp.file', err));
+            }
+            if (fd == 0) {
+                return reject(new NestedError("Temporary package file unavailable"));
+            }
+
+            resolve(<tmp.SynchrounousResult>{ name: path, fd: fd, removeCallback: cleanupCallback });
+        });
+    });
+
+    return {
+        fd: tmpFile.fd,
+        name: tmpFile.name,
+        dispose: tmpFile.removeCallback
+    };
+}
+
+export async function CreateTmpDir(unsafeCleanup: boolean): Promise<TmpAsset> {
+    const tmpDir = await new Promise<tmp.SynchrounousResult>((resolve, reject) => {
+        tmp.dir({ unsafeCleanup }, (err, path, cleanupCallback) => {
+            if (err) {
+                return reject(new NestedError('Error from tmp.dir', err));
+            }
+
+            resolve(<tmp.SynchrounousResult>{ name: path, removeCallback: cleanupCallback });
+        });
+    });
+
+    return {
+        fd: tmpDir.fd,
+        name: tmpDir.name,
+        dispose: () => {
+            if (unsafeCleanup) {
+                rimraf(tmpDir.name);//to delete directories that have folders inside them
+            }
+            else {
+                tmpDir.removeCallback();
+            }
+        }
+    };
+}
+
+export interface TmpAsset {
+    fd: number;
+    name: string;
+    dispose: () => void;
+}

src/NestedError.ts

@@ -0,0 +1,10 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+export class NestedError extends Error {
+    constructor(public message: string, public err: Error = null) {
+        super(message);
+    }
+}

src/NetworkSettings.ts

@@ -0,0 +1,24 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { vscode } from "./vscodeAdapter";
+
+export default class NetworkSettings {
+    constructor(public readonly proxy: string, public readonly strictSSL: boolean) {
+    }
+}
+
+export interface NetworkSettingsProvider {
+    (): NetworkSettings;
+}
+
+export function vscodeNetworkSettingsProvider(vscode: vscode): NetworkSettingsProvider {
+    return () => {
+        const config = vscode.workspace.getConfiguration();
+        const proxy = config.get<string>('http.proxy');
+        const strictSSL = config.get('http.proxyStrictSSL', true);
+        return new NetworkSettings(proxy, strictSSL);
+    };
+}

src/main.ts

@@ -28,6 +28,7 @@ import TelemetryReporter from 'vscode-extension-telemetry';
 import { addJSONProviders } from './features/json/jsonContributions';
 import { ProjectStatusBarObserver } from './observers/ProjectStatusBarObserver';
 import CSharpExtensionExports from './CSharpExtensionExports';
+import { vscodeNetworkSettingsProvider, NetworkSettingsProvider } from './NetworkSettings';
 
 export async function activate(context: vscode.ExtensionContext): Promise<CSharpExtensionExports> {
 
@@ -90,10 +91,11 @@ export async function activate(context: vscode.ExtensionContext): Promise<CSharp
     let telemetryObserver = new TelemetryObserver(platformInfo, () => reporter);
     eventStream.subscribe(telemetryObserver.post);
 
-    let runtimeDependenciesExist = await ensureRuntimeDependencies(extension, eventStream, platformInfo);
-
+    let networkSettingsProvider = vscodeNetworkSettingsProvider(vscode);
+    let runtimeDependenciesExist = await ensureRuntimeDependencies(extension, eventStream, platformInfo, networkSettingsProvider);
+
     // activate language services
-    let omniSharpPromise = OmniSharp.activate(context, eventStream, extension.packageJSON, platformInfo);
+    let omniSharpPromise = OmniSharp.activate(context, eventStream, extension.packageJSON, platformInfo, networkSettingsProvider);
 
     // register JSON completion & hover providers for project.json
     context.subscriptions.push(addJSONProviders());
@@ -116,11 +118,11 @@ export async function activate(context: vscode.ExtensionContext): Promise<CSharp
     };
 }
 
-async function ensureRuntimeDependencies(extension: vscode.Extension<CSharpExtensionExports>, eventStream: EventStream, platformInfo: PlatformInformation): Promise<boolean> {
+async function ensureRuntimeDependencies(extension: vscode.Extension<CSharpExtensionExports>, eventStream: EventStream, platformInfo: PlatformInformation, networkSettingsProvider : NetworkSettingsProvider): Promise<boolean> {
     return util.installFileExists(util.InstallFileType.Lock)
         .then(exists => {
             if (!exists) {
-                const downloader = new CSharpExtDownloader(vscode, eventStream, extension.packageJSON, platformInfo);
+                const downloader = new CSharpExtDownloader(networkSettingsProvider, eventStream, extension.packageJSON, platformInfo);
                 return downloader.installRuntimeDependencies();
             } else {
                 return true;

src/observers/BaseChannelObserver.ts

@@ -14,7 +14,7 @@ export abstract class BaseChannelObserver {
     abstract post: (event: BaseEvent) => void;
 
     public showChannel(preserveFocusOrColumn?: boolean) {
-        this.channel.show(preserveFocusOrColumn as boolean);
+        this.channel.show(preserveFocusOrColumn);
     }
 
     public clearChannel() {

src/observers/BaseLoggerObserver.ts

@@ -11,7 +11,7 @@ export abstract class BaseLoggerObserver {
     public logger: Logger;
     constructor(channel: vscode.OutputChannel | Logger) {
         if (channel instanceof Logger) {
-            this.logger = channel as Logger;
+            this.logger = channel;
         }
         else {
             this.logger = new Logger((message) => channel.append(message));

src/observers/CsharpLoggerObserver.ts

@@ -3,9 +3,10 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { PackageError } from "../packages";
 import { BaseLoggerObserver } from "./BaseLoggerObserver";
 import * as Event from "../omnisharp/loggingEvents";
+import { PackageError } from "../packageManager/PackageError";
+
 export class CsharpLoggerObserver extends BaseLoggerObserver {
     private dots: number;
 
@@ -27,8 +28,8 @@ export class CsharpLoggerObserver extends BaseLoggerObserver {
                 this.logger.appendLine('Finished');
                 this.logger.appendLine();
                 break;
-            case Event.InstallationProgress.name:
-                this.handleInstallationProgress(<Event.InstallationProgress>event);
+            case Event.InstallationStart.name:
+                this.handleInstallationStart(<Event.InstallationStart>event);
                 break;
             case Event.DownloadStart.name:
                 this.handleDownloadStart(<Event.DownloadStart>event);
@@ -51,6 +52,9 @@ export class CsharpLoggerObserver extends BaseLoggerObserver {
             case Event.DownloadSizeObtained.name:
                 this.handleDownloadSizeObtained(<Event.DownloadSizeObtained>event);
                 break;
+            case Event.LatestBuildDownloadStart.name:
+                this.logger.appendLine("Getting latest OmniSharp version information");
+                break;
         }
     }
 
@@ -104,7 +108,7 @@ export class CsharpLoggerObserver extends BaseLoggerObserver {
         this.dots = 0;
     }
 
-    private handleInstallationProgress(event: Event.InstallationProgress) {
+    private handleInstallationStart(event: Event.InstallationStart) {
         this.logger.appendLine(`Installing package '${event.packageDescription}'`);
         this.logger.appendLine();
     }

src/observers/OmnisharpStatusBarObserver.ts

@@ -3,7 +3,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { OmnisharpServerOnServerError, BaseEvent, OmnisharpOnBeforeServerInstall, OmnisharpOnBeforeServerStart, OmnisharpServerOnStop, OmnisharpServerOnStart, DownloadStart, InstallationProgress, DownloadProgress } from "../omnisharp/loggingEvents";
+import { OmnisharpServerOnServerError, BaseEvent, OmnisharpOnBeforeServerInstall, OmnisharpOnBeforeServerStart, OmnisharpServerOnStop, OmnisharpServerOnStart, DownloadStart, InstallationStart, DownloadProgress } from "../omnisharp/loggingEvents";
 import { BaseStatusBarItemObserver } from './BaseStatusBarItemObserver';
 
 export class OmnisharpStatusBarObserver extends BaseStatusBarItemObserver {
@@ -27,8 +27,8 @@ export class OmnisharpStatusBarObserver extends BaseStatusBarItemObserver {
             case DownloadStart.name:
                 this.SetAndShowStatusBar("$(cloud-download) Downloading packages", '', '', `Downloading package '${(<DownloadStart>event).packageDescription}...' `);
                 break;
-            case InstallationProgress.name:
-                this.SetAndShowStatusBar("$(desktop-download) Installing packages...", '', '', `Installing package '${(<InstallationProgress>event).packageDescription}'`);
+            case InstallationStart.name:
+                this.SetAndShowStatusBar("$(desktop-download) Installing packages...", '', '', `Installing package '${(<InstallationStart>event).packageDescription}'`);
                 break;
             case DownloadProgress.name:
                 let progressEvent = <DownloadProgress>event;

src/observers/TelemetryObserver.ts

@@ -3,9 +3,9 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { PackageError } from "../packages";
 import { PlatformInformation } from "../platform";
 import { BaseEvent, PackageInstallation, InstallationFailure, InstallationSuccess, OmnisharpDelayTrackerEventMeasures, OmnisharpStart, TestExecutionCountReport, TelemetryEventWithMeasures } from "../omnisharp/loggingEvents";
+import { PackageError } from "../packageManager/PackageError";
 
 export interface ITelemetryReporter {
     sendTelemetryEvent(eventName: string, properties?: { [key: string]: string }, measures?: { [key: string]: number }): void;