Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Storage] Name properties on clients - Emulator support #5557

Merged
merged 8 commits into from
Oct 17, 2019
42 changes: 31 additions & 11 deletions sdk/storage/storage-blob/src/BlobClient.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,14 @@ import { newPipeline, StoragePipelineOptions, Pipeline } from "./Pipeline";
import {
DEFAULT_MAX_DOWNLOAD_RETRY_REQUESTS,
URLConstants,
DEFAULT_BLOB_DOWNLOAD_BLOCK_BYTES
DEFAULT_BLOB_DOWNLOAD_BLOCK_BYTES,
DevelopmentConnectionString
} from "./utils/constants";
import {
setURLParameter,
extractConnectionStringParts,
appendToURLPath
appendToURLPath,
getValueInConnString
} from "./utils/utils.common";
import { readStreamToLocalFile } from "./utils/utils.node";
import { AppendBlobClient, StorageClient, CommonOptions } from "./internal";
Expand Down Expand Up @@ -1595,21 +1597,39 @@ export class BlobClient extends StorageClient {
}

private getBlobAndContainerNamesFromUrl(): { blobName: string; containerName: string } {
// URL may look like the following
// "https://myaccount.blob.core.windows.net/mycontainer/blob?sasString";
// "https://myaccount.blob.core.windows.net/mycontainer/blob";
// "https://myaccount.blob.core.windows.net/mycontainer/blob/a.txt?sasString";
// "https://myaccount.blob.core.windows.net/mycontainer/blob/a.txt";

let containerName;
let blobName;
try {
// URL may look like the following
// "https://myaccount.blob.core.windows.net/mycontainer/blob?sasString";
// "https://myaccount.blob.core.windows.net/mycontainer/blob";
// "https://myaccount.blob.core.windows.net/mycontainer/blob/a.txt?sasString";
// "https://myaccount.blob.core.windows.net/mycontainer/blob/a.txt";
// or an emulator URL that starts with the endpoint `http://127.0.0.1:10000/devstoreaccount1`

let urlWithoutSAS = this.url.split("?")[0]; // removing the sas part of url if present
urlWithoutSAS = urlWithoutSAS.endsWith("/") ? urlWithoutSAS.slice(0, -1) : urlWithoutSAS; // Slicing off '/' at the end if exists

const partsOfUrl = urlWithoutSAS.match("([^/]*)://([^/]*)/([^/]*)(/(.*))?");
// http://127.0.0.1:10000/devstoreaccount1
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IP style URL is not fixed to 127.0.0.1. Besides Azurite, Azure Storage test servers are also using IP style url/endpoints, but IP is not fixed. For example, https://52.132.33.11:81/accountname. Or even IPv6 style in the future. Alghorithm in SDK should cover these scenarios.

Some non IP style url like http://localhost:10000/accountname also have account name in the path. Or some devs may define the host file pointing to a local deployed xstore service like http://customized.server:10000/accountname

It's bit complex to come up with an alghorithm to cover all scenarios in the SDK side though.

Copy link
Member Author

@HarshaNalluru HarshaNalluru Oct 17, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, logged an issue here #5604.
Since the new changes would only be additional support, we are postponing and will most likely be done after the p.5.

const emulatorBlobEndpoint = getValueInConnString(
DevelopmentConnectionString,
"BlobEndpoint"
);

if (this.url.startsWith(emulatorBlobEndpoint)) {
// Emulator URL starts with `http://127.0.0.1:10000/devstoreaccount1`
const partsOfUrl = urlWithoutSAS.match(emulatorBlobEndpoint + "/([^/]*)(/(.*))?");
containerName = partsOfUrl![1];
blobName = partsOfUrl![3];
} else {
const partsOfUrl = urlWithoutSAS.match("([^/]*)://([^/]*)/([^/]*)(/(.*))?");
containerName = partsOfUrl![3];
blobName = partsOfUrl![5];
}

// decode the encoded blobName, containerName - to get all the special characters that might be present in them
const containerName = decodeURIComponent(partsOfUrl![3]);
let blobName = decodeURIComponent(partsOfUrl![5]);
containerName = decodeURIComponent(containerName);
blobName = decodeURIComponent(blobName);

// Azure Storage Server will replace "\" with "/" in the blob names
// doing the same in the SDK side so that the user doesn't have to replace "\" instances in the blobName
Expand Down
33 changes: 26 additions & 7 deletions sdk/storage/storage-blob/src/ContainerClient.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,14 @@ import { AbortSignalLike } from "@azure/abort-controller";
import * as Models from "./generated/src/models";
import { Container } from "./generated/src/operations";
import { ContainerAccessConditions, Metadata } from "./models";
import { DevelopmentConnectionString } from "./utils/constants";
import { newPipeline, StoragePipelineOptions, Pipeline } from "./Pipeline";
import { ETagNone } from "./utils/constants";
import {
appendToURLPath,
truncatedISO8061Date,
extractConnectionStringParts
extractConnectionStringParts,
getValueInConnString
} from "./utils/utils.common";
import {
AppendBlobClient,
Expand Down Expand Up @@ -1554,17 +1556,34 @@ export class ContainerClient extends StorageClient {
}

private getContainerNameFromUrl(): string {
// URL may look like the following
// "https://myaccount.blob.core.windows.net/mycontainer?sasString";
// "https://myaccount.blob.core.windows.net/mycontainer";
let containerName;
try {
// URL may look like the following
// "https://myaccount.blob.core.windows.net/mycontainer?sasString";
// "https://myaccount.blob.core.windows.net/mycontainer";
// or an emulator URL that starts with the endpoint `http://127.0.0.1:10000/devstoreaccount1`

let urlWithoutSAS = this.url.split("?")[0]; // removing the sas part of url if present
urlWithoutSAS = urlWithoutSAS.endsWith("/") ? urlWithoutSAS.slice(0, -1) : urlWithoutSAS; // Slicing off '/' at the end if exists

const partsOfUrl = urlWithoutSAS.match("([^/]*)://([^/]*)/([^/]*)");
// http://127.0.0.1:10000/devstoreaccount1
const emulatorBlobEndpoint = getValueInConnString(
DevelopmentConnectionString,
"BlobEndpoint"
);

if (this.url.startsWith(emulatorBlobEndpoint)) {
// Emulator URL starts with `http://127.0.0.1:10000/devstoreaccount1`

// decode the encoded containerName - to get all the special characters that might be present in it
const containerName = decodeURIComponent(partsOfUrl![3]);
const partsOfUrl = urlWithoutSAS.match(emulatorBlobEndpoint + "/([^/]*)");
containerName = partsOfUrl![1];
} else {
const partsOfUrl = urlWithoutSAS.match("([^/]*)://([^/]*)/([^/]*)");

// decode the encoded containerName - to get all the special characters that might be present in it
containerName = partsOfUrl![3];
}
containerName = decodeURIComponent(containerName);

if (!containerName) {
throw new Error("Provided containerName is invalid.");
Expand Down
2 changes: 1 addition & 1 deletion sdk/storage/storage-blob/src/utils/utils.common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ function getProxyUriFromDevConnString(connectionString: string): string {
return proxyUri;
}

function getValueInConnString(
export function getValueInConnString(
connectionString: string,
argument:
| "BlobEndpoint"
Expand Down
28 changes: 28 additions & 0 deletions sdk/storage/storage-blob/test/node/emulator-tests.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,12 @@ describe("Emulator Tests", () => {
b: "b"
};
await newClient.setMetadata(metadata);
assert.equal(
containerName,
newClient.containerName,
"Container name didn't match with the provided one."
);
assert.equal(newClient.blobName, blobName, "Blob name didn't match with the provided one.");
const result = await newClient.getProperties();
assert.deepStrictEqual(result.metadata, metadata);
});
Expand All @@ -71,6 +77,12 @@ describe("Emulator Tests", () => {
blobName
);

assert.equal(
newClient.containerName,
containerName,
"Container name didn't match with the provided one."
);
assert.equal(newClient.blobName, blobName, "Blob name didn't match with the provided one.");
const body: string = "randomstring";
await newClient.upload(body, body.length);
const result = await newClient.download(0);
Expand All @@ -86,6 +98,11 @@ describe("Emulator Tests", () => {

const result = await newClient.getProperties();

assert.equal(
newClient.containerName,
containerName,
"Container name didn't match with the provided one."
);
assert.ok(result.eTag!.length > 0);
assert.ok(result.lastModified);
assert.ok(!result.leaseDuration);
Expand All @@ -102,6 +119,11 @@ describe("Emulator Tests", () => {

const result = await newClient.getProperties();

assert.equal(
newClient.containerName,
containerName,
"Container name didn't match with the provided one."
);
assert.ok(result.eTag!.length > 0);
assert.ok(result.lastModified);
assert.ok(result.requestId);
Expand All @@ -114,6 +136,12 @@ describe("Emulator Tests", () => {
blobName
);

assert.equal(
newClient.containerName,
containerName,
"Container name didn't match with the provided one."
);
assert.equal(newClient.blobName, blobName, "Blob name didn't match with the provided one.");
await newClient.create(512);
const result = await newClient.download(0);
assert.deepStrictEqual(await bodyToString(result, 512), "\u0000".repeat(512));
Expand Down
25 changes: 21 additions & 4 deletions sdk/storage/storage-queue/src/QueueClient.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,13 @@ import {
appendToURLPath,
extractConnectionStringParts,
truncatedISO8061Date,
getValueInConnString,
getStorageClientContext
} from "./utils/utils.common";
import { SharedKeyCredential } from "./credentials/SharedKeyCredential";
import { AnonymousCredential } from "./credentials/AnonymousCredential";
import { createSpan } from "./utils/tracing";
import { DevelopmentConnectionString } from "./utils/constants";
import { Metadata } from "./models";

/**
Expand Down Expand Up @@ -979,14 +981,29 @@ export class QueueClient extends StorageClient {
}

private getQueueNameFromUrl(): string {
// URL may look like the following
// "https://myaccount.queue.core.windows.net/myqueue?sasString".
// "https://myaccount.queue.core.windows.net/myqueue".
let queueName;
try {
// URL may look like the following
// "https://myaccount.queue.core.windows.net/myqueue?sasString".
// "https://myaccount.queue.core.windows.net/myqueue".
// or an emulator URL that starts with the endpoint `http://127.0.0.1:10001/devstoreaccount1`

let urlWithoutSAS = this.url.split("?")[0]; // removing the sas part of url if present
urlWithoutSAS = urlWithoutSAS.endsWith("/") ? urlWithoutSAS.slice(0, -1) : urlWithoutSAS; // Slicing off '/' at the end if exists

const queueName = urlWithoutSAS.match("([^/]*)://([^/]*)/([^/]*)")![3];
// http://127.0.0.1:10001/devstoreaccount1
const emulatorQueueEndpoint = getValueInConnString(
DevelopmentConnectionString,
"QueueEndpoint"
);

if (this.url.startsWith(emulatorQueueEndpoint)) {
// Emulator URL starts with `http://127.0.0.1:10001/devstoreaccount1`
queueName = urlWithoutSAS.match(emulatorQueueEndpoint + "/([^/]*)")![1];
} else {
queueName = urlWithoutSAS.match("([^/]*)://([^/]*)/([^/]*)")![3];
}

if (!queueName) {
throw new Error("Provided queueName is invalid.");
} else {
Expand Down
2 changes: 1 addition & 1 deletion sdk/storage/storage-queue/src/utils/utils.common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ function getProxyUriFromDevConnString(connectionString: string): string {
return proxyUri;
}

function getValueInConnString(
export function getValueInConnString(
connectionString: string,
argument:
| "QueueEndpoint"
Expand Down
1 change: 1 addition & 0 deletions sdk/storage/storage-queue/test/node/emulator-tests.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ describe("Emulator Tests", () => {

const result = await newClient.getProperties();

assert.equal(newClient.queueName, queueName, "Queue name didn't match with the provided one.");
assert.ok(result.requestId);
assert.ok(result.version);
assert.ok(result.date);
Expand Down