Skip to content
This repository has been archived by the owner on Jul 20, 2023. It is now read-only.

feat(samples): adds samples for enhanced version of library #16

Merged
merged 39 commits into from
Jan 13, 2021
Merged
Changes from 21 commits
Commits
Show all changes
39 commits
Select commit Hold shift + click to select a range
ff70678
feat: generates enhanced library via synthtool
telpirion Nov 18, 2020
2b599bc
feat: adds methods to enhanced types
telpirion Nov 18, 2020
c3b8ec9
fix: lint
telpirion Nov 18, 2020
3325360
feat: adds enhanced library samples
telpirion Nov 18, 2020
ba6849f
Merge branch 'master' into encl-samples
telpirion Jan 7, 2021
ff30eb5
fix: finishing merge
telpirion Jan 7, 2021
c35a5b7
fix: final merge conflict
telpirion Jan 7, 2021
f469857
Merge branch 'master' into encl-samples
telpirion Jan 7, 2021
6b422e2
Merge branch 'master' into encl-samples
telpirion Jan 7, 2021
46d909b
chore: add tests for samples
telpirion Jan 7, 2021
5c9d4ec
fix: lint
telpirion Jan 7, 2021
4b94439
fix: lint
telpirion Jan 7, 2021
2a52217
fix: lint
telpirion Jan 8, 2021
0b35029
fix: lint
telpirion Jan 8, 2021
2f5f18e
fix: changed cleanup for tests
telpirion Jan 8, 2021
1350f5b
fix: changes batch prediction jobs to use API rather than samples
telpirion Jan 8, 2021
24b1e14
fix: broken return statement
telpirion Jan 8, 2021
76d0bc3
fix: import jobServiceClient
telpirion Jan 8, 2021
0c85285
fix: attempt to fix tests
telpirion Jan 8, 2021
95883fd
fix: tests
telpirion Jan 8, 2021
93ab9aa
fix: lint
telpirion Jan 8, 2021
e216c89
Merge branch 'master' into encl-samples
telpirion Jan 11, 2021
367c084
fix: sample tests
telpirion Jan 11, 2021
b3abd86
Merge branch 'encl-samples' of https://github.com/googleapis/nodejs-a…
telpirion Jan 11, 2021
62037bb
fix: tests
telpirion Jan 11, 2021
a3190d3
fix: tests
telpirion Jan 11, 2021
7104596
fix: tests
telpirion Jan 11, 2021
006505f
fix: project ID in tests
telpirion Jan 11, 2021
573d02a
fix: lint
telpirion Jan 11, 2021
2eb0276
fix: sample tests
telpirion Jan 11, 2021
d46eb72
fix: tests and lint
telpirion Jan 12, 2021
f09d8de
fix: added quotes to text tests
telpirion Jan 12, 2021
91de4d8
fix: tabular classification test
telpirion Jan 12, 2021
5d0fd54
Merge branch 'master' into encl-samples
telpirion Jan 12, 2021
7df4ba4
fix: per reviewer
telpirion Jan 12, 2021
697a099
Merge branch 'encl-samples' of https://github.com/googleapis/nodejs-a…
telpirion Jan 12, 2021
a39db41
fix: per reviewer
telpirion Jan 13, 2021
a938114
fix: per reviewer
telpirion Jan 13, 2021
fb1ec49
docs: update README
bcoe Jan 13, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -12,3 +12,4 @@ system-test/*key.json
.DS_Store
package-lock.json
__pycache__
.vscode
2 changes: 1 addition & 1 deletion .kokoro/presubmit/node12/samples-test.cfg

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

249 changes: 249 additions & 0 deletions samples/create-batch-prediction-job-video-classification.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,249 @@
/*
* Copyright 2020 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

'use strict';

async function main(
batchPredictionDisplayName,
modelId,
gcsSourceUri,
gcsDestinationOutputUriPrefix,
project,
location = 'us-central1'
) {
// [START aiplatform_create_batch_prediction_job_video_classification]
/**
* TODO(developer): Uncomment these variables before running the sample.\
* (Not necessary if passing values as arguments)
*/

// const batchPredictionDisplayName = 'YOUR_BATCH_PREDICTION_DISPLAY_NAME';
// const modelId = 'YOUR_MODEL_ID';
// const gcsSourceUri = 'YOUR_GCS_SOURCE_URI';
// const gcsDestinationOutputUriPrefix = 'YOUR_GCS_DEST_OUTPUT_URI_PREFIX';
// eg. "gs://<your-gcs-bucket>/destination_path"
// const project = 'YOUR_PROJECT_ID';
// const location = 'YOUR_PROJECT_LOCATION';
const aiplatform = require('@google-cloud/aiplatform');
const {
params,
} = aiplatform.protos.google.cloud.aiplatform.v1beta1.schema.predict;

// Imports the Google Cloud Job Service Client library
const {JobServiceClient} = require('@google-cloud/aiplatform');

// Specifies the location of the api endpoint
const clientOptions = {
apiEndpoint: 'us-central1-aiplatform.googleapis.com',
};

// Instantiates a client
const jobServiceClient = new JobServiceClient(clientOptions);

async function createBatchPredictionJobVideoClassification() {
// Configure the parent resource
const parent = `projects/${project}/locations/${location}`;
const modelName = `projects/${project}/locations/${location}/models/${modelId}`;

const modelParamsObj = new params.VideoClassificationPredictionParams({
confidenceThreshold: 0.5,
maxPredictions: 1000,
segmentClassification: true,
shotClassification: true,
oneSecIntervalClassification: true,
});

const modelParameters = modelParamsObj.toValue();

const inputConfig = {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

there's enough setup here, that I would be tempted to add a couple comments that include links to the corresponding documentation.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.

instancesFormat: 'jsonl',
gcsSource: {uris: [gcsSourceUri]},
};
const outputConfig = {
predictionsFormat: 'jsonl',
gcsDestination: {outputUriPrefix: gcsDestinationOutputUriPrefix},
};
const batchPredictionJob = {
displayName: batchPredictionDisplayName,
model: modelName,
modelParameters,
inputConfig,
outputConfig,
};
const request = {
parent,
batchPredictionJob,
};

// Create batch prediction job request
const [response] = await jobServiceClient.createBatchPredictionJob(request);

console.log(`Create batch prediction job video classification response`);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
console.log(`Create batch prediction job video classification response`);
console.log('Create batch prediction job video classification response');

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a problem across all of these samples. I've addressed them in spots, where I've found them, but we need to root them all out.

I've logged issue #38 .

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually, I think we can mark #38 as fixed with this PR. I've removed all of the console.log statements with the unneeded back ticks.

console.log(`\tName : ${response.name}`);
console.log(`\tDisplay name: ${response.displayName}`);
console.log(`\tModel : ${response.model}`);
console.log(`\tModel parameters : ${response.modelParameters}`);
console.log(`\tGenerate explanation : ${response.generateExplanation}`);
console.log(`\tState : ${response.state}`);
console.log(`\tCreate time : ${JSON.stringify(response.createTime)}`);
console.log(`\tStart time : ${JSON.stringify(response.startTime)}`);
console.log(`\tEnd time : ${JSON.stringify(response.endTime)}`);
console.log(`\tUpdate time : ${JSON.stringify(response.updateTime)}`);
console.log(`\tLabels : ${JSON.stringify(response.labels)}`);

const inputConfiguration = response.inputConfig;
console.log(`\tInput config`);
console.log(`\t\tInstances format : ${inputConfiguration.instancesFormat}`);

const gcsSource = inputConfiguration.gcsSource;
console.log(`\t\tGcs source`);
console.log(`\t\t\tUris : ${gcsSource.uris}`);

const bigquerySource = inputConfiguration.bigquerySource;
console.log(`\t\tBigQuery source`);
if (bigquerySource === null) {
console.log(`\t\t\tInput uri : {}`);
} else {
console.log(`\t\t\tInput uri : ${bigquerySource.inputUri}`);
}

const outputConfiguration = response.outputConfig;
console.log(`\t\tOutput config`);
console.log(
`\t\tPredictions format : ${outputConfiguration.predictionsFormat}`
);

const gcsDestination = outputConfiguration.gcsDestination;
console.log(`\t\tGcs destination`);
console.log(`\t\t\tOutput uri prefix : ${gcsDestination.outputUriPrefix}`);

const bigqueryDestination = outputConfiguration.bigqueryDestination;
if (bigqueryDestination === null) {
console.log(`\t\tBigquery destination`);
console.log(`\t\t\tOutput uri : {}`);
} else {
console.log(`\t\tBigquery destination`);
console.log(`\t\t\tOutput uri : ${bigqueryDestination.outputUri}`);
}

const dedicatedResource = response.dedicatedResource;
console.log(`\tDedicated resources`);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There's a lot of logic here for the printing logic, I'd be tempted to see if I could abstract it a bit into a helper, that's also between the START/END blocks, something like:

function printDedicatedResource(dedicatedResource) {
  
}

function printCompletionStats () {

}

...

If you moved these down to the bottom of the sample, I think you would draw better attention to the more important part of samples which is the calls to the API.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Removed. Earlier versions of the Python canonicals had more verbose responses printed out. Now they're much shorter.

if (dedicatedResource === null) {
console.log(`\t\tStarting replica count : {}`);
console.log(`\t\tMax replica count : {}`);
} else {
console.log(
`\ttStarting replica count : \
${dedicatedResource.startingReplicaCount}`
);
console.log(
`\ttMax replica count : ${dedicatedResource.maxReplicaCount}`
);

const machineSpec = dedicatedResource.machineSpec;
console.log(`\t\tMachine spec`);
if (machineSpec === null) {
console.log(`\t\t\tMachine type : {}`);
console.log(`\t\t\tAccelerator type : {}`);
console.log(`\t\t\tAccelerator count : {}`);
} else {
console.log(`\t\t\tMachine type : ${machineSpec.machineType}`);
console.log(`\t\t\tAccelerator type : ${machineSpec.acceleratorType}`);
console.log(
`\t\t\tAccelerator count : ${machineSpec.acceleratorCount}`
);
}
}

const manualBatchTuningParameters = response.manualBatchTuningParameters;
console.log(`\tManual batch tuning parameters`);
if (manualBatchTuningParameters === null) {
console.log(`\t\tBatch size : {}`);
} else {
console.log(`\t\tBatch size : ${manualBatchTuningParameters.batchSize}`);
}

const outputInfo = response.outputInfo;
if (outputInfo === null) {
console.log(`\tOutput info`);
console.log(`\t\tGcs output directory : {}`);
console.log(`\t\tBigquery output dataset : {}`);
} else {
console.log(`\tOutput info`);
console.log(
`\t\tGcs output directory : ${outputInfo.gcsOutputDirectory}`
);
console.log(`\t\tBigquery output dataset :
${outputInfo.bigqueryOutputDataset}`);
}

const error = response.error;
if (error === null) {
console.log(`\tError`);
console.log(`\t\tCode : {}`);
console.log(`\t\tMessage : {}`);
} else {
console.log(`\tError`);
console.log(`\t\tCode : ${error.code}`);
console.log(`\t\tMessage : ${error.message}`);

const details = error.details;
if (details === null) {
console.log(`\t\tDetails : {}`);
} else {
console.log(`\t\tDetails : ${details}`);
}
}

const partialFailures = response.partialFailures;
if (partialFailures === null) {
console.log(`\tPartial failure`);
} else {
for (const partialFailure of partialFailures) {
console.log(`\tPartial failure`);
console.log(`\t\tCode : ${partialFailure.code}`);
console.log(`\t\tMessage : ${partialFailure.message}`);
}
}

const resourcesConsumed = response.resourcesConsumed;
console.log(`\tResource consumed`);
if (resourcesConsumed === null) {
console.log(`\t\tReplica hours: {}`);
} else {
console.log(`\t\tReplica hours: ${resourcesConsumed.replicaHours}`);
}

const completionStats = response.completionStats;
console.log(`\tCompletion status`);
if (completionStats === null) {
console.log(`\t\tSuccessful count: {}`);
console.log(`\t\tFailed count: {}`);
console.log(`\t\tIncomplete count: {}`);
} else {
console.log(`\t\tSuccessful count: ${completionStats.successfulCount}`);
console.log(`\t\tFailed count: ${completionStats.failedCount}`);
console.log(`\t\tIncomplete count: ${completionStats.incompleteCount}`);
}
}
// [END aiplatform_create_batch_prediction_job_video_classification]
await createBatchPredictionJobVideoClassification();
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we don't have the createBatchPredictionJobVideoClassification inside the snippet, we'll show example code that doesn't actually invoke to the customer, instead do:

createBatchPredictionJobVideoClassification();

Dropping the await.

And you can add this:

rocess.on('unhandledRejection', err => {
  console.error(err.message);
  process.exitCode = 1;
});

Just above your invocation of main.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done.

}

main(...process.argv.slice(2)).catch(err => {
console.error(err);
process.exitCode = 1;
});
Loading