-
Notifications
You must be signed in to change notification settings - Fork 14
feat(samples): adds samples for enhanced version of library #16
Changes from 21 commits
ff70678
2b599bc
c3b8ec9
3325360
ba6849f
ff30eb5
c35a5b7
f469857
6b422e2
46d909b
5c9d4ec
4b94439
2a52217
0b35029
2f5f18e
1350f5b
24b1e14
76d0bc3
0c85285
95883fd
93ab9aa
e216c89
367c084
b3abd86
62037bb
a3190d3
7104596
006505f
573d02a
2eb0276
d46eb72
f09d8de
91de4d8
5d0fd54
7df4ba4
697a099
a39db41
a938114
fb1ec49
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -12,3 +12,4 @@ system-test/*key.json | |
.DS_Store | ||
package-lock.json | ||
__pycache__ | ||
.vscode |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
@@ -0,0 +1,249 @@ | ||||||
/* | ||||||
* Copyright 2020 Google LLC | ||||||
* | ||||||
* Licensed under the Apache License, Version 2.0 (the "License"); | ||||||
* you may not use this file except in compliance with the License. | ||||||
* You may obtain a copy of the License at | ||||||
* | ||||||
* https://www.apache.org/licenses/LICENSE-2.0 | ||||||
* | ||||||
* Unless required by applicable law or agreed to in writing, software | ||||||
* distributed under the License is distributed on an "AS IS" BASIS, | ||||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||||||
* See the License for the specific language governing permissions and | ||||||
* limitations under the License. | ||||||
*/ | ||||||
|
||||||
'use strict'; | ||||||
|
||||||
async function main( | ||||||
batchPredictionDisplayName, | ||||||
modelId, | ||||||
gcsSourceUri, | ||||||
gcsDestinationOutputUriPrefix, | ||||||
project, | ||||||
location = 'us-central1' | ||||||
) { | ||||||
// [START aiplatform_create_batch_prediction_job_video_classification] | ||||||
/** | ||||||
* TODO(developer): Uncomment these variables before running the sample.\ | ||||||
* (Not necessary if passing values as arguments) | ||||||
*/ | ||||||
|
||||||
// const batchPredictionDisplayName = 'YOUR_BATCH_PREDICTION_DISPLAY_NAME'; | ||||||
// const modelId = 'YOUR_MODEL_ID'; | ||||||
// const gcsSourceUri = 'YOUR_GCS_SOURCE_URI'; | ||||||
// const gcsDestinationOutputUriPrefix = 'YOUR_GCS_DEST_OUTPUT_URI_PREFIX'; | ||||||
// eg. "gs://<your-gcs-bucket>/destination_path" | ||||||
// const project = 'YOUR_PROJECT_ID'; | ||||||
// const location = 'YOUR_PROJECT_LOCATION'; | ||||||
const aiplatform = require('@google-cloud/aiplatform'); | ||||||
const { | ||||||
params, | ||||||
} = aiplatform.protos.google.cloud.aiplatform.v1beta1.schema.predict; | ||||||
|
||||||
// Imports the Google Cloud Job Service Client library | ||||||
const {JobServiceClient} = require('@google-cloud/aiplatform'); | ||||||
|
||||||
// Specifies the location of the api endpoint | ||||||
const clientOptions = { | ||||||
apiEndpoint: 'us-central1-aiplatform.googleapis.com', | ||||||
}; | ||||||
|
||||||
// Instantiates a client | ||||||
const jobServiceClient = new JobServiceClient(clientOptions); | ||||||
|
||||||
async function createBatchPredictionJobVideoClassification() { | ||||||
// Configure the parent resource | ||||||
const parent = `projects/${project}/locations/${location}`; | ||||||
const modelName = `projects/${project}/locations/${location}/models/${modelId}`; | ||||||
|
||||||
const modelParamsObj = new params.VideoClassificationPredictionParams({ | ||||||
confidenceThreshold: 0.5, | ||||||
maxPredictions: 1000, | ||||||
segmentClassification: true, | ||||||
shotClassification: true, | ||||||
oneSecIntervalClassification: true, | ||||||
}); | ||||||
|
||||||
const modelParameters = modelParamsObj.toValue(); | ||||||
|
||||||
const inputConfig = { | ||||||
instancesFormat: 'jsonl', | ||||||
gcsSource: {uris: [gcsSourceUri]}, | ||||||
}; | ||||||
const outputConfig = { | ||||||
predictionsFormat: 'jsonl', | ||||||
gcsDestination: {outputUriPrefix: gcsDestinationOutputUriPrefix}, | ||||||
}; | ||||||
const batchPredictionJob = { | ||||||
displayName: batchPredictionDisplayName, | ||||||
model: modelName, | ||||||
modelParameters, | ||||||
inputConfig, | ||||||
outputConfig, | ||||||
}; | ||||||
const request = { | ||||||
parent, | ||||||
batchPredictionJob, | ||||||
}; | ||||||
|
||||||
// Create batch prediction job request | ||||||
const [response] = await jobServiceClient.createBatchPredictionJob(request); | ||||||
|
||||||
console.log(`Create batch prediction job video classification response`); | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is a problem across all of these samples. I've addressed them in spots, where I've found them, but we need to root them all out. I've logged issue #38 . There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Actually, I think we can mark #38 as fixed with this PR. I've removed all of the |
||||||
console.log(`\tName : ${response.name}`); | ||||||
console.log(`\tDisplay name: ${response.displayName}`); | ||||||
console.log(`\tModel : ${response.model}`); | ||||||
console.log(`\tModel parameters : ${response.modelParameters}`); | ||||||
console.log(`\tGenerate explanation : ${response.generateExplanation}`); | ||||||
console.log(`\tState : ${response.state}`); | ||||||
console.log(`\tCreate time : ${JSON.stringify(response.createTime)}`); | ||||||
console.log(`\tStart time : ${JSON.stringify(response.startTime)}`); | ||||||
console.log(`\tEnd time : ${JSON.stringify(response.endTime)}`); | ||||||
console.log(`\tUpdate time : ${JSON.stringify(response.updateTime)}`); | ||||||
console.log(`\tLabels : ${JSON.stringify(response.labels)}`); | ||||||
|
||||||
const inputConfiguration = response.inputConfig; | ||||||
console.log(`\tInput config`); | ||||||
console.log(`\t\tInstances format : ${inputConfiguration.instancesFormat}`); | ||||||
|
||||||
const gcsSource = inputConfiguration.gcsSource; | ||||||
console.log(`\t\tGcs source`); | ||||||
console.log(`\t\t\tUris : ${gcsSource.uris}`); | ||||||
|
||||||
const bigquerySource = inputConfiguration.bigquerySource; | ||||||
console.log(`\t\tBigQuery source`); | ||||||
if (bigquerySource === null) { | ||||||
console.log(`\t\t\tInput uri : {}`); | ||||||
} else { | ||||||
console.log(`\t\t\tInput uri : ${bigquerySource.inputUri}`); | ||||||
} | ||||||
|
||||||
const outputConfiguration = response.outputConfig; | ||||||
console.log(`\t\tOutput config`); | ||||||
console.log( | ||||||
`\t\tPredictions format : ${outputConfiguration.predictionsFormat}` | ||||||
); | ||||||
|
||||||
const gcsDestination = outputConfiguration.gcsDestination; | ||||||
console.log(`\t\tGcs destination`); | ||||||
console.log(`\t\t\tOutput uri prefix : ${gcsDestination.outputUriPrefix}`); | ||||||
|
||||||
const bigqueryDestination = outputConfiguration.bigqueryDestination; | ||||||
if (bigqueryDestination === null) { | ||||||
console.log(`\t\tBigquery destination`); | ||||||
console.log(`\t\t\tOutput uri : {}`); | ||||||
} else { | ||||||
console.log(`\t\tBigquery destination`); | ||||||
console.log(`\t\t\tOutput uri : ${bigqueryDestination.outputUri}`); | ||||||
} | ||||||
|
||||||
const dedicatedResource = response.dedicatedResource; | ||||||
console.log(`\tDedicated resources`); | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. There's a lot of logic here for the printing logic, I'd be tempted to see if I could abstract it a bit into a helper, that's also between the function printDedicatedResource(dedicatedResource) {
}
function printCompletionStats () {
}
... If you moved these down to the bottom of the sample, I think you would draw better attention to the more important part of samples which is the calls to the API. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Removed. Earlier versions of the Python canonicals had more verbose responses printed out. Now they're much shorter. |
||||||
if (dedicatedResource === null) { | ||||||
console.log(`\t\tStarting replica count : {}`); | ||||||
console.log(`\t\tMax replica count : {}`); | ||||||
} else { | ||||||
console.log( | ||||||
`\ttStarting replica count : \ | ||||||
${dedicatedResource.startingReplicaCount}` | ||||||
); | ||||||
console.log( | ||||||
`\ttMax replica count : ${dedicatedResource.maxReplicaCount}` | ||||||
); | ||||||
|
||||||
const machineSpec = dedicatedResource.machineSpec; | ||||||
console.log(`\t\tMachine spec`); | ||||||
if (machineSpec === null) { | ||||||
console.log(`\t\t\tMachine type : {}`); | ||||||
console.log(`\t\t\tAccelerator type : {}`); | ||||||
console.log(`\t\t\tAccelerator count : {}`); | ||||||
} else { | ||||||
console.log(`\t\t\tMachine type : ${machineSpec.machineType}`); | ||||||
console.log(`\t\t\tAccelerator type : ${machineSpec.acceleratorType}`); | ||||||
console.log( | ||||||
`\t\t\tAccelerator count : ${machineSpec.acceleratorCount}` | ||||||
); | ||||||
} | ||||||
} | ||||||
|
||||||
const manualBatchTuningParameters = response.manualBatchTuningParameters; | ||||||
console.log(`\tManual batch tuning parameters`); | ||||||
if (manualBatchTuningParameters === null) { | ||||||
console.log(`\t\tBatch size : {}`); | ||||||
} else { | ||||||
console.log(`\t\tBatch size : ${manualBatchTuningParameters.batchSize}`); | ||||||
} | ||||||
|
||||||
const outputInfo = response.outputInfo; | ||||||
if (outputInfo === null) { | ||||||
console.log(`\tOutput info`); | ||||||
console.log(`\t\tGcs output directory : {}`); | ||||||
console.log(`\t\tBigquery output dataset : {}`); | ||||||
} else { | ||||||
console.log(`\tOutput info`); | ||||||
console.log( | ||||||
`\t\tGcs output directory : ${outputInfo.gcsOutputDirectory}` | ||||||
); | ||||||
console.log(`\t\tBigquery output dataset : | ||||||
${outputInfo.bigqueryOutputDataset}`); | ||||||
} | ||||||
|
||||||
const error = response.error; | ||||||
if (error === null) { | ||||||
console.log(`\tError`); | ||||||
console.log(`\t\tCode : {}`); | ||||||
console.log(`\t\tMessage : {}`); | ||||||
} else { | ||||||
console.log(`\tError`); | ||||||
console.log(`\t\tCode : ${error.code}`); | ||||||
console.log(`\t\tMessage : ${error.message}`); | ||||||
|
||||||
const details = error.details; | ||||||
if (details === null) { | ||||||
console.log(`\t\tDetails : {}`); | ||||||
} else { | ||||||
console.log(`\t\tDetails : ${details}`); | ||||||
} | ||||||
} | ||||||
|
||||||
const partialFailures = response.partialFailures; | ||||||
if (partialFailures === null) { | ||||||
console.log(`\tPartial failure`); | ||||||
} else { | ||||||
for (const partialFailure of partialFailures) { | ||||||
console.log(`\tPartial failure`); | ||||||
console.log(`\t\tCode : ${partialFailure.code}`); | ||||||
console.log(`\t\tMessage : ${partialFailure.message}`); | ||||||
} | ||||||
} | ||||||
|
||||||
const resourcesConsumed = response.resourcesConsumed; | ||||||
console.log(`\tResource consumed`); | ||||||
if (resourcesConsumed === null) { | ||||||
console.log(`\t\tReplica hours: {}`); | ||||||
} else { | ||||||
console.log(`\t\tReplica hours: ${resourcesConsumed.replicaHours}`); | ||||||
} | ||||||
|
||||||
const completionStats = response.completionStats; | ||||||
console.log(`\tCompletion status`); | ||||||
if (completionStats === null) { | ||||||
console.log(`\t\tSuccessful count: {}`); | ||||||
console.log(`\t\tFailed count: {}`); | ||||||
console.log(`\t\tIncomplete count: {}`); | ||||||
} else { | ||||||
console.log(`\t\tSuccessful count: ${completionStats.successfulCount}`); | ||||||
console.log(`\t\tFailed count: ${completionStats.failedCount}`); | ||||||
console.log(`\t\tIncomplete count: ${completionStats.incompleteCount}`); | ||||||
} | ||||||
} | ||||||
// [END aiplatform_create_batch_prediction_job_video_classification] | ||||||
await createBatchPredictionJobVideoClassification(); | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If we don't have the
Dropping the And you can add this: rocess.on('unhandledRejection', err => {
console.error(err.message);
process.exitCode = 1;
}); Just above your invocation of There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done. |
||||||
} | ||||||
|
||||||
main(...process.argv.slice(2)).catch(err => { | ||||||
console.error(err); | ||||||
process.exitCode = 1; | ||||||
}); |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
there's enough setup here, that I would be tempted to add a couple comments that include links to the corresponding documentation.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Done.