Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add controls for AILabImage component stories - Half-Fix #36

Merged
merged 2 commits into from
Dec 23, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 92 additions & 17 deletions examples/ai-lab-example/storybook/AILabImage.stories.tsx
Original file line number Diff line number Diff line change
@@ -1,30 +1,105 @@
import React from 'react';
import { AILabImage } from 'ai-lab';
import { ComponentStory, ComponentMeta } from '@storybook/react';

export default {
title: 'Example/AILabImage',
title: 'Example/AILabImage/SSD Model',
component: AILabImage,
argTypes: {
imageSource: {
options: ['dinner', 'cat', 'beach'],
control: { type: 'select' },
},
threshold: {
control: { type: 'range', min: 0, max: 1, step: 0.01 },
},
maxBoxes: {
control: { type: 'range', min: 0, max: 50, step: 1 },
},
iouThreshold: {
control: { type: 'range', min: 0, max: 1, step: 0.01 },
},
},
} as ComponentMeta<typeof AILabImage>;

const gimmeImage = (src: string) => {
switch (src) {
case 'dinner':
return require('./dinner.jpg');
case 'cat':
return require('./cat.jpeg');
default:
return require('./beach.jpeg');
}
};

// @ts-ignore
const imageStory = args => {
const theImage = require('./dinner.jpg');
////////////////////////////////////////////////////////////////
// Image Defaults
////////////////////////////////////////////////////////////////

const imageStory: ComponentStory<typeof AILabImage> = (args, { loaded }) => {
//@ts-ignore
const theImage = gimmeImage(args.imageSource);
return (
<AILabImage perf={args.perf} src={theImage} style={{ width: '100%' }} />
<AILabImage
model={loaded.SSDModel}
src={theImage}
style={{ height: '100%' }}
/>
);
};

export const withAnImage = imageStory.bind({});
export const withAnImageDefaults = imageStory.bind({});
// @ts-ignore
withAnImage.args = {
withAnImageDefaults.args = {
imageSource: 'cat',
};
withAnImageDefaults.parameters = {
controls: { include: ['imageSource'] },
};

////////////////////////////////////////////////////////////////
// Image with Params
////////////////////////////////////////////////////////////////
const imageParamsStory: ComponentStory<typeof AILabImage> = (
args,
{ loaded }
) => {
//@ts-ignore
const theImage = gimmeImage(args.imageSource);
return (
<AILabImage
model={loaded.SSDModel}
perf={args.perf}
src={theImage}
style={{ height: '100%' }}
modelInfo={{
modelType: 'ssd',
threshold: args.threshold,
nmsActive: args.nmsActive,
maxBoxes: args.maxBoxes,
}}
/>
);
};
export const withImageAndCustomizedSettings = imageParamsStory.bind({});
withImageAndCustomizedSettings.args = {
perf: true,
// restrain: false,
// modelInfo: {
// model: 'this is the model',
// objectDetection: true,
// labels: ['dog', 'cat'],
// threshold: 0.4,
// max: 20,
// IOU: 0.5,
// },
imageSource: 'dinner',
threshold: 0.4,
iouThreshold: 0.5,
maxBoxes: 20,
nmsActive: true,
};

withImageAndCustomizedSettings.parameters = {
controls: {
include: [
'threshold',
'imageSource',
'perf',
'nmsActive',
'maxBoxes',
'iouThreshold',
],
},
};
16 changes: 14 additions & 2 deletions examples/ai-lab-example/storybook/preview.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,21 @@
import * as tf from '@tensorflow/tfjs';

export const parameters = {
actions: { argTypesRegex: "^on[A-Z].*" },
actions: { argTypesRegex: '^on[A-Z].*' },
controls: {
matchers: {
color: /(background|color)$/i,
date: /Date$/,
},
},
}
};

// Load models to feed into all components
// @ts-ignore
export const loaders = [
async () => ({
SSDModel: await tf.loadGraphModel(
'https://storage.googleapis.com/tfhub-tfjs-modules/tensorflow/tfjs-model/ssd_mobilenet_v2/1/default/1/model.json'
),
}),
];
69 changes: 45 additions & 24 deletions packages/ai-lab/src/components/AILabImage/AILabImage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,32 @@ import {
PerformanceProps,
} from '../../performance';

export type ImageProps = React.ImgHTMLAttributes<HTMLImageElement> & PerformanceProps
export interface ImageLabProps {
model: tf.GraphModel;
modelInfo?: {
modelType: 'classification' | 'ssd',
threshold?: number,
maxBoxes?: number,
iouThreshold?: number,
nmsActive?: Boolean
};
}

export type ImageProps = React.ImgHTMLAttributes<HTMLImageElement> & PerformanceProps & ImageLabProps

const defaultModelConfig = {
modelType: 'ssd',
threshold: 0.4,
maxBoxes: 20,
iouThreshold: 0.5,
nmsActive: true
}

export const AILabImage = ({
model,
perf,
perfCallback,
modelInfo,
src,
...props
}: ImageProps) => {
Expand All @@ -22,8 +43,8 @@ export const AILabImage = ({
const [drawingTime, setDrawingTime] = useState(0);
const [perfProps, setPerfProps] = useState<PerformanceInfo>();

const modelPath =
'https://storage.googleapis.com/tfhub-tfjs-modules/tensorflow/tfjs-model/ssd_mobilenet_v2/1/default/1/model.json';
// const modelPath =
// 'https://storage.googleapis.com/tfhub-tfjs-modules/tensorflow/tfjs-model/ssd_mobilenet_v2/1/default/1/model.json';

const tensorFlowIt = async (model: tf.GraphModel) => {
const image = imgRef.current;
Expand All @@ -42,10 +63,9 @@ export const AILabImage = ({
ctx!.font = '16px sans-serif';
ctx!.textBaseline = 'top';

// Merge defaults & props
const {threshold:detectionThreshold, maxBoxes, iouThreshold, nmsActive} = {...defaultModelConfig, ...modelInfo};
// Get a clean tensor of top indices
const detectionThreshold = 0.4;
const iouThreshold = 0.4;
const maxBoxes = 20;
const prominentDetection = tf.topk((results as tf.Tensor<tf.Rank>[])[0]);
const justBoxes = (results as tf.Tensor<tf.Rank>[])[1].squeeze<tf.Tensor<tf.Rank.R2>>();
const justValues =
Expand All @@ -60,29 +80,31 @@ export const AILabImage = ({

// https://arxiv.org/pdf/1704.04503.pdf, use Async to keep visuals
const nmsDetections = await tf.image.nonMaxSuppressionWithScoreAsync(
justBoxes, // [numBoxes, 4]
justValues, // [numBoxes]
justBoxes,
justValues,
maxBoxes,
iouThreshold,
detectionThreshold,
1 // 0 is normal NMS, 1 is Soft-NMS for overlapping support
nmsActive ? 1 : 0 // 0 is normal NMS, 1 is Soft-NMS for overlapping support
);

const chosen = await nmsDetections.selectedIndices.data();
// Mega Clean
// tf.dispose([
// results[0],
// results[1],
// model,
// nmsDetections.selectedIndices,
// nmsDetections.selectedScores,
// prominentDetection.indices,
// prominentDetection.values,
// tensor,
// readyfied,
// justBoxes,
// justValues,
// ]);
tf.dispose([
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure why this dispose was commented out, but it needs to be brought back in to protect from mem-leaks

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You are right! I was getting type errors and since fast refresh disposed tensors I commented those out then forgot to brought them back even after the type errors are fixed in different components.

//@ts-ignore
results[0],
//@ts-ignore
results[1],
model,
nmsDetections.selectedIndices,
nmsDetections.selectedScores,
prominentDetection.indices,
prominentDetection.values,
tensor,
readyfied,
justBoxes,
justValues,
]);

//Drawing starts
let start = performance.now();
Expand Down Expand Up @@ -125,7 +147,6 @@ export const AILabImage = ({

useEffect(() => {
const setupTFJS = async () => {
const model = await tf.loadGraphModel(modelPath);
if (perf || perfCallback) {
const perfMetrics = await perfInfo(async () => {
await tensorFlowIt(model);
Expand All @@ -148,7 +169,7 @@ export const AILabImage = ({

return (
<div style={{ position: 'relative' }}>
<img ref={imgRef} src={src} alt="image" {...props} />
<img ref={imgRef} src={src} {...props} />
<canvas
ref={canvasRef}
style={{ position: 'absolute', left: 0, right: 0, bottom: 0, top: 0 }}
Expand Down