zilliztech · shanghaikid · Sep 29, 2022 · Sep 27, 2022 · Sep 28, 2022 · Sep 29, 2022
diff --git a/.gitignore b/.gitignore
@@ -12,4 +12,10 @@ dist
 *.csv
 *.index
 *.pyc
-test/output/*
+test/output/*
+test/testFederView/data
+test/testFederView/lib
+dev/data
+dev/bundle.js
+case/oneServer/data
+case/twoServer/data
diff --git a/README.md b/README.md
@@ -93,22 +93,64 @@ feder
 
 We prepare a simple case, which is the visualizations of the `hnsw` and `ivf_flat` with 17,000+ vectors that embedded from [VOC 2012](http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar)).
 
-Only need enable a web service.
-
 ```shell
 git clone git@github.com:zilliztech/feder.git
-cd test
-python -m http.server
+cd feder
+yarn install
+yarn dev
 ```
 
-Then open http://localhost:8000/
+Then open http://localhost:12355/
 
 It will show 4 visualizations:
 - `hnsw` overview
 - `hnsw` search view
 - `ivf_flat` overview
 - `ivf_flat` search view
 
+## Feder for Large Index
+
+`Feder` consists of three components: 
+- `FederIndex` - parse the index file. It requires a lot of memory.
+- `FederLayout` - layout calculations. It consumes a lot of computational resources.
+- `FederView` - render and interaction.
+
+In case of excessive amount of data, we support separating the computation part and running it on a node server.
+We have two solutions for you:
+- oneServer
+  - federServer (with `FederIndex` and `FederLayout`).
+- twoServer
+  - indexServer (with `FederIndex`)
+  - layoutServer (with `FederLayout`)
+
+Referring to **case/oneServer** and **case/twoServer**.
+
+### Example with One Server
+1. launch the server
+```shell
+yarn test_one_server_backend
+```
+2. launch the front web service
+```shell
+yarn test_one_server_front
+```
+3. open http://localhost:8000
+
+### Example with Two Servers
+1. launch the FederIndex server
+```shell
+yarn test_two_server_feder_index
+```
+2. launch the FederLayout server
+```shell
+yarn test_two_server_feder_layout
+```
+3. launch the front web service
+```shell
+yarn test_two_server_front
+```
+4. open http://localhost:8000
+
 ## Pipeline - explore a new dataset with feder
 
 ### Step 1. Dataset preparation
@@ -141,12 +183,13 @@ import * as d3 from 'd3';
 
 const domSelector = '#container';
 const filePath = [index_file_path];
+const source = "hnswlib"; // "hnswlib" or "faiss"
 
 const mediaCallback = (rowId) => mediaUrl;
 
 const feder = new Feder({
   filePath,
-  source: 'hnswlib',
+  source,
   domSelector,
   viewParams: {
     mediaType: 'img',

diff --git a/test_old/index.html → case/noServer/index.html b/test_old/index.html → case/noServer/index.html
@@ -6,7 +6,7 @@
   <meta name="viewport" content="width=device-width,initial-scale=1" />
   <meta name="theme-color" content="#ffffff" />
   <title>Feder view test</title>
-  <script src="./bundle.js"></script>
+  <script type="module" src="./index.js"></script>
 </head>
 
 <body style="background: #000">

diff --git a/case/noServer/index.js b/case/noServer/index.js
@@ -0,0 +1,18 @@
+import { Feder } from 'https://unpkg.com/@zilliz/feder';
+// import { Feder } from '@zilliz/feder';
+
+window.addEventListener('DOMContentLoaded', () => {
+  const feder = new Feder({
+    source: 'hnswlib',
+    filePath:
+      'https://assets.zilliz.com/hnswlib_hnsw_voc_17k_1f1dfd63a9.index',
+    domSelector: '#container',
+    viewParams: {
+      width: 800,
+      height: 400,
+      mediaContent: (id) => id
+    },
+  });
+  feder.overview();
+  feder.searchRandTestVec();
+});
diff --git a/case/oneServer/config.js b/case/oneServer/config.js
@@ -0,0 +1,14 @@
+export const federIndexLayoutServerPort = 12359;
+export const federIndexLayoutUrl = `http://localhost:${federIndexLayoutServerPort}`;
+
+export const federIndexFuncs = {
+  getIndexType: 'getIndexType',
+  getIndexMeta: 'getIndexMeta',
+  getSearchRecords: 'getSearchRecords',
+  getVectorById: 'getVectorById',
+  getVectorsCount: 'getVectorsCount',
+};
+
+export const federLayoutFuncs = {
+  getVisData: 'getVisData',
+};
diff --git a/test/index.html → case/oneServer/index.html b/test/index.html → case/oneServer/index.html
@@ -5,8 +5,8 @@
   <meta charset="utf-8" />
   <meta name="viewport" content="width=device-width,initial-scale=1" />
   <meta name="theme-color" content="#ffffff" />
-  <title>Feder2 view test</title>
-  <script src="./bundle.js"></script>
+  <title>Feder view test</title>
+  <script type="module" src="./index.js"></script>
 </head>
 
 <body style="background: #000">

diff --git a/case/oneServer/index.js b/case/oneServer/index.js
@@ -0,0 +1,68 @@
+import { FederView } from 'https://unpkg.com/@zilliz/feder';
+import {
+  federIndexLayoutUrl,
+  federIndexFuncs,
+  federLayoutFuncs,
+} from './config.js';
+
+const requestData = (url, path, params = {}) => {
+  return fetch(
+    url +
+      '/' +
+      path +
+      '?' +
+      new URLSearchParams({ params: JSON.stringify(params) })
+  )
+    .then((res) => res.json())
+    .then((res) => {
+      if (res.status === 'succeed') return res.data;
+      else throw new Error(`Wrong: ${path}`);
+    });
+};
+
+const federIndex = {
+  [federIndexFuncs.getVectorById]: (id) =>
+    requestData(federIndexLayoutUrl, federIndexFuncs.getVectorById, { id }),
+};
+
+const federLayout = {
+  [federLayoutFuncs.getVisData]: (params) =>
+    requestData(federIndexLayoutUrl, federLayoutFuncs.getVisData, params),
+};
+
+window.addEventListener('DOMContentLoaded', async () => {
+  const container = document.querySelector('#container');
+
+  const viewParams = {
+    // mediaType: 'image',
+    // mediaContent: rowId2imgUrl,
+    // width: 1000,
+    // height: 1000,
+    mediaType: 'text',
+    mediaContent: (id) => `this is content of No.${id}`,
+    mediaContentCount: 5,
+  };
+  const visDataOverview = await federLayout.getVisData({
+    actionType: 'overview',
+    layoutParams: viewParams,
+  });
+  const overview = new FederView(visDataOverview, viewParams);
+  overview.render();
+  container.appendChild(overview.node);
+
+  const targetId = 1492;
+  const target = await federIndex.getVectorById(targetId);
+  const targetMedia = 'this is the content of the target.';
+  const visDataSearchView = await federLayout.getVisData({
+    actionType: 'search',
+    actionData: {
+      target,
+      targetMedia,
+      searchParams: { nprobe: 7, k: 5 },
+    },
+    layoutParams: viewParams,
+  });
+  const searchView = new FederView(visDataSearchView, viewParams);
+  searchView.render();
+  container.appendChild(searchView.node);
+});
diff --git a/case/oneServer/initSymLink.js b/case/oneServer/initSymLink.js
@@ -0,0 +1,8 @@
+import fs from 'fs';
+
+// data
+try {
+  fs.accessSync('data');
+} catch (e) {
+  fs.symlinkSync('../../test/data', 'data', 'dir');
+}
diff --git a/case/oneServer/server.js b/case/oneServer/server.js
@@ -0,0 +1,72 @@
+import fs from 'fs';
+import { FederIndex, FederLayout } from '../../dist/index.js';
+import {
+  federIndexFuncs,
+  federLayoutFuncs,
+  federIndexLayoutServerPort,
+} from './config.js';
+import {
+  isLocal,
+  ivfflatSource,
+  ivfflatIndexFilePath,
+  hnswSource,
+  hnswIndexFilePath,
+} from './data/dataConfig.js';
+import express from 'express';
+import cors from 'cors';
+import fetch from 'node-fetch';
+
+const getFederIndex = async (sourceType, filePath) => {
+  const arraybuffer = isLocal
+    ? fs.readFileSync(filePath).buffer
+    : await fetch(filePath).then((res) => res.arrayBuffer());
+  const federIndex = new FederIndex(sourceType, arraybuffer);
+  return federIndex;
+};
+
+const federIndex = await getFederIndex(ivfflatSource, ivfflatIndexFilePath);
+// const federIndex = await getFederIndex(hnswSource, hnswIndexFilePath);
+
+const federLayout = new FederLayout(federIndex);
+
+const resData = (data = null) =>
+  data ? { status: 'succeed', data } : { status: 'failed' };
+
+const app = express();
+app.use(cors());
+
+app.get('/' + federIndexFuncs.getVectorsCount, async (req, res) => {
+  let data = null;
+  try {
+    data = await federIndex.getVectorsCount();
+  } catch (e) {
+    console.log(e);
+  }
+  res.json(resData(data));
+});
+
+app.get('/' + federIndexFuncs.getVectorById, async (req, res) => {
+  let data = null;
+  const { id } = JSON.parse(req.query.params);
+  try {
+    data = await federIndex.getVectorById(id);
+  } catch (e) {
+    console.log(e);
+  }
+  res.json(resData(data));
+});
+
+app.get('/' + federLayoutFuncs.getVisData, async (req, res) => {
+  let data = null;
+  const params = JSON.parse(req.query.params.toString());
+  try {
+    data = await federLayout.getVisData(params);
+  } catch (e) {
+    console.log(e);
+  }
+  res.json(resData(data));
+});
+
+app.listen(federIndexLayoutServerPort, () => {
+  console.log('listening on port', federIndexLayoutServerPort);
+});
diff --git a/case/twoServer/config.js b/case/twoServer/config.js
@@ -0,0 +1,16 @@
+export const federIndexServerPort = 12357;
+export const federIndexUrl = `http://localhost:${federIndexServerPort}`;
+export const federLayoutServerPort = 12358;
+export const federLayoutUrl = `http://localhost:${federLayoutServerPort}`;
+
+export const federIndexFuncs = {
+  getIndexType: 'getIndexType',
+  getIndexMeta: 'getIndexMeta',
+  getSearchRecords: 'getSearchRecords',
+  getVectorById: 'getVectorById',
+  getVectorsCount: 'getVectorsCount',
+};
+
+export const federLayoutFuncs = {
+  getVisData: 'getVisData',
+};