From 82409d6ccc721660dff26089e0260f77eb367e09 Mon Sep 17 00:00:00 2001 From: Balazs Date: Wed, 10 Aug 2022 07:27:02 +0200 Subject: [PATCH 1/6] https://github.com/denodrivers/mongo/issues/364 --- src/gridfs/indexes.ts | 129 +++++++++++++-------------------------- tests/cases/06_gridfs.ts | 18 ++++++ 2 files changed, 59 insertions(+), 88 deletions(-) diff --git a/src/gridfs/indexes.ts b/src/gridfs/indexes.ts index 7250699..4067c9e 100644 --- a/src/gridfs/indexes.ts +++ b/src/gridfs/indexes.ts @@ -1,34 +1,48 @@ -import { Document } from "../../deps.ts"; import { Collection } from "../collection/collection.ts"; import { Chunk, File } from "../types/gridfs.ts"; +import { IndexOptions } from "../types.ts"; -export function createFileIndex(collection: Collection) { - const index = { filename: 1, uploadDate: 1 }; - - return collection.createIndexes({ - indexes: [ - { - name: "gridFSFiles", - key: index, - background: false, - }, - ], +async function ensureIndex(index: IndexOptions, collection: Collection): Promise["createIndexes"]>> { + // We need to check collection emptiness (ns not found error for listIndexes on empty collection) + const doc = await collection.findOne({}, {projection: {_id: 1}}); + if (doc === undefined) { + return collection.createIndexes({indexes: [index]}); + } + const keys = Object.keys(index.key); + const indexes = await collection.listIndexes().toArray(); + const existing = indexes.find(({key}) => { + const currentKeys = Object.keys(key); + return currentKeys.length === keys.length && currentKeys.every(k => keys.includes(k)); }); + if (existing === undefined) { + return collection.createIndexes({indexes: [index]}); + } else { + return { + ok: 1, + createdCollectionAutomatically: false, + numIndexesBefore: indexes.length, + numIndexesAfter: indexes.length + }; + } } -export function createChunksIndex(collection: Collection) { - // deno-lint-ignore camelcase - const index = { files_id: 1, n: 1 }; - return collection.createIndexes({ - indexes: [ - { - name: "gridFSFiles", - key: index, - unique: true, - background: false, - }, - ], - }); +const fileIndexSpec = { + name: "gridFSFiles", + key: {filename: 1, uploadDate: 1}, + background: false +}; +export function createFileIndex(collection: Collection) { + return ensureIndex(fileIndexSpec, collection); +} + +const chunkIndexSpec = { + name: "gridFSFiles", + key: {files_id: 1, n: 1}, + unique: true, + background: false +}; +export function createChunksIndex(collection: Collection) { + return ensureIndex(chunkIndexSpec, collection); } export async function checkIndexes( @@ -36,68 +50,7 @@ export async function checkIndexes( chunksCollection: Collection, hasCheckedIndexes: (value: boolean) => void, ) { - const filesCollectionIsEmpty = !(await filesCollection.findOne( - {}, - { - projection: { _id: 1 }, - }, - )); - - const chunksCollectionIsEmpty = !(await chunksCollection.findOne( - {}, - { - projection: { _id: 1 }, - }, - )); - - if (filesCollectionIsEmpty || chunksCollectionIsEmpty) { - // At least one collection is empty so we create indexes - await createFileIndex(filesCollection); - await createChunksIndex(chunksCollection); - hasCheckedIndexes(true); - return; - } - - // Now check that the right indexes are there - const fileIndexes = await filesCollection.listIndexes().toArray(); - let hasFileIndex = false; - - if (fileIndexes) { - fileIndexes.forEach((index) => { - const keys = Object.keys(index.key); - if ( - keys.length === 2 && - index.key.filename === 1 && - index.key.uploadDate === 1 - ) { - hasFileIndex = true; - } - }); - } - - if (!hasFileIndex) { - await createFileIndex(filesCollection); - } - - const chunkIndexes = await chunksCollection.listIndexes().toArray(); - let hasChunksIndex = false; - - if (chunkIndexes) { - chunkIndexes.forEach((index: Document) => { - const keys = Object.keys(index.key); - if ( - keys.length === 2 && - index.key.filename === 1 && - index.key.uploadDate === 1 && - index.options.unique - ) { - hasChunksIndex = true; - } - }); - } - - if (!hasChunksIndex) { - await createChunksIndex(chunksCollection); - } + await createFileIndex(filesCollection); + await createChunksIndex(chunksCollection); hasCheckedIndexes(true); } diff --git a/tests/cases/06_gridfs.ts b/tests/cases/06_gridfs.ts index 1154401..10d8b99 100644 --- a/tests/cases/06_gridfs.ts +++ b/tests/cases/06_gridfs.ts @@ -113,3 +113,21 @@ testWithClient( assert(!file[0]); }, ); + +// https://www.mongodb.com/docs/manual/reference/command/createIndexes/#considerations +testWithClient( + "GridFS: Creating indexes - skip index creation on same index keys", + async (client) => { + const addAsset = async (index: number) => { + const bucket = new GridFSBucket(client.database("test"), { + bucketName: "sameKeys" + }); + const upstream = await bucket.openUploadStream(`test-asset-${index}`); + const writer = upstream.getWriter(); + await writer.write(new TextEncoder().encode(`[asset${index}]`)); + await writer.close(); + } + await addAsset(0); + await addAsset(1); + }, +); From c21210edb0a1471e12f59d18e204ebbba914feaf Mon Sep 17 00:00:00 2001 From: Balazs Date: Wed, 10 Aug 2022 07:36:43 +0200 Subject: [PATCH 2/6] Fixing deno fmt --- src/gridfs/indexes.ts | 26 +++++++++++++++----------- tests/cases/06_gridfs.ts | 28 ++++++++++++++-------------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/gridfs/indexes.ts b/src/gridfs/indexes.ts index 4067c9e..8d33193 100644 --- a/src/gridfs/indexes.ts +++ b/src/gridfs/indexes.ts @@ -2,34 +2,38 @@ import { Collection } from "../collection/collection.ts"; import { Chunk, File } from "../types/gridfs.ts"; import { IndexOptions } from "../types.ts"; -async function ensureIndex(index: IndexOptions, collection: Collection): Promise["createIndexes"]>> { +async function ensureIndex( + index: IndexOptions, + collection: Collection, +): Promise["createIndexes"]>> { // We need to check collection emptiness (ns not found error for listIndexes on empty collection) - const doc = await collection.findOne({}, {projection: {_id: 1}}); + const doc = await collection.findOne({}, { projection: { _id: 1 } }); if (doc === undefined) { - return collection.createIndexes({indexes: [index]}); + return collection.createIndexes({ indexes: [index] }); } const keys = Object.keys(index.key); const indexes = await collection.listIndexes().toArray(); - const existing = indexes.find(({key}) => { + const existing = indexes.find(({ key }) => { const currentKeys = Object.keys(key); - return currentKeys.length === keys.length && currentKeys.every(k => keys.includes(k)); + return currentKeys.length === keys.length && + currentKeys.every((k) => keys.includes(k)); }); if (existing === undefined) { - return collection.createIndexes({indexes: [index]}); + return collection.createIndexes({ indexes: [index] }); } else { return { ok: 1, createdCollectionAutomatically: false, numIndexesBefore: indexes.length, - numIndexesAfter: indexes.length + numIndexesAfter: indexes.length, }; } } const fileIndexSpec = { name: "gridFSFiles", - key: {filename: 1, uploadDate: 1}, - background: false + key: { filename: 1, uploadDate: 1 }, + background: false, }; export function createFileIndex(collection: Collection) { return ensureIndex(fileIndexSpec, collection); @@ -37,9 +41,9 @@ export function createFileIndex(collection: Collection) { const chunkIndexSpec = { name: "gridFSFiles", - key: {files_id: 1, n: 1}, + key: { files_id: 1, n: 1 }, unique: true, - background: false + background: false, }; export function createChunksIndex(collection: Collection) { return ensureIndex(chunkIndexSpec, collection); diff --git a/tests/cases/06_gridfs.ts b/tests/cases/06_gridfs.ts index 10d8b99..ccc52f8 100644 --- a/tests/cases/06_gridfs.ts +++ b/tests/cases/06_gridfs.ts @@ -116,18 +116,18 @@ testWithClient( // https://www.mongodb.com/docs/manual/reference/command/createIndexes/#considerations testWithClient( - "GridFS: Creating indexes - skip index creation on same index keys", - async (client) => { - const addAsset = async (index: number) => { - const bucket = new GridFSBucket(client.database("test"), { - bucketName: "sameKeys" - }); - const upstream = await bucket.openUploadStream(`test-asset-${index}`); - const writer = upstream.getWriter(); - await writer.write(new TextEncoder().encode(`[asset${index}]`)); - await writer.close(); - } - await addAsset(0); - await addAsset(1); - }, + "GridFS: Creating indexes - skip index creation on same index keys", + async (client) => { + const addAsset = async (index: number) => { + const bucket = new GridFSBucket(client.database("test"), { + bucketName: "sameKeys", + }); + const upstream = await bucket.openUploadStream(`test-asset-${index}`); + const writer = upstream.getWriter(); + await writer.write(new TextEncoder().encode(`[asset${index}]`)); + await writer.close(); + }; + await addAsset(0); + await addAsset(1); + }, ); From 71f00bfdba2ee67eee83c39c8a5dc4cfaf44b040 Mon Sep 17 00:00:00 2001 From: Balazs Date: Wed, 10 Aug 2022 07:42:41 +0200 Subject: [PATCH 3/6] Change deno logo (deno_logo.png not found) --- tests/cases/06_gridfs.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cases/06_gridfs.ts b/tests/cases/06_gridfs.ts index ccc52f8..271c404 100644 --- a/tests/cases/06_gridfs.ts +++ b/tests/cases/06_gridfs.ts @@ -35,14 +35,14 @@ testWithClient("GridFS: Echo large Image", async (client) => { chunkSizeBytes: 255 * 8, }); - await (await fetch("https://deno.land/images/deno_logo.png")).body!.pipeTo( + await (await fetch("https://deno.land/images/artwork/snow_den.png")).body!.pipeTo( upstream, ); const getId = (await bucket.find({ filename: "deno_logo.png" }).toArray())[0]._id; - const expected = await fetch("https://deno.land/images/deno_logo.png"); + const expected = await fetch("https://deno.land/images/artwork/snow_den.png"); const actual = await new Response(await bucket.openDownloadStream(getId)) .arrayBuffer(); From 195b3fb0874b681d287fec52e5a8f29cb20d111a Mon Sep 17 00:00:00 2001 From: Balazs Date: Wed, 10 Aug 2022 07:44:25 +0200 Subject: [PATCH 4/6] fmt fix --- tests/cases/06_gridfs.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/cases/06_gridfs.ts b/tests/cases/06_gridfs.ts index 271c404..7573799 100644 --- a/tests/cases/06_gridfs.ts +++ b/tests/cases/06_gridfs.ts @@ -35,9 +35,10 @@ testWithClient("GridFS: Echo large Image", async (client) => { chunkSizeBytes: 255 * 8, }); - await (await fetch("https://deno.land/images/artwork/snow_den.png")).body!.pipeTo( - upstream, - ); + await (await fetch("https://deno.land/images/artwork/snow_den.png")).body! + .pipeTo( + upstream, + ); const getId = (await bucket.find({ filename: "deno_logo.png" }).toArray())[0]._id; From bb276f9920a6af36e0ad243f1b588a0ef2be108d Mon Sep 17 00:00:00 2001 From: Balazs Date: Wed, 10 Aug 2022 08:49:36 +0200 Subject: [PATCH 5/6] Revert deno logo (because of pr conflict) --- tests/cases/06_gridfs.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/cases/06_gridfs.ts b/tests/cases/06_gridfs.ts index 7573799..ccc52f8 100644 --- a/tests/cases/06_gridfs.ts +++ b/tests/cases/06_gridfs.ts @@ -35,15 +35,14 @@ testWithClient("GridFS: Echo large Image", async (client) => { chunkSizeBytes: 255 * 8, }); - await (await fetch("https://deno.land/images/artwork/snow_den.png")).body! - .pipeTo( - upstream, - ); + await (await fetch("https://deno.land/images/deno_logo.png")).body!.pipeTo( + upstream, + ); const getId = (await bucket.find({ filename: "deno_logo.png" }).toArray())[0]._id; - const expected = await fetch("https://deno.land/images/artwork/snow_den.png"); + const expected = await fetch("https://deno.land/images/deno_logo.png"); const actual = await new Response(await bucket.openDownloadStream(getId)) .arrayBuffer(); From ddb98114a9ad3ac3e74b6ba13f0c033bd07fc65b Mon Sep 17 00:00:00 2001 From: Balazs Date: Wed, 10 Aug 2022 08:58:09 +0200 Subject: [PATCH 6/6] Add assertion for skipping index creation on same keys (gridfs). --- tests/cases/06_gridfs.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/cases/06_gridfs.ts b/tests/cases/06_gridfs.ts index ccc52f8..1b34b01 100644 --- a/tests/cases/06_gridfs.ts +++ b/tests/cases/06_gridfs.ts @@ -119,15 +119,19 @@ testWithClient( "GridFS: Creating indexes - skip index creation on same index keys", async (client) => { const addAsset = async (index: number) => { - const bucket = new GridFSBucket(client.database("test"), { + const db = client.database("test"); + const bucket = new GridFSBucket(db, { bucketName: "sameKeys", }); const upstream = await bucket.openUploadStream(`test-asset-${index}`); const writer = upstream.getWriter(); await writer.write(new TextEncoder().encode(`[asset${index}]`)); await writer.close(); + return { + files: await db.collection("sameKeys.files").listIndexes().toArray(), + chunks: await db.collection("sameKeys.chunks").listIndexes().toArray(), + }; }; - await addAsset(0); - await addAsset(1); + assertEquals(await addAsset(0), await addAsset(1)); }, );