From d16a4e46fc293b0dea3b90c1ade69652236c2836 Mon Sep 17 00:00:00 2001
From: achingbrain <alex@achingbrain.net>
Date: Wed, 28 Nov 2018 15:57:01 +0000
Subject: [PATCH] perf: do not list directory contents when statting files

Restrict how deeply directory trees are listed

License: MIT
Signed-off-by: achingbrain <alex@achingbrain.net>
---
 src/cli/read.js             |  4 ++--
 src/core/stat.js            |  6 +-----
 src/core/utils/add-link.js  | 19 ++++++++++++++++---
 src/core/utils/constants.js |  3 ++-
 4 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/src/cli/read.js b/src/cli/read.js
index ae0cc7ee63..d629aa5a4c 100644
--- a/src/cli/read.js
+++ b/src/cli/read.js
@@ -2,7 +2,7 @@
 
 const pull = require('pull-stream/pull')
 const through = require('pull-stream/throughs/through')
-const collect = require('pull-stream/sinks/collect')
+const onEnd = require('pull-stream/sinks/on-end')
 const {
   print
 } = require('./utils')
@@ -43,7 +43,7 @@ module.exports = {
           through(buffer => {
             print(buffer, false)
           }),
-          collect((error) => {
+          onEnd((error) => {
             if (error) {
               return reject(error)
             }
diff --git a/src/core/stat.js b/src/core/stat.js
index cafa91e7fb..05416f4070 100644
--- a/src/core/stat.js
+++ b/src/core/stat.js
@@ -10,7 +10,6 @@ const waterfall = require('async/waterfall')
 const pull = require('pull-stream/pull')
 const collect = require('pull-stream/sinks/collect')
 const asyncMap = require('pull-stream/throughs/async-map')
-const filter = require('pull-stream/throughs/filter')
 const exporter = require('ipfs-unixfs-exporter')
 const log = require('debug')('ipfs:mfs:stat')
 
@@ -37,12 +36,9 @@ module.exports = (context) => {
       ({ mfsPath, depth }, cb) => {
         pull(
           exporter(mfsPath, context.ipld, {
-            maxDepth: depth + 1,
-            fullPath: true
+            maxDepth: depth
           }),
-          filter(node => node.depth === depth),
 
-          // load DAGNodes for each file
           asyncMap((file, cb) => {
             loadNode(context, {
               cid: file.hash
diff --git a/src/core/utils/add-link.js b/src/core/utils/add-link.js
index e9c7ceb92d..0c8aceb925 100644
--- a/src/core/utils/add-link.js
+++ b/src/core/utils/add-link.js
@@ -72,7 +72,7 @@ const addLink = (context, options, callback) => {
     return addToShardedDirectory(context, options, callback)
   }
 
-  if (options.parent.links.length === options.shardSplitThreshold) {
+  if (options.parent.links.length >= options.shardSplitThreshold) {
     log('Converting directory to sharded directory')
 
     return convertToShardedDirectory(context, options, callback)
@@ -151,8 +151,21 @@ const addToShardedDirectory = async (context, options, callback) => {
     })
   }
 
+  const existingFile = options.parent.links
+    .filter(link => link.name.substring(2) === options.name)
+    .pop()
+
+  if (existingFile) {
+    log(`Updating file ${existingFile.name}`)
+
+    return addToDirectory(context, {
+      ...options,
+      name: existingFile.name
+    }, callback)
+  }
+
   const existingUnshardedFile = options.parent.links
-    .filter(link => link.name.substring(0, 2) === prefix || link.name.substring(2) === options.name)
+    .filter(link => link.name.substring(0, 2) === prefix)
     .pop()
 
   if (existingUnshardedFile) {
@@ -194,7 +207,7 @@ const addToShardedDirectory = async (context, options, callback) => {
     })
   }
 
-  log(`Updating or appending ${prefix + options.name} to shard`)
+  log(`Appending ${prefix + options.name} to shard`)
 
   return addToDirectory(context, {
     ...options,
diff --git a/src/core/utils/constants.js b/src/core/utils/constants.js
index 483f0ce1b2..96a8e36d32 100644
--- a/src/core/utils/constants.js
+++ b/src/core/utils/constants.js
@@ -4,7 +4,8 @@ const Key = require('interface-datastore').Key
 
 const FILE_TYPES = {
   file: 0,
-  directory: 1
+  directory: 1,
+  'hamt-sharded-directory': 1
 }
 
 module.exports = {