fs: uv_fs_{open,read,close}_dir

This is the same changes as joyent/libuv#1574.
This commit is just the start of getting them to work in libuv/libuv.

Failing tests will be fixed asap.

Fixes libuv#170.

Makefile.am

@@ -157,6 +157,7 @@ test_run_tests_SOURCES = test/blackhole-server.c \
                          test/test-fs-event.c \
                          test/test-fs-poll.c \
                          test/test-fs.c \
+                         test/test-fs-readdir.c \
                          test/test-get-currentexe.c \
                          test/test-get-loadavg.c \
                          test/test-get-memory.c \

include/uv-unix.h

@@ -145,6 +145,13 @@ typedef gid_t uv_gid_t;
 typedef uid_t uv_uid_t;
 
 typedef struct dirent uv__dirent_t;
+/*
+ * "dirent" is used to hold a buffer large enough for any dirent in the
+ * directory being read. Avoids allocating for each directory entry.
+ */
+#define UV_DIR_PRIVATE_FIELDS \
+  uv__dirent_t* dirent;       \
+  DIR* dir;
 
 #if defined(DT_UNKNOWN)
 # define HAVE_DIRENT_TYPES

include/uv-win.h

@@ -251,6 +251,29 @@ typedef struct uv__dirent_s {
   char d_name[1];
 } uv__dirent_t;
 
+/*
+ * "handle" is the actual directory handle that is used to perform calls
+ * to FindFirstFile/FindNextFile.
+ *
+ * "find_data" is needed to store the result of the FindFirstFile call that
+ * happened in uv_fs_opendir so that it can be used in the subsequent
+ * uv_fs_readdir call.
+ *
+ * "need_find_call" is a boolean determining if the next call to uv_fs_readdir
+ * must call FindNextFile. In uv_fs_opendir, FindFirstFile reads the first
+ * entry of the directory, and thus the subsequent call to uv_fs_readdir must
+ * not call FindNextFile, or otherwise it would miss the first directory entry.
+ * The next uv_fs_readdir calls after the first one will use FindNextFile.
+ *
+ * "dirent" is used to hold a buffer large enough for any dirent in the
+ * directory being read. It avoids allocating for each directory entry.
+ */
+#define UV_DIR_PRIVATE_FIELDS \
+  HANDLE dir_handle;          \
+  WIN32_FIND_DATAW find_data; \
+  BOOL need_find_call;        \
+  uv__dirent_t* dirent;
+
 #define HAVE_DIRENT_TYPES
 #define UV__DT_DIR     UV_DIRENT_DIR
 #define UV__DT_FILE    UV_DIRENT_FILE

include/uv.h

@@ -194,6 +194,7 @@ typedef enum {
 /* Handle types. */
 typedef struct uv_loop_s uv_loop_t;
 typedef struct uv_handle_s uv_handle_t;
+typedef struct uv_dir_s uv_dir_t;
 typedef struct uv_stream_s uv_stream_t;
 typedef struct uv_tcp_s uv_tcp_t;
 typedef struct uv_udp_s uv_udp_t;
@@ -1092,6 +1093,9 @@ typedef enum {
   UV_FS_MKDTEMP,
   UV_FS_RENAME,
   UV_FS_SCANDIR,
+  UV_FS_OPENDIR,
+  UV_FS_READDIR,
+  UV_FS_CLOSEDIR,
   UV_FS_LINK,
   UV_FS_SYMLINK,
   UV_FS_READLINK,
@@ -1100,6 +1104,10 @@ typedef enum {
   UV_FS_REALPATH
 } uv_fs_type;
 
+struct uv_dir_s {
+  UV_DIR_PRIVATE_FIELDS
+};
+
 /* uv_fs_t is a subclass of uv_req_t. */
 struct uv_fs_s {
   UV_REQ_FIELDS
@@ -1110,6 +1118,9 @@ struct uv_fs_s {
   void* ptr;
   const char* path;
   uv_stat_t statbuf;  /* Stores the result of uv_fs_stat() and uv_fs_fstat(). */
+  const uv_dir_t* dir; /* Stores the result of uv_fs_opendir() */
+  uv_dirent_t* dirents;
+  size_t nentries;
   UV_FS_PRIVATE_FIELDS
 };
 
@@ -1162,6 +1173,22 @@ UV_EXTERN int uv_fs_scandir(uv_loop_t* loop,
                             uv_fs_cb cb);
 UV_EXTERN int uv_fs_scandir_next(uv_fs_t* req,
                                  uv_dirent_t* ent);
+
+UV_EXTERN int uv_fs_opendir(uv_loop_t* loop,
+                            uv_fs_t* req,
+                            const char* path,
+                            uv_fs_cb cb);
+UV_EXTERN int uv_fs_readdir(uv_loop_t* loop,
+                            uv_fs_t* req,
+                            const uv_dir_t* dir,
+                            uv_dirent_t dirents[],
+                            size_t nentries,
+                            uv_fs_cb cb);
+UV_EXTERN int uv_fs_closedir(uv_loop_t* loop,
+                             uv_fs_t* req,
+                             const uv_dir_t* dir,
+                             uv_fs_cb cb);
+
 UV_EXTERN int uv_fs_stat(uv_loop_t* loop,
                          uv_fs_t* req,
                          const char* path,

src/unix/fs.c

@@ -136,6 +136,73 @@ static ssize_t uv__fs_fdatasync(uv_fs_t* req) {
 }
 
 
+/*
+ * Computes the required size in bytes for directory entries read from the
+ * directory identified by the handle "dir".
+ *
+ * This code does not trust values of NAME_MAX that are less than
+ * 255, since some systems (including at least HP-UX) incorrectly
+ * define it to be a smaller value.
+ *
+ * This code was copied from
+ * http://womble.decadent.org.uk/readdir_r-advisory.html
+ * and slightly adapted.
+ */
+static size_t uv__fs_direntry_size(const DIR* dir) {
+  long name_max;
+  size_t name_end;
+
+  name_max = -1;
+  name_end = -1;
+
+#if defined(HAVE_FPATHCONF) && defined(HAVE_DIRFD) \
+    && defined(_PC_NAME_MAX)
+  name_max = fpathconf(dirfd(dirp), _PC_NAME_MAX);
+  if (name_max == -1)
+#   if defined(NAME_MAX)
+      name_max = (NAME_MAX > 255) ? NAME_MAX : 255;
+#   else
+      return (size_t)(-1);
+#   endif /* defined(NAME_MAX)
+#else
+# if defined(NAME_MAX)
+    name_max = (NAME_MAX > 255) ? NAME_MAX : 255;
+# else
+#   error "buffer size for readdir_r cannot be determined"
+# endif /* defined(NAME_MAX) */
+#endif
+
+  /*
+   * Most of the time, struct dirent is defined by operating systems' header
+   * files as either one of the following possible implementations:
+   * 1)
+   *
+   * struct dirent {
+   *   other stuff;
+   *   char d_name[ NAME_MAX + 1 ];
+   * };
+   *
+   * 2)
+   *
+   * struct dirent {
+   *   other stuff;
+   *   char d_name[1];
+   * };
+   *
+   * In case 1, using sizeof(struct dirent) to compute the size for the buffer
+   * would account for the path name. In case 2), it would not.
+   *
+   * The following test ensures that this function always allocate enough
+   * memory to hold the longest path names that can exist for a given
+   * directory, regardless of how system headers implement struct dirent.
+   */
+  name_end = (size_t)offsetof(struct dirent, d_name) + name_max + 1;
+  if (name_end > sizeof(struct dirent))
+    return name_end;
+  return sizeof(struct dirent);
+}
+
+
 static ssize_t uv__fs_futime(uv_fs_t* req) {
 #if defined(__linux__)
   /* utimesat() has nanosecond resolution but we stick to microseconds
@@ -370,10 +437,10 @@ static ssize_t uv__fs_scandir(uv_fs_t* req) {
   if (dents != NULL) {
     int i;
 
-    /* Memory was allocated using the system allocator, so use free() here. */
+    /* Memory was allocated using the system allocator, so use uv__free() here. */
     for (i = 0; i < n; i++)
-      free(dents[i]);
-    free(dents);
+      uv__free(dents[i]);
+    uv__free(dents);
   }
   errno = saved_errno;
 
@@ -382,6 +449,116 @@ static ssize_t uv__fs_scandir(uv_fs_t* req) {
   return n;
 }
 
+static int uv__fs_opendir(uv_fs_t* req) {
+  DIR *dir_stream;
+  uv__dirent_t *dirent;
+  uv_dir_t *dir;
+  size_t len;
+
+  assert(req);
+
+  dir_stream = opendir(req->path);
+  if (dir_stream == NULL)
+    return -1;
+
+  len = uv__fs_direntry_size(dir_stream);
+  if (len == (size_t)-1)
+    return -1;
+
+  /*
+  * Allocate the space for each directory entry just once instead of
+  * once per directory entry.
+  */
+  dirent = uv__malloc(len + 1);
+  if (dirent == NULL) {
+   errno = ENOMEM;
+   goto failed;
+  }
+
+  dir = uv__malloc(sizeof(*dir));
+  if (dir == NULL) {
+    errno = ENOMEM;
+    goto failed;
+  }
+
+  dir->dir = dir_stream;
+  dir->dirent = dirent;
+
+  req->dir = dir;
+  req->ptr = (void*)req->dir;
+
+  return 0;
+
+failed:
+  if (errno == ENOMEM) {
+    if (dirent != NULL)
+      uv__free(dirent);
+    if (dir != NULL)
+      uv__free(dir);
+  }
+
+  return -1;
+}
+
+static int uv__fs_readdir(uv_fs_t* req) {
+  uv_dirent_t *dirents;
+  struct dirent *res;
+  int r;
+  unsigned int dirent_idx;
+
+  assert(req);
+
+  assert(req->dir);
+  assert(req->dir->dir);
+  assert(req->dir->dirent);
+
+  assert(req->dirents);
+  assert(req->nentries > 0);
+
+  dirents = req->dirents;
+  req->ptr = req->dirents;
+
+  for (dirent_idx = 0; dirent_idx < req->nentries; ++dirent_idx) {
+    r = readdir_r(req->dir->dir, req->dir->dirent, &res);
+
+    if (r == 0) {
+      if (res != NULL) {
+        dirents[dirent_idx].name = uv__strdup(req->dir->dirent->d_name);
+        if (dirents[dirent_idx].name != NULL) {
+          dirents[dirent_idx].type = uv__fs_get_dirent_type(req->dir->dirent);
+        } else {
+          errno = ENOMEM;
+          return -1;
+        }
+      } else {
+        return dirent_idx;
+      }
+    }
+  }
+
+  return dirent_idx;
+}
+
+static int uv__fs_closedir(uv_fs_t* req) {
+  assert(req);
+  assert(req->dir);
+
+  if (req->dir) {
+    if (req->dir->dir) {
+      closedir(req->dir->dir);
+      ((uv_dir_t*)req->dir)->dir = NULL;
+    }
+
+    if (req->dir->dirent) {
+      uv__free(req->dir->dirent);
+      ((uv_dir_t*)req->dir)->dirent = NULL;
+    }
+  }
+
+  req->ptr = NULL;
+  return 0;
+}
+
 
 static ssize_t uv__fs_pathmax_size(const char* path) {
   ssize_t pathmax;
@@ -924,6 +1101,9 @@ static void uv__fs_work(struct uv__work* w) {
     X(OPEN, uv__fs_open(req));
     X(READ, uv__fs_buf_iter(req, uv__fs_read));
     X(SCANDIR, uv__fs_scandir(req));
+    X(OPENDIR, uv__fs_opendir(req));
+    X(READDIR, uv__fs_readdir(req));
+    X(CLOSEDIR, uv__fs_closedir(req));
     X(READLINK, uv__fs_readlink(req));
     X(REALPATH, uv__fs_realpath(req));
     X(RENAME, rename(req->path, req->new_path));
@@ -1185,6 +1365,40 @@ int uv_fs_scandir(uv_loop_t* loop,
   POST;
 }
 
+int uv_fs_opendir(uv_loop_t* loop,
+                  uv_fs_t* req,
+                  const char* path,
+                  uv_fs_cb cb) {
+
+  INIT(OPENDIR);
+  PATH;
+  POST;
+}
+
+int uv_fs_readdir(uv_loop_t* loop,
+                  uv_fs_t* req,
+                  const uv_dir_t* dir,
+                  uv_dirent_t dirents[],
+                  size_t nentries,
+                  uv_fs_cb cb) {
+  INIT(READDIR);
+
+  req->dir = dir;
+  req->dirents = dirents;
+  req->nentries = nentries;
+
+  POST;
+}
+
+int uv_fs_closedir(uv_loop_t* loop,
+                   uv_fs_t* req,
+                   const uv_dir_t* dir,
+                   uv_fs_cb cb) {
+  INIT(CLOSEDIR);
+  req->dir = dir;
+
+  POST;
+}
 
 int uv_fs_readlink(uv_loop_t* loop,
                    uv_fs_t* req,
@@ -1319,15 +1533,25 @@ void uv_fs_req_cleanup(uv_fs_t* req) {
    * exception to the rule, it always allocates memory.
    */
   if (req->path != NULL && (req->cb != NULL || req->fs_type == UV_FS_MKDTEMP))
-    uv__free((void*) req->path);  /* Memory is shared with req->new_path. */
+    uv__free(req->path);  /* Memory is shared with req->new_path. */
 
   req->path = NULL;
   req->new_path = NULL;
 
   if (req->fs_type == UV_FS_SCANDIR && req->ptr != NULL)
     uv__fs_scandir_cleanup(req);
 
-  if (req->ptr != &req->statbuf)
+  if (req->fs_type == UV_FS_READDIR)
+    uv__fs_readdir_cleanup(req);
+
+  /*
+   * Do not free the results of OPENDIR or READDIR requests,
+   * as they point to data that needs to be reused for subsequent
+   * calls.
+   */
+  if (req->fs_type != UV_FS_READDIR &&
+      req->fs_type != UV_FS_OPENDIR &&
+      req->ptr != &req->statbuf)
     uv__free(req->ptr);
   req->ptr = NULL;
 }