Merge pull request #53 from ECP-VeloC/filelist

add interface to list encoded files

src/redset.c

@@ -1240,7 +1240,7 @@ static int redset_from_dir(
 #endif
 
 /* returns a list of files added by redundancy descriptor */
-redset_filelist redset_filelist_get(
+redset_filelist redset_filelist_enc_get(
   const char* name,
   const redset dvp)
 {
@@ -1257,16 +1257,16 @@ redset_filelist redset_filelist_get(
   /* get files added by redundancy method */
   switch (d->type) {
   case REDSET_COPY_SINGLE:
-    tmp = redset_filelist_get_single(name, d);
+    tmp = redset_filelist_enc_get_single(name, d);
     break;
   case REDSET_COPY_PARTNER:
-    tmp = redset_filelist_get_partner(name, d);
+    tmp = redset_filelist_enc_get_partner(name, d);
     break;
   case REDSET_COPY_XOR:
-    tmp = redset_filelist_get_xor(name, d);
+    tmp = redset_filelist_enc_get_xor(name, d);
     break;
   case REDSET_COPY_RS:
-    tmp = redset_filelist_get_rs(name, d);
+    tmp = redset_filelist_enc_get_rs(name, d);
     break;
   }
 
@@ -1304,3 +1304,33 @@ redset_filelist redset_filelist_get(
 
   return list;
 }
+
+/* returns a list of original files encoded by redundancy descriptor */
+redset_filelist redset_filelist_orig_get(
+  const char* name,
+  const redset dvp)
+{
+  /* get pointer to redset structure */
+  redset_base* d = (redset_base*) dvp;
+
+  /* create a temporary file list to record files from redundancy scheme */
+  redset_list* tmp = NULL;
+
+  /* get files added by redundancy method */
+  switch (d->type) {
+  case REDSET_COPY_SINGLE:
+    tmp = redset_filelist_orig_get_single(name, d);
+    break;
+  case REDSET_COPY_PARTNER:
+    tmp = redset_filelist_orig_get_partner(name, d);
+    break;
+  case REDSET_COPY_XOR:
+    tmp = redset_filelist_orig_get_xor(name, d);
+    break;
+  case REDSET_COPY_RS:
+    tmp = redset_filelist_orig_get_rs(name, d);
+    break;
+  }
+
+  return tmp;
+}

src/redset.h

@@ -156,12 +156,18 @@ int redset_unapply(
 );
 
 /** return list of files added by redundancy scheme */
-redset_filelist redset_filelist_get(
+redset_filelist redset_filelist_enc_get(
   const char* name, /**< [IN] - path/filename prefix to prepend to redset metadata files */
   const redset d    /**< [IN] - redundancy descriptor associated with above path */
 );
 
-/** free file list allocated by call to redset_filelist_get */
+/** return list of original files encoded by redundancy scheme */
+redset_filelist redset_filelist_orig_get(
+  const char* name, /**< [IN] - path/filename prefix to prepend to redset metadata files */
+  const redset d    /**< [IN] - redundancy descriptor associated with above path */
+);
+
+/** free file list allocated by call to redset_filelist_*_get */
 int redset_filelist_release(
   redset_filelist* plist /**< [INOUT] - address of pointer to list to be freed, sets pointer to NULL */
 );

src/redset_internal.h

@@ -267,26 +267,46 @@ int redset_unapply_rs(
 );
 
 
-redset_list* redset_filelist_get_single(
+redset_list* redset_filelist_enc_get_single(
   const char* name,
   redset_base* d
 );
 
-redset_list* redset_filelist_get_partner(
+redset_list* redset_filelist_enc_get_partner(
   const char* name,
   redset_base* d
 );
 
-redset_list* redset_filelist_get_xor(
+redset_list* redset_filelist_enc_get_xor(
   const char* name,
   redset_base* d
 );
 
-redset_list* redset_filelist_get_rs(
+redset_list* redset_filelist_enc_get_rs(
   const char* name,
   redset_base* d
 );
 
+redset_list* redset_filelist_orig_get_single(
+  const char* name,
+  const redset_base* d
+);
+
+redset_list* redset_filelist_orig_get_partner(
+  const char* name,
+  const redset_base* d
+);
+
+redset_list* redset_filelist_orig_get_xor(
+  const char* name,
+  const redset_base* d
+);
+
+redset_list* redset_filelist_orig_get_rs(
+  const char* name,
+  const redset_base* d
+);
+
 #ifdef HAVE_PTHREADS
 int redset_xor_encode_pthreads(
   const redset_base* d,

src/redset_lofi.c

@@ -243,15 +243,15 @@ int redset_lofi_check_mapped(const kvtree* hash, const kvtree* map)
     kvtree* index_hash = kvtree_getf(files_hash, "%d", i);
     kvtree_elem* elem = kvtree_elem_first(index_hash);
     const char* file = kvtree_elem_key(elem);
-  
+
     /* lookup hash for this file */
     kvtree* file_hash = kvtree_getf(files_hash, "%d %s", i, file);
     if (file_hash == NULL) {
       /* failed to find file name recorded */
       have_my_files = 0;
       continue;
     }
-  
+
     /* get name of file we're opening */
     char* file_name_mapped = (char*) file;
     if (map != NULL) {
@@ -269,10 +269,10 @@ int redset_lofi_check_mapped(const kvtree* hash, const kvtree* map)
       have_my_files = 0;
       continue;
     }
-  
+
     /* get file size of this file */
     unsigned long file_size = redset_file_size(file_name_mapped);
-  
+
     /* lookup expected file size and compare to actual size */
     unsigned long file_size_saved;
     if (kvtree_util_get_bytecount(file_hash, "SIZE", &file_size_saved) == KVTREE_SUCCESS) {
@@ -343,7 +343,7 @@ int redset_lofi_open_mapped(const kvtree* hash, const kvtree* map, int flags, mo
     kvtree* index_hash = kvtree_getf(files_hash, "%d", i);
     kvtree_elem* elem = kvtree_elem_first(index_hash);
     const char* file_name = kvtree_elem_key(elem);
-  
+
     /* lookup hash for this file */
     kvtree* file_hash = kvtree_getf(files_hash, "%d %s", i, file_name);
 
@@ -399,7 +399,7 @@ int redset_lofi_open_mapped(const kvtree* hash, const kvtree* map, int flags, mo
   rsf->bytes     = bytes;
   rsf->fds       = fds;
   rsf->filenames = filenames;
-  rsf->filesizes = filesizes;  
+  rsf->filesizes = filesizes;
 
   return rc;
 }
@@ -509,7 +509,7 @@ int redset_lofi_apply_meta_mapped(kvtree* hash, const kvtree* map)
     kvtree* index_hash = kvtree_getf(files_hash, "%d", i);
     kvtree_elem* elem = kvtree_elem_first(index_hash);
     const char* file_name = kvtree_elem_key(elem);
-  
+
     /* lookup hash for this file */
     kvtree* file_hash = kvtree_getf(files_hash, "%d %s", i, file_name);
 
@@ -537,3 +537,42 @@ int redset_lofi_apply_meta(kvtree* hash)
   int rc = redset_lofi_apply_meta_mapped(hash, NULL);
   return rc;
 }
+
+/* given a hash that defines a set of files, return list of files */
+redset_filelist redset_lofi_filelist(const kvtree* hash)
+{
+  /* lookup number of files this process wrote */
+  int num_files = 0;
+  if (kvtree_util_get_int(hash, "FILES", &num_files) != REDSET_SUCCESS) {
+    redset_abort(-1, "Failed to read number of files from hash @ %s:%d",
+      __FILE__, __LINE__
+    );
+  }
+
+  /* allocate arrays to hold filenames */
+  const char** filenames = (const char**) REDSET_MALLOC(num_files * sizeof(char*));
+
+  /* open each of our files */
+  int i;
+  kvtree* files_hash = kvtree_get(hash, "FILE");
+  for (i = 0; i < num_files; i++) {
+    /* get file name of this file */
+    kvtree* index_hash = kvtree_getf(files_hash, "%d", i);
+    kvtree_elem* elem = kvtree_elem_first(index_hash);
+    const char* file_name = kvtree_elem_key(elem);
+
+    /* copy the full filename */
+    filenames[i] = strdup(file_name);
+    if (filenames[i] == NULL) {
+      redset_abort(-1, "Failed to copy filename %s @ %s:%d",
+        file_name, __FILE__, __LINE__
+      );
+    }
+  }
+
+  redset_list* list = (redset_list*) REDSET_MALLOC(sizeof(redset_list));
+  list->count = num_files;
+  list->files = filenames;
+
+  return list;
+}

src/redset_lofi.h

@@ -52,6 +52,9 @@ int redset_lofi_apply_meta_mapped(kvtree* hash, const kvtree* map);
 /* given a hash that defines a set of files, apply metadata recorded to each file */
 int redset_lofi_apply_meta(kvtree* hash);
 
+/* given a hash that defines a set of files, return list of files */
+redset_filelist redset_lofi_filelist(const kvtree* hash);
+
 #ifdef __cplusplus
 } /* extern C */
 #endif

src/redset_partner.c

@@ -24,7 +24,7 @@
 static void redset_build_partner_filename(
   const char* name,
   const redset_base* d,
-  char* file, 
+  char* file,
   size_t len)
 {
   int rank_world;
@@ -1034,7 +1034,7 @@ int redset_unapply_partner(
 }
 
 /* returns a list of files added by redundancy descriptor */
-redset_list* redset_filelist_get_partner(
+redset_list* redset_filelist_enc_get_partner(
   const char* name,
   redset_base* d)
 {
@@ -1046,3 +1046,22 @@ redset_list* redset_filelist_get_partner(
   list->files[0] = strdup(file);
   return list;
 }
+
+/* returns a list of original files encoded by redundancy descriptor */
+redset_list* redset_filelist_orig_get_partner(
+  const char* name,
+  const redset_base* d)
+{
+  redset_list* list = NULL;
+
+  /* check whether we have our files and our partner's files */
+  kvtree* header = kvtree_new();
+  if (redset_read_partner_file(name, d, header) == REDSET_SUCCESS) {
+    /* get pointer to hash for this rank */
+    kvtree* current_hash = kvtree_getf(header, "%s %d", REDSET_KEY_COPY_PARTNER_DESC, d->rank);
+    list = redset_lofi_filelist(current_hash);
+  }
+  kvtree_delete(&header);
+
+  return list;
+}

src/redset_reedsolomon.c

@@ -34,7 +34,7 @@ Distribute and file rebuild functions
 static void redset_build_rs_filename(
   const char* name,
   const redset_base* d,
-  char* file, 
+  char* file,
   size_t len)
 {
   int rank_world;
@@ -265,7 +265,7 @@ int redset_encode_reddesc_rs(
      * receive incoming hash from left neighbors */
     kvtree* partner_hash = kvtree_new();
     kvtree_sendrecv(send_hash, rhs_rank, partner_hash, lhs_rank, d->comm);
-     
+
     /* store partner hash in our under its name */
     kvtree_merge(hash, partner_hash);
     kvtree_delete(&partner_hash);
@@ -606,7 +606,7 @@ int redset_reedsolomon_decode(
 
   /* make a copy of the matrix coeficients */
   unsigned int* mcopy = (unsigned int*) REDSET_MALLOC(missing * missing * sizeof(unsigned int));
-  
+
   /* during the reduce-scatter phase, each process has 1 outstanding send/recv at a time,
    * at the end, each process sends data to each failed rank and failed ranks receive a
    * message from all ranks, this allocation is more than needed */
@@ -1128,7 +1128,7 @@ int redset_unapply_rs(
 }
 
 /* returns a list of files added by redundancy descriptor */
-redset_list* redset_filelist_get_rs(
+redset_list* redset_filelist_enc_get_rs(
   const char* name,
   redset_base* d)
 {
@@ -1142,3 +1142,22 @@ redset_list* redset_filelist_get_rs(
 
   return list;
 }
+
+/* returns a list of original files encoded by redundancy descriptor */
+redset_list* redset_filelist_orig_get_rs(
+  const char* name,
+  const redset_base* d)
+{
+  redset_list* list = NULL;
+
+  /* check whether we have our files and our partner's files */
+  kvtree* header = kvtree_new();
+  if (redset_read_rs_file(name, d, header) == REDSET_SUCCESS) {
+    /* get pointer to hash for this rank */
+    kvtree* current_hash = kvtree_getf(header, "%s %d", REDSET_KEY_COPY_RS_DESC, d->rank);
+    list = redset_lofi_filelist(current_hash);
+  }
+  kvtree_delete(&header);
+
+  return list;
+}

src/redset_single.c

@@ -37,7 +37,7 @@
 static void redset_build_single_filename(
   const char* name,
   const redset_base* d,
-  char* file, 
+  char* file,
   size_t len)
 {
   int rank_world;
@@ -170,7 +170,7 @@ int redset_unapply_single(
 }
 
 /* returns a list of files added by redundancy descriptor */
-redset_list* redset_filelist_get_single(
+redset_list* redset_filelist_enc_get_single(
   const char* name,
   redset_base* d)
 {
@@ -182,3 +182,22 @@ redset_list* redset_filelist_get_single(
   list->files[0] = strdup(file);
   return list;
 }
+
+/* returns a list of original files encoded by redundancy descriptor */
+redset_list* redset_filelist_orig_get_single(
+  const char* name,
+  const redset_base* d)
+{
+  redset_list* list = NULL;
+
+  /* check whether we have our files and our partner's files */
+  kvtree* header = kvtree_new();
+  if (redset_read_single_file(name, d, header) == REDSET_SUCCESS) {
+    /* get pointer to hash for this rank */
+    kvtree* current_hash = kvtree_getf(header, "%s %d", REDSET_KEY_COPY_SINGLE_DESC, d->rank);
+    list = redset_lofi_filelist(current_hash);
+  }
+  kvtree_delete(&header);
+
+  return list;
+}