2.6.0 (#59)

* removed ldb duplicated code.
* solve memory segfault during attribution mining.
* update Makefile.
* remove md5 calc duplicated functions.

Makefile

@@ -6,19 +6,11 @@ CCFLAGS?=-g -Wall -I./inc -I./external/inc -D_LARGEFILE64_SOURCE -D_GNU_SOURCE
 # Linker flags
 LDFLAGS=-lz -lldb -lpthread -ldl
 
-LDB_CURRENT_VERSION := $(shell ldb -v | sed 's/ldb-//' | head -c 3)
-LDB_TARGET_VERSION := 3.2
-
-VERSION_IS_LESS := $(shell echo $(LDB_CURRENT_VERSION) \< $(LDB_TARGET_VERSION) | bc)
-ifeq ($(VERSION_IS_LESS),1)
-	LDFLAGS += -lcrypto
-endif
-
 BUILD_DIR =build
 SOURCES=$(wildcard src/*.c) $(wildcard src/**/*.c)  $(wildcard external/*.c) $(wildcard external/**/*.c)
 
 SOURCES_MINR=$(filter-out src/mz_main.c, $(SOURCES))
-OBJECTS_MIRN=$(SOURCES_MINR:.c=.o)
+OBJECTS_MINR=$(SOURCES_MINR:.c=.o)
 
 SOURCES_MZ=$(filter-out src/main.c, $(SOURCES))
 OBJECTS_MZ=$(SOURCES_MZ:.c=.o)
@@ -28,11 +20,17 @@ TARGET_MZ=mz
 
 VERSION=$(shell ./version.sh)
 
+LDB_CURRENT_VERSION := $(shell ldb -v | sed 's/ldb-//' | head -c 3)
+LDB_TARGET_VERSION := 4.1
+VERSION_IS_LESS := $(shell echo $(LDB_CURRENT_VERSION) \< $(LDB_TARGET_VERSION) | bc)
+
 all: clean $(TARGET_MINR) $(TARGET_MZ)
 
-$(TARGET_MINR): $(OBJECTS_MIRN)
-	@echo "Current version: $(LDB_CURRENT_VERSION)"
-	@echo "LDFLAGS: $(LDFLAGS)"
+$(TARGET_MINR): $(OBJECTS_MINR)
+ifeq ($(VERSION_IS_LESS),1)
+	@echo "Current LDB version: $(LDB_CURRENT_VERSION) is too old, please update to the lastest version to continue."
+	exit 1
+endif
 	$(CC) -o $@ $^ $(LDFLAGS)
 
 $(TARGET_MZ): $(OBJECTS_MZ)

external/src/winnowing.c

@@ -25,7 +25,6 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
-#include <openssl/md5.h>
 #include "crc32c.h"
 #include "winnowing.h"
 

inc/crc32c.h

@@ -1,6 +1,6 @@
 #ifndef __CRC32C_H
     #define __CRC32C_H
-
+#include <stdint.h>
 uint32_t calc_crc32c (char *data, size_t len);
 
 #endif

inc/file.h

@@ -5,8 +5,6 @@
 #include <stdbool.h>
 #include <stdio.h>
 
-
-//void file_md5(char *filepath, uint8_t *md5_result);
 uint64_t get_file_size(char *path);
 void read_file(char *out, char *path, uint64_t maxlen);
 bool is_file(char *path);

inc/import.h

@@ -1,5 +1,5 @@
-#ifndef __IMPORT_H
-#define __IMPORT_H
+#ifndef __MINR_IMPORT_H
+#define __MINR_IMPORT_H
 
 #include <stdbool.h>
 

inc/minr.h

@@ -29,7 +29,7 @@
 #include <string.h>
 
 /* Definitions */
-#define MINR_VERSION "2.5.20"
+#define MINR_VERSION "2.6.0"
 #define FILE_FILES 256
 #define MAX_ARG_LEN 1024
 #define MIN_FILE_REC_LEN 70

inc/minr_log.h

@@ -2,7 +2,6 @@
 #define __MINR_LOG_H
 #include <stdio.h>
 
-extern char log_file[FILENAME_MAX];
 void minr_log(const char *fmt, ...);
-
+void minr_log_path(char * path);
 #endif

inc/mz.h

@@ -14,12 +14,5 @@ typedef enum
 
 void mz_optimise(struct mz_job *job, mz_optimise_mode_t mode);
 void mz_extract(struct mz_job *job);
-void mz_list(struct mz_job *job);
-
-#ifndef MZ_DEFLATE
-#define MZ_DEFLATE_LOCAL
-void mz_deflate2(struct mz_job *job);
-#define MZ_DEFLATE(job) mz_deflate2(job)
-#endif
 
 #endif

src/attributions.c

@@ -239,7 +239,7 @@ void mine_attribution_notice(struct minr_job *job, char *path)
 
 	/* Compress data */
 	job->zsrc_ln = compressBound(job->src_ln + 1);
-	job->zsrc =calloc((job->zsrc_ln + 1), 1);
+	job->zsrc =calloc((job->zsrc_ln + 128), 1);
 
 	/* Save the first bytes of zsrc to accomodate the MZ header */
 	compress(job->zsrc + MZ_HEAD, &job->zsrc_ln, (uint8_t *)job->src, job->src_ln + 1);

src/file.c

@@ -27,7 +27,6 @@
   */
 
 #include <sys/stat.h>
-#include <openssl/md5.h>
 #include <stdlib.h>
 #include <ftw.h>
 #include <string.h>

src/import.c

@@ -38,7 +38,6 @@
 #include "file.h"
 #include "hex.h"
 #include "ignorelist.h"
-#include "join.h"
 #include "minr_log.h"
 
 

src/main.c

@@ -29,7 +29,6 @@
 #include <dirent.h>
 #include <errno.h>
 #include <fcntl.h>
-#include <openssl/md5.h>
 #include <stdbool.h>
 #include <stdint.h>
 #include <stdio.h>
@@ -261,7 +260,7 @@ int main(int argc, char *argv[])
 				strcpy(job.metadata, optarg);
 				break;
 			case 'V':
-				strcpy(log_file, optarg);
+				minr_log_path(optarg);
 				break;
 			case 's':
 				job.skip_sort = true;

src/minr.c

@@ -33,11 +33,10 @@
 #include <zlib.h>
 #include "attributions.h"
 #include "file.h"
-#include "md5.h"
 #include "hex.h"
 #include "ignorelist.h"
 #include "ignored_files.h"
-#include "ldb.h"
+#include <ldb.h>
 #include "crypto.h"
 #include "minr_log.h"
 
@@ -171,7 +170,7 @@ char *downloaded_file(char *tmp_dir)
  */
 void load_urlid(struct minr_job *job, char *tmp_file)
 {
-	uint8_t *bin_md5 = file_md5(tmp_file);
+	uint8_t *bin_md5 = md5_file(tmp_file);
 	char *hex_md5 = bin_to_hex(bin_md5, 16);
 	strcpy(job->urlid, hex_md5);
 	free(hex_md5);
@@ -334,7 +333,7 @@ int load_file(struct minr_job *job, char *path)
 	fclose(fp);
 
 	/* Calculate file MD5 */
-	uint8_t * md5 = file_md5(path);
+	uint8_t * md5 = md5_file(path);
 	memcpy(job->md5, md5, sizeof(job->md5));
 	free(md5);
 	ldb_bin_to_hex(job->md5, MD5_LEN, job->fileid);

src/minr_log.c

@@ -1,8 +1,14 @@
 #include <sys/time.h>
 #include <stdarg.h>
+#include <string.h>
 #include "minr_log.h"
 
-char log_file[FILENAME_MAX] = "\0";
+static char log_file[FILENAME_MAX] = "\0";
+
+void minr_log_path(char * path)
+{
+	strcpy(log_file, path);
+}
 
 /**
  * @brief Print the logs in stderr