-
Notifications
You must be signed in to change notification settings - Fork 10
/
H5Zlz4.c
233 lines (202 loc) · 7.66 KB
/
H5Zlz4.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
/*
* This file is an example of an HDF5 filter plugin.
* The plugin can be used with the HDF5 library vesrion 1.8.11+ to read
* HDF5 datasets compressed with lz4.
*/
#include <sys/types.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <stdio.h>
#if defined(_WIN32)
#include <Winsock2.h>
#else
#include <arpa/inet.h>
#endif
#include <H5PLextern.h>
#include <lz4.h>
#include "lz4_h5filter.h"
#define PUSH_ERR(func, minor, str) \
H5Epush1(__FILE__, func, __LINE__, H5E_PLINE, minor, str)
static size_t H5Z_filter_lz4(unsigned int flags, size_t cd_nelmts,
const unsigned int cd_values[], size_t nbytes,
size_t *buf_size, void **buf);
#define H5Z_FILTER_LZ4 32004
#define htonll(x) ( ( (uint64_t)(htonl( (uint32_t)((x << 32) >> 32)))<< 32) | htonl( ((uint32_t)(x >> 32)) ))
#define ntohll(x) htonll(x)
#define htobe16t(x) htons(x)
#define htobe32t(x) htonl(x)
#define htobe64t(x) htonll(x)
#define be16toht(x) ntohs(x)
#define be32toht(x) ntohl(x)
#define be64toht(x) ntohll(x)
#define DEFAULT_BLOCK_SIZE 1<<30; /* 1GB. LZ4 needs blocks < 1.9GB. */
const H5Z_class2_t H5Z_LZ4[1] = {{
H5Z_CLASS_T_VERS, /* H5Z_class_t version */
(H5Z_filter_t)H5Z_FILTER_LZ4, /* Filter id number */
1, /* encoder_present flag (set to true) */
1, /* decoder_present flag (set to true) */
"HDF5 lz4 filter; see http://www.hdfgroup.org/services/contributions.html",
/* Filter name for debugging */
NULL, /* The "can apply" callback */
NULL, /* The "set local" callback */
(H5Z_func_t)H5Z_filter_lz4, /* The actual filter function */
}};
static size_t H5Z_filter_lz4(unsigned int flags, size_t cd_nelmts,
const unsigned int cd_values[], size_t nbytes,
size_t *buf_size, void **buf)
{
void * outBuf = NULL;
size_t ret_value;
if (flags & H5Z_FLAG_REVERSE)
{
uint32_t *i32Buf;
uint32_t blockSize;
char *roBuf; /* pointer to current write position */
uint64_t decompSize;
const char* rpos = (char*)*buf; /* pointer to current read position */
const uint64_t * const i64Buf = (uint64_t *) rpos;
const uint64_t origSize = (uint64_t)(be64toht(*i64Buf));/* is saved in be format */
rpos += 8; /* advance the pointer */
i32Buf = (uint32_t*)rpos;
blockSize = (uint32_t)(be32toht(*i32Buf));
rpos += 4;
if(blockSize>origSize)
blockSize = origSize;
if (NULL==(outBuf = H5allocate_memory(origSize, false)))
{
printf("error calling H5allocate_memory\n");
goto error;
}
roBuf = (char*)outBuf; /* pointer to current write position */
decompSize = 0;
/// start with the first block ///
while(decompSize < origSize)
{
uint32_t compressedBlockSize; /// is saved in be format
if(origSize-decompSize < blockSize) /* the last block can be smaller than blockSize. */
blockSize = origSize-decompSize;
i32Buf = (uint32_t*)rpos;
compressedBlockSize = be32toht(*i32Buf); /// is saved in be format
rpos += 4;
if(compressedBlockSize == blockSize) /* there was no compression */
{
memcpy(roBuf, rpos, blockSize);
}
else /* do the decompression */
{
#if LZ4_VERSION_NUMBER > 10300
int compressedBytes = LZ4_decompress_fast(rpos, roBuf, blockSize);
#else
int compressedBytes = LZ4_uncompress(rpos, roBuf, blockSize);
#endif
if(compressedBytes != compressedBlockSize)
{
printf("decompressed size not the same: %d, != %d\n", compressedBytes, compressedBlockSize);
goto error;
}
}
rpos += compressedBlockSize; /* advance the read pointer to the next block */
roBuf += blockSize; /* advance the write pointer */
decompSize += blockSize;
}
H5free_memory(*buf);
*buf = outBuf;
outBuf = NULL;
ret_value = (size_t)origSize; // should always work, as orig_size cannot be > 2GB (sizeof(size_t) < 4GB)
}
else /* forward filter */
{
size_t blockSize;
size_t nBlocks;
size_t outSize; /* size of the output buffer. Header size (12 bytes) is included */
size_t block;
uint64_t *i64Buf;
uint32_t *i32Buf;
size_t maxDestSize;
char *rpos; /* pointer to current read position */
char *roBuf; /* pointer to current write position */
if (nbytes > INT32_MAX)
{
/* can only compress chunks up to 2GB */
goto error;
}
if(cd_nelmts > 0 && cd_values[0] > 0)
{
blockSize = cd_values[0];
}
else
{
blockSize = DEFAULT_BLOCK_SIZE;
}
if(blockSize > nbytes)
{
blockSize = nbytes;
}
nBlocks = (nbytes-1)/blockSize +1;
maxDestSize = nBlocks * LZ4_compressBound(blockSize) + 4 + 8 + nBlocks*4;
outBuf = H5allocate_memory(maxDestSize, false);
if (NULL == outBuf)
{
goto error;
}
rpos = (char*)*buf; /* pointer to current read position */
roBuf = (char*)outBuf; /* pointer to current write position */
/* header */
i64Buf = (uint64_t *) (roBuf);
i64Buf[0] = htobe64t((uint64_t)nbytes); /* Store decompressed size in be format */
roBuf += 8;
i32Buf = (uint32_t *) (roBuf);
i32Buf[0] = htobe32t((uint32_t)blockSize); /* Store the block size in be format */
roBuf += 4;
outSize = 12; /* size of the output buffer. Header size (12 bytes) is included */
for(block = 0; block < nBlocks; ++block)
{
uint32_t compBlockSize; /// reserve space for compBlockSize
size_t origWritten = block*blockSize;
if(nbytes - origWritten < blockSize) /* the last block may be < blockSize */
blockSize = nbytes - origWritten;
#if LZ4_VERSION_NUMBER > 10300
compBlockSize = LZ4_compress_default(rpos, roBuf+4, blockSize, LZ4_compressBound(blockSize)); /// reserve space for compBlockSize
#else
compBlockSize = LZ4_compress(rpos, roBuf+4, blockSize); /// reserve space for compBlockSize
#endif
if(!compBlockSize)
goto error;
if(compBlockSize >= blockSize) /* compression did not save any space, do a memcpy instead */
{
compBlockSize = blockSize;
memcpy(roBuf+4, rpos, blockSize);
}
i32Buf = (uint32_t *) (roBuf);
i32Buf[0] = htobe32t((uint32_t)compBlockSize); /* write blocksize */
roBuf += 4;
rpos += blockSize; /* advance read pointer */
roBuf += compBlockSize; /* advance write pointer */
outSize += compBlockSize + 4;
}
H5free_memory(*buf);
*buf = outBuf;
*buf_size = outSize;
outBuf = NULL;
ret_value = outSize;
}
/* done: */
if(outBuf)
H5free_memory(outBuf);
return ret_value;
error:
if(outBuf)
H5free_memory(outBuf);
outBuf = NULL;
return 0;
}
int lz4_register_h5filter(void){
int retval;
retval = H5Zregister(H5Z_LZ4);
if(retval<0){
PUSH_ERR("lz4_register_h5filter",
H5E_CANTREGISTER, "Can't register lz4 filter");
}
return retval;
}