From 8d7d3fb545b4273cf9d1a61bf7ea3bfdde8a1199 Mon Sep 17 00:00:00 2001 From: Jeff Knupp Date: Sun, 23 Jul 2017 12:20:57 -0400 Subject: [PATCH] BUG: Use size_t to avoid array index overflow; add missing malloc of error_msg Fix a few locations where a parser's `error_msg` buffer is written to without having been previously allocated. This manifested as a double free during exception handling code making use of the `error_msg`. Additionally, use `size_t/ssize_t` where array indices or lengths will be stored. Previously, int32_t was used and would overflow on columns with very large amounts of data (i.e. greater than INTMAX bytes). xref #14696 closes #16798 Author: Jeff Knupp Author: Jeff Knupp Closes #17040 from jeffknupp/16790-core-on-large-csv and squashes the following commits: 6a1ba23 [Jeff Knupp] Clear up prose a5d5677 [Jeff Knupp] Fix linting issues 4380c53 [Jeff Knupp] Fix linting issues 7b1cd8d [Jeff Knupp] Fix linting issues e3cb9c1 [Jeff Knupp] Add unit test plus '--high-memory' option, *off by default*. 2ab4971 [Jeff Knupp] Remove debugging code 2930eaa [Jeff Knupp] Fix line length to conform to linter rules e4dfd19 [Jeff Knupp] Revert printf format strings; fix more comment alignment 3171674 [Jeff Knupp] Fix some leftover size_t references 0985cf3 [Jeff Knupp] Remove debugging code; fix type cast 669d99b [Jeff Knupp] Fix linting errors re: line length 1f24847 [Jeff Knupp] Fix comment alignment; add whatsnew entry e04d12a [Jeff Knupp] Switch to use int64_t rather than size_t due to portability concerns. d5c75e8 [Jeff Knupp] BUG: Use size_t to avoid array index overflow; add missing malloc of error_msg --- doc/source/whatsnew/v0.21.0.txt | 3 +- pandas/_libs/parsers.pyx | 145 ++++++++++++++----------- pandas/_libs/src/parser/tokenizer.c | 111 ++++++++++--------- pandas/_libs/src/parser/tokenizer.h | 42 +++---- pandas/conftest.py | 9 +- pandas/tests/io/parser/test_parsers.py | 15 +++ setup.cfg | 1 + 7 files changed, 187 insertions(+), 139 deletions(-) diff --git a/doc/source/whatsnew/v0.21.0.txt b/doc/source/whatsnew/v0.21.0.txt index a64f30b6e97a5..096040bb85a10 100644 --- a/doc/source/whatsnew/v0.21.0.txt +++ b/doc/source/whatsnew/v0.21.0.txt @@ -264,7 +264,8 @@ I/O ^^^ - Bug in :func:`read_csv` in which non integer values for the header argument generated an unhelpful / unrelated error message (:issue:`16338`) - +- Bug in :func:`read_csv` in which memory management issues in exception handling, under certain conditions, would cause the interpreter to segfault (:issue:`14696, :issue:`16798`). +- Bug in :func:`read_csv` when called with ``low_memory=False`` in which a CSV with at least one column > 2GB in size would incorrectly raise a ``MemoryError`` (:issue:`16798`). - Bug in :func:`read_stata` where value labels could not be read when using an iterator (:issue:`16923`) - Bug in :func:`read_html` where import check fails when run in multiple threads (:issue:`16928`) diff --git a/pandas/_libs/parsers.pyx b/pandas/_libs/parsers.pyx index 7375a2197c6b7..c512a9fd39e95 100644 --- a/pandas/_libs/parsers.pyx +++ b/pandas/_libs/parsers.pyx @@ -121,30 +121,30 @@ cdef extern from "parser/tokenizer.h": io_callback cb_io io_cleanup cb_cleanup - int chunksize # Number of bytes to prepare for each chunk - char *data # pointer to data to be processed - int datalen # amount of data available - int datapos + int64_t chunksize # Number of bytes to prepare for each chunk + char *data # pointer to data to be processed + int64_t datalen # amount of data available + int64_t datapos # where to write out tokenized data char *stream - int stream_len - int stream_cap + int64_t stream_len + int64_t stream_cap # Store words in (potentially ragged) matrix for now, hmm char **words - int *word_starts # where we are in the stream - int words_len - int words_cap + int64_t *word_starts # where we are in the stream + int64_t words_len + int64_t words_cap - char *pword_start # pointer to stream start of current field - int word_start # position start of current field + char *pword_start # pointer to stream start of current field + int64_t word_start # position start of current field - int *line_start # position in words for start of line - int *line_fields # Number of fields in each line - int lines # Number of lines observed - int file_lines # Number of file lines observed (with bad/skipped) - int lines_cap # Vector capacity + int64_t *line_start # position in words for start of line + int64_t *line_fields # Number of fields in each line + int64_t lines # Number of lines observed + int64_t file_lines # Number of lines observed (with bad/skipped) + int64_t lines_cap # Vector capacity # Tokenizing stuff ParserState state @@ -177,14 +177,14 @@ cdef extern from "parser/tokenizer.h": # thousands separator (comma, period) char thousands - int header # Boolean: 1: has header, 0: no header - int header_start # header row start - int header_end # header row end + int header # Boolean: 1: has header, 0: no header + int64_t header_start # header row start + int64_t header_end # header row end void *skipset PyObject *skipfunc int64_t skip_first_N_rows - int skipfooter + int64_t skipfooter # pick one, depending on whether the converter requires GIL double (*double_converter_nogil)(const char *, char **, char, char, char, int) nogil @@ -195,12 +195,12 @@ cdef extern from "parser/tokenizer.h": char *warn_msg char *error_msg - int skip_empty_lines + int64_t skip_empty_lines ctypedef struct coliter_t: char **words - int *line_start - int col + int64_t *line_start + int64_t col ctypedef struct uint_state: int seen_sint @@ -210,7 +210,8 @@ cdef extern from "parser/tokenizer.h": void uint_state_init(uint_state *self) int uint64_conflict(uint_state *self) - void coliter_setup(coliter_t *it, parser_t *parser, int i, int start) nogil + void coliter_setup(coliter_t *it, parser_t *parser, + int64_t i, int64_t start) nogil void COLITER_NEXT(coliter_t, const char *) nogil parser_t* parser_new() @@ -289,14 +290,14 @@ cdef class TextReader: object true_values, false_values object handle bint na_filter, verbose, has_usecols, has_mi_columns - int parser_start + int64_t parser_start list clocks char *c_encoding kh_str_t *false_set kh_str_t *true_set cdef public: - int leading_cols, table_width, skipfooter, buffer_lines + int64_t leading_cols, table_width, skipfooter, buffer_lines object allow_leading_cols object delimiter, converters, delim_whitespace object na_values @@ -730,7 +731,8 @@ cdef class TextReader: Py_ssize_t i, start, field_count, passed_count, unnamed_count # noqa char *word object name - int status, hr, data_line + int status + int64_t hr, data_line char *errors = "strict" cdef StringPath path = _string_path(self.c_encoding) @@ -949,8 +951,8 @@ cdef class TextReader: cdef _read_rows(self, rows, bint trim): cdef: - int buffered_lines - int irows, footer = 0 + int64_t buffered_lines + int64_t irows, footer = 0 self._start_clock() @@ -1018,12 +1020,13 @@ cdef class TextReader: def _convert_column_data(self, rows=None, upcast_na=False, footer=0): cdef: - Py_ssize_t i, nused + int64_t i + int nused kh_str_t *na_hashset = NULL - int start, end + int64_t start, end object name, na_flist, col_dtype = None bint na_filter = 0 - Py_ssize_t num_cols + int64_t num_cols start = self.parser_start @@ -1195,7 +1198,7 @@ cdef class TextReader: return col_res, na_count cdef _convert_with_dtype(self, object dtype, Py_ssize_t i, - int start, int end, + int64_t start, int64_t end, bint na_filter, bint user_dtype, kh_str_t *na_hashset, @@ -1275,7 +1278,7 @@ cdef class TextReader: raise TypeError("the dtype %s is not " "supported for parsing" % dtype) - cdef _string_convert(self, Py_ssize_t i, int start, int end, + cdef _string_convert(self, Py_ssize_t i, int64_t start, int64_t end, bint na_filter, kh_str_t *na_hashset): cdef StringPath path = _string_path(self.c_encoding) @@ -1336,6 +1339,7 @@ cdef class TextReader: kh_destroy_str(table) cdef _get_column_name(self, Py_ssize_t i, Py_ssize_t nused): + cdef int64_t j if self.has_usecols and self.names is not None: if (not callable(self.usecols) and len(self.names) == len(self.usecols)): @@ -1427,8 +1431,8 @@ cdef inline StringPath _string_path(char *encoding): # ---------------------------------------------------------------------- # Type conversions / inference support code -cdef _string_box_factorize(parser_t *parser, int col, - int line_start, int line_end, +cdef _string_box_factorize(parser_t *parser, int64_t col, + int64_t line_start, int64_t line_end, bint na_filter, kh_str_t *na_hashset): cdef: int error, na_count = 0 @@ -1480,8 +1484,8 @@ cdef _string_box_factorize(parser_t *parser, int col, return result, na_count -cdef _string_box_utf8(parser_t *parser, int col, - int line_start, int line_end, +cdef _string_box_utf8(parser_t *parser, int64_t col, + int64_t line_start, int64_t line_end, bint na_filter, kh_str_t *na_hashset): cdef: int error, na_count = 0 @@ -1533,8 +1537,8 @@ cdef _string_box_utf8(parser_t *parser, int col, return result, na_count -cdef _string_box_decode(parser_t *parser, int col, - int line_start, int line_end, +cdef _string_box_decode(parser_t *parser, int64_t col, + int64_t line_start, int64_t line_end, bint na_filter, kh_str_t *na_hashset, char *encoding): cdef: @@ -1592,8 +1596,8 @@ cdef _string_box_decode(parser_t *parser, int col, @cython.boundscheck(False) -cdef _categorical_convert(parser_t *parser, int col, - int line_start, int line_end, +cdef _categorical_convert(parser_t *parser, int64_t col, + int64_t line_start, int64_t line_end, bint na_filter, kh_str_t *na_hashset, char *encoding): "Convert column data into codes, categories" @@ -1663,8 +1667,8 @@ cdef _categorical_convert(parser_t *parser, int col, kh_destroy_str(table) return np.asarray(codes), result, na_count -cdef _to_fw_string(parser_t *parser, int col, int line_start, - int line_end, size_t width): +cdef _to_fw_string(parser_t *parser, int64_t col, int64_t line_start, + int64_t line_end, int64_t width): cdef: Py_ssize_t i coliter_t it @@ -1680,11 +1684,11 @@ cdef _to_fw_string(parser_t *parser, int col, int line_start, return result -cdef inline void _to_fw_string_nogil(parser_t *parser, int col, - int line_start, int line_end, +cdef inline void _to_fw_string_nogil(parser_t *parser, int64_t col, + int64_t line_start, int64_t line_end, size_t width, char *data) nogil: cdef: - Py_ssize_t i + int64_t i coliter_t it const char *word = NULL @@ -1699,7 +1703,8 @@ cdef char* cinf = b'inf' cdef char* cposinf = b'+inf' cdef char* cneginf = b'-inf' -cdef _try_double(parser_t *parser, int col, int line_start, int line_end, +cdef _try_double(parser_t *parser, int64_t col, + int64_t line_start, int64_t line_end, bint na_filter, kh_str_t *na_hashset, object na_flist): cdef: int error, na_count = 0 @@ -1808,7 +1813,8 @@ cdef inline int _try_double_nogil(parser_t *parser, return 0 -cdef _try_uint64(parser_t *parser, int col, int line_start, int line_end, +cdef _try_uint64(parser_t *parser, int64_t col, + int64_t line_start, int64_t line_end, bint na_filter, kh_str_t *na_hashset): cdef: int error @@ -1842,8 +1848,9 @@ cdef _try_uint64(parser_t *parser, int col, int line_start, int line_end, return result -cdef inline int _try_uint64_nogil(parser_t *parser, int col, int line_start, - int line_end, bint na_filter, +cdef inline int _try_uint64_nogil(parser_t *parser, int64_t col, + int64_t line_start, + int64_t line_end, bint na_filter, const kh_str_t *na_hashset, uint64_t *data, uint_state *state) nogil: cdef: @@ -1879,7 +1886,8 @@ cdef inline int _try_uint64_nogil(parser_t *parser, int col, int line_start, return 0 -cdef _try_int64(parser_t *parser, int col, int line_start, int line_end, +cdef _try_int64(parser_t *parser, int64_t col, + int64_t line_start, int64_t line_end, bint na_filter, kh_str_t *na_hashset): cdef: int error, na_count = 0 @@ -1906,8 +1914,9 @@ cdef _try_int64(parser_t *parser, int col, int line_start, int line_end, return result, na_count -cdef inline int _try_int64_nogil(parser_t *parser, int col, int line_start, - int line_end, bint na_filter, +cdef inline int _try_int64_nogil(parser_t *parser, int64_t col, + int64_t line_start, + int64_t line_end, bint na_filter, const kh_str_t *na_hashset, int64_t NA, int64_t *data, int *na_count) nogil: cdef: @@ -1944,7 +1953,8 @@ cdef inline int _try_int64_nogil(parser_t *parser, int col, int line_start, return 0 -cdef _try_bool(parser_t *parser, int col, int line_start, int line_end, +cdef _try_bool(parser_t *parser, int64_t col, + int64_t line_start, int64_t line_end, bint na_filter, kh_str_t *na_hashset): cdef: int na_count @@ -1966,8 +1976,9 @@ cdef _try_bool(parser_t *parser, int col, int line_start, int line_end, return None, None return result.view(np.bool_), na_count -cdef inline int _try_bool_nogil(parser_t *parser, int col, int line_start, - int line_end, bint na_filter, +cdef inline int _try_bool_nogil(parser_t *parser, int64_t col, + int64_t line_start, + int64_t line_end, bint na_filter, const kh_str_t *na_hashset, uint8_t NA, uint8_t *data, int *na_count) nogil: cdef: @@ -2006,7 +2017,8 @@ cdef inline int _try_bool_nogil(parser_t *parser, int col, int line_start, data += 1 return 0 -cdef _try_bool_flex(parser_t *parser, int col, int line_start, int line_end, +cdef _try_bool_flex(parser_t *parser, int64_t col, + int64_t line_start, int64_t line_end, bint na_filter, const kh_str_t *na_hashset, const kh_str_t *true_hashset, const kh_str_t *false_hashset): @@ -2032,8 +2044,9 @@ cdef _try_bool_flex(parser_t *parser, int col, int line_start, int line_end, return None, None return result.view(np.bool_), na_count -cdef inline int _try_bool_flex_nogil(parser_t *parser, int col, int line_start, - int line_end, bint na_filter, +cdef inline int _try_bool_flex_nogil(parser_t *parser, int64_t col, + int64_t line_start, + int64_t line_end, bint na_filter, const kh_str_t *na_hashset, const kh_str_t *true_hashset, const kh_str_t *false_hashset, @@ -2251,8 +2264,8 @@ for k in list(na_values): na_values[np.dtype(k)] = na_values[k] -cdef _apply_converter(object f, parser_t *parser, int col, - int line_start, int line_end, +cdef _apply_converter(object f, parser_t *parser, int64_t col, + int64_t line_start, int64_t line_end, char* c_encoding): cdef: int error @@ -2296,7 +2309,7 @@ def _to_structured_array(dict columns, object names, object usecols): object name, fnames, field_type Py_ssize_t i, offset, nfields, length - int stride, elsize + int64_t stride, elsize char *buf if names is None: @@ -2344,10 +2357,10 @@ def _to_structured_array(dict columns, object names, object usecols): return recs -cdef _fill_structured_column(char *dst, char* src, int elsize, - int stride, int length, bint incref): +cdef _fill_structured_column(char *dst, char* src, int64_t elsize, + int64_t stride, int64_t length, bint incref): cdef: - Py_ssize_t i + int64_t i if incref: util.transfer_object_column(dst, src, stride, length) diff --git a/pandas/_libs/src/parser/tokenizer.c b/pandas/_libs/src/parser/tokenizer.c index be23ebb023383..ab92290f87719 100644 --- a/pandas/_libs/src/parser/tokenizer.c +++ b/pandas/_libs/src/parser/tokenizer.c @@ -69,9 +69,9 @@ static void free_if_not_null(void **ptr) { */ -static void *grow_buffer(void *buffer, int length, int *capacity, int space, - int elsize, int *error) { - int cap = *capacity; +static void *grow_buffer(void *buffer, size_t length, size_t *capacity, + size_t space, size_t elsize, int *error) { + size_t cap = *capacity; void *newbuffer = buffer; // Can we fit potentially nbytes tokens (+ null terminators) in the stream? @@ -169,7 +169,7 @@ int parser_cleanup(parser_t *self) { } int parser_init(parser_t *self) { - int sz; + size_t sz; /* Initialize data buffers @@ -196,14 +196,14 @@ int parser_init(parser_t *self) { sz = STREAM_INIT_SIZE / 10; sz = sz ? sz : 1; self->words = (char **)malloc(sz * sizeof(char *)); - self->word_starts = (int *)malloc(sz * sizeof(int)); + self->word_starts = (size_t *)malloc(sz * sizeof(size_t)); self->words_cap = sz; self->words_len = 0; // line pointers and metadata - self->line_start = (int *)malloc(sz * sizeof(int)); + self->line_start = (size_t *)malloc(sz * sizeof(size_t)); - self->line_fields = (int *)malloc(sz * sizeof(int)); + self->line_fields = (size_t *)malloc(sz * sizeof(size_t)); self->lines_cap = sz; self->lines = 0; @@ -247,7 +247,8 @@ void parser_del(parser_t *self) { } static int make_stream_space(parser_t *self, size_t nbytes) { - int i, status, cap; + size_t i, cap; + int status; void *orig_ptr, *newptr; // Can we fit potentially nbytes tokens (+ null terminators) in the stream? @@ -304,11 +305,11 @@ static int make_stream_space(parser_t *self, size_t nbytes) { "self->words_cap=%d\n", nbytes, self->words_cap)) newptr = safe_realloc((void *)self->word_starts, - sizeof(int) * self->words_cap); + sizeof(int64_t) * self->words_cap); if (newptr == NULL) { return PARSER_OUT_OF_MEMORY; } else { - self->word_starts = (int *)newptr; + self->word_starts = (int64_t *)newptr; } } @@ -317,8 +318,8 @@ static int make_stream_space(parser_t *self, size_t nbytes) { */ cap = self->lines_cap; self->line_start = - (int *)grow_buffer((void *)self->line_start, self->lines + 1, - &self->lines_cap, nbytes, sizeof(int), &status); + (int64_t *)grow_buffer((void *)self->line_start, self->lines + 1, + &self->lines_cap, nbytes, sizeof(int64_t), &status); TRACE(( "make_stream_space: grow_buffer(self->line_start, %zu, %zu, %zu, %d)\n", self->lines + 1, self->lines_cap, nbytes, status)) @@ -331,11 +332,11 @@ static int make_stream_space(parser_t *self, size_t nbytes) { TRACE(("make_stream_space: cap != self->lines_cap, nbytes = %d\n", nbytes)) newptr = safe_realloc((void *)self->line_fields, - sizeof(int) * self->lines_cap); + sizeof(int64_t) * self->lines_cap); if (newptr == NULL) { return PARSER_OUT_OF_MEMORY; } else { - self->line_fields = (int *)newptr; + self->line_fields = (int64_t *)newptr; } } @@ -350,7 +351,7 @@ static int push_char(parser_t *self, char c) { ("push_char: ERROR!!! self->stream_len(%d) >= " "self->stream_cap(%d)\n", self->stream_len, self->stream_cap)) - int bufsize = 100; + size_t bufsize = 100; self->error_msg = (char *)malloc(bufsize); snprintf(self->error_msg, bufsize, "Buffer overflow caught - possible malformed input file.\n"); @@ -367,7 +368,7 @@ int P_INLINE end_field(parser_t *self) { ("end_field: ERROR!!! self->words_len(%zu) >= " "self->words_cap(%zu)\n", self->words_len, self->words_cap)) - int bufsize = 100; + size_t bufsize = 100; self->error_msg = (char *)malloc(bufsize); snprintf(self->error_msg, bufsize, "Buffer overflow caught - possible malformed input file.\n"); @@ -399,8 +400,8 @@ int P_INLINE end_field(parser_t *self) { } static void append_warning(parser_t *self, const char *msg) { - int ex_length; - int length = strlen(msg); + size_t ex_length; + size_t length = strlen(msg); void *newptr; if (self->warn_msg == NULL) { @@ -420,12 +421,13 @@ static int end_line(parser_t *self) { char *msg; int fields; int ex_fields = self->expected_fields; - int bufsize = 100; // for error or warning messages + size_t bufsize = 100; // for error or warning messages fields = self->line_fields[self->lines]; TRACE(("end_line: Line end, nfields: %d\n", fields)); + TRACE(("end_line: lines: %d\n", self->lines)); if (self->lines > 0) { if (self->expected_fields >= 0) { ex_fields = self->expected_fields; @@ -433,6 +435,7 @@ static int end_line(parser_t *self) { ex_fields = self->line_fields[self->lines - 1]; } } + TRACE(("end_line: ex_fields: %d\n", ex_fields)); if (self->state == START_FIELD_IN_SKIP_LINE || self->state == IN_FIELD_IN_SKIP_LINE || @@ -450,7 +453,7 @@ static int end_line(parser_t *self) { return 0; } - if (!(self->lines <= self->header_end + 1) && + if (!(self->lines <= (int64_t) self->header_end + 1) && (self->expected_fields < 0 && fields > ex_fields) && !(self->usecols)) { // increment file line count self->file_lines++; @@ -485,10 +488,13 @@ static int end_line(parser_t *self) { } } else { // missing trailing delimiters - if ((self->lines >= self->header_end + 1) && fields < ex_fields) { + if ((self->lines >= (int64_t) self->header_end + 1) && + fields < ex_fields) { // might overrun the buffer when closing fields if (make_stream_space(self, ex_fields - fields) < 0) { - self->error_msg = "out of memory"; + size_t bufsize = 100; + self->error_msg = (char *)malloc(bufsize); + snprintf(self->error_msg, bufsize, "out of memory"); return -1; } @@ -507,7 +513,7 @@ static int end_line(parser_t *self) { TRACE(( "end_line: ERROR!!! self->lines(%zu) >= self->lines_cap(%zu)\n", self->lines, self->lines_cap)) - int bufsize = 100; + size_t bufsize = 100; self->error_msg = (char *)malloc(bufsize); snprintf(self->error_msg, bufsize, "Buffer overflow caught - " @@ -568,7 +574,7 @@ static int parser_buffer_bytes(parser_t *self, size_t nbytes) { self->datalen = bytes_read; if (status != REACHED_EOF && self->data == NULL) { - int bufsize = 200; + size_t bufsize = 200; self->error_msg = (char *)malloc(bufsize); if (status == CALLING_READ_FAILED) { @@ -599,7 +605,7 @@ static int parser_buffer_bytes(parser_t *self, size_t nbytes) { if (slen >= self->stream_cap) { \ TRACE(("PUSH_CHAR: ERROR!!! slen(%d) >= stream_cap(%d)\n", slen, \ self->stream_cap)) \ - int bufsize = 100; \ + size_t bufsize = 100; \ self->error_msg = (char *)malloc(bufsize); \ snprintf(self->error_msg, bufsize, \ "Buffer overflow caught - possible malformed input file.\n");\ @@ -626,7 +632,7 @@ static int parser_buffer_bytes(parser_t *self, size_t nbytes) { stream = self->stream + self->stream_len; \ slen = self->stream_len; \ self->state = STATE; \ - if (line_limit > 0 && self->lines == start_lines + (int)line_limit) { \ + if (line_limit > 0 && self->lines == start_lines + (size_t)line_limit) { \ goto linelimit; \ } @@ -641,7 +647,7 @@ static int parser_buffer_bytes(parser_t *self, size_t nbytes) { stream = self->stream + self->stream_len; \ slen = self->stream_len; \ self->state = STATE; \ - if (line_limit > 0 && self->lines == start_lines + (int)line_limit) { \ + if (line_limit > 0 && self->lines == start_lines + (size_t)line_limit) { \ goto linelimit; \ } @@ -712,15 +718,17 @@ int skip_this_line(parser_t *self, int64_t rownum) { } } -int tokenize_bytes(parser_t *self, size_t line_limit, int start_lines) { - int i, slen; +int tokenize_bytes(parser_t *self, size_t line_limit, int64_t start_lines) { + int64_t i, slen; int should_skip; char c; char *stream; char *buf = self->data + self->datapos; if (make_stream_space(self, self->datalen - self->datapos) < 0) { - self->error_msg = "out of memory"; + size_t bufsize = 100; + self->error_msg = (char *)malloc(bufsize); + snprintf(self->error_msg, bufsize, "out of memory"); return -1; } @@ -1025,7 +1033,7 @@ int tokenize_bytes(parser_t *self, size_t line_limit, int start_lines) { PUSH_CHAR(c); self->state = IN_FIELD; } else { - int bufsize = 100; + size_t bufsize = 100; self->error_msg = (char *)malloc(bufsize); snprintf(self->error_msg, bufsize, "delimiter expected after quote in quote"); @@ -1079,7 +1087,7 @@ int tokenize_bytes(parser_t *self, size_t line_limit, int start_lines) { --i; buf--; // let's try this character again (HACK!) if (line_limit > 0 && - self->lines == start_lines + (int)line_limit) { + self->lines == start_lines + line_limit) { goto linelimit; } } @@ -1121,7 +1129,7 @@ int tokenize_bytes(parser_t *self, size_t line_limit, int start_lines) { } static int parser_handle_eof(parser_t *self) { - int bufsize = 100; + size_t bufsize = 100; TRACE( ("handling eof, datalen: %d, pstate: %d\n", self->datalen, self->state)) @@ -1165,9 +1173,9 @@ static int parser_handle_eof(parser_t *self) { } int parser_consume_rows(parser_t *self, size_t nrows) { - int i, offset, word_deletions, char_count; + size_t i, offset, word_deletions, char_count; - if ((int)nrows > self->lines) { + if (nrows > self->lines) { nrows = self->lines; } @@ -1204,7 +1212,7 @@ int parser_consume_rows(parser_t *self, size_t nrows) { self->word_start -= char_count; /* move line metadata */ - for (i = 0; i < self->lines - (int)nrows + 1; ++i) { + for (i = 0; i < self->lines - nrows + 1; ++i) { offset = i + nrows; self->line_start[i] = self->line_start[offset] - word_deletions; self->line_fields[i] = self->line_fields[offset]; @@ -1227,11 +1235,11 @@ int parser_trim_buffers(parser_t *self) { size_t new_cap; void *newptr; - int i; + int64_t i; /* trim words, word_starts */ new_cap = _next_pow2(self->words_len) + 1; - if ((int)new_cap < self->words_cap) { + if (new_cap < self->words_cap) { TRACE(("parser_trim_buffers: new_cap < self->words_cap\n")); newptr = safe_realloc((void *)self->words, new_cap * sizeof(char *)); if (newptr == NULL) { @@ -1239,11 +1247,12 @@ int parser_trim_buffers(parser_t *self) { } else { self->words = (char **)newptr; } - newptr = safe_realloc((void *)self->word_starts, new_cap * sizeof(int)); + newptr = safe_realloc((void *)self->word_starts, + new_cap * sizeof(int64_t)); if (newptr == NULL) { return PARSER_OUT_OF_MEMORY; } else { - self->word_starts = (int *)newptr; + self->word_starts = (int64_t *)newptr; self->words_cap = new_cap; } } @@ -1254,7 +1263,7 @@ int parser_trim_buffers(parser_t *self) { ("parser_trim_buffers: new_cap = %zu, stream_cap = %zu, lines_cap = " "%zu\n", new_cap, self->stream_cap, self->lines_cap)); - if ((int)new_cap < self->stream_cap) { + if (new_cap < self->stream_cap) { TRACE( ("parser_trim_buffers: new_cap < self->stream_cap, calling " "safe_realloc\n")); @@ -1282,19 +1291,21 @@ int parser_trim_buffers(parser_t *self) { /* trim line_start, line_fields */ new_cap = _next_pow2(self->lines) + 1; - if ((int)new_cap < self->lines_cap) { + if (new_cap < self->lines_cap) { TRACE(("parser_trim_buffers: new_cap < self->lines_cap\n")); - newptr = safe_realloc((void *)self->line_start, new_cap * sizeof(int)); + newptr = safe_realloc((void *)self->line_start, + new_cap * sizeof(int64_t)); if (newptr == NULL) { return PARSER_OUT_OF_MEMORY; } else { - self->line_start = (int *)newptr; + self->line_start = (int64_t *)newptr; } - newptr = safe_realloc((void *)self->line_fields, new_cap * sizeof(int)); + newptr = safe_realloc((void *)self->line_fields, + new_cap * sizeof(int64_t)); if (newptr == NULL) { return PARSER_OUT_OF_MEMORY; } else { - self->line_fields = (int *)newptr; + self->line_fields = (int64_t *)newptr; self->lines_cap = new_cap; } } @@ -1303,7 +1314,7 @@ int parser_trim_buffers(parser_t *self) { } void debug_print_parser(parser_t *self) { - int j, line; + int64_t j, line; char *token; for (line = 0; line < self->lines; ++line) { @@ -1324,7 +1335,7 @@ void debug_print_parser(parser_t *self) { int _tokenize_helper(parser_t *self, size_t nrows, int all) { int status = 0; - int start_lines = self->lines; + int64_t start_lines = self->lines; if (self->state == FINISHED) { return 0; @@ -1332,10 +1343,10 @@ int _tokenize_helper(parser_t *self, size_t nrows, int all) { TRACE(( "_tokenize_helper: Asked to tokenize %d rows, datapos=%d, datalen=%d\n", - (int)nrows, self->datapos, self->datalen)); + nrows, self->datapos, self->datalen)); while (1) { - if (!all && self->lines - start_lines >= (int)nrows) break; + if (!all && self->lines - start_lines >= nrows) break; if (self->datapos == self->datalen) { status = parser_buffer_bytes(self, self->chunksize); diff --git a/pandas/_libs/src/parser/tokenizer.h b/pandas/_libs/src/parser/tokenizer.h index b4344e8a6c070..9462608a26814 100644 --- a/pandas/_libs/src/parser/tokenizer.h +++ b/pandas/_libs/src/parser/tokenizer.h @@ -137,30 +137,30 @@ typedef struct parser_t { io_callback cb_io; io_cleanup cb_cleanup; - int chunksize; // Number of bytes to prepare for each chunk - char *data; // pointer to data to be processed - int datalen; // amount of data available - int datapos; + int64_t chunksize; // Number of bytes to prepare for each chunk + char *data; // pointer to data to be processed + int64_t datalen; // amount of data available + int64_t datapos; // where to write out tokenized data char *stream; - int stream_len; - int stream_cap; + int64_t stream_len; + int64_t stream_cap; // Store words in (potentially ragged) matrix for now, hmm char **words; - int *word_starts; // where we are in the stream - int words_len; - int words_cap; + int64_t *word_starts; // where we are in the stream + int64_t words_len; + int64_t words_cap; - char *pword_start; // pointer to stream start of current field - int word_start; // position start of current field + char *pword_start; // pointer to stream start of current field + int64_t word_start; // position start of current field - int *line_start; // position in words for start of line - int *line_fields; // Number of fields in each line - int lines; // Number of (good) lines observed - int file_lines; // Number of file lines observed (including bad or skipped) - int lines_cap; // Vector capacity + int64_t *line_start; // position in words for start of line + int64_t *line_fields; // Number of fields in each line + int64_t lines; // Number of (good) lines observed + int64_t file_lines; // Number of lines (including bad or skipped) + int64_t lines_cap; // Vector capacity // Tokenizing stuff ParserState state; @@ -193,9 +193,9 @@ typedef struct parser_t { // thousands separator (comma, period) char thousands; - int header; // Boolean: 1: has header, 0: no header - int header_start; // header row start - int header_end; // header row end + int header; // Boolean: 1: has header, 0: no header + int64_t header_start; // header row start + int64_t header_end; // header row end void *skipset; PyObject *skipfunc; @@ -216,7 +216,7 @@ typedef struct parser_t { typedef struct coliter_t { char **words; - int *line_start; + int64_t *line_start; int col; } coliter_t; @@ -225,7 +225,7 @@ coliter_t *coliter_new(parser_t *self, int i); #define COLITER_NEXT(iter, word) \ do { \ - const int i = *iter.line_start++ + iter.col; \ + const int64_t i = *iter.line_start++ + iter.col; \ word = i < *iter.line_start ? iter.words[i] : ""; \ } while (0) diff --git a/pandas/conftest.py b/pandas/conftest.py index 8a3ffe22242ac..bae45743bbcfb 100644 --- a/pandas/conftest.py +++ b/pandas/conftest.py @@ -9,7 +9,9 @@ def pytest_addoption(parser): parser.addoption("--skip-slow", action="store_true", help="skip slow tests") parser.addoption("--skip-network", action="store_true", - help="run network tests") + help="skip network tests") + parser.addoption("--run-highmemory", action="store_true", + help="run high memory tests") parser.addoption("--only-slow", action="store_true", help="run only slow tests") @@ -24,6 +26,11 @@ def pytest_runtest_setup(item): if 'network' in item.keywords and item.config.getoption("--skip-network"): pytest.skip("skipping due to --skip-network") + if 'high_memory' in item.keywords and not item.config.getoption( + "--run-highmemory"): + pytest.skip( + "skipping high memory test since --run-highmemory was not set") + # Configurations for all tests and all test modules diff --git a/pandas/tests/io/parser/test_parsers.py b/pandas/tests/io/parser/test_parsers.py index 8d59e3acb3230..f23bd24f5cbe3 100644 --- a/pandas/tests/io/parser/test_parsers.py +++ b/pandas/tests/io/parser/test_parsers.py @@ -1,6 +1,9 @@ # -*- coding: utf-8 -*- import os +from io import StringIO + +import pytest import pandas.util.testing as tm @@ -25,6 +28,18 @@ from .dtypes import DtypeTests +@pytest.mark.high_memory +def test_bytes_exceed_2gb(): + """Read from a "CSV" that has a column larger than 2GB. + + GH 16798 + """ + csv = StringIO('strings\n' + '\n'.join( + ['x' * (1 << 20) for _ in range(2100)])) + df = read_csv(csv, low_memory=False) + assert not df.empty + + class BaseParser(CommentTests, CompressionTests, ConverterTests, DialectTests, HeaderTests, IndexColTests, diff --git a/setup.cfg b/setup.cfg index 05d4c84ca56c4..0123078523b6f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,3 +27,4 @@ markers = single: mark a test as single cpu only slow: mark a test as slow network: mark a test as network + highmemory: mark a test as a high-memory only