From b2d766069f13624c58feb9b69469d414f8d75778 Mon Sep 17 00:00:00 2001 From: Andreas Ntalakas Date: Thu, 7 Nov 2019 20:54:05 +0200 Subject: [PATCH] Adds example for reading ranges, fixes adding ranges, GetRanges function --- .gitignore | 3 +- examples/range_test.go | 12 +- examples/reading_range_test.go | 197 +++++++++++++++++++++++++++++++++ query.go | 153 ++++++++++++++++--------- 4 files changed, 305 insertions(+), 60 deletions(-) create mode 100644 examples/reading_range_test.go diff --git a/.gitignore b/.gitignore index b3efc391..621072e7 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ # Output of the go coverage tool, specifically when used with LiteIDE *.out -.idea \ No newline at end of file +.idea +.vscode \ No newline at end of file diff --git a/examples/range_test.go b/examples/range_test.go index 6fe86b83..7990dafc 100644 --- a/examples/range_test.go +++ b/examples/range_test.go @@ -126,22 +126,16 @@ func addRange() { query, err := tiledb.NewQuery(ctx, array) checkError(err) - var startFloat float32 = 1 - var endFloat float32 = 3 - // Try with invalid dimension types - err = query.AddRange(0, &startFloat, &endFloat) + err = query.AddRange(0, float32(1), float32(3)) fmt.Println(err) - var start int32 = 1 - var end int32 = 3 - // Try with invalid dimension index - err = query.AddRange(2, &start, &end) + err = query.AddRange(2, int32(1), int32(3)) fmt.Println(err) // Try using valid index, range - err = query.AddRange(0, &start, &end) + err = query.AddRange(0, int32(1), int32(3)) checkError(err) err = array.Close() diff --git a/examples/reading_range_test.go b/examples/reading_range_test.go new file mode 100644 index 00000000..d2cf185b --- /dev/null +++ b/examples/reading_range_test.go @@ -0,0 +1,197 @@ +/** + * @file rerading_range_test.go + * + * @section LICENSE + * + * The MIT License + * + * @copyright Copyright (c) 2019 TileDB, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * @section DESCRIPTION + * + * This is a part of the TileDB reading_range tutorial: + * https://docs.tiledb.io/en/latest/reading_range.html + * + * When run, this program will create a simple 2D dense array, write some data + * to it, and read a slice of the data back in the layout of the user's choice + * (passed as an argument to the program: "row", "col", or "global"). + * + */ + +package examples + +import ( + "fmt" + "os" + + tiledb "github.com/TileDB-Inc/TileDB-Go" +) + +// Name of array. +var readRangeArrayName = "read_range_array" + +func createReadRangeArray() { + // Create a TileDB context. + ctx, err := tiledb.NewContext(nil) + checkError(err) + + // The array will be 4x4 with dimensions "rows" and "cols", with domain [1,4]. + domain, err := tiledb.NewDomain(ctx) + checkError(err) + rowDim, err := tiledb.NewDimension(ctx, "rows", []int32{1, 4}, int32(4)) + checkError(err) + colDim, err := tiledb.NewDimension(ctx, "cols", []int32{1, 4}, int32(4)) + checkError(err) + err = domain.AddDimensions(rowDim, colDim) + checkError(err) + + // The array will be dense. + schema, err := tiledb.NewArraySchema(ctx, tiledb.TILEDB_DENSE) + err = schema.SetDomain(domain) + checkError(err) + err = schema.SetCellOrder(tiledb.TILEDB_ROW_MAJOR) + checkError(err) + err = schema.SetTileOrder(tiledb.TILEDB_ROW_MAJOR) + checkError(err) + + // Add a single attribute "a" so each (i,j) cell can store an integer. + a, err := tiledb.NewAttribute(ctx, "a", tiledb.TILEDB_INT32) + checkError(err) + err = schema.AddAttributes(a) + checkError(err) + + // Create the (empty) array on disk. + array, err := tiledb.NewArray(ctx, readRangeArrayName) + checkError(err) + err = array.Create(schema) + checkError(err) +} + +func writeRearRangeArray() { + ctx, err := tiledb.NewContext(nil) + checkError(err) + + // Prepare some data for the array + data := []int32{ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16} + + // Open the array for writing and create the query. + array, err := tiledb.NewArray(ctx, readRangeArrayName) + checkError(err) + err = array.Open(tiledb.TILEDB_WRITE) + checkError(err) + query, err := tiledb.NewQuery(ctx, array) + checkError(err) + err = query.SetLayout(tiledb.TILEDB_ROW_MAJOR) + checkError(err) + _, err = query.SetBuffer("a", data) + checkError(err) + + // Perform the write and close the array. + err = query.Submit() + checkError(err) + err = array.Close() + checkError(err) +} + +func readReadRangeArray(dimIdx uint32) { + ctx, err := tiledb.NewContext(nil) + checkError(err) + + // Prepare the array for reading + array, err := tiledb.NewArray(ctx, readRangeArrayName) + checkError(err) + err = array.Open(tiledb.TILEDB_READ) + checkError(err) + + // Prepare the vector that will hold the result (of size 6 elements) + data := make([]int32, 12) + + // Prepare the query + query, err := tiledb.NewQuery(ctx, array) + checkError(err) + + err = query.AddRange(dimIdx, int32(1), int32(1)) + checkError(err) + err = query.AddRange(dimIdx, int32(3), int32(4)) + checkError(err) + + numOfRanges, err := query.GetRangeNum(dimIdx) + checkError(err) + fmt.Printf("Num of Ranges: %d\n", *numOfRanges) + + var I uint64 + for I = 0; I < *numOfRanges; I++ { + start, end, err := query.GetRange(dimIdx, I) + checkError(err) + fmt.Printf("Range for dimension: %d, start: %v, end: %v\n", dimIdx, start, end) + } + + ranges, err := query.GetRanges() + checkError(err) + + fmt.Printf("Ranges: %v\n", ranges) + + _, err = query.SetBuffer("a", data) + checkError(err) + + // Submit the query and close the array. + err = query.Submit() + checkError(err) + err = array.Close() + checkError(err) + + // Print out the results. + fmt.Println(data) +} + +// ExampleReadRangeArray shows and example creation, writing and range reading +// of a dense array +func ExampleReadRangeArray() { + createReadRangeArray() + writeRearRangeArray() + // Rows + readReadRangeArray(0) + // Columns + readReadRangeArray(1) + + // Cleanup example so unit tests are clean + if _, err := os.Stat(readRangeArrayName); err == nil { + err = os.RemoveAll(readRangeArrayName) + checkError(err) + } + + // Output: Num of Ranges: 2 + // Range for dimension: 0, start: 1, end: 1 + // Range for dimension: 0, start: 3, end: 4 + // Ranges: [[{1 1} {3 4}] [{1 4}]] + // [1 2 3 4 9 10 11 12 13 14 15 16] + // Num of Ranges: 2 + // Range for dimension: 1, start: 1, end: 1 + // Range for dimension: 1, start: 3, end: 4 + // Ranges: [[{1 4}] [{1 1} {3 4}]] + // [1 3 4 5 7 8 9 11 12 13 15 16] +} + +// 1 2 3 4 +// 5 6 7 8 +// 9 10 11 12 +// 13 14 15 16 diff --git a/query.go b/query.go index 2a423e70..29c2952e 100644 --- a/query.go +++ b/query.go @@ -27,6 +27,12 @@ type Query struct { resultBufferElements map[string][2]*uint64 } +// RangeLimits defines a query range +type RangeLimits struct { + start interface{} + end interface{} +} + /* NewQuery Creates a TileDB query object. @@ -367,9 +373,9 @@ func (q *Query) SetBuffer(attribute string, buffer interface{}) (*uint64, } // AddRange adds a 1D range along a subarray dimension, which is in the form -// (start, end, stride). All arguments should be references. The datatype of -// the range components must be the same as the type of the domain of the -// array in the query. The stride is currently unsupportedand set to nil. +// (start, end, stride). The datatype of the range components must be the same +// as the type of the domain of the array in the query. +// The stride is currently unsupported and set to nil. func (q *Query) AddRange(dimIdx uint32, start interface{}, end interface{}) error { startReflectValue := reflect.ValueOf(start) endReflectValue := reflect.ValueOf(end) @@ -384,69 +390,69 @@ func (q *Query) AddRange(dimIdx uint32, start interface{}, end interface{}) erro var endBuffer unsafe.Pointer startReflectType := reflect.TypeOf(start) - startType := startReflectType.Elem().Kind() + startType := startReflectType.Kind() switch startType { case reflect.Int: - tStart := start.(*int) - tEnd := end.(*int) - startBuffer = unsafe.Pointer(tStart) - endBuffer = unsafe.Pointer(tEnd) + tStart := start.(int) + tEnd := end.(int) + startBuffer = unsafe.Pointer(&tStart) + endBuffer = unsafe.Pointer(&tEnd) case reflect.Int8: - tStart := start.(*int8) - tEnd := end.(*int8) - startBuffer = unsafe.Pointer(tStart) - endBuffer = unsafe.Pointer(tEnd) + tStart := start.(int8) + tEnd := end.(int8) + startBuffer = unsafe.Pointer(&tStart) + endBuffer = unsafe.Pointer(&tEnd) case reflect.Int16: - tStart := start.(*int16) - tEnd := end.(*int16) - startBuffer = unsafe.Pointer(tStart) - endBuffer = unsafe.Pointer(tEnd) + tStart := start.(int16) + tEnd := end.(int16) + startBuffer = unsafe.Pointer(&tStart) + endBuffer = unsafe.Pointer(&tEnd) case reflect.Int32: - tStart := start.(*int32) - tEnd := end.(*int32) - startBuffer = unsafe.Pointer(tStart) - endBuffer = unsafe.Pointer(tEnd) + tStart := start.(int32) + tEnd := end.(int32) + startBuffer = unsafe.Pointer(&tStart) + endBuffer = unsafe.Pointer(&tEnd) case reflect.Int64: - tStart := start.(*int64) - tEnd := end.(*int64) - startBuffer = unsafe.Pointer(tStart) - endBuffer = unsafe.Pointer(tEnd) + tStart := start.(int64) + tEnd := end.(int64) + startBuffer = unsafe.Pointer(&tStart) + endBuffer = unsafe.Pointer(&tEnd) case reflect.Uint: - tStart := start.(*uint) - tEnd := end.(*uint) - startBuffer = unsafe.Pointer(tStart) - endBuffer = unsafe.Pointer(tEnd) + tStart := start.(uint) + tEnd := end.(uint) + startBuffer = unsafe.Pointer(&tStart) + endBuffer = unsafe.Pointer(&tEnd) case reflect.Uint8: - tStart := start.(*uint8) - tEnd := end.(*uint8) - startBuffer = unsafe.Pointer(tStart) - endBuffer = unsafe.Pointer(tEnd) + tStart := start.(uint8) + tEnd := end.(uint8) + startBuffer = unsafe.Pointer(&tStart) + endBuffer = unsafe.Pointer(&tEnd) case reflect.Uint16: - tStart := start.(*uint16) - tEnd := end.(*uint16) + tStart := start.(uint16) + tEnd := end.(uint16) startBuffer = unsafe.Pointer(&tStart) endBuffer = unsafe.Pointer(&tEnd) case reflect.Uint32: - tStart := start.(*uint32) - tEnd := end.(*uint32) - startBuffer = unsafe.Pointer(tStart) - endBuffer = unsafe.Pointer(tEnd) + tStart := start.(uint32) + tEnd := end.(uint32) + startBuffer = unsafe.Pointer(&tStart) + endBuffer = unsafe.Pointer(&tEnd) case reflect.Uint64: - tStart := start.(*uint64) - tEnd := end.(*uint64) - startBuffer = unsafe.Pointer(tStart) - endBuffer = unsafe.Pointer(tEnd) + tStart := start.(uint64) + tEnd := end.(uint64) + startBuffer = unsafe.Pointer(&tStart) + endBuffer = unsafe.Pointer(&tEnd) case reflect.Float32: - tStart := start.(*float32) - tEnd := end.(*float32) - startBuffer = unsafe.Pointer(tStart) - endBuffer = unsafe.Pointer(tEnd) + tStart := start.(float32) + tEnd := end.(float32) + startBuffer = unsafe.Pointer(&tStart) + endBuffer = unsafe.Pointer(&tEnd) case reflect.Float64: - tStart := start.(*float64) - tEnd := end.(*float64) - startBuffer = unsafe.Pointer(tStart) - endBuffer = unsafe.Pointer(tEnd) + tStart := start.(float64) + tEnd := end.(float64) + startBuffer = unsafe.Pointer(&tStart) + endBuffer = unsafe.Pointer(&tEnd) default: return fmt.Errorf("Unrecognized type of range component passed: %s", startType.String()) @@ -547,6 +553,53 @@ func (q *Query) GetRange(dimIdx uint32, rangeNum uint64) (interface{}, interface return start, end, nil } +// GetRanges gets the number of dimensions from the array under current query +// and builds an array of dimensions that have as memmbers arrays of ranges +func (q *Query) GetRanges() ([][]RangeLimits, error) { + // We need to infer the datatype of the dimension represented by index + // dimIdx. That said: + // Get array schema + schema, err := q.array.Schema() + if err != nil { + return nil, err + } + + // Get the domain object + domain, err := schema.Domain() + if err != nil { + return nil, err + } + + // Use the index to retrieve the dimension object + nDim, err := domain.NDim() + if err != nil { + return nil, err + } + + var dimIdx uint32 + + ranges := make([][]RangeLimits, nDim) + for dimIdx = 0; dimIdx < uint32(nDim); dimIdx++ { + numOfRanges, err := q.GetRangeNum(dimIdx) + if err != nil { + return nil, err + } + + var I uint64 + ranges[dimIdx] = make([]RangeLimits, 0) + for I = 0; I < *numOfRanges; I++ { + start, end, err := q.GetRange(dimIdx, I) + if err != nil { + return nil, err + } + + ranges[dimIdx] = append(ranges[dimIdx], RangeLimits{start: start, end: end}) + } + } + + return ranges, err +} + // GetRangeNum retrieves the number of ranges of the query subarray // along a given dimension. func (q *Query) GetRangeNum(dimIdx uint32) (*uint64, error) {