From a8bda52fd686ba2d950efa9dad3022e3a0cc9a93 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Mon, 30 Mar 2020 03:40:07 +0530 Subject: [PATCH 01/62] Initial Commit(setup) --- src/ArchGDAL.jl | 6 ++++-- src/Tables.jl | 57 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 src/Tables.jl diff --git a/src/ArchGDAL.jl b/src/ArchGDAL.jl index 8b4c0259..6c9fc2da 100644 --- a/src/ArchGDAL.jl +++ b/src/ArchGDAL.jl @@ -1,7 +1,8 @@ module ArchGDAL import GDAL, GeoInterface - import DataStreams: Data + import Tables + #import DataStreams: Data import GeoInterface: coordinates, geotype using Dates @@ -26,7 +27,8 @@ module ArchGDAL include("context.jl") include("base/iterators.jl") include("base/display.jl") - include("datastreams.jl") + include("tables.jl") + #include("datastreams.jl") include("geointerface.jl") mutable struct DriverManager diff --git a/src/Tables.jl b/src/Tables.jl new file mode 100644 index 00000000..e0a9791c --- /dev/null +++ b/src/Tables.jl @@ -0,0 +1,57 @@ +# #Since tables doesn't define a common "Source" type + +# mutable struct Source <: TableSource +# schema::Tables.Schema +# featurelayer::AbstractFeatureLayer +# feature::ArchGDAL.Feature +# ngeom::Int +# end + +# function Source(layer::AbstractFeatureLayer) +# featuredefn = layerdefn(layer) +# ngeometries = ngeom(featuredefn) +# nfld = nfield(featuredefn) +# header = [ +# ["geometry$(i-1)" for i in 1:ngeometries]; +# [getname(getfielddefn(featuredefn,i-1)) for i in 1:nfld] +# ] +# types = [ +# [IGeometry for i in 1:ngeometries]; +# [_FIELDTYPE[gettype(getfielddefn(featuredefn,i-1))] for i in 1:nfld] +# ] +# ArchGDAL.Source( +# Data.Schema(types, header, nfeature(layer)), +# layer, +# unsafe_nextfeature(layer), +# ngeometries +# ) +# end +# Data.schema(source::ArchGDAL.Source) = source.schema +# Data.isdone(s::ArchGDAL.Source, row, col) = s.feature.ptr == C_NULL +# Data.streamtype(::Type{ArchGDAL.Source}, ::Type{Data.Field}) = true +# Data.accesspattern(source::ArchGDAL.Source) = Data.Sequential +# Data.reset!(source::ArchGDAL.Source) = resetreading!(source.featurelayer) + +# function Data.streamfrom( +# source::ArchGDAL.Source, +# ::Type{Data.Field}, +# ::Type{T}, +# row, +# col +# ) where T +# val = if col <= source.ngeom +# @assert T <: IGeometry +# getgeom(source.feature, col-1) +# else +# T(getfield(source.feature, col - source.ngeom - 1)) +# end +# if col == source.schema.cols +# destroy(source.feature) +# source.feature.ptr = GDAL.ogr_l_getnextfeature(source.featurelayer.ptr) +# if row == source.schema.rows +# @assert source.feature.ptr == C_NULL +# resetreading!(source.featurelayer) +# end +# end +# val +# end From 08c3e55f165b7ea4570e7093a8f79f0fcc01d6a7 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Mon, 30 Mar 2020 05:31:53 +0530 Subject: [PATCH 02/62] Add Tables dependency --- Project.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Project.toml b/Project.toml index 5cc42077..88d18148 100644 --- a/Project.toml +++ b/Project.toml @@ -6,6 +6,7 @@ desc = "A high level API for GDAL - Geospatial Data Abstraction Library" version = "0.3.1" [deps] +Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" DataStreams = "9a8bc11e-79be-5b39-94d7-1ccc349a1a85" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" GDAL = "add2ef01-049f-52c4-9ee2-e494f65e021a" From e7b0edc9505802e266baeaf042901b3084e30b60 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Mon, 30 Mar 2020 09:05:06 +0530 Subject: [PATCH 03/62] Setup GeoTable, reading and parsing files[broken] --- src/Tables.jl | 113 ++++++++++++++++++++++++++------------------------ 1 file changed, 59 insertions(+), 54 deletions(-) diff --git a/src/Tables.jl b/src/Tables.jl index e0a9791c..7444c349 100644 --- a/src/Tables.jl +++ b/src/Tables.jl @@ -1,57 +1,62 @@ -# #Since tables doesn't define a common "Source" type +struct GeoTable{AbstractVector} <: Tables.AbstractColumns + names::Array + geometry :: AG.IGeometry[] +end -# mutable struct Source <: TableSource -# schema::Tables.Schema -# featurelayer::AbstractFeatureLayer -# feature::ArchGDAL.Feature -# ngeom::Int -# end -# function Source(layer::AbstractFeatureLayer) -# featuredefn = layerdefn(layer) -# ngeometries = ngeom(featuredefn) -# nfld = nfield(featuredefn) -# header = [ -# ["geometry$(i-1)" for i in 1:ngeometries]; -# [getname(getfielddefn(featuredefn,i-1)) for i in 1:nfld] -# ] -# types = [ -# [IGeometry for i in 1:ngeometries]; -# [_FIELDTYPE[gettype(getfielddefn(featuredefn,i-1))] for i in 1:nfld] -# ] -# ArchGDAL.Source( -# Data.Schema(types, header, nfeature(layer)), -# layer, -# unsafe_nextfeature(layer), -# ngeometries -# ) -# end -# Data.schema(source::ArchGDAL.Source) = source.schema -# Data.isdone(s::ArchGDAL.Source, row, col) = s.feature.ptr == C_NULL -# Data.streamtype(::Type{ArchGDAL.Source}, ::Type{Data.Field}) = true -# Data.accesspattern(source::ArchGDAL.Source) = Data.Sequential -# Data.reset!(source::ArchGDAL.Source) = resetreading!(source.featurelayer) +function geotable(dataset::ArchGDAL.IDataset) + layer = getlayer(dataset, 0) + featuredefn = layerdefn(layer) + ngeometries = ngeom(featuredefn) + nfld = nfield(featuredefn) + featuredefn = layerdefn(layer) + + d = Dict{String, Vector}() + featuredefn = AG.layerdefn(layer) + for field_no in 0:nfield-1 + field = AG.getfielddefn(featuredefn, field_no) + name = AG.getname(field) + typ = AG._FIELDTYPE[AG.gettype(field)] + d[name] = typ[] + end + + d["geometry"] = AG.IGeometry[] -# function Data.streamfrom( -# source::ArchGDAL.Source, -# ::Type{Data.Field}, -# ::Type{T}, -# row, -# col -# ) where T -# val = if col <= source.ngeom -# @assert T <: IGeometry -# getgeom(source.feature, col-1) -# else -# T(getfield(source.feature, col - source.ngeom - 1)) -# end -# if col == source.schema.cols -# destroy(source.feature) -# source.feature.ptr = GDAL.ogr_l_getnextfeature(source.featurelayer.ptr) -# if row == source.schema.rows -# @assert source.feature.ptr == C_NULL -# resetreading!(source.featurelayer) -# end -# end -# val -# end + for fid in 0:nfeat-1 + AG.getfeature(layer, fid) do feature + for (k, v) in pairs(d) + if k == "geometry" + val = AG.getgeom(feature, 0) + else + val = AG.getfield(feature, k) + end + push!(v, val) + end + end + end + + + # names = [ + # ["geometry$(i-1)" for i in 1:ngeometries]; #in case there are more than one geometries + # [AG.getname(AG.getfielddefn(featuredefn,i-1)) for i in 1:nfld] + # ] + + + # types = [ + # [AG.IGeometry for i in 1:ngeometries]; + # [AG._FIELDTYPE[AG.gettype(AG.getfielddefn(featuredefn,i-1))] for i in 1:nfld] + # ] + + # val = if nfld <= ngeom + # getgeom(feature, col-1) + # else + # T(getfield(feature, col - ngeom - 1)) + # end + + Tables.istable(::Type{<:GeoTable}) = true + # getter methods to avoid getproperty clash + names(g::GeoTable) = getfield(g, :names) + + geometry(g::GeoTable) = getfield(g, :geometry) + # schema is column names and types + Tables.schema(g::GeoTable{T}) where {T} = Tables.Schema(names(g), ) From 5fbda99ed57515355f2085f0473e2c81eae32148 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Tue, 31 Mar 2020 20:36:48 +0530 Subject: [PATCH 04/62] Create function for returning a NamedTuple --- Project.toml | 4 ++- src/ArchGDAL.jl | 9 ++++--- src/{Tables.jl => tables.jl} | 50 +++++++++++++++++++----------------- 3 files changed, 34 insertions(+), 29 deletions(-) rename src/{Tables.jl => tables.jl} (54%) diff --git a/Project.toml b/Project.toml index 88d18148..60789e92 100644 --- a/Project.toml +++ b/Project.toml @@ -6,11 +6,12 @@ desc = "A high level API for GDAL - Geospatial Data Abstraction Library" version = "0.3.1" [deps] -Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" DataStreams = "9a8bc11e-79be-5b39-94d7-1ccc349a1a85" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" GDAL = "add2ef01-049f-52c4-9ee2-e494f65e021a" GeoInterface = "cf35fbd7-0cd7-5166-be24-54bfbe79505f" +Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +NamedTupleTools = "d9ec5142-1e00-5aa0-9d6a-321866360f50" [compat] DataStreams = "0.4.2" @@ -26,3 +27,4 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] test = ["Dates", "Statistics", "Test", "BinaryProvider"] + diff --git a/src/ArchGDAL.jl b/src/ArchGDAL.jl index 6c9fc2da..8a30a8ab 100644 --- a/src/ArchGDAL.jl +++ b/src/ArchGDAL.jl @@ -1,10 +1,10 @@ module ArchGDAL import GDAL, GeoInterface - import Tables - #import DataStreams: Data + import DataStreams: Data import GeoInterface: coordinates, geotype using Dates + using NamedTupleTools include("utils.jl") include("types.jl") @@ -27,9 +27,9 @@ module ArchGDAL include("context.jl") include("base/iterators.jl") include("base/display.jl") - include("tables.jl") - #include("datastreams.jl") + include("datastreams.jl") include("geointerface.jl") + include("tables.jl") mutable struct DriverManager function DriverManager() @@ -47,3 +47,4 @@ module ArchGDAL end end # module + diff --git a/src/Tables.jl b/src/tables.jl similarity index 54% rename from src/Tables.jl rename to src/tables.jl index 7444c349..0571938d 100644 --- a/src/Tables.jl +++ b/src/tables.jl @@ -1,40 +1,42 @@ -struct GeoTable{AbstractVector} <: Tables.AbstractColumns - names::Array - geometry :: AG.IGeometry[] -end - - function geotable(dataset::ArchGDAL.IDataset) layer = getlayer(dataset, 0) featuredefn = layerdefn(layer) ngeometries = ngeom(featuredefn) - nfld = nfield(featuredefn) + nfield = ArchGDAL.nfield(featuredefn) featuredefn = layerdefn(layer) - + nfeat = nfeature(layer) + + d = Dict{String, Vector}() - featuredefn = AG.layerdefn(layer) + for field_no in 0:nfield-1 - field = AG.getfielddefn(featuredefn, field_no) - name = AG.getname(field) - typ = AG._FIELDTYPE[AG.gettype(field)] + field = getfielddefn(featuredefn, field_no) + name = getname(field) + typ = _FIELDTYPE[gettype(field)] d[name] = typ[] end - - d["geometry"] = AG.IGeometry[] + # key = String[] + # vals = [] + + d["geometry"] = IGeometry[] + # push!(key, "Geometry") for fid in 0:nfeat-1 - AG.getfeature(layer, fid) do feature + getfeature(layer, fid) do feature for (k, v) in pairs(d) if k == "geometry" - val = AG.getgeom(feature, 0) + val = getgeom(feature, 0) else - val = AG.getfield(feature, k) + val = getfield(feature, k) end push!(v, val) end end end - + + # Named_Tuple = namedtuple(Symbol.(keys(d)), values(d)) + return namedtuple(Symbol.(keys(d)), values(d)) +end # names = [ # ["geometry$(i-1)" for i in 1:ngeometries]; #in case there are more than one geometries @@ -53,10 +55,10 @@ function geotable(dataset::ArchGDAL.IDataset) # T(getfield(feature, col - ngeom - 1)) # end - Tables.istable(::Type{<:GeoTable}) = true - # getter methods to avoid getproperty clash - names(g::GeoTable) = getfield(g, :names) + # Tables.istable(::Type{<:GeoTable}) = true + # # getter methods to avoid getproperty clash + # names(g::GeoTable) = getfield(g, :names) - geometry(g::GeoTable) = getfield(g, :geometry) - # schema is column names and types - Tables.schema(g::GeoTable{T}) where {T} = Tables.Schema(names(g), ) + # geometry(g::GeoTable) = getfield(g, :geometry) + # # schema is column names and types + # Tables.schema(g::GeoTable{T}) where {T} = Tables.Schema(names(g), ) \ No newline at end of file From fee14043e3a3f83958b0c7f9a4bca851b27ca06d Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Wed, 1 Apr 2020 05:37:48 +0530 Subject: [PATCH 05/62] Remove extra dependency --- Project.toml | 2 +- src/ArchGDAL.jl | 2 +- src/tables.jl | 36 +++++++++++------------------------- 3 files changed, 13 insertions(+), 27 deletions(-) diff --git a/Project.toml b/Project.toml index 60789e92..fff22a5a 100644 --- a/Project.toml +++ b/Project.toml @@ -11,7 +11,7 @@ Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" GDAL = "add2ef01-049f-52c4-9ee2-e494f65e021a" GeoInterface = "cf35fbd7-0cd7-5166-be24-54bfbe79505f" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" -NamedTupleTools = "d9ec5142-1e00-5aa0-9d6a-321866360f50" + [compat] DataStreams = "0.4.2" diff --git a/src/ArchGDAL.jl b/src/ArchGDAL.jl index 8a30a8ab..dd698d38 100644 --- a/src/ArchGDAL.jl +++ b/src/ArchGDAL.jl @@ -4,7 +4,7 @@ module ArchGDAL import DataStreams: Data import GeoInterface: coordinates, geotype using Dates - using NamedTupleTools + include("utils.jl") include("types.jl") diff --git a/src/tables.jl b/src/tables.jl index 0571938d..c8c49303 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -15,12 +15,9 @@ function geotable(dataset::ArchGDAL.IDataset) typ = _FIELDTYPE[gettype(field)] d[name] = typ[] end - # key = String[] - # vals = [] - + d["geometry"] = IGeometry[] - # push!(key, "Geometry") - + for fid in 0:nfeat-1 getfeature(layer, fid) do feature for (k, v) in pairs(d) @@ -33,32 +30,21 @@ function geotable(dataset::ArchGDAL.IDataset) end end end - - # Named_Tuple = namedtuple(Symbol.(keys(d)), values(d)) - return namedtuple(Symbol.(keys(d)), values(d)) -end - - # names = [ - # ["geometry$(i-1)" for i in 1:ngeometries]; #in case there are more than one geometries - # [AG.getname(AG.getfielddefn(featuredefn,i-1)) for i in 1:nfld] - # ] - + keys_tup = () + for _key in keys(d) + keys_tup = (keys_tup..., Symbol(_key)) + end - # types = [ - # [AG.IGeometry for i in 1:ngeometries]; - # [AG._FIELDTYPE[AG.gettype(AG.getfielddefn(featuredefn,i-1))] for i in 1:nfld] - # ] + vals_tup = Tuple(values(d)) - # val = if nfld <= ngeom - # getgeom(feature, col-1) - # else - # T(getfield(feature, col - ngeom - 1)) - # end + return NamedTuple{keys_tup}(vals_tup) +end + # Tables.istable(::Type{<:GeoTable}) = true # # getter methods to avoid getproperty clash # names(g::GeoTable) = getfield(g, :names) # geometry(g::GeoTable) = getfield(g, :geometry) # # schema is column names and types - # Tables.schema(g::GeoTable{T}) where {T} = Tables.Schema(names(g), ) \ No newline at end of file + # Tables.schema(g::GeoTable{T}) where {T} = Tables.Schema(names(g), ) From 04489982f6cb9e219bf1c7683a13607e7ff36e1f Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Fri, 3 Apr 2020 05:49:43 +0530 Subject: [PATCH 06/62] Implement Tables interface --- src/ArchGDAL.jl | 3 +-- src/tables.jl | 41 +++++++++++++++++++++++++++++------------ 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/src/ArchGDAL.jl b/src/ArchGDAL.jl index dd698d38..67de810d 100644 --- a/src/ArchGDAL.jl +++ b/src/ArchGDAL.jl @@ -1,11 +1,10 @@ module ArchGDAL import GDAL, GeoInterface - import DataStreams: Data + import Tables, DataStreams: Data import GeoInterface: coordinates, geotype using Dates - include("utils.jl") include("types.jl") include("driver.jl") diff --git a/src/tables.jl b/src/tables.jl index c8c49303..fa5f7575 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -1,8 +1,7 @@ -function geotable(dataset::ArchGDAL.IDataset) - layer = getlayer(dataset, 0) +function geotable(layer::Union{IFeatureLayer, FeatureLayer}, i::Int) featuredefn = layerdefn(layer) ngeometries = ngeom(featuredefn) - nfield = ArchGDAL.nfield(featuredefn) + nfield = nfield(featuredefn) featuredefn = layerdefn(layer) nfeat = nfeature(layer) @@ -34,17 +33,35 @@ function geotable(dataset::ArchGDAL.IDataset) for _key in keys(d) keys_tup = (keys_tup..., Symbol(_key)) end - vals_tup = Tuple(values(d)) - return NamedTuple{keys_tup}(vals_tup) + #Using the tables interface + RowTable = rowtable(NamedTuple{keys_tup}(vals_tup)) + return reshape(RowTable, (1,length(RowTable))) end +# Base.eltype(#type) where {T} = GeoTableRow{T} +# Base.iterate(#type, st=1) = st > length(m) ? nothing : (#logic) +# Base.length(#type) = length(geotable(#type)) + + + +# Base.IteratorSize(::Type{<:#type}) = Base.HasLength() +# Base.IteratorEltype(::Type{<:#type}) = Base.HasEltype() + +# function Base.iterate(t::Table, st = 1) +# st > length(t) && return nothing +# geom = @inbounds getshp(t).shapes[st] +# record = DBFTables.Row(getdbf(t), st) +# return Row(geom, record), st + 1 +# end + +# #Implementing the tables interface + +# Tables.istable(::Type{<:geotable}) = true +# Tables.rowaccess(::Type{<:geotable}) = true +# Tables.rows(g::geotable) = g + +# Tables.schema(g::geotable) = Tables.Schema() - # Tables.istable(::Type{<:GeoTable}) = true - # # getter methods to avoid getproperty clash - # names(g::GeoTable) = getfield(g, :names) - - # geometry(g::GeoTable) = getfield(g, :geometry) - # # schema is column names and types - # Tables.schema(g::GeoTable{T}) where {T} = Tables.Schema(names(g), ) + From 8f3368729dc028f68e1a58636741f79b648d97fe Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Mon, 6 Apr 2020 10:09:58 +0530 Subject: [PATCH 07/62] add struct GeoTable --- src/tables.jl | 39 ++++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index fa5f7575..92678a8d 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -1,11 +1,14 @@ -function geotable(layer::Union{IFeatureLayer, FeatureLayer}, i::Int) +struct GeoTable{T <: NamedTuple} <:AbstractVector{T} + parsed_shapefile::T +end + +function geotable(layer::Union{IFeatureLayer, FeatureLayer}, i::Int) featuredefn = layerdefn(layer) ngeometries = ngeom(featuredefn) - nfield = nfield(featuredefn) + nfield = ArchGDAL.nfield(featuredefn) featuredefn = layerdefn(layer) nfeat = nfeature(layer) - d = Dict{String, Vector}() for field_no in 0:nfield-1 @@ -37,30 +40,28 @@ function geotable(layer::Union{IFeatureLayer, FeatureLayer}, i::Int) #Using the tables interface RowTable = rowtable(NamedTuple{keys_tup}(vals_tup)) - return reshape(RowTable, (1,length(RowTable))) + return GeoTable(reshape(RowTable, (1,length(RowTable)))) end -# Base.eltype(#type) where {T} = GeoTableRow{T} -# Base.iterate(#type, st=1) = st > length(m) ? nothing : (#logic) -# Base.length(#type) = length(geotable(#type)) +# "Struct representing a singe record in a shapefile" +# struct Row{T} +# ###### +# end +Tables.istable(::Type{<:GeoTable}) = true +Tables.rowaccess(::Type{<:GeoTable}) = true +Tables.rows(g::GeoTable) = g +# Base.IteratorSize(::Type{<:GeoTable}) = Base.HasLength() +# Base.length(fc::GeoTable) = length(geotable(g)) +# Base.IteratorEltype(::Type{<:GeoTable}) = Base.HasEltype() -# Base.IteratorSize(::Type{<:#type}) = Base.HasLength() -# Base.IteratorEltype(::Type{<:#type}) = Base.HasEltype() -# function Base.iterate(t::Table, st = 1) -# st > length(t) && return nothing -# geom = @inbounds getshp(t).shapes[st] -# record = DBFTables.Row(getdbf(t), st) -# return Row(geom, record), st + 1 -# end +# "Iterate over the rows of a Shapefile.Table, yielding a Shapefile.Row for each row" +# Base.iterate(g::GeoTable, st=1) = st > length(g) ? nothing : (Row(st, g), st + 1 + -# #Implementing the tables interface -# Tables.istable(::Type{<:geotable}) = true -# Tables.rowaccess(::Type{<:geotable}) = true -# Tables.rows(g::geotable) = g # Tables.schema(g::geotable) = Tables.Schema() From 59a3f29db4d4594292c2f0cec9718ed970aad5e1 Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Sun, 12 Apr 2020 21:54:28 +0200 Subject: [PATCH 08/62] work towards a row based Tables interface --- Project.toml | 8 ++--- src/ArchGDAL.jl | 6 ++-- src/datastreams.jl | 55 ---------------------------------- test/runtests.jl | 2 +- test/test_datastreams.jl | 19 ------------ test/test_tables.jl | 64 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 69 insertions(+), 85 deletions(-) delete mode 100644 src/datastreams.jl delete mode 100644 test/test_datastreams.jl create mode 100644 test/test_tables.jl diff --git a/Project.toml b/Project.toml index fff22a5a..87d491b3 100644 --- a/Project.toml +++ b/Project.toml @@ -6,25 +6,21 @@ desc = "A high level API for GDAL - Geospatial Data Abstraction Library" version = "0.3.1" [deps] -DataStreams = "9a8bc11e-79be-5b39-94d7-1ccc349a1a85" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" GDAL = "add2ef01-049f-52c4-9ee2-e494f65e021a" GeoInterface = "cf35fbd7-0cd7-5166-be24-54bfbe79505f" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" - [compat] -DataStreams = "0.4.2" GDAL = "1.0.2" GeoInterface = "0.4, 0.5" +Tables = "1" julia = "1" [extras] BinaryProvider = "b99e7846-7c00-51b0-8f62-c81ae34c0232" -Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Dates", "Statistics", "Test", "BinaryProvider"] - +test = ["BinaryProvider", "Statistics", "Test"] diff --git a/src/ArchGDAL.jl b/src/ArchGDAL.jl index 67de810d..2047a16d 100644 --- a/src/ArchGDAL.jl +++ b/src/ArchGDAL.jl @@ -1,7 +1,7 @@ module ArchGDAL import GDAL, GeoInterface - import Tables, DataStreams: Data + using Tables: Tables import GeoInterface: coordinates, geotype using Dates @@ -26,9 +26,8 @@ module ArchGDAL include("context.jl") include("base/iterators.jl") include("base/display.jl") - include("datastreams.jl") - include("geointerface.jl") include("tables.jl") + include("geointerface.jl") mutable struct DriverManager function DriverManager() @@ -46,4 +45,3 @@ module ArchGDAL end end # module - diff --git a/src/datastreams.jl b/src/datastreams.jl deleted file mode 100644 index 7c813206..00000000 --- a/src/datastreams.jl +++ /dev/null @@ -1,55 +0,0 @@ -mutable struct Source <: Data.Source - schema::Data.Schema - featurelayer::AbstractFeatureLayer - feature::ArchGDAL.Feature - ngeom::Int -end - -function Source(layer::AbstractFeatureLayer) - featuredefn = layerdefn(layer) - ngeometries = ngeom(featuredefn) - nfld = nfield(featuredefn) - header = [ - ["geometry$(i-1)" for i in 1:ngeometries]; - [getname(getfielddefn(featuredefn,i-1)) for i in 1:nfld] - ] - types = [ - [IGeometry for i in 1:ngeometries]; - [_FIELDTYPE[gettype(getfielddefn(featuredefn,i-1))] for i in 1:nfld] - ] - ArchGDAL.Source( - Data.Schema(types, header, nfeature(layer)), - layer, - unsafe_nextfeature(layer), - ngeometries - ) -end -Data.schema(source::ArchGDAL.Source) = source.schema -Data.isdone(s::ArchGDAL.Source, row, col) = s.feature.ptr == C_NULL -Data.streamtype(::Type{ArchGDAL.Source}, ::Type{Data.Field}) = true -Data.accesspattern(source::ArchGDAL.Source) = Data.Sequential -Data.reset!(source::ArchGDAL.Source) = resetreading!(source.featurelayer) - -function Data.streamfrom( - source::ArchGDAL.Source, - ::Type{Data.Field}, - ::Type{T}, - row, - col - ) where T - val = if col <= source.ngeom - @assert T <: IGeometry - getgeom(source.feature, col-1) - else - T(getfield(source.feature, col - source.ngeom - 1)) - end - if col == source.schema.cols - destroy(source.feature) - source.feature.ptr = GDAL.ogr_l_getnextfeature(source.featurelayer.ptr) - if row == source.schema.rows - @assert source.feature.ptr == C_NULL - resetreading!(source.featurelayer) - end - end - val -end diff --git a/test/runtests.jl b/test/runtests.jl index 72fafc0a..a247c651 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -8,7 +8,7 @@ include("remotefiles.jl") @testset "ArchGDAL" begin cd(dirname(@__FILE__)) do isdir("tmp") || mkpath("tmp") - include("test_datastreams.jl") + include("test_tables.jl") include("test_gdal_tutorials.jl") include("test_geometry.jl") include("test_types.jl") diff --git a/test/test_datastreams.jl b/test/test_datastreams.jl deleted file mode 100644 index 4ff980ad..00000000 --- a/test/test_datastreams.jl +++ /dev/null @@ -1,19 +0,0 @@ -using Test -import ArchGDAL; const AG = ArchGDAL -import DataStreams - -@testset "DataStream Support" begin - df = AG.read("data/point.geojson") do dataset - DataStreams.Data.close!(DataStreams.Data.stream!( - AG.Source(AG.getlayer(dataset,0)), DataStreams.Data.Table - )) - end - @test df.FID == [2.0, 3.0, 0.0, 3.0] - @test df.pointname == ["point-a", "point-b", "a", "b"] - @test AG.toWKT.(df.geometry0) == [ - "POINT (100 0)", - "POINT (100.2785 0.0893)", - "POINT (100 0)", - "POINT (100.2785 0.0893)" - ] -end diff --git a/test/test_tables.jl b/test/test_tables.jl new file mode 100644 index 00000000..0abe41f6 --- /dev/null +++ b/test/test_tables.jl @@ -0,0 +1,64 @@ +using Test +import ArchGDAL; const AG = ArchGDAL +using Tables + +# @testset "DataStream Support" begin +# df = AG.read("data/point.geojson") do dataset +# DataStreams.Data.close!(DataStreams.Data.stream!( +# AG.Source(AG.getlayer(dataset,0)), DataStreams.Data.Table +# )) +# end +# @test df.FID == [2.0, 3.0, 0.0, 3.0] +# @test df.pointname == ["point-a", "point-b", "a", "b"] +# @test AG.toWKT.(df.geometry0) == [ +# "POINT (100 0)", +# "POINT (100.2785 0.0893)", +# "POINT (100 0)", +# "POINT (100.2785 0.0893)" +# ] +# end + +dataset = AG.read(joinpath(@__DIR__, "data/point.geojson")) +layer = AG.getlayer(dataset, 0) + +nfeat = AG.nfeature(layer) +nfield = AG.nfield(layer) +featuredefn = AG.layerdefn(layer) +ngeometries = AG.ngeom(featuredefn) + +Tables.istable(layer::AG.AbstractFeatureLayer) = true +Tables.rowaccess(layer::AG.AbstractFeatureLayer) = true + +function Tables.schema(layer::AG.AbstractFeatureLayer) + # TODO include names and types of geometry columns + featuredefn = AG.layerdefn(layer) + fielddefns = (AG.getfielddefn(featuredefn, i) for i in 0:nfield-1) + names = Tuple(AG.getname(fielddefn) for fielddefn in fielddefns) + types = Tuple(AG._FIELDTYPE[AG.gettype(fielddefn)] for fielddefn in fielddefns) + Tables.Schema(names, types) +end + +schema = Tables.schema(layer) + +function Tables.rows(layer::AG.AbstractFeatureLayer) + # TODO return an iterator rather than a vector of NamedTuples + schema = Tables.schema(layer) + T = NamedTuple{schema.names, Tuple{schema.types...}} + AG.resetreading!(layer) + nfeat = AG.nfeature(layer) + nfield = AG.nfield(layer) + rows = T[] + for _ in 1:nfeat + AG.nextfeature(layer) do feature + # AG.getgeom(feature, 0) # TODO + push!(rows, T(AG.getfield(feature, j) for j in 0:nfield-1)) + end + end + return rows +end + +Tables.rows(layer) + +# TODO getcolumn support +# using DataFrames +# DataFrame(layer) From fb98c7bf2f5828373c4c466129d1fb5734d66711 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Mon, 20 Apr 2020 20:43:33 +0530 Subject: [PATCH 09/62] Reset local fork and local repo made changes --- Project.toml | 4 +- src/ArchGDAL.jl | 2 +- src/datastreams.jl | 55 -------------------- src/tables.jl | 108 +++++++++++++++++++++++++++++++++++++++ test/runtests.jl | 2 +- test/test_datastreams.jl | 19 ------- test/test_tables.jl | 33 ++++++++++++ 7 files changed, 145 insertions(+), 78 deletions(-) delete mode 100644 src/datastreams.jl create mode 100644 src/tables.jl delete mode 100644 test/test_datastreams.jl create mode 100644 test/test_tables.jl diff --git a/Project.toml b/Project.toml index 6f4cd234..43d71848 100644 --- a/Project.toml +++ b/Project.toml @@ -6,14 +6,14 @@ desc = "A high level API for GDAL - Geospatial Data Abstraction Library" version = "0.3.2" [deps] -DataStreams = "9a8bc11e-79be-5b39-94d7-1ccc349a1a85" +Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" GDAL = "add2ef01-049f-52c4-9ee2-e494f65e021a" GeoFormatTypes = "68eda718-8dee-11e9-39e7-89f7f65f511f" GeoInterface = "cf35fbd7-0cd7-5166-be24-54bfbe79505f" [compat] -DataStreams = "0.4.2" +Tables = "1" GDAL = "1.0.2" GeoInterface = "0.4, 0.5" GeoFormatTypes = "0.3" diff --git a/src/ArchGDAL.jl b/src/ArchGDAL.jl index 4698e9b3..2ab76fae 100644 --- a/src/ArchGDAL.jl +++ b/src/ArchGDAL.jl @@ -30,7 +30,7 @@ module ArchGDAL include("context.jl") include("base/iterators.jl") include("base/display.jl") - include("datastreams.jl") + include("tables.jl") include("geointerface.jl") include("convert.jl") diff --git a/src/datastreams.jl b/src/datastreams.jl deleted file mode 100644 index 7c813206..00000000 --- a/src/datastreams.jl +++ /dev/null @@ -1,55 +0,0 @@ -mutable struct Source <: Data.Source - schema::Data.Schema - featurelayer::AbstractFeatureLayer - feature::ArchGDAL.Feature - ngeom::Int -end - -function Source(layer::AbstractFeatureLayer) - featuredefn = layerdefn(layer) - ngeometries = ngeom(featuredefn) - nfld = nfield(featuredefn) - header = [ - ["geometry$(i-1)" for i in 1:ngeometries]; - [getname(getfielddefn(featuredefn,i-1)) for i in 1:nfld] - ] - types = [ - [IGeometry for i in 1:ngeometries]; - [_FIELDTYPE[gettype(getfielddefn(featuredefn,i-1))] for i in 1:nfld] - ] - ArchGDAL.Source( - Data.Schema(types, header, nfeature(layer)), - layer, - unsafe_nextfeature(layer), - ngeometries - ) -end -Data.schema(source::ArchGDAL.Source) = source.schema -Data.isdone(s::ArchGDAL.Source, row, col) = s.feature.ptr == C_NULL -Data.streamtype(::Type{ArchGDAL.Source}, ::Type{Data.Field}) = true -Data.accesspattern(source::ArchGDAL.Source) = Data.Sequential -Data.reset!(source::ArchGDAL.Source) = resetreading!(source.featurelayer) - -function Data.streamfrom( - source::ArchGDAL.Source, - ::Type{Data.Field}, - ::Type{T}, - row, - col - ) where T - val = if col <= source.ngeom - @assert T <: IGeometry - getgeom(source.feature, col-1) - else - T(getfield(source.feature, col - source.ngeom - 1)) - end - if col == source.schema.cols - destroy(source.feature) - source.feature.ptr = GDAL.ogr_l_getnextfeature(source.featurelayer.ptr) - if row == source.schema.rows - @assert source.feature.ptr == C_NULL - resetreading!(source.featurelayer) - end - end - val -end diff --git a/src/tables.jl b/src/tables.jl new file mode 100644 index 00000000..4d87dc32 --- /dev/null +++ b/src/tables.jl @@ -0,0 +1,108 @@ +const AG = ArchGDAL + +struct GeoTable{T<:Union{IFeatureLayer, FeatureLayer}} + layer::T +end + +function GeoTable(layer) + featuredefn = layerdefn(layer) + ngeometries = ngeom(featuredefn) + + if(ngeometries == 0 ) + error("Layer is not a valid ArchGDAL layer") + end + GeoTable(layer) +end + +struct FeatureRow + Row::Array + feature_number::Int +end + +function Tables.schema(layer::AG.AbstractFeatureLayer) + nfield = AG.nfield(layer) + featuredefn = AG.layerdefn(layer) + ngeom = ArchGDAL.ngeom(featuredefn) + + fielddefns = (AG.getfielddefn(featuredefn, i) for i in 0:nfield-1) + geomdefns = (ArchGDAL.getgeomdefn(featuredefn, i) for i in 0:ngeom-1) + + names_fields = Tuple(AG.getname(fielddefn) for fielddefn in fielddefns) + geom_names = Tuple(ArchGDAL.getname(geomdefn) for geomdefn in geomdefns) + names = (names_fields..., geom_names...) + + types_fields = Tuple(AG._FIELDTYPE[AG.gettype(fielddefn)] for fielddefn in fielddefns) + geom_types = Tuple(ArchGDAL.gettype(geomdefn) for geomdefn in geomdefns) + types = (types_fields..., geom_types...) + + Tables.Schema(names, types) +end + +function Base.iterate(gt::GeoTable, st = 0) + layer = gt.layer + AG.resetreading!(layer) + nfeat = AG.nfeature(layer) + nfield = AG.nfield(layer) + ngeom = AG.ngeom(layer) + featuredefn = layerdefn(layer) + + d = Dict{String, Vector}() + + for field_no in 0:nfield-1 + field = getfielddefn(featuredefn, field_no) + name = getname(field) + typ = _FIELDTYPE[gettype(field)] + d[name] = typ[] + end + d["geometry"] = IGeometry[] + + st >= nfeat && return nothing + AG.nextfeature(layer) do feature + for (k, v) in pairs(d) + if k == "geometry" + val = getgeom(feature, 0) + else + val = getfield(feature, k) + end + push!(v, val) + end + end + + keys_tup = () + for _key in keys(d) + keys_tup = (keys_tup..., Symbol(_key)) + end + vals_tup = Tuple(values(d)) + + #Using the tables interface + Row = Tables.rowtable(NamedTuple{keys_tup}(vals_tup)) + + return FeatureRow(Row, st), st + 1 +end + + +Tables.istable(::Type{<:GeoTable}) = true +Tables.rowaccess(::Type{<:GeoTable}) = true +Tables.rows(gt::GeoTable) = gt + +Base.IteratorSize(::Type{<:GeoTable}) = Base.HasLength() +Base.size(gt::GeoTable) = nfeature(gt.layer) +Base.length(gt::GeoTable) = (Base.size(gt::GeoTable)) * (nfield(gt.layer) + ngeom(gt.layer)) +Base.IteratorEltype(::Type{<:GeoTable}) = Base.HasEltype() + +Base.propertynames(fr::FeatureRow, gt::GeoTable) = (Tables.schema(gt.layer)).names +#geometry +# geometry(fr::FeatureRow) = (fr.Row) + + +function Base.show(io::IO, gt::GeoTable) + println(io, "GeoTable with $(ArchGDAL.nfeature(gt.layer)) Features") +end +# function Base.show(io::IO, fr::FeatureRow) +# println(io, "Feature with properties $(propertynames(fr,gt))") +# end +Base.show(io::IO, ::MIME"text/plain", gt::GeoTable) = show(io, gt) +Base.show(io::IO, ::MIME"text/plain", fr::FeatureRow) = show(io, fr) + + + diff --git a/test/runtests.jl b/test/runtests.jl index 10a86edb..ea18593a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -9,7 +9,7 @@ include("remotefiles.jl") cd(dirname(@__FILE__)) do isdir("tmp") || mkpath("tmp") include("test_convert.jl") - include("test_datastreams.jl") + include("test_tables.jl") include("test_gdal_tutorials.jl") include("test_geometry.jl") include("test_types.jl") diff --git a/test/test_datastreams.jl b/test/test_datastreams.jl deleted file mode 100644 index 4ff980ad..00000000 --- a/test/test_datastreams.jl +++ /dev/null @@ -1,19 +0,0 @@ -using Test -import ArchGDAL; const AG = ArchGDAL -import DataStreams - -@testset "DataStream Support" begin - df = AG.read("data/point.geojson") do dataset - DataStreams.Data.close!(DataStreams.Data.stream!( - AG.Source(AG.getlayer(dataset,0)), DataStreams.Data.Table - )) - end - @test df.FID == [2.0, 3.0, 0.0, 3.0] - @test df.pointname == ["point-a", "point-b", "a", "b"] - @test AG.toWKT.(df.geometry0) == [ - "POINT (100 0)", - "POINT (100.2785 0.0893)", - "POINT (100 0)", - "POINT (100.2785 0.0893)" - ] -end diff --git a/test/test_tables.jl b/test/test_tables.jl new file mode 100644 index 00000000..ee0adfc0 --- /dev/null +++ b/test/test_tables.jl @@ -0,0 +1,33 @@ +using Test +import ArchGDAL; const AG = ArchGDAL +using Tables + +@testset "Tables Support" begin + # dataset = AG.read("point.geojson") + dataset = AG.read(joinpath(@__DIR__, "data/point.geojson")) + layer = AG.getlayer(dataset, 0) + + nfeat = AG.nfeature(layer) + nfield = AG.nfield(layer) + featuredefn = AG.layerdefn(layer) + ngeometries = AG.ngeom(featuredefn) + + #testing the main GeoTable function + @test sprint(print, AG.GeoTable(layer)) == "GeoTable with 4 Features\n" + @test sprint(print, AG.GeoTable(layer)) != "Layer is not a valid ArchGDAL layer\n" + + #testing Tables.schema + @test sprint(print, AG.Tables.schema(layer)) == "Tables.Schema:\n :FID Float64 \n :pointname String \n Symbol(\"\") GDAL.wkbPoint" + #testing Base.iterate + # @test sprint(print, AG.Base.iterate(AG.GeoTable(layer))) == "(ArchGDAL.FeatureRow(NamedTuple{(:pointname, :geometry, :FID),Tuple{String,ArchGDAL.IGeometry,Float64}}[(pointname = \"point-a\", geometry = Geometry: POINT (100 0), FID = 2.0)], 0), 1)" + @test Tables.istable(AG.GeoTable(layer)) == true + @test Tables.rowaccess(AG.GeoTable(layer)) == true + @test Tables.rows(AG.GeoTable(layer)) == AG.GeoTable(layer) + + @test Base.size(AG.GeoTable(layer)) == 4 + @test Base.length(AG.GeoTable(layer)) == 12 + +end + + + From 3746305ca57e6351c6a1e57e381c2e2a1632ecda Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Mon, 20 Apr 2020 20:58:43 +0530 Subject: [PATCH 10/62] Import Tables instead of DataStreams --- src/ArchGDAL.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ArchGDAL.jl b/src/ArchGDAL.jl index 2ab76fae..fb300131 100644 --- a/src/ArchGDAL.jl +++ b/src/ArchGDAL.jl @@ -1,7 +1,7 @@ module ArchGDAL import GDAL, GeoInterface, GeoFormatTypes - import DataStreams: Data + import Tables: Tables import GeoInterface: coordinates, geotype import Base: convert From 31cad0be8068cd4fa3e61970cfe976cfc8f8d0dd Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Tue, 21 Apr 2020 13:15:36 +0530 Subject: [PATCH 11/62] Add Base.iterate test --- test/test_tables.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/test_tables.jl b/test/test_tables.jl index ee0adfc0..63321972 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -19,7 +19,8 @@ using Tables #testing Tables.schema @test sprint(print, AG.Tables.schema(layer)) == "Tables.Schema:\n :FID Float64 \n :pointname String \n Symbol(\"\") GDAL.wkbPoint" #testing Base.iterate - # @test sprint(print, AG.Base.iterate(AG.GeoTable(layer))) == "(ArchGDAL.FeatureRow(NamedTuple{(:pointname, :geometry, :FID),Tuple{String,ArchGDAL.IGeometry,Float64}}[(pointname = \"point-a\", geometry = Geometry: POINT (100 0), FID = 2.0)], 0), 1)" + @test sprint(print, AG.Base.iterate(AG.GeoTable(layer))) == "(ArchGDAL.FeatureRow(NamedTuple{(:pointname, :geometry, :FID),Tuple{String,ArchGDAL.IGeometry,Float64}}[(pointname = \"point-a\", geometry = Geometry: POINT (100 0), FID = 2.0)], 0), 1)" + @test Tables.istable(AG.GeoTable(layer)) == true @test Tables.rowaccess(AG.GeoTable(layer)) == true @test Tables.rows(AG.GeoTable(layer)) == AG.GeoTable(layer) From 67b43362a70f303f9fa20341458c4eba90483ff5 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Tue, 21 Apr 2020 13:26:00 +0530 Subject: [PATCH 12/62] Modify tests --- test/test_tables.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_tables.jl b/test/test_tables.jl index 63321972..5ed5ebd3 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -16,10 +16,10 @@ using Tables @test sprint(print, AG.GeoTable(layer)) == "GeoTable with 4 Features\n" @test sprint(print, AG.GeoTable(layer)) != "Layer is not a valid ArchGDAL layer\n" - #testing Tables.schema - @test sprint(print, AG.Tables.schema(layer)) == "Tables.Schema:\n :FID Float64 \n :pointname String \n Symbol(\"\") GDAL.wkbPoint" - #testing Base.iterate - @test sprint(print, AG.Base.iterate(AG.GeoTable(layer))) == "(ArchGDAL.FeatureRow(NamedTuple{(:pointname, :geometry, :FID),Tuple{String,ArchGDAL.IGeometry,Float64}}[(pointname = \"point-a\", geometry = Geometry: POINT (100 0), FID = 2.0)], 0), 1)" + # #testing Tables.schema + # @test sprint(print, AG.Tables.schema(layer)) == "Tables.Schema:\n :FID Float64 \n :pointname String \n Symbol(\"\") GDAL.wkbPoint" + # #testing Base.iterate + # @test sprint(print, AG.Base.iterate(AG.GeoTable(layer))) == "(ArchGDAL.FeatureRow(NamedTuple{(:pointname, :geometry, :FID),Tuple{String,ArchGDAL.IGeometry,Float64}}[(pointname = \"point-a\", geometry = Geometry: POINT (100 0), FID = 2.0)], 0), 1)" @test Tables.istable(AG.GeoTable(layer)) == true @test Tables.rowaccess(AG.GeoTable(layer)) == true From 82ee8a0a6046aeac0fea64f4e0ee72eb71c1772e Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Mon, 10 Aug 2020 10:53:16 +0530 Subject: [PATCH 13/62] Resolve conflict --- Project.toml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Project.toml b/Project.toml index 66b74cb9..57364964 100644 --- a/Project.toml +++ b/Project.toml @@ -3,27 +3,27 @@ uuid = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3" keywords = ["GDAL", "IO"] license = "MIT" desc = "A high level API for GDAL - Geospatial Data Abstraction Library" -version = "0.3.2" +version = "0.4.1" [deps] -Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +DataStreams = "9a8bc11e-79be-5b39-94d7-1ccc349a1a85" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" GDAL = "add2ef01-049f-52c4-9ee2-e494f65e021a" GeoFormatTypes = "68eda718-8dee-11e9-39e7-89f7f65f511f" GeoInterface = "cf35fbd7-0cd7-5166-be24-54bfbe79505f" -Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" [compat] -Tables = "1" -GDAL = "1.0.2" +DataStreams = "0.4.2" +GDAL = "1.1.3" GeoInterface = "0.4, 0.5" GeoFormatTypes = "0.3" -julia = "1" +julia = "1.3" [extras] BinaryProvider = "b99e7846-7c00-51b0-8f62-c81ae34c0232" +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["BinaryProvider", "Statistics", "Test"] +test = ["Dates", "Statistics", "Test", "BinaryProvider"] From 86f588bee11ca947f89a3e89a081c95df0b726fa Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Mon, 10 Aug 2020 12:27:33 +0530 Subject: [PATCH 14/62] Minor Refeactor --- Project.toml | 2 ++ src/tables.jl | 14 +++++++------- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Project.toml b/Project.toml index 57364964..2fa98996 100644 --- a/Project.toml +++ b/Project.toml @@ -11,11 +11,13 @@ Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" GDAL = "add2ef01-049f-52c4-9ee2-e494f65e021a" GeoFormatTypes = "68eda718-8dee-11e9-39e7-89f7f65f511f" GeoInterface = "cf35fbd7-0cd7-5166-be24-54bfbe79505f" +Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" [compat] DataStreams = "0.4.2" GDAL = "1.1.3" GeoInterface = "0.4, 0.5" +Tables = "1" GeoFormatTypes = "0.3" julia = "1.3" diff --git a/src/tables.jl b/src/tables.jl index 4d87dc32..bfc7fff6 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -15,7 +15,7 @@ function GeoTable(layer) end struct FeatureRow - Row::Array + Geometry::Array feature_number::Int end @@ -76,7 +76,7 @@ function Base.iterate(gt::GeoTable, st = 0) #Using the tables interface Row = Tables.rowtable(NamedTuple{keys_tup}(vals_tup)) - + println(Row) return FeatureRow(Row, st), st + 1 end @@ -91,18 +91,18 @@ Base.length(gt::GeoTable) = (Base.size(gt::GeoTable)) * (nfield(gt.layer) + ngeo Base.IteratorEltype(::Type{<:GeoTable}) = Base.HasEltype() Base.propertynames(fr::FeatureRow, gt::GeoTable) = (Tables.schema(gt.layer)).names -#geometry -# geometry(fr::FeatureRow) = (fr.Row) - +geometry(fr::FeatureRow) = (fr.Row) function Base.show(io::IO, gt::GeoTable) println(io, "GeoTable with $(ArchGDAL.nfeature(gt.layer)) Features") end + # function Base.show(io::IO, fr::FeatureRow) -# println(io, "Feature with properties $(propertynames(fr,gt))") +# println(io, "Feature with properties $(propertynames(fr))") # end + Base.show(io::IO, ::MIME"text/plain", gt::GeoTable) = show(io, gt) -Base.show(io::IO, ::MIME"text/plain", fr::FeatureRow) = show(io, fr) +# Base.show(io::IO, ::MIME"text/plain", fr::FeatureRow) = show(io, fr) From 4f5d9c5bc143198fe2f27e7d3fd4ae1e4a2f472a Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Mon, 10 Aug 2020 12:36:17 +0530 Subject: [PATCH 15/62] Clean up --- src/tables.jl | 6 ------ test/test_tables.jl | 8 ++------ 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index bfc7fff6..d54ccc83 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -96,13 +96,7 @@ geometry(fr::FeatureRow) = (fr.Row) function Base.show(io::IO, gt::GeoTable) println(io, "GeoTable with $(ArchGDAL.nfeature(gt.layer)) Features") end - -# function Base.show(io::IO, fr::FeatureRow) -# println(io, "Feature with properties $(propertynames(fr))") -# end - Base.show(io::IO, ::MIME"text/plain", gt::GeoTable) = show(io, gt) -# Base.show(io::IO, ::MIME"text/plain", fr::FeatureRow) = show(io, fr) diff --git a/test/test_tables.jl b/test/test_tables.jl index 5ed5ebd3..af54000d 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -3,7 +3,6 @@ import ArchGDAL; const AG = ArchGDAL using Tables @testset "Tables Support" begin - # dataset = AG.read("point.geojson") dataset = AG.read(joinpath(@__DIR__, "data/point.geojson")) layer = AG.getlayer(dataset, 0) @@ -12,14 +11,11 @@ using Tables featuredefn = AG.layerdefn(layer) ngeometries = AG.ngeom(featuredefn) - #testing the main GeoTable function @test sprint(print, AG.GeoTable(layer)) == "GeoTable with 4 Features\n" @test sprint(print, AG.GeoTable(layer)) != "Layer is not a valid ArchGDAL layer\n" - # #testing Tables.schema - # @test sprint(print, AG.Tables.schema(layer)) == "Tables.Schema:\n :FID Float64 \n :pointname String \n Symbol(\"\") GDAL.wkbPoint" - # #testing Base.iterate - # @test sprint(print, AG.Base.iterate(AG.GeoTable(layer))) == "(ArchGDAL.FeatureRow(NamedTuple{(:pointname, :geometry, :FID),Tuple{String,ArchGDAL.IGeometry,Float64}}[(pointname = \"point-a\", geometry = Geometry: POINT (100 0), FID = 2.0)], 0), 1)" + @test sprint(print, AG.Tables.schema(layer)) == "Tables.Schema:\n :FID Float64 \n :pointname String \n Symbol(\"\") GDAL.wkbPoint" + @test sprint(print, AG.Base.iterate(AG.GeoTable(layer))) == "(ArchGDAL.FeatureRow(NamedTuple{(:pointname, :geometry, :FID),Tuple{String,ArchGDAL.IGeometry,Float64}}[(pointname = \"point-a\", geometry = Geometry: POINT (100 0), FID = 2.0)], 0), 1)" @test Tables.istable(AG.GeoTable(layer)) == true @test Tables.rowaccess(AG.GeoTable(layer)) == true From e6b0e9e47a2f4162fdec6b8a41a1bedc67dd8b4a Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Mon, 10 Aug 2020 12:49:09 +0530 Subject: [PATCH 16/62] Update test --- test/test_tables.jl | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/test/test_tables.jl b/test/test_tables.jl index af54000d..5a20c1d2 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -5,7 +5,6 @@ using Tables @testset "Tables Support" begin dataset = AG.read(joinpath(@__DIR__, "data/point.geojson")) layer = AG.getlayer(dataset, 0) - nfeat = AG.nfeature(layer) nfield = AG.nfield(layer) featuredefn = AG.layerdefn(layer) @@ -13,17 +12,13 @@ using Tables @test sprint(print, AG.GeoTable(layer)) == "GeoTable with 4 Features\n" @test sprint(print, AG.GeoTable(layer)) != "Layer is not a valid ArchGDAL layer\n" - - @test sprint(print, AG.Tables.schema(layer)) == "Tables.Schema:\n :FID Float64 \n :pointname String \n Symbol(\"\") GDAL.wkbPoint" + @test sprint(print, AG.Tables.schema(layer)) == "Tables.Schema:\n :FID Float64 \n :pointname String \n Symbol(\"\") wkbPoint::OGRwkbGeometryType = 0x00000001" @test sprint(print, AG.Base.iterate(AG.GeoTable(layer))) == "(ArchGDAL.FeatureRow(NamedTuple{(:pointname, :geometry, :FID),Tuple{String,ArchGDAL.IGeometry,Float64}}[(pointname = \"point-a\", geometry = Geometry: POINT (100 0), FID = 2.0)], 0), 1)" - @test Tables.istable(AG.GeoTable(layer)) == true @test Tables.rowaccess(AG.GeoTable(layer)) == true @test Tables.rows(AG.GeoTable(layer)) == AG.GeoTable(layer) - @test Base.size(AG.GeoTable(layer)) == 4 @test Base.length(AG.GeoTable(layer)) == 12 - end From 23cc9f1af2f21afb3a06c598b3ffddac4736f8cc Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Mon, 10 Aug 2020 14:53:50 +0530 Subject: [PATCH 17/62] Schema test for 1.5 --- test/test_tables.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_tables.jl b/test/test_tables.jl index 5a20c1d2..afb737e2 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -12,7 +12,7 @@ using Tables @test sprint(print, AG.GeoTable(layer)) == "GeoTable with 4 Features\n" @test sprint(print, AG.GeoTable(layer)) != "Layer is not a valid ArchGDAL layer\n" - @test sprint(print, AG.Tables.schema(layer)) == "Tables.Schema:\n :FID Float64 \n :pointname String \n Symbol(\"\") wkbPoint::OGRwkbGeometryType = 0x00000001" + @test sprint(print, AG.Tables.schema(layer)) == "Tables.Schema:\n :FID Float64\n :pointname String\n Symbol(\"\") wkbPoint::OGRwkbGeometryType = 0x00000001" @test sprint(print, AG.Base.iterate(AG.GeoTable(layer))) == "(ArchGDAL.FeatureRow(NamedTuple{(:pointname, :geometry, :FID),Tuple{String,ArchGDAL.IGeometry,Float64}}[(pointname = \"point-a\", geometry = Geometry: POINT (100 0), FID = 2.0)], 0), 1)" @test Tables.istable(AG.GeoTable(layer)) == true @test Tables.rowaccess(AG.GeoTable(layer)) == true From cf68ff150542a35e87b7caceca6c836467f988c1 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Tue, 11 Aug 2020 11:02:33 +0530 Subject: [PATCH 18/62] Make schema test more concrete --- src/tables.jl | 1 - test/test_tables.jl | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index d54ccc83..ed06b394 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -76,7 +76,6 @@ function Base.iterate(gt::GeoTable, st = 0) #Using the tables interface Row = Tables.rowtable(NamedTuple{keys_tup}(vals_tup)) - println(Row) return FeatureRow(Row, st), st + 1 end diff --git a/test/test_tables.jl b/test/test_tables.jl index afb737e2..43229c78 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -12,8 +12,8 @@ using Tables @test sprint(print, AG.GeoTable(layer)) == "GeoTable with 4 Features\n" @test sprint(print, AG.GeoTable(layer)) != "Layer is not a valid ArchGDAL layer\n" - @test sprint(print, AG.Tables.schema(layer)) == "Tables.Schema:\n :FID Float64\n :pointname String\n Symbol(\"\") wkbPoint::OGRwkbGeometryType = 0x00000001" - @test sprint(print, AG.Base.iterate(AG.GeoTable(layer))) == "(ArchGDAL.FeatureRow(NamedTuple{(:pointname, :geometry, :FID),Tuple{String,ArchGDAL.IGeometry,Float64}}[(pointname = \"point-a\", geometry = Geometry: POINT (100 0), FID = 2.0)], 0), 1)" + @test getproperty(AG.Tables.schema(layer), :types) == (Float64, String, GDAL.wkbPoint) + @test getproperty(AG.Tables.schema(layer), :names) == (:FID, :pointname, Symbol("")) @test Tables.istable(AG.GeoTable(layer)) == true @test Tables.rowaccess(AG.GeoTable(layer)) == true @test Tables.rows(AG.GeoTable(layer)) == AG.GeoTable(layer) From 78fc8fa8a6b38200ca31c55ef89310e80ae82a98 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Tue, 11 Aug 2020 11:12:22 +0530 Subject: [PATCH 19/62] Refactor test dependencies --- test/test_tables.jl | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test/test_tables.jl b/test/test_tables.jl index 43229c78..3c4f6e18 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -1,6 +1,5 @@ using Test import ArchGDAL; const AG = ArchGDAL -using Tables @testset "Tables Support" begin dataset = AG.read(joinpath(@__DIR__, "data/point.geojson")) @@ -12,11 +11,11 @@ using Tables @test sprint(print, AG.GeoTable(layer)) == "GeoTable with 4 Features\n" @test sprint(print, AG.GeoTable(layer)) != "Layer is not a valid ArchGDAL layer\n" - @test getproperty(AG.Tables.schema(layer), :types) == (Float64, String, GDAL.wkbPoint) - @test getproperty(AG.Tables.schema(layer), :names) == (:FID, :pointname, Symbol("")) - @test Tables.istable(AG.GeoTable(layer)) == true - @test Tables.rowaccess(AG.GeoTable(layer)) == true - @test Tables.rows(AG.GeoTable(layer)) == AG.GeoTable(layer) + @test getproperty(Tables.schema(layer), :types) == (Float64, String, AG.GDAL.wkbPoint) + @test getproperty(Tables.schema(layer), :names) == (:FID, :pointname, Symbol("")) + @test AG.Tables.istable(AG.GeoTable(layer)) == true + @test AG.Tables.rowaccess(AG.GeoTable(layer)) == true + @test AG.Tables.rows(AG.GeoTable(layer)) == AG.GeoTable(layer) @test Base.size(AG.GeoTable(layer)) == 4 @test Base.length(AG.GeoTable(layer)) == 12 end From 5c35700d7b3e5e400255de76701886c1189d2e6f Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Tue, 11 Aug 2020 11:18:01 +0530 Subject: [PATCH 20/62] Tables dependency. :( --- test/test_tables.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/test_tables.jl b/test/test_tables.jl index 3c4f6e18..ac735735 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -1,5 +1,6 @@ using Test import ArchGDAL; const AG = ArchGDAL +using Tables @testset "Tables Support" begin dataset = AG.read(joinpath(@__DIR__, "data/point.geojson")) @@ -13,9 +14,9 @@ import ArchGDAL; const AG = ArchGDAL @test sprint(print, AG.GeoTable(layer)) != "Layer is not a valid ArchGDAL layer\n" @test getproperty(Tables.schema(layer), :types) == (Float64, String, AG.GDAL.wkbPoint) @test getproperty(Tables.schema(layer), :names) == (:FID, :pointname, Symbol("")) - @test AG.Tables.istable(AG.GeoTable(layer)) == true - @test AG.Tables.rowaccess(AG.GeoTable(layer)) == true - @test AG.Tables.rows(AG.GeoTable(layer)) == AG.GeoTable(layer) + @test Tables.istable(AG.GeoTable(layer)) == true + @test Tables.rowaccess(AG.GeoTable(layer)) == true + @test Tables.rows(AG.GeoTable(layer)) == AG.GeoTable(layer) @test Base.size(AG.GeoTable(layer)) == 4 @test Base.length(AG.GeoTable(layer)) == 12 end From 31019045d4a3c49e57389f6e3877590509a28514 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Tue, 11 Aug 2020 17:00:27 +0530 Subject: [PATCH 21/62] Improve iterate method --- src/tables.jl | 6 +++--- test/test_tables.jl | 26 ++++++++++++++------------ 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index ed06b394..011ca641 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -86,11 +86,11 @@ Tables.rows(gt::GeoTable) = gt Base.IteratorSize(::Type{<:GeoTable}) = Base.HasLength() Base.size(gt::GeoTable) = nfeature(gt.layer) -Base.length(gt::GeoTable) = (Base.size(gt::GeoTable)) * (nfield(gt.layer) + ngeom(gt.layer)) +Base.length(gt::GeoTable) = Base.size(gt) Base.IteratorEltype(::Type{<:GeoTable}) = Base.HasEltype() +Base.propertynames(gt::GeoTable) = (Tables.schema(gt.layer)).names -Base.propertynames(fr::FeatureRow, gt::GeoTable) = (Tables.schema(gt.layer)).names -geometry(fr::FeatureRow) = (fr.Row) +geometry(gt::GeoTable) = [iterate(gt, i)[1].geometry for i in 0:length(gt)-1] function Base.show(io::IO, gt::GeoTable) println(io, "GeoTable with $(ArchGDAL.nfeature(gt.layer)) Features") diff --git a/test/test_tables.jl b/test/test_tables.jl index ac735735..f2858b00 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -5,21 +5,23 @@ using Tables @testset "Tables Support" begin dataset = AG.read(joinpath(@__DIR__, "data/point.geojson")) layer = AG.getlayer(dataset, 0) + gt = AG.GeoTable(layer) + fr = iterate(gt, 1)[1] nfeat = AG.nfeature(layer) nfield = AG.nfield(layer) featuredefn = AG.layerdefn(layer) ngeometries = AG.ngeom(featuredefn) - @test sprint(print, AG.GeoTable(layer)) == "GeoTable with 4 Features\n" - @test sprint(print, AG.GeoTable(layer)) != "Layer is not a valid ArchGDAL layer\n" + @test sprint(print, gt) == "GeoTable with 4 Features\n" + @test sprint(print, gt) != "Layer is not a valid ArchGDAL layer\n" @test getproperty(Tables.schema(layer), :types) == (Float64, String, AG.GDAL.wkbPoint) - @test getproperty(Tables.schema(layer), :names) == (:FID, :pointname, Symbol("")) - @test Tables.istable(AG.GeoTable(layer)) == true - @test Tables.rowaccess(AG.GeoTable(layer)) == true - @test Tables.rows(AG.GeoTable(layer)) == AG.GeoTable(layer) - @test Base.size(AG.GeoTable(layer)) == 4 - @test Base.length(AG.GeoTable(layer)) == 12 -end - - - + @test getproperty(Tables.schema(layer), :names) == propertynames(gt) + @test Tables.istable(AG.GeoTable) == true + @test Tables.rowaccess(AG.GeoTable) == true + @test Tables.rows(gt) == AG.GeoTable(layer) + @test Base.size(gt) == 4 + @test Base.length(gt) == 4 + @test propertynames(iterate(gt, 1)[1]) == (:pointname, :geometry, :FID) + @test AG.geometry(gt) isa Array{AG.IGeometry} + @test iterate(gt, 5) === nothing +end \ No newline at end of file From 649c941dfca52a715a742f860e069b3d6a2b38ab Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Tue, 11 Aug 2020 17:00:47 +0530 Subject: [PATCH 22/62] Work out extra methods; clean up stuff --- src/tables.jl | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 011ca641..5dd25a3f 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -14,11 +14,6 @@ function GeoTable(layer) GeoTable(layer) end -struct FeatureRow - Geometry::Array - feature_number::Int -end - function Tables.schema(layer::AG.AbstractFeatureLayer) nfield = AG.nfield(layer) featuredefn = AG.layerdefn(layer) @@ -57,7 +52,7 @@ function Base.iterate(gt::GeoTable, st = 0) d["geometry"] = IGeometry[] st >= nfeat && return nothing - AG.nextfeature(layer) do feature + AG.getfeature(layer, st) do feature for (k, v) in pairs(d) if k == "geometry" val = getgeom(feature, 0) @@ -76,7 +71,7 @@ function Base.iterate(gt::GeoTable, st = 0) #Using the tables interface Row = Tables.rowtable(NamedTuple{keys_tup}(vals_tup)) - return FeatureRow(Row, st), st + 1 + return Row..., st + 1 end From 0c9f982d299d6e4cd6774e7ba7bcefa00e777c20 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Tue, 11 Aug 2020 17:23:24 +0530 Subject: [PATCH 23/62] Pass through the geotable function first --- src/tables.jl | 3 +-- test/test_tables.jl | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 5dd25a3f..1fc1cbc3 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -4,10 +4,9 @@ struct GeoTable{T<:Union{IFeatureLayer, FeatureLayer}} layer::T end -function GeoTable(layer) +function geotable(layer) featuredefn = layerdefn(layer) ngeometries = ngeom(featuredefn) - if(ngeometries == 0 ) error("Layer is not a valid ArchGDAL layer") end diff --git a/test/test_tables.jl b/test/test_tables.jl index f2858b00..57636775 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -5,7 +5,7 @@ using Tables @testset "Tables Support" begin dataset = AG.read(joinpath(@__DIR__, "data/point.geojson")) layer = AG.getlayer(dataset, 0) - gt = AG.GeoTable(layer) + gt = AG.geotable(layer) fr = iterate(gt, 1)[1] nfeat = AG.nfeature(layer) nfield = AG.nfield(layer) @@ -18,7 +18,7 @@ using Tables @test getproperty(Tables.schema(layer), :names) == propertynames(gt) @test Tables.istable(AG.GeoTable) == true @test Tables.rowaccess(AG.GeoTable) == true - @test Tables.rows(gt) == AG.GeoTable(layer) + @test Tables.rows(gt) == AG.geotable(layer) @test Base.size(gt) == 4 @test Base.length(gt) == 4 @test propertynames(iterate(gt, 1)[1]) == (:pointname, :geometry, :FID) From ade232fd4449260008c28cc0f6487e3d17d95aa6 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Wed, 12 Aug 2020 12:50:37 +0530 Subject: [PATCH 24/62] Minor Refactor --- src/tables.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 1fc1cbc3..d223cec3 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -45,8 +45,7 @@ function Base.iterate(gt::GeoTable, st = 0) for field_no in 0:nfield-1 field = getfielddefn(featuredefn, field_no) name = getname(field) - typ = _FIELDTYPE[gettype(field)] - d[name] = typ[] + d[name] = Vector{_FIELDTYPE[gettype(field)]}() end d["geometry"] = IGeometry[] From 1a0537ccf1be2e0a3237214161428063a50fc166 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Wed, 12 Aug 2020 14:43:40 +0530 Subject: [PATCH 25/62] Take Geometry Out of the table --- src/tables.jl | 45 ++++++++++++++++++++++++++++----------------- test/test_tables.jl | 9 +++++---- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index d223cec3..17c95966 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -8,7 +8,7 @@ function geotable(layer) featuredefn = layerdefn(layer) ngeometries = ngeom(featuredefn) if(ngeometries == 0 ) - error("Layer is not a valid ArchGDAL layer") + println("NULL Geometry found") end GeoTable(layer) end @@ -19,17 +19,10 @@ function Tables.schema(layer::AG.AbstractFeatureLayer) ngeom = ArchGDAL.ngeom(featuredefn) fielddefns = (AG.getfielddefn(featuredefn, i) for i in 0:nfield-1) - geomdefns = (ArchGDAL.getgeomdefn(featuredefn, i) for i in 0:ngeom-1) - names_fields = Tuple(AG.getname(fielddefn) for fielddefn in fielddefns) - geom_names = Tuple(ArchGDAL.getname(geomdefn) for geomdefn in geomdefns) - names = (names_fields..., geom_names...) - types_fields = Tuple(AG._FIELDTYPE[AG.gettype(fielddefn)] for fielddefn in fielddefns) - geom_types = Tuple(ArchGDAL.gettype(geomdefn) for geomdefn in geomdefns) - types = (types_fields..., geom_types...) - Tables.Schema(names, types) + Tables.Schema(names_fields, types_fields) end function Base.iterate(gt::GeoTable, st = 0) @@ -37,7 +30,6 @@ function Base.iterate(gt::GeoTable, st = 0) AG.resetreading!(layer) nfeat = AG.nfeature(layer) nfield = AG.nfield(layer) - ngeom = AG.ngeom(layer) featuredefn = layerdefn(layer) d = Dict{String, Vector}() @@ -47,16 +39,11 @@ function Base.iterate(gt::GeoTable, st = 0) name = getname(field) d[name] = Vector{_FIELDTYPE[gettype(field)]}() end - d["geometry"] = IGeometry[] st >= nfeat && return nothing AG.getfeature(layer, st) do feature for (k, v) in pairs(d) - if k == "geometry" - val = getgeom(feature, 0) - else - val = getfield(feature, k) - end + val = getfield(feature, k) push!(v, val) end end @@ -83,7 +70,31 @@ Base.length(gt::GeoTable) = Base.size(gt) Base.IteratorEltype(::Type{<:GeoTable}) = Base.HasEltype() Base.propertynames(gt::GeoTable) = (Tables.schema(gt.layer)).names -geometry(gt::GeoTable) = [iterate(gt, i)[1].geometry for i in 0:length(gt)-1] +""" +returns the nth geometry starting from 1 to n +""" +function geometry(gt::GeoTable, n::Int) + if n > length(gt) + return "NULL Geometry" + else + layer = gt.layer + AG.getfeature(layer, n-1) do feature + getgeom(feature) + end + end +end + +function geometry(gt::GeoTable) + layer = gt.layer + AG.resetreading!(layer) + arr = AG.IGeometry[] + for i in 1:length(gt) + AG.nextfeature(layer) do feature + push!(arr, getgeom(feature)) + end + end + return arr +end function Base.show(io::IO, gt::GeoTable) println(io, "GeoTable with $(ArchGDAL.nfeature(gt.layer)) Features") diff --git a/test/test_tables.jl b/test/test_tables.jl index 57636775..4b3908cb 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -11,17 +11,18 @@ using Tables nfield = AG.nfield(layer) featuredefn = AG.layerdefn(layer) ngeometries = AG.ngeom(featuredefn) + f2 = AG.geometry(gt) @test sprint(print, gt) == "GeoTable with 4 Features\n" - @test sprint(print, gt) != "Layer is not a valid ArchGDAL layer\n" - @test getproperty(Tables.schema(layer), :types) == (Float64, String, AG.GDAL.wkbPoint) + @test getproperty(Tables.schema(layer), :types) == (Float64, String) @test getproperty(Tables.schema(layer), :names) == propertynames(gt) @test Tables.istable(AG.GeoTable) == true @test Tables.rowaccess(AG.GeoTable) == true @test Tables.rows(gt) == AG.geotable(layer) @test Base.size(gt) == 4 @test Base.length(gt) == 4 - @test propertynames(iterate(gt, 1)[1]) == (:pointname, :geometry, :FID) - @test AG.geometry(gt) isa Array{AG.IGeometry} + @test propertynames(iterate(gt, 1)[1]) == (:pointname, :FID) + @test f2 isa Array{AG.IGeometry} + @test typeof(f2) === typeof([AG.geometry(gt, i) for i in 1:length(gt)]) @test iterate(gt, 5) === nothing end \ No newline at end of file From 77742569a4b3b7b1016ea22f4f1504c01eb6e1f8 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Sat, 15 Aug 2020 22:44:24 +0530 Subject: [PATCH 26/62] (no branch): Remove no geom error --- src/tables.jl | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 17c95966..7b3573fa 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -7,9 +7,6 @@ end function geotable(layer) featuredefn = layerdefn(layer) ngeometries = ngeom(featuredefn) - if(ngeometries == 0 ) - println("NULL Geometry found") - end GeoTable(layer) end @@ -21,8 +18,10 @@ function Tables.schema(layer::AG.AbstractFeatureLayer) fielddefns = (AG.getfielddefn(featuredefn, i) for i in 0:nfield-1) names_fields = Tuple(AG.getname(fielddefn) for fielddefn in fielddefns) types_fields = Tuple(AG._FIELDTYPE[AG.gettype(fielddefn)] for fielddefn in fielddefns) + geom_types = Tuple(ArchGDAL.gettype(geomdefn) for geomdefn in geomdefns) + types = (types_fields..., geom_types...) - Tables.Schema(names_fields, types_fields) + Tables.Schema(names, types) end function Base.iterate(gt::GeoTable, st = 0) @@ -30,33 +29,32 @@ function Base.iterate(gt::GeoTable, st = 0) AG.resetreading!(layer) nfeat = AG.nfeature(layer) nfield = AG.nfield(layer) + ngeom = AG.ngeom(layer) featuredefn = layerdefn(layer) - d = Dict{String, Vector}() - + name = [] + v = [] for field_no in 0:nfield-1 - field = getfielddefn(featuredefn, field_no) - name = getname(field) - d[name] = Vector{_FIELDTYPE[gettype(field)]}() + field = AG.getfielddefn(featuredefn, field_no) + push!(name, AG.getname(field)) end + d["geometry"] = IGeometry[] st >= nfeat && return nothing AG.getfeature(layer, st) do feature for (k, v) in pairs(d) - val = getfield(feature, k) + if k == "geometry" + val = getgeom(feature, 0) + else + val = getfield(feature, k) + end push!(v, val) end end - keys_tup = () - for _key in keys(d) - keys_tup = (keys_tup..., Symbol(_key)) - end - vals_tup = Tuple(values(d)) - - #Using the tables interface - Row = Tables.rowtable(NamedTuple{keys_tup}(vals_tup)) - return Row..., st + 1 + + Row = NamedTuple{Tuple(Symbol.(name))}(v) + return Row, st + 1 end From 0730dce795be2e991d26225d29650ea7e95955f8 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Sat, 15 Aug 2020 22:46:53 +0530 Subject: [PATCH 27/62] Accept null geometries --- src/tables.jl | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 7b3573fa..a7be6804 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -18,10 +18,7 @@ function Tables.schema(layer::AG.AbstractFeatureLayer) fielddefns = (AG.getfielddefn(featuredefn, i) for i in 0:nfield-1) names_fields = Tuple(AG.getname(fielddefn) for fielddefn in fielddefns) types_fields = Tuple(AG._FIELDTYPE[AG.gettype(fielddefn)] for fielddefn in fielddefns) - geom_types = Tuple(ArchGDAL.gettype(geomdefn) for geomdefn in geomdefns) - types = (types_fields..., geom_types...) - - Tables.Schema(names, types) + Tables.Schema(names_fields, types_fields) end function Base.iterate(gt::GeoTable, st = 0) @@ -30,7 +27,7 @@ function Base.iterate(gt::GeoTable, st = 0) nfeat = AG.nfeature(layer) nfield = AG.nfield(layer) ngeom = AG.ngeom(layer) - featuredefn = layerdefn(layer) + featuredefn = AG.layerdefn(layer) name = [] v = [] @@ -38,15 +35,15 @@ function Base.iterate(gt::GeoTable, st = 0) field = AG.getfielddefn(featuredefn, field_no) push!(name, AG.getname(field)) end - d["geometry"] = IGeometry[] + push!(name, "geometry") st >= nfeat && return nothing AG.getfeature(layer, st) do feature - for (k, v) in pairs(d) + for k in name if k == "geometry" - val = getgeom(feature, 0) + val = AG.getgeom(feature, 0) else - val = getfield(feature, k) + val = AG.getfield(feature, k) end push!(v, val) end From 4bf1f0de7b1c4114aa8f5826a6f7b655577a15b0 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Mon, 17 Aug 2020 00:47:17 +0530 Subject: [PATCH 28/62] Support mutiple geomtries; rename to geotable to Table --- src/tables.jl | 83 +++++++++++++++++++-------------------------------- 1 file changed, 31 insertions(+), 52 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index a7be6804..543a4483 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -1,47 +1,53 @@ const AG = ArchGDAL -struct GeoTable{T<:Union{IFeatureLayer, FeatureLayer}} +struct Table{T<:Union{IFeatureLayer, FeatureLayer}} layer::T end -function geotable(layer) +function Table(layer) featuredefn = layerdefn(layer) ngeometries = ngeom(featuredefn) - GeoTable(layer) + if ngeometries == 0 + print("NULL Geometry found") + end + Table(layer) end function Tables.schema(layer::AG.AbstractFeatureLayer) nfield = AG.nfield(layer) featuredefn = AG.layerdefn(layer) - ngeom = ArchGDAL.ngeom(featuredefn) - fielddefns = (AG.getfielddefn(featuredefn, i) for i in 0:nfield-1) names_fields = Tuple(AG.getname(fielddefn) for fielddefn in fielddefns) types_fields = Tuple(AG._FIELDTYPE[AG.gettype(fielddefn)] for fielddefn in fielddefns) Tables.Schema(names_fields, types_fields) end -function Base.iterate(gt::GeoTable, st = 0) - layer = gt.layer - AG.resetreading!(layer) +function Base.iterate(t::Table, st = 0) + layer = t.layer + if iszero(st) + AG.resetreading!(layer) + end + nfeat = AG.nfeature(layer) nfield = AG.nfield(layer) ngeom = AG.ngeom(layer) featuredefn = AG.layerdefn(layer) - + geomdefns = (ArchGDAL.getgeomdefn(featuredefn, i) for i in 0:ngeom-1) + name = [] v = [] for field_no in 0:nfield-1 field = AG.getfielddefn(featuredefn, field_no) push!(name, AG.getname(field)) end - push!(name, "geometry") - + geom_names = [ArchGDAL.getname(geomdefn) for geomdefn in geomdefns] + push!(name, geom_names...) + st >= nfeat && return nothing - AG.getfeature(layer, st) do feature + AG.nextfeature(layer) do feature for k in name - if k == "geometry" - val = AG.getgeom(feature, 0) + if k in geom_names + val = AG.getgeom(feature, findfirst(a->a==k, geom_names)-1) else val = AG.getfield(feature, k) end @@ -49,52 +55,25 @@ function Base.iterate(gt::GeoTable, st = 0) end end - Row = NamedTuple{Tuple(Symbol.(name))}(v) return Row, st + 1 end -Tables.istable(::Type{<:GeoTable}) = true -Tables.rowaccess(::Type{<:GeoTable}) = true -Tables.rows(gt::GeoTable) = gt +Tables.istable(::Type{<:Table}) = true +Tables.rowaccess(::Type{<:Table}) = true +Tables.rows(t::Table) = t -Base.IteratorSize(::Type{<:GeoTable}) = Base.HasLength() -Base.size(gt::GeoTable) = nfeature(gt.layer) -Base.length(gt::GeoTable) = Base.size(gt) -Base.IteratorEltype(::Type{<:GeoTable}) = Base.HasEltype() -Base.propertynames(gt::GeoTable) = (Tables.schema(gt.layer)).names - -""" -returns the nth geometry starting from 1 to n -""" -function geometry(gt::GeoTable, n::Int) - if n > length(gt) - return "NULL Geometry" - else - layer = gt.layer - AG.getfeature(layer, n-1) do feature - getgeom(feature) - end - end -end - -function geometry(gt::GeoTable) - layer = gt.layer - AG.resetreading!(layer) - arr = AG.IGeometry[] - for i in 1:length(gt) - AG.nextfeature(layer) do feature - push!(arr, getgeom(feature)) - end - end - return arr -end +Base.IteratorSize(::Type{<:Table}) = Base.HasLength() +Base.size(t::Table) = nfeature(t.layer) +Base.length(t::Table) = Base.size(t) +Base.IteratorEltype(::Type{<:Table}) = Base.HasEltype() +Base.propertynames(t::Table) = Tables.schema(t.layer).names -function Base.show(io::IO, gt::GeoTable) - println(io, "GeoTable with $(ArchGDAL.nfeature(gt.layer)) Features") +function Base.show(io::IO, t::Table) + println(io, "Table with $(ArchGDAL.nfeature(t.layer)) Features") end -Base.show(io::IO, ::MIME"text/plain", gt::GeoTable) = show(io, gt) +Base.show(io::IO, ::MIME"text/plain", t::Table) = show(io, t) From 385d80ebeeb6d7f64dc10efa90c12ff8e321dd7e Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Mon, 17 Aug 2020 00:47:24 +0530 Subject: [PATCH 29/62] Add tests --- test/remotefiles.jl | 1 + test/test_tables.jl | 32 +++++++++++++++++++++++--------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/test/remotefiles.jl b/test/remotefiles.jl index f9096ea8..601b27ad 100644 --- a/test/remotefiles.jl +++ b/test/remotefiles.jl @@ -9,6 +9,7 @@ REPO_URL = "https://github.com/yeesian/ArchGDALDatasets/blob/master/" # remote files with SHA-2 256 hash remotefiles = [ + ("data/multi_geom.csv", "00520017658b66ff21e40cbf553672fa8e280cddae6e7a5d1f8bd36bcd521770"), ("data/point.geojson", "8744593479054a67c784322e0c198bfa880c9388b39a2ddd4c56726944711bd9"), ("data/utmsmall.tif", "f40dae6e8b5e18f3648e9f095e22a0d7027014bb463418d32f732c3756d8c54f"), ("gdalworkshop/world.tif", "b376dc8af62f9894b5050a6a9273ac0763ae2990b556910d35d4a8f4753278bb"), diff --git a/test/test_tables.jl b/test/test_tables.jl index 4b3908cb..c93c7c88 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -4,25 +4,39 @@ using Tables @testset "Tables Support" begin dataset = AG.read(joinpath(@__DIR__, "data/point.geojson")) + dataset1 = AG.read(joinpath(@__DIR__, "data/multi_geom.csv"), options = ["GEOM_POSSIBLE_NAMES=point,linestring", "KEEP_GEOM_COLUMNS=NO"]) layer = AG.getlayer(dataset, 0) - gt = AG.geotable(layer) + layer1 = AG.getlayer(dataset1, 0) + gt = AG.Table(layer) + gt1 = AG.Table(layer1) fr = iterate(gt, 1)[1] + fr1 = iterate(gt, 1)[1] nfeat = AG.nfeature(layer) + nfeat1 = AG.nfeature(layer1) nfield = AG.nfield(layer) + nfield1 = AG.nfield(layer1) featuredefn = AG.layerdefn(layer) + featuredefn1 = AG.layerdefn(layer1) ngeometries = AG.ngeom(featuredefn) - f2 = AG.geometry(gt) + ngeometries1 = AG.ngeom(featuredefn1) - @test sprint(print, gt) == "GeoTable with 4 Features\n" + @test sprint(print, gt) == "Table with 4 Features\n" + @test sprint(print, gt1) == "Table with 2 Features\n" @test getproperty(Tables.schema(layer), :types) == (Float64, String) + @test getproperty(Tables.schema(layer1), :types) == (String, String, String) @test getproperty(Tables.schema(layer), :names) == propertynames(gt) - @test Tables.istable(AG.GeoTable) == true - @test Tables.rowaccess(AG.GeoTable) == true - @test Tables.rows(gt) == AG.geotable(layer) + @test getproperty(Tables.schema(layer1), :names) == propertynames(gt1) + + + @test Tables.istable(AG.Table) == true + @test Tables.rows(gt) == AG.Table(layer) + @test Tables.rows(gt1) == AG.Table(layer1) @test Base.size(gt) == 4 + @test Base.size(gt1) == 2 @test Base.length(gt) == 4 - @test propertynames(iterate(gt, 1)[1]) == (:pointname, :FID) - @test f2 isa Array{AG.IGeometry} - @test typeof(f2) === typeof([AG.geometry(gt, i) for i in 1:length(gt)]) + @test Base.length(gt1) == 2 + @test propertynames(iterate(gt, 1)[1]) == (:FID, :pointname, Symbol("")) + @test propertynames(iterate(gt1, 1)[1]) == (:id, :zoom, :location, :point, :linestring) @test iterate(gt, 5) === nothing + @test iterate(gt1, 3) === nothing end \ No newline at end of file From 45196aab8b155448e53ddd0007416066af98c393 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Mon, 17 Aug 2020 12:58:44 +0530 Subject: [PATCH 30/62] Add docs for tables --- docs/Project.toml | 2 +- docs/make.jl | 1 + docs/src/reference.md | 6 +++++ docs/src/tables.md | 58 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 docs/src/tables.md diff --git a/docs/Project.toml b/docs/Project.toml index 7417328a..84b65338 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -3,6 +3,6 @@ ArchGDAL = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3" BinaryProvider = "b99e7846-7c00-51b0-8f62-c81ae34c0232" DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" - +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" [compat] Documenter = "0.25.1" diff --git a/docs/make.jl b/docs/make.jl index 3e6b2632..b8634a47 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -16,6 +16,7 @@ makedocs( "GDAL Datasets" => "datasets.md", "Feature Data" => "features.md", "Raster Data" => "rasters.md", + "Tables Interface" => "tables.md", "Geometric Operations" => "geometries.md", "Spatial Projections" => "projections.md", # "Working with Spatialite" => "spatialite.md", diff --git a/docs/src/reference.md b/docs/src/reference.md index db9a5577..0c16580e 100644 --- a/docs/src/reference.md +++ b/docs/src/reference.md @@ -20,6 +20,12 @@ Pages = ["dataset.jl"] Modules = [ArchGDAL] Pages = ["feature.jl", "featuredefn.jl", "featurelayer.jl", "fielddefn.jl", "geometry.jl", "styletable.jl", "context.jl"] ``` +## [Tables Interface](@id API-Tables-Interface) + +```@autodocs +Modules = [ArchGDAL] +Pages = ["tables.jl"] +``` ## [Raster Data](@id API-Raster-Data) diff --git a/docs/src/tables.md b/docs/src/tables.md new file mode 100644 index 00000000..c8efd19d --- /dev/null +++ b/docs/src/tables.md @@ -0,0 +1,58 @@ +# Tabular Interface + +```@setup tables +using ArchGDAL +using DataFrames +``` + +ArchGDAL now brings in greater flexibilty in terms of raster data handling via the [Tables.jl](https://github.com/JuliaData/Tables.jl) API, that aims to provide a fast and responsive tabular interface to data, which was earlier handled by the [DataStreams](https://github.com/JuliaData/DataStreams.jl) API. + +In this section, we revisit the [`data/point.geojson`](https://github.com/yeesian/ArchGDALDatasets/blob/307f8f0e584a39a050c042849004e6a2bd674f99/data/point.geojson) dataset. + +```@example tables +dataset = ArchGDAL.read("data/point.geojson") +``` + +Each layer can be represented as a separate Table. + +```@example tables +layer = ArchGDAL.getlayer(dataset, 0) +``` + +The [`ArchGDAL.Table`](@ref) method accepts an `ArchGDAL.FeatureLayer`. +```@example tables +table = ArchGDAL.Table(layer) +``` + +Individual rows can be retrieved using the `iterate(t::ArchGDAL.Table, n::Int)` method. + +```@example tables +row = Base.iterate(table, 1) +``` + +Layers are retrievable! +One can get back the layer that a Table is made up of. +```@example tables +lyr = table.layer +``` + +The Tables interface also support multiple geomtries per layer. + +Here, we visit the [`data/multi_geom.csv`](https://github.com/yeesian/ArchGDALDatasets/blob/master/data/multi_geom.csv) dataset. + +```@example tables +dataset1 = ArchGDAL.read("data/multi_geom.csv", options = ["GEOM_POSSIBLE_NAMES=point,linestring", "KEEP_GEOM_COLUMNS=NO"]) + +layer = ArchGDAL.getlayer(dataset, 0) +table = ArchGDAL.Table(layer) +``` + +Exatracting a row from the table, we see that the row/feature is made up of two geomtries viz. `point` and `linestring`. +```@example tables +row = Base.iterate(table, 0) +``` + +Finally layers can be viewed as DataFrames to perform miscellaneous spatial operations. +```@example tables +df = DataFrame(table) +``` From 86a2882f73b271ff3eb58782f01215c31593f596 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Mon, 17 Aug 2020 15:23:08 +0530 Subject: [PATCH 31/62] CSV doc --- docs/src/datasets.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/src/datasets.md b/docs/src/datasets.md index 21359878..3722ddf7 100644 --- a/docs/src/datasets.md +++ b/docs/src/datasets.md @@ -101,3 +101,21 @@ dataset = ArchGDAL.(...) !!! note This pattern of using `do`-blocks to manage context plays a big way into the way we handle memory in this package. For details, see the section on Memory Management. + +The [`ArchGDAL.read`](@ref) method accepts keyword arguments(`kwargs`) viz. the GDAL [open-options](https://gdal.org/drivers/vector/csv.html#open-options) for reading `.csv` spatial datasets. + +Example: In a CSV the data is stored as `String`. + +```@example datasets +dataset1 = ArchGDAL.read("data/multi_geom.csv") +layer1 = ArchGDAL.getlayer(dataset1, 0) +``` + +Well this is weird, the CSV driver recognises our point and linestring geomtries as `String`. Now if you have a .csvt file of the same name with the geometry types as `WKT`, they types will be recognized, else, GDAL offers open-options to tweak the read parameters that are passed as `kwargs`. + +So for the above CSV, we want the driver to detect our geometries, so according to [open-options](https://gdal.org/drivers/vector/csv.html#open-options) we should use the `"GEOM_POSSIBLE_NAMES=point,linestring"` option. Also we want that the geometry columns should not be kept as regular `String` columns, so we add a `"KEEP_GEOM_COLUMNS=NO"` option too. + +```@example datasets +dataset2 = ArchGDAL.read("data/multi_geom.csv", options = ["GEOM_POSSIBLE_NAMES=point,linestring", "KEEP_GEOM_COLUMNS=NO"]) +layer2 = ArchGDAL.getlayer(dataset2, 0) +``` From 3ccdf96c62499855319fafb5970a48520857a8bf Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Mon, 17 Aug 2020 15:27:01 +0530 Subject: [PATCH 32/62] Add sha details --- docs/src/datasets.md | 1 + test/remotefiles.jl | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/docs/src/datasets.md b/docs/src/datasets.md index 3722ddf7..745e09f1 100644 --- a/docs/src/datasets.md +++ b/docs/src/datasets.md @@ -117,5 +117,6 @@ So for the above CSV, we want the driver to detect our geometries, so according ```@example datasets dataset2 = ArchGDAL.read("data/multi_geom.csv", options = ["GEOM_POSSIBLE_NAMES=point,linestring", "KEEP_GEOM_COLUMNS=NO"]) + layer2 = ArchGDAL.getlayer(dataset2, 0) ``` diff --git a/test/remotefiles.jl b/test/remotefiles.jl index 601b27ad..96d64d47 100644 --- a/test/remotefiles.jl +++ b/test/remotefiles.jl @@ -8,6 +8,14 @@ const testdatadir = @__DIR__ REPO_URL = "https://github.com/yeesian/ArchGDALDatasets/blob/master/" # remote files with SHA-2 256 hash +""" +While adding more files, follow the below steps to generate the SHA +``` +julia> using SHA +julia> open(filepath/filename) do f + bytes2hex(sha256(f)) + end +""" remotefiles = [ ("data/multi_geom.csv", "00520017658b66ff21e40cbf553672fa8e280cddae6e7a5d1f8bd36bcd521770"), ("data/point.geojson", "8744593479054a67c784322e0c198bfa880c9388b39a2ddd4c56726944711bd9"), From fbdef4929e9294b51ccb91c8c3a90e546a4b149c Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Tue, 18 Aug 2020 17:28:00 +0530 Subject: [PATCH 33/62] Pass through the Table function --- src/tables.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 543a4483..880baa46 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -1,16 +1,16 @@ const AG = ArchGDAL -struct Table{T<:Union{IFeatureLayer, FeatureLayer}} +struct Table{T} layer::T end -function Table(layer) +function Table(layer::T) where {T<:Union{IFeatureLayer, FeatureLayer}} featuredefn = layerdefn(layer) ngeometries = ngeom(featuredefn) if ngeometries == 0 print("NULL Geometry found") end - Table(layer) + Table{T}(layer) end function Tables.schema(layer::AG.AbstractFeatureLayer) From c1b7ac42047322487c6e1a9270bb551ea8dad1f6 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Wed, 19 Aug 2020 09:59:38 +0530 Subject: [PATCH 34/62] nit's --- src/tables.jl | 20 ++++++++++++-------- test/remotefiles.jl | 2 +- test/test_tables.jl | 7 +++---- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 880baa46..abdd2409 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -1,5 +1,11 @@ const AG = ArchGDAL +""" +Constructs `Table` out of `FeatureLayer`, where every row is a `Feature` consisting of Geometry and attributes. +``` +ArchGDAL.Table(T::Union{IFeatureLayer, FeatureLayer}) +``` +""" struct Table{T} layer::T end @@ -33,14 +39,16 @@ function Base.iterate(t::Table, st = 0) ngeom = AG.ngeom(layer) featuredefn = AG.layerdefn(layer) geomdefns = (ArchGDAL.getgeomdefn(featuredefn, i) for i in 0:ngeom-1) + typ = Tables.schema(layer).types + geom_types = Tuple(Type{AG.gettype(geomdefn)} for geomdefn in geomdefns) - name = [] - v = [] + name = String[] + v = Union{typ..., AG.IGeometry}[] for field_no in 0:nfield-1 field = AG.getfielddefn(featuredefn, field_no) push!(name, AG.getname(field)) end - geom_names = [ArchGDAL.getname(geomdefn) for geomdefn in geomdefns] + geom_names = [AG.getname(geomdefn) for geomdefn in geomdefns] push!(name, geom_names...) st >= nfeat && return nothing @@ -59,7 +67,6 @@ function Base.iterate(t::Table, st = 0) return Row, st + 1 end - Tables.istable(::Type{<:Table}) = true Tables.rowaccess(::Type{<:Table}) = true Tables.rows(t::Table) = t @@ -71,9 +78,6 @@ Base.IteratorEltype(::Type{<:Table}) = Base.HasEltype() Base.propertynames(t::Table) = Tables.schema(t.layer).names function Base.show(io::IO, t::Table) - println(io, "Table with $(ArchGDAL.nfeature(t.layer)) Features") + println(io, "Table with $(ArchGDAL.nfeature(t.layer)) features") end Base.show(io::IO, ::MIME"text/plain", t::Table) = show(io, t) - - - diff --git a/test/remotefiles.jl b/test/remotefiles.jl index 9536e6f9..1ea95647 100644 --- a/test/remotefiles.jl +++ b/test/remotefiles.jl @@ -11,7 +11,7 @@ REPO_URL = "https://github.com/yeesian/ArchGDALDatasets/blob/master/" # remote files with SHA-2 256 hash """ -While adding more files, follow the below steps to generate the SHA +To add more files, follow the below steps to generate the SHA ``` julia> using SHA julia> open(filepath/filename) do f diff --git a/test/test_tables.jl b/test/test_tables.jl index c93c7c88..eee7c8bc 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -20,14 +20,13 @@ using Tables ngeometries = AG.ngeom(featuredefn) ngeometries1 = AG.ngeom(featuredefn1) - @test sprint(print, gt) == "Table with 4 Features\n" - @test sprint(print, gt1) == "Table with 2 Features\n" + @test sprint(print, gt) == "Table with 4 features\n" + @test sprint(print, gt1) == "Table with 2 features\n" @test getproperty(Tables.schema(layer), :types) == (Float64, String) @test getproperty(Tables.schema(layer1), :types) == (String, String, String) @test getproperty(Tables.schema(layer), :names) == propertynames(gt) @test getproperty(Tables.schema(layer1), :names) == propertynames(gt1) - @test Tables.istable(AG.Table) == true @test Tables.rows(gt) == AG.Table(layer) @test Tables.rows(gt1) == AG.Table(layer1) @@ -39,4 +38,4 @@ using Tables @test propertynames(iterate(gt1, 1)[1]) == (:id, :zoom, :location, :point, :linestring) @test iterate(gt, 5) === nothing @test iterate(gt1, 3) === nothing -end \ No newline at end of file +end From e9949d4bcad687d8be41b5ea70a141c167602faa Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Sat, 22 Aug 2020 15:37:07 +0530 Subject: [PATCH 35/62] Refactor Base.iterate --- docs/src/tables.md | 2 +- src/tables.jl | 39 ++++++++++++++------------------------- 2 files changed, 15 insertions(+), 26 deletions(-) diff --git a/docs/src/tables.md b/docs/src/tables.md index c8efd19d..ebc0ba7a 100644 --- a/docs/src/tables.md +++ b/docs/src/tables.md @@ -5,7 +5,7 @@ using ArchGDAL using DataFrames ``` -ArchGDAL now brings in greater flexibilty in terms of raster data handling via the [Tables.jl](https://github.com/JuliaData/Tables.jl) API, that aims to provide a fast and responsive tabular interface to data, which was earlier handled by the [DataStreams](https://github.com/JuliaData/DataStreams.jl) API. +ArchGDAL now brings in greater flexibilty in terms of raster data handling via the [Tables.jl](https://github.com/JuliaData/Tables.jl) API, that aims to provide a fast and responsive tabular interface to data. In this section, we revisit the [`data/point.geojson`](https://github.com/yeesian/ArchGDALDatasets/blob/307f8f0e584a39a050c042849004e6a2bd674f99/data/point.geojson) dataset. diff --git a/src/tables.jl b/src/tables.jl index abdd2409..eb31d1f9 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -10,14 +10,7 @@ struct Table{T} layer::T end -function Table(layer::T) where {T<:Union{IFeatureLayer, FeatureLayer}} - featuredefn = layerdefn(layer) - ngeometries = ngeom(featuredefn) - if ngeometries == 0 - print("NULL Geometry found") - end - Table{T}(layer) -end +Table(layer::T) where {T<:Union{IFeatureLayer, FeatureLayer}} = Table{T}(layer) function Tables.schema(layer::AG.AbstractFeatureLayer) nfield = AG.nfield(layer) @@ -34,36 +27,32 @@ function Base.iterate(t::Table, st = 0) AG.resetreading!(layer) end - nfeat = AG.nfeature(layer) - nfield = AG.nfield(layer) ngeom = AG.ngeom(layer) featuredefn = AG.layerdefn(layer) geomdefns = (ArchGDAL.getgeomdefn(featuredefn, i) for i in 0:ngeom-1) - typ = Tables.schema(layer).types - geom_types = Tuple(Type{AG.gettype(geomdefn)} for geomdefn in geomdefns) - name = String[] - v = Union{typ..., AG.IGeometry}[] + field_names = String[] + typ = Tables.schema(layer).types + nfield = AG.nfield(layer) for field_no in 0:nfield-1 field = AG.getfielddefn(featuredefn, field_no) - push!(name, AG.getname(field)) + push!(field_names, AG.getname(field)) end geom_names = [AG.getname(geomdefn) for geomdefn in geomdefns] - push!(name, geom_names...) - + nfeat = AG.nfeature(layer) st >= nfeat && return nothing + v = Union{typ..., AG.IGeometry}[] AG.nextfeature(layer) do feature - for k in name - if k in geom_names - val = AG.getgeom(feature, findfirst(a->a==k, geom_names)-1) - else - val = AG.getfield(feature, k) - end + for name in field_names + val = AG.getfield(feature, name) push!(v, val) end + for idx in 1:length(geom_names) + val = AG.getgeom(feature, idx-1) + push!(v, val) + end end - - Row = NamedTuple{Tuple(Symbol.(name))}(v) + Row = NamedTuple{(Symbol.(field_names)..., Symbol.(geom_names)...)}(v) return Row, st + 1 end From 91f6489729a79a314a4f7bd41a47c5b6c2d2dbd3 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Sat, 22 Aug 2020 23:34:12 +0530 Subject: [PATCH 36/62] Remove AG; refactor methods --- src/tables.jl | 47 ++++++++++++++++++++--------------------------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index eb31d1f9..8760765c 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -1,5 +1,3 @@ -const AG = ArchGDAL - """ Constructs `Table` out of `FeatureLayer`, where every row is a `Feature` consisting of Geometry and attributes. ``` @@ -12,44 +10,39 @@ end Table(layer::T) where {T<:Union{IFeatureLayer, FeatureLayer}} = Table{T}(layer) -function Tables.schema(layer::AG.AbstractFeatureLayer) - nfield = AG.nfield(layer) - featuredefn = AG.layerdefn(layer) - fielddefns = (AG.getfielddefn(featuredefn, i) for i in 0:nfield-1) - names_fields = Tuple(AG.getname(fielddefn) for fielddefn in fielddefns) - types_fields = Tuple(AG._FIELDTYPE[AG.gettype(fielddefn)] for fielddefn in fielddefns) +function Tables.schema(layer::AbstractFeatureLayer) + featuredefn = layerdefn(layer) + fielddefns = (getfielddefn(featuredefn, i) for i in 0:nfield(layer)-1) + names_fields = Tuple(getname(fielddefn) for fielddefn in fielddefns) + types_fields = Tuple(_FIELDTYPE[gettype(fielddefn)] for fielddefn in fielddefns) Tables.Schema(names_fields, types_fields) end function Base.iterate(t::Table, st = 0) layer = t.layer if iszero(st) - AG.resetreading!(layer) + resetreading!(layer) end - ngeom = AG.ngeom(layer) - featuredefn = AG.layerdefn(layer) - geomdefns = (ArchGDAL.getgeomdefn(featuredefn, i) for i in 0:ngeom-1) + ngeom = ArchGDAL.ngeom(layer) + featuredefn = layerdefn(layer) + geomdefns = (getgeomdefn(featuredefn, i) for i in 0:ngeom-1) field_names = String[] - typ = Tables.schema(layer).types - nfield = AG.nfield(layer) - for field_no in 0:nfield-1 - field = AG.getfielddefn(featuredefn, field_no) - push!(field_names, AG.getname(field)) + for field_no in 0:nfield(layer)-1 + field = getfielddefn(featuredefn, field_no) + push!(field_names, getname(field)) end - geom_names = [AG.getname(geomdefn) for geomdefn in geomdefns] - nfeat = AG.nfeature(layer) - st >= nfeat && return nothing - v = Union{typ..., AG.IGeometry}[] - AG.nextfeature(layer) do feature + geom_names = [getname(geomdefn) for geomdefn in geomdefns] + + st >= nfeature(layer) && return nothing + v = Union{Tables.schema(layer).types..., IGeometry}[] + nextfeature(layer) do feature for name in field_names - val = AG.getfield(feature, name) - push!(v, val) + push!(v, getfield(feature, name)) end for idx in 1:length(geom_names) - val = AG.getgeom(feature, idx-1) - push!(v, val) + push!(v, getgeom(feature, idx-1)) end end Row = NamedTuple{(Symbol.(field_names)..., Symbol.(geom_names)...)}(v) @@ -67,6 +60,6 @@ Base.IteratorEltype(::Type{<:Table}) = Base.HasEltype() Base.propertynames(t::Table) = Tables.schema(t.layer).names function Base.show(io::IO, t::Table) - println(io, "Table with $(ArchGDAL.nfeature(t.layer)) features") + println(io, "Table with $(nfeature(t.layer)) features") end Base.show(io::IO, ::MIME"text/plain", t::Table) = show(io, t) From b9d0d6bb11fefcf8254e74fe4b1116423c5da5e5 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Sat, 22 Aug 2020 23:39:38 +0530 Subject: [PATCH 37/62] inline ngeom --- src/tables.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 8760765c..504c73d3 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -24,9 +24,8 @@ function Base.iterate(t::Table, st = 0) resetreading!(layer) end - ngeom = ArchGDAL.ngeom(layer) featuredefn = layerdefn(layer) - geomdefns = (getgeomdefn(featuredefn, i) for i in 0:ngeom-1) + geomdefns = (getgeomdefn(featuredefn, i) for i in 0:ArchGDAL.ngeom(layer)-1) field_names = String[] for field_no in 0:nfield(layer)-1 From a4bf268f8e58708ac9576c15da2da70cd3c27cc9 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Sun, 23 Aug 2020 12:02:24 +0530 Subject: [PATCH 38/62] Minor refactor --- src/tables.jl | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 504c73d3..44dac781 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -24,15 +24,9 @@ function Base.iterate(t::Table, st = 0) resetreading!(layer) end - featuredefn = layerdefn(layer) - geomdefns = (getgeomdefn(featuredefn, i) for i in 0:ArchGDAL.ngeom(layer)-1) - - field_names = String[] - for field_no in 0:nfield(layer)-1 - field = getfielddefn(featuredefn, field_no) - push!(field_names, getname(field)) - end - geom_names = [getname(geomdefn) for geomdefn in geomdefns] + featuredefn = layerdefn(layer) + field_names = [getname(getfielddefn(featuredefn, i-1)) for i in 1:nfield(layer)] + geom_names = [getname(getgeomdefn(featuredefn, i-1)) for i in 1:ngeom(layer)] st >= nfeature(layer) && return nothing v = Union{Tables.schema(layer).types..., IGeometry}[] From 803691b8363a36cb1b6dbc5561360e0e842f7c7f Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Sun, 23 Aug 2020 12:19:58 +0530 Subject: [PATCH 39/62] Typos fix --- docs/src/datasets.md | 2 +- docs/src/tables.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/datasets.md b/docs/src/datasets.md index 745e09f1..366a62e7 100644 --- a/docs/src/datasets.md +++ b/docs/src/datasets.md @@ -111,7 +111,7 @@ dataset1 = ArchGDAL.read("data/multi_geom.csv") layer1 = ArchGDAL.getlayer(dataset1, 0) ``` -Well this is weird, the CSV driver recognises our point and linestring geomtries as `String`. Now if you have a .csvt file of the same name with the geometry types as `WKT`, they types will be recognized, else, GDAL offers open-options to tweak the read parameters that are passed as `kwargs`. +Well this is weird, the CSV driver recognises our point and linestring geometries as `String`. Now if you have a .csvt file of the same name with the geometry types as `WKT`, they types will be recognized, else, GDAL offers open-options to tweak the read parameters that are passed as `kwargs`. So for the above CSV, we want the driver to detect our geometries, so according to [open-options](https://gdal.org/drivers/vector/csv.html#open-options) we should use the `"GEOM_POSSIBLE_NAMES=point,linestring"` option. Also we want that the geometry columns should not be kept as regular `String` columns, so we add a `"KEEP_GEOM_COLUMNS=NO"` option too. diff --git a/docs/src/tables.md b/docs/src/tables.md index ebc0ba7a..9bd37163 100644 --- a/docs/src/tables.md +++ b/docs/src/tables.md @@ -36,7 +36,7 @@ One can get back the layer that a Table is made up of. lyr = table.layer ``` -The Tables interface also support multiple geomtries per layer. +The Tables interface also support multiple geometries per layer. Here, we visit the [`data/multi_geom.csv`](https://github.com/yeesian/ArchGDALDatasets/blob/master/data/multi_geom.csv) dataset. @@ -47,7 +47,7 @@ layer = ArchGDAL.getlayer(dataset, 0) table = ArchGDAL.Table(layer) ``` -Exatracting a row from the table, we see that the row/feature is made up of two geomtries viz. `point` and `linestring`. +Exatracting a row from the table, we see that the row/feature is made up of two geometries viz. `point` and `linestring`. ```@example tables row = Base.iterate(table, 0) ``` From 1655a18bc7eb9525170e37371048439c76810c6f Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Sun, 23 Aug 2020 17:26:18 +0200 Subject: [PATCH 40/62] remove DataStreams from deps --- Project.toml | 6 ++---- docs/Project.toml | 3 ++- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/Project.toml b/Project.toml index a1fa7fba..6640812b 100644 --- a/Project.toml +++ b/Project.toml @@ -6,21 +6,19 @@ desc = "A high level API for GDAL - Geospatial Data Abstraction Library" version = "0.5.0" [deps] -DataStreams = "9a8bc11e-79be-5b39-94d7-1ccc349a1a85" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" GDAL = "add2ef01-049f-52c4-9ee2-e494f65e021a" GeoFormatTypes = "68eda718-8dee-11e9-39e7-89f7f65f511f" GeoInterface = "cf35fbd7-0cd7-5166-be24-54bfbe79505f" -Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" [compat] -DataStreams = "0.4.2" DiskArrays = "0.2.4" GDAL = "1.1.3" +GeoFormatTypes = "0.3" GeoInterface = "0.4, 0.5" Tables = "1" -GeoFormatTypes = "0.3" julia = "1.3" [extras] diff --git a/docs/Project.toml b/docs/Project.toml index 101cb169..b19bbae3 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,7 +1,8 @@ [deps] ArchGDAL = "c9ce4bd3-c3d5-55b8-8973-c0e20141b8c3" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" + [compat] Documenter = "0.25.1" From a0450f7d7862f71e099ba50ffa6831868292e61b Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Sun, 23 Aug 2020 17:27:03 +0200 Subject: [PATCH 41/62] wrap long lines in docs --- docs/src/tables.md | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/docs/src/tables.md b/docs/src/tables.md index 9bd37163..10860f27 100644 --- a/docs/src/tables.md +++ b/docs/src/tables.md @@ -5,9 +5,13 @@ using ArchGDAL using DataFrames ``` -ArchGDAL now brings in greater flexibilty in terms of raster data handling via the [Tables.jl](https://github.com/JuliaData/Tables.jl) API, that aims to provide a fast and responsive tabular interface to data. +ArchGDAL now brings in greater flexibilty in terms of raster data handling via the +[Tables.jl](https://github.com/JuliaData/Tables.jl) API, that aims to provide a fast and +responsive tabular interface to data. -In this section, we revisit the [`data/point.geojson`](https://github.com/yeesian/ArchGDALDatasets/blob/307f8f0e584a39a050c042849004e6a2bd674f99/data/point.geojson) dataset. +In this section, we revisit the +[`data/point.geojson`](https://github.com/yeesian/ArchGDALDatasets/blob/307f8f0e584a39a050c042849004e6a2bd674f99/data/point.geojson) +dataset. ```@example tables dataset = ArchGDAL.read("data/point.geojson") @@ -38,7 +42,9 @@ lyr = table.layer The Tables interface also support multiple geometries per layer. -Here, we visit the [`data/multi_geom.csv`](https://github.com/yeesian/ArchGDALDatasets/blob/master/data/multi_geom.csv) dataset. +Here, we visit the +[`data/multi_geom.csv`](https://github.com/yeesian/ArchGDALDatasets/blob/master/data/multi_geom.csv) +dataset. ```@example tables dataset1 = ArchGDAL.read("data/multi_geom.csv", options = ["GEOM_POSSIBLE_NAMES=point,linestring", "KEEP_GEOM_COLUMNS=NO"]) @@ -47,12 +53,13 @@ layer = ArchGDAL.getlayer(dataset, 0) table = ArchGDAL.Table(layer) ``` -Exatracting a row from the table, we see that the row/feature is made up of two geometries viz. `point` and `linestring`. +Exatracting a row from the table, we see that the row/feature is made up of two geometries +viz. `point` and `linestring`. ```@example tables -row = Base.iterate(table, 0) +row = Base.iterate(table) ``` -Finally layers can be viewed as DataFrames to perform miscellaneous spatial operations. +Finally layers can be converted to DataFrames to perform miscellaneous spatial operations. ```@example tables df = DataFrame(table) ``` From 2156aec5ef7a4f7328ffce110b1fe746d8800711 Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Sun, 23 Aug 2020 17:28:01 +0200 Subject: [PATCH 42/62] using instead of import, to force explicit override which were already done, hence this change could be made --- src/ArchGDAL.jl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/ArchGDAL.jl b/src/ArchGDAL.jl index fb300131..1bb05c96 100644 --- a/src/ArchGDAL.jl +++ b/src/ArchGDAL.jl @@ -1,11 +1,10 @@ module ArchGDAL - import GDAL, GeoInterface, GeoFormatTypes - import Tables: Tables - import GeoInterface: coordinates, geotype - import Base: convert - using Dates + using GDAL: GDAL + using GeoFormatTypes: GeoFormatTypes + using GeoInterface: GeoInterface + using Tables: Tables const GFT = GeoFormatTypes From 2aa7499e1780ae16dbda458301f656a1a37f8bd7 Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Sun, 23 Aug 2020 17:34:13 +0200 Subject: [PATCH 43/62] extend getlayer for Table In preparation for overloading `getproperty` for Table, since `table.layer` won't work anymore. --- docs/src/tables.md | 2 +- src/dataset.jl | 4 +++- src/tables.jl | 13 +++++-------- test/test_tables.jl | 1 + 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/src/tables.md b/docs/src/tables.md index 10860f27..e0948bbb 100644 --- a/docs/src/tables.md +++ b/docs/src/tables.md @@ -37,7 +37,7 @@ row = Base.iterate(table, 1) Layers are retrievable! One can get back the layer that a Table is made up of. ```@example tables -lyr = table.layer +lyr = ArchGDAL.getlayer(table) ``` The Tables interface also support multiple geometries per layer. diff --git a/src/dataset.jl b/src/dataset.jl index d976ae67..25a9c3ea 100644 --- a/src/dataset.jl +++ b/src/dataset.jl @@ -460,8 +460,10 @@ unsafe_getlayer(dataset::AbstractDataset, i::Integer) = """ getlayer(dataset::AbstractDataset, name::AbstractString) + getlayer(table::Table) -Fetch the feature layer corresponding to the given name. +Fetch the feature layer corresponding to the given name. If it is called on a Table, which +supports only one layer, a name is not needed. The returned layer remains owned by the GDALDataset and should not be deleted by the application. diff --git a/src/tables.jl b/src/tables.jl index 44dac781..8f0eec9f 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -8,7 +8,7 @@ struct Table{T} layer::T end -Table(layer::T) where {T<:Union{IFeatureLayer, FeatureLayer}} = Table{T}(layer) +getlayer(t::Table) = Base.getfield(t, :layer) function Tables.schema(layer::AbstractFeatureLayer) featuredefn = layerdefn(layer) @@ -19,9 +19,7 @@ function Tables.schema(layer::AbstractFeatureLayer) end function Base.iterate(t::Table, st = 0) - layer = t.layer - if iszero(st) - resetreading!(layer) + layer = getlayer(t) end featuredefn = layerdefn(layer) @@ -47,12 +45,11 @@ Tables.rowaccess(::Type{<:Table}) = true Tables.rows(t::Table) = t Base.IteratorSize(::Type{<:Table}) = Base.HasLength() -Base.size(t::Table) = nfeature(t.layer) -Base.length(t::Table) = Base.size(t) +Base.size(t::Table) = nfeature(getlayer(t)) Base.IteratorEltype(::Type{<:Table}) = Base.HasEltype() -Base.propertynames(t::Table) = Tables.schema(t.layer).names +Base.propertynames(t::Table) = Tables.schema(getlayer(t)).names function Base.show(io::IO, t::Table) - println(io, "Table with $(nfeature(t.layer)) features") + println(io, "Table with $(nfeature(getlayer(t))) features") end Base.show(io::IO, ::MIME"text/plain", t::Table) = show(io, t) diff --git a/test/test_tables.jl b/test/test_tables.jl index eee7c8bc..46868dc8 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -8,6 +8,7 @@ using Tables layer = AG.getlayer(dataset, 0) layer1 = AG.getlayer(dataset1, 0) gt = AG.Table(layer) + @test getlayer(gt) === layer gt1 = AG.Table(layer1) fr = iterate(gt, 1)[1] fr1 = iterate(gt, 1)[1] From c196b3a1faf4ec599588585662e832b77f6ca4f5 Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Sun, 23 Aug 2020 17:37:24 +0200 Subject: [PATCH 44/62] support passing field name as symbol to getfield and findfieldindex ccall converts these successfully is needed for next commit --- src/ogr/feature.jl | 6 +++--- src/ogr/featuredefn.jl | 4 ++-- src/ogr/featurelayer.jl | 4 ++-- test/test_featurelayer.jl | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/ogr/feature.jl b/src/ogr/feature.jl index 858e731e..6d0f0a7b 100644 --- a/src/ogr/feature.jl +++ b/src/ogr/feature.jl @@ -95,7 +95,7 @@ getfielddefn(feature::Feature, i::Integer) = IFieldDefnView(GDAL.ogr_f_getfielddefnref(feature.ptr, i)) """ - findfieldindex(feature::Feature, name::AbstractString) + findfieldindex(feature::Feature, name::Union{AbstractString, Symbol}) Fetch the field index given field name. @@ -109,7 +109,7 @@ the field index, or -1 if no matching field is found. ### Remarks This is a cover for the `OGRFeatureDefn::GetFieldIndex()` method. """ -findfieldindex(feature::Feature, name::AbstractString) = +findfieldindex(feature::Feature, name::Union{AbstractString, Symbol}) = GDAL.ogr_f_getfieldindex(feature.ptr, name) """ @@ -381,7 +381,7 @@ function getfield(feature::Feature, i::Integer) end end -getfield(feature::Feature, name::AbstractString) = +getfield(feature::Feature, name::Union{AbstractString, Symbol}) = getfield(feature, findfieldindex(feature, name)) """ diff --git a/src/ogr/featuredefn.jl b/src/ogr/featuredefn.jl index fec37009..3d3a792a 100644 --- a/src/ogr/featuredefn.jl +++ b/src/ogr/featuredefn.jl @@ -91,7 +91,7 @@ getfielddefn(featuredefn::IFeatureDefnView, i::Integer) = IFieldDefnView(GDAL.ogr_fd_getfielddefn(featuredefn.ptr, i)) """ - findfieldindex(featuredefn::AbstractFeatureDefn, name::AbstractString) + findfieldindex(featuredefn::AbstractFeatureDefn, name::Union{AbstractString, Symbol}) Find field by name. @@ -101,7 +101,7 @@ the field index, or -1 if no match found. ### Remarks This uses the OGRFeatureDefn::GetFieldIndex() method. """ -findfieldindex(featuredefn::AbstractFeatureDefn, name::AbstractString) = +findfieldindex(featuredefn::AbstractFeatureDefn, name::Union{AbstractString, Symbol}) = GDAL.ogr_fd_getfieldindex(featuredefn.ptr, name) """ diff --git a/src/ogr/featurelayer.jl b/src/ogr/featurelayer.jl index 2962d437..aa48dc8d 100644 --- a/src/ogr/featurelayer.jl +++ b/src/ogr/featurelayer.jl @@ -524,7 +524,7 @@ layerdefn(layer::AbstractFeatureLayer) = IFeatureDefnView(GDAL.ogr_l_getlayerdefn(layer.ptr)) """ - findfieldindex(layer::AbstractFeatureLayer, field::AbstractString, exactmatch::Bool) + findfieldindex(layer::AbstractFeatureLayer, field::Union{AbstractString, Symbol}, exactmatch::Bool) Find the index of the field in a layer, or -1 if the field doesn't exist. @@ -534,7 +534,7 @@ the layer was created (eg. like `LAUNDER` in the OCI driver). """ function findfieldindex( layer::AbstractFeatureLayer, - field::AbstractString, + field::Union{AbstractString, Symbol}, exactmatch::Bool ) return GDAL.ogr_l_findfieldindex(layer.ptr, field, exactmatch) diff --git a/test/test_featurelayer.jl b/test/test_featurelayer.jl index c3b69788..df1205c0 100644 --- a/test/test_featurelayer.jl +++ b/test/test_featurelayer.jl @@ -148,7 +148,7 @@ import ArchGDAL; const AG = ArchGDAL end end @test AG.findfieldindex(layer,"FID", true) == 0 - @test AG.findfieldindex(layer,"FID", false) == 0 + @test AG.findfieldindex(layer,:FID, false) == 0 @test AG.findfieldindex(layer,"pointname", true) == 1 @test AG.findfieldindex(layer,"pointname", false) == 1 @test AG.findfieldindex(layer,"geom", true) == -1 From d933f09149b2266a9ec3c4fa81cc344acc90cf95 Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Sun, 23 Aug 2020 17:40:23 +0200 Subject: [PATCH 45/62] allow only creating tables from layers --- src/tables.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tables.jl b/src/tables.jl index 8f0eec9f..45e88b94 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -4,7 +4,7 @@ Constructs `Table` out of `FeatureLayer`, where every row is a `Feature` consist ArchGDAL.Table(T::Union{IFeatureLayer, FeatureLayer}) ``` """ -struct Table{T} +struct Table{T<:Union{IFeatureLayer, FeatureLayer}} layer::T end From e84491da028c209d818bb15d2257e52ed63f403f Mon Sep 17 00:00:00 2001 From: Martijn Visser Date: Sun, 23 Aug 2020 17:42:09 +0200 Subject: [PATCH 46/62] various other fixups Notably, avoid a few allocations on `iterate`, by directly creating tuples, not vectors first. --- src/tables.jl | 15 +++++++++------ test/remotefiles.jl | 1 + test/test_tables.jl | 10 +++++----- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 45e88b94..4ffc5a66 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -20,11 +20,13 @@ end function Base.iterate(t::Table, st = 0) layer = getlayer(t) + if iszero(st) + resetreading!(layer) end - featuredefn = layerdefn(layer) - field_names = [getname(getfielddefn(featuredefn, i-1)) for i in 1:nfield(layer)] - geom_names = [getname(getgeomdefn(featuredefn, i-1)) for i in 1:ngeom(layer)] + featuredefn = layerdefn(layer) + field_names = Tuple(Symbol(getname(getfielddefn(featuredefn, i-1))) for i in 1:nfield(layer)) + geom_names = Tuple(Symbol(getname(getgeomdefn(featuredefn, i-1))) for i in 1:ngeom(layer)) st >= nfeature(layer) && return nothing v = Union{Tables.schema(layer).types..., IGeometry}[] @@ -33,11 +35,11 @@ function Base.iterate(t::Table, st = 0) push!(v, getfield(feature, name)) end for idx in 1:length(geom_names) - push!(v, getgeom(feature, idx-1)) + push!(v, getgeom(feature, idx-1)) end end - Row = NamedTuple{(Symbol.(field_names)..., Symbol.(geom_names)...)}(v) - return Row, st + 1 + row = NamedTuple{(field_names..., geom_names...)}(v) + return row, st + 1 end Tables.istable(::Type{<:Table}) = true @@ -46,6 +48,7 @@ Tables.rows(t::Table) = t Base.IteratorSize(::Type{<:Table}) = Base.HasLength() Base.size(t::Table) = nfeature(getlayer(t)) +Base.length(t::Table) = size(t) Base.IteratorEltype(::Type{<:Table}) = Base.HasEltype() Base.propertynames(t::Table) = Tables.schema(getlayer(t)).names diff --git a/test/remotefiles.jl b/test/remotefiles.jl index 1ea95647..3c3631d9 100644 --- a/test/remotefiles.jl +++ b/test/remotefiles.jl @@ -17,6 +17,7 @@ julia> using SHA julia> open(filepath/filename) do f bytes2hex(sha256(f)) end +``` """ remotefiles = [ ("data/multi_geom.csv", "00520017658b66ff21e40cbf553672fa8e280cddae6e7a5d1f8bd36bcd521770"), diff --git a/test/test_tables.jl b/test/test_tables.jl index 46868dc8..f67a1662 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -8,7 +8,7 @@ using Tables layer = AG.getlayer(dataset, 0) layer1 = AG.getlayer(dataset1, 0) gt = AG.Table(layer) - @test getlayer(gt) === layer + @test AG.getlayer(gt) === layer gt1 = AG.Table(layer1) fr = iterate(gt, 1)[1] fr1 = iterate(gt, 1)[1] @@ -23,10 +23,10 @@ using Tables @test sprint(print, gt) == "Table with 4 features\n" @test sprint(print, gt1) == "Table with 2 features\n" - @test getproperty(Tables.schema(layer), :types) == (Float64, String) - @test getproperty(Tables.schema(layer1), :types) == (String, String, String) - @test getproperty(Tables.schema(layer), :names) == propertynames(gt) - @test getproperty(Tables.schema(layer1), :names) == propertynames(gt1) + @test Tables.schema(layer).types == (Float64, String) + @test Tables.schema(layer1).types == (String, String, String) + @test Tables.schema(layer).names == propertynames(gt) + @test Tables.schema(layer1).names == propertynames(gt1) @test Tables.istable(AG.Table) == true @test Tables.rows(gt) == AG.Table(layer) From 90e148f265c83f38630075088cff259fa758e56c Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Sun, 23 Aug 2020 23:52:59 +0530 Subject: [PATCH 47/62] Refactor schema to include geometry names/types --- src/tables.jl | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 4ffc5a66..ba488b6b 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -13,9 +13,17 @@ getlayer(t::Table) = Base.getfield(t, :layer) function Tables.schema(layer::AbstractFeatureLayer) featuredefn = layerdefn(layer) fielddefns = (getfielddefn(featuredefn, i) for i in 0:nfield(layer)-1) - names_fields = Tuple(getname(fielddefn) for fielddefn in fielddefns) - types_fields = Tuple(_FIELDTYPE[gettype(fielddefn)] for fielddefn in fielddefns) - Tables.Schema(names_fields, types_fields) + names_fields = (getname(fielddefn) for fielddefn in fielddefns) + types_fields = (_FIELDTYPE[gettype(fielddefn)] for fielddefn in fielddefns) + + ngeom = ArchGDAL.ngeom(featuredefn) + geomdefns = (ArchGDAL.getgeomdefn(featuredefn, i) for i in 0:ngeom-1) + geom_names = (ArchGDAL.getname(geomdefn) for geomdefn in geomdefns) + geom_types = (ArchGDAL.gettype(geomdefn) for geomdefn in geomdefns) + + names = (names_fields..., geom_names...) + types = (types_fields..., geom_types...) + Tables.Schema(names, types) end function Base.iterate(t::Table, st = 0) @@ -26,10 +34,12 @@ function Base.iterate(t::Table, st = 0) featuredefn = layerdefn(layer) field_names = Tuple(Symbol(getname(getfielddefn(featuredefn, i-1))) for i in 1:nfield(layer)) + fielddefns = (getfielddefn(featuredefn, i) for i in 0:nfield(layer)-1) + types_fields = (_FIELDTYPE[gettype(fielddefn)] for fielddefn in fielddefns) geom_names = Tuple(Symbol(getname(getgeomdefn(featuredefn, i-1))) for i in 1:ngeom(layer)) st >= nfeature(layer) && return nothing - v = Union{Tables.schema(layer).types..., IGeometry}[] + v = Union{types_fields..., IGeometry}[] nextfeature(layer) do feature for name in field_names push!(v, getfield(feature, name)) From 924df4766b908f3c0d7af5e75e0620735bfbe409 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Sun, 23 Aug 2020 23:53:15 +0530 Subject: [PATCH 48/62] Add getindex method --- src/tables.jl | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/tables.jl b/src/tables.jl index ba488b6b..c7b386c0 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -61,6 +61,29 @@ Base.size(t::Table) = nfeature(getlayer(t)) Base.length(t::Table) = size(t) Base.IteratorEltype(::Type{<:Table}) = Base.HasEltype() Base.propertynames(t::Table) = Tables.schema(getlayer(t)).names +Base.getproperty(t::Table, s::Symbol) = [getproperty(iterate(t, i)[1], s) for i in 0:size(t)-1] + +function Base.getindex(t::Table, idx::Int) + layer = getlayer(t) + setnextbyindex!(layer, idx) + featuredefn = layerdefn(layer) + field_names = Tuple(Symbol(getname(getfielddefn(featuredefn, i-1))) for i in 1:nfield(layer)) + fielddefns = (getfielddefn(featuredefn, i) for i in 0:nfield(layer)-1) + types_fields = (_FIELDTYPE[gettype(fielddefn)] for fielddefn in fielddefns) + geom_names = Tuple(Symbol(getname(getgeomdefn(featuredefn, i-1))) for i in 1:ngeom(layer)) + v = Union{types_fields..., IGeometry}[] + + nextfeature(layer) do feature + for name in field_names + push!(v, getfield(feature, name)) + end + for idx in 1:length(geom_names) + push!(v, getgeom(feature, idx-1)) + end + end + row = NamedTuple{(field_names..., geom_names...)}(v) +end + function Base.show(io::IO, t::Table) println(io, "Table with $(nfeature(getlayer(t))) features") From cee6cecb0afedf02e539ff117af8e13bc2dbd8b3 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Sun, 23 Aug 2020 23:53:24 +0530 Subject: [PATCH 49/62] Update docs --- docs/src/tables.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/tables.md b/docs/src/tables.md index e0948bbb..8ee530a3 100644 --- a/docs/src/tables.md +++ b/docs/src/tables.md @@ -28,10 +28,10 @@ The [`ArchGDAL.Table`](@ref) method accepts an `ArchGDAL.FeatureLayer`. table = ArchGDAL.Table(layer) ``` -Individual rows can be retrieved using the `iterate(t::ArchGDAL.Table, n::Int)` method. +Individual rows can be retrieved using the `Base.getindex(t::ArchGDAL.Table, idx::Int)` method. ```@example tables -row = Base.iterate(table, 1) +row = Base.getindex(table, 1) ``` Layers are retrievable! @@ -56,7 +56,7 @@ table = ArchGDAL.Table(layer) Exatracting a row from the table, we see that the row/feature is made up of two geometries viz. `point` and `linestring`. ```@example tables -row = Base.iterate(table) +row = Base.getindex(table, 1) ``` Finally layers can be converted to DataFrames to perform miscellaneous spatial operations. From 4bd8c5a45fec2c76c7860f9eeb74dd02f081ba45 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Sun, 23 Aug 2020 23:53:34 +0530 Subject: [PATCH 50/62] Add/update tests --- test/test_tables.jl | 67 ++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/test/test_tables.jl b/test/test_tables.jl index f67a1662..015171de 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -3,40 +3,45 @@ import ArchGDAL; const AG = ArchGDAL using Tables @testset "Tables Support" begin + dataset = AG.read(joinpath(@__DIR__, "data/point.geojson")) dataset1 = AG.read(joinpath(@__DIR__, "data/multi_geom.csv"), options = ["GEOM_POSSIBLE_NAMES=point,linestring", "KEEP_GEOM_COLUMNS=NO"]) + @test dataset isa ArchGDAL.IDataset + @test dataset1 isa ArchGDAL.IDataset layer = AG.getlayer(dataset, 0) layer1 = AG.getlayer(dataset1, 0) - gt = AG.Table(layer) - @test AG.getlayer(gt) === layer - gt1 = AG.Table(layer1) - fr = iterate(gt, 1)[1] - fr1 = iterate(gt, 1)[1] - nfeat = AG.nfeature(layer) - nfeat1 = AG.nfeature(layer1) - nfield = AG.nfield(layer) - nfield1 = AG.nfield(layer1) - featuredefn = AG.layerdefn(layer) - featuredefn1 = AG.layerdefn(layer1) - ngeometries = AG.ngeom(featuredefn) - ngeometries1 = AG.ngeom(featuredefn1) - @test sprint(print, gt) == "Table with 4 features\n" - @test sprint(print, gt1) == "Table with 2 features\n" - @test Tables.schema(layer).types == (Float64, String) - @test Tables.schema(layer1).types == (String, String, String) - @test Tables.schema(layer).names == propertynames(gt) - @test Tables.schema(layer1).names == propertynames(gt1) - - @test Tables.istable(AG.Table) == true - @test Tables.rows(gt) == AG.Table(layer) - @test Tables.rows(gt1) == AG.Table(layer1) - @test Base.size(gt) == 4 - @test Base.size(gt1) == 2 - @test Base.length(gt) == 4 - @test Base.length(gt1) == 2 - @test propertynames(iterate(gt, 1)[1]) == (:FID, :pointname, Symbol("")) - @test propertynames(iterate(gt1, 1)[1]) == (:id, :zoom, :location, :point, :linestring) - @test iterate(gt, 5) === nothing - @test iterate(gt1, 3) === nothing + @testset "read layer to table" begin + gt = AG.Table(layer) + gt1 = AG.Table(layer1) + @test AG.getlayer(gt) === layer + @test AG.getlayer(gt1) === layer1 + @test sprint(print, gt) == "Table with 4 features\n" + @test sprint(print, gt1) == "Table with 2 features\n" + end + + @testset "Tables methods" begin + @test Tables.schema(layer).names == propertynames(gt) + @test Tables.schema(layer1).names == propertynames(gt1) + @test Tables.istable(AG.Table) == true + @test Tables.rows(gt) == AG.Table(layer) + @test Tables.rows(gt1) == AG.Table(layer1) + end + + @testset "Misc. methods" begin + @test Base.size(gt) == 4 + @test Base.size(gt1) == 2 + @test Base.length(gt) == 4 + @test Base.length(gt1) == 2 + @test Base.IteratorSize(typeof(gt)) == Base.HasLength() + @test Base.IteratorEltype(typeof(gt1)) == Base.HasEltype() + @test propertynames(gt) == (:FID, :pointname, Symbol("")) + @test propertynames(gt1) == (:id, :zoom, :location, :point, :linestring) + @test getproperty(gt, :FID) == [iterate(gt, i)[1].FID for i in 0:size(gt)-1] + @test getproperty(gt1, :zoom) == [iterate(gt1, i)[1].zoom for i in 0:size(gt1)-1] + @test iterate(gt, 5) === nothing + @test iterate(gt1, 3) === nothing + @test typeof([getindex(gt, i) for i in 0:size(gt)-1]) == typeof([iterate(gt, i)[1] for i in 0:size(gt)-1]) + @test typeof([getindex(gt1, i) for i in 0:size(gt1)-1]) == typeof([iterate(gt1, i)[1] for i in 0:size(gt1)-1]) + end end From bcec31e23223c0423f16266c2b87dc4655cdae46 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Sun, 23 Aug 2020 23:58:03 +0530 Subject: [PATCH 51/62] Fix scope in tests --- test/test_tables.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/test_tables.jl b/test/test_tables.jl index 015171de..cc8b3934 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -10,10 +10,11 @@ using Tables @test dataset1 isa ArchGDAL.IDataset layer = AG.getlayer(dataset, 0) layer1 = AG.getlayer(dataset1, 0) + gt = AG.Table(layer) + gt1 = AG.Table(layer1) + @testset "read layer to table" begin - gt = AG.Table(layer) - gt1 = AG.Table(layer1) @test AG.getlayer(gt) === layer @test AG.getlayer(gt1) === layer1 @test sprint(print, gt) == "Table with 4 features\n" From d245b1268724e77e9c4c277bb7e56d832c1b6d7a Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Mon, 24 Aug 2020 01:12:23 +0530 Subject: [PATCH 52/62] Clean up methods --- src/tables.jl | 47 +++++++++++++++------------------------------ test/test_tables.jl | 1 - 2 files changed, 15 insertions(+), 33 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index c7b386c0..c3c4b103 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -19,43 +19,24 @@ function Tables.schema(layer::AbstractFeatureLayer) ngeom = ArchGDAL.ngeom(featuredefn) geomdefns = (ArchGDAL.getgeomdefn(featuredefn, i) for i in 0:ngeom-1) geom_names = (ArchGDAL.getname(geomdefn) for geomdefn in geomdefns) - geom_types = (ArchGDAL.gettype(geomdefn) for geomdefn in geomdefns) + geom_types = (IGeometry for i in 1:ngeom) - names = (names_fields..., geom_names...) - types = (types_fields..., geom_types...) - Tables.Schema(names, types) + Tables.Schema((names_fields..., geom_names...), (types_fields..., geom_types...)) end +Tables.istable(::Type{<:Table}) = true +Tables.rowaccess(::Type{<:Table}) = true +Tables.rows(t::Table) = t + function Base.iterate(t::Table, st = 0) layer = getlayer(t) + st >= nfeature(layer) && return nothing if iszero(st) resetreading!(layer) end - - featuredefn = layerdefn(layer) - field_names = Tuple(Symbol(getname(getfielddefn(featuredefn, i-1))) for i in 1:nfield(layer)) - fielddefns = (getfielddefn(featuredefn, i) for i in 0:nfield(layer)-1) - types_fields = (_FIELDTYPE[gettype(fielddefn)] for fielddefn in fielddefns) - geom_names = Tuple(Symbol(getname(getgeomdefn(featuredefn, i-1))) for i in 1:ngeom(layer)) - - st >= nfeature(layer) && return nothing - v = Union{types_fields..., IGeometry}[] - nextfeature(layer) do feature - for name in field_names - push!(v, getfield(feature, name)) - end - for idx in 1:length(geom_names) - push!(v, getgeom(feature, idx-1)) - end - end - row = NamedTuple{(field_names..., geom_names...)}(v) - return row, st + 1 + return get_row(layer), st + 1 end -Tables.istable(::Type{<:Table}) = true -Tables.rowaccess(::Type{<:Table}) = true -Tables.rows(t::Table) = t - Base.IteratorSize(::Type{<:Table}) = Base.HasLength() Base.size(t::Table) = nfeature(getlayer(t)) Base.length(t::Table) = size(t) @@ -66,13 +47,16 @@ Base.getproperty(t::Table, s::Symbol) = [getproperty(iterate(t, i)[1], s) for i function Base.getindex(t::Table, idx::Int) layer = getlayer(t) setnextbyindex!(layer, idx) + return get_row(layer) +end + +function get_row(layer::IFeatureLayer) featuredefn = layerdefn(layer) - field_names = Tuple(Symbol(getname(getfielddefn(featuredefn, i-1))) for i in 1:nfield(layer)) fielddefns = (getfielddefn(featuredefn, i) for i in 0:nfield(layer)-1) - types_fields = (_FIELDTYPE[gettype(fielddefn)] for fielddefn in fielddefns) - geom_names = Tuple(Symbol(getname(getgeomdefn(featuredefn, i-1))) for i in 1:ngeom(layer)) - v = Union{types_fields..., IGeometry}[] + field_names = Tuple(Symbol(getname(fielddefn)) for fielddefn in fielddefns) + geom_names = (Symbol(getname(getgeomdefn(featuredefn, i-1))) for i in 1:ngeom(layer)) + v = Union{Tables.schema(layer).types...}[] nextfeature(layer) do feature for name in field_names push!(v, getfield(feature, name)) @@ -83,7 +67,6 @@ function Base.getindex(t::Table, idx::Int) end row = NamedTuple{(field_names..., geom_names...)}(v) end - function Base.show(io::IO, t::Table) println(io, "Table with $(nfeature(getlayer(t))) features") diff --git a/test/test_tables.jl b/test/test_tables.jl index cc8b3934..b139f9d1 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -3,7 +3,6 @@ import ArchGDAL; const AG = ArchGDAL using Tables @testset "Tables Support" begin - dataset = AG.read(joinpath(@__DIR__, "data/point.geojson")) dataset1 = AG.read(joinpath(@__DIR__, "data/multi_geom.csv"), options = ["GEOM_POSSIBLE_NAMES=point,linestring", "KEEP_GEOM_COLUMNS=NO"]) @test dataset isa ArchGDAL.IDataset From 513fe6581fa05f2deaf930bed0a17c55560cf8df Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Mon, 24 Aug 2020 03:57:03 +0530 Subject: [PATCH 53/62] Refaactors; pretty code :) --- docs/src/tables.md | 2 +- src/tables.jl | 25 ++++++++++--------------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/docs/src/tables.md b/docs/src/tables.md index 8ee530a3..10e3d476 100644 --- a/docs/src/tables.md +++ b/docs/src/tables.md @@ -31,7 +31,7 @@ table = ArchGDAL.Table(layer) Individual rows can be retrieved using the `Base.getindex(t::ArchGDAL.Table, idx::Int)` method. ```@example tables -row = Base.getindex(table, 1) +row = table[1] ``` Layers are retrievable! diff --git a/src/tables.jl b/src/tables.jl index c3c4b103..f00d64d5 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -37,18 +37,18 @@ function Base.iterate(t::Table, st = 0) return get_row(layer), st + 1 end -Base.IteratorSize(::Type{<:Table}) = Base.HasLength() -Base.size(t::Table) = nfeature(getlayer(t)) -Base.length(t::Table) = size(t) -Base.IteratorEltype(::Type{<:Table}) = Base.HasEltype() -Base.propertynames(t::Table) = Tables.schema(getlayer(t)).names -Base.getproperty(t::Table, s::Symbol) = [getproperty(iterate(t, i)[1], s) for i in 0:size(t)-1] - function Base.getindex(t::Table, idx::Int) layer = getlayer(t) setnextbyindex!(layer, idx) return get_row(layer) end + +Base.IteratorSize(::Type{<:Table}) = Base.HasLength() +Base.size(t::Table) = nfeature(getlayer(t)) +Base.length(t::Table) = size(t) +Base.IteratorEltype(::Type{<:Table}) = Base.HasEltype() +Base.propertynames(t::Table) = Tables.schema(getlayer(t)).names +Base.getproperty(t::Table, s::Symbol) = [getproperty(row, s) for row in t] function get_row(layer::IFeatureLayer) featuredefn = layerdefn(layer) @@ -56,16 +56,11 @@ function get_row(layer::IFeatureLayer) field_names = Tuple(Symbol(getname(fielddefn)) for fielddefn in fielddefns) geom_names = (Symbol(getname(getgeomdefn(featuredefn, i-1))) for i in 1:ngeom(layer)) - v = Union{Tables.schema(layer).types...}[] nextfeature(layer) do feature - for name in field_names - push!(v, getfield(feature, name)) - end - for idx in 1:length(geom_names) - push!(v, getgeom(feature, idx-1)) - end + prop = (getfield(feature, name) for name in field_names) + geom = (getgeom(feature, idx-1) for idx in 1:length(geom_names)) + row = NamedTuple{(field_names..., geom_names...)}((prop..., geom...)) end - row = NamedTuple{(field_names..., geom_names...)}(v) end function Base.show(io::IO, t::Table) From fadaa78260ead12284a9528cac2e5458591131d2 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Wed, 26 Aug 2020 14:27:20 +0530 Subject: [PATCH 54/62] nit's --- src/tables.jl | 40 ++++++++++++++++++++++------------------ test/test_tables.jl | 15 ++++++++++++--- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index f00d64d5..78a01dbb 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -11,17 +11,12 @@ end getlayer(t::Table) = Base.getfield(t, :layer) function Tables.schema(layer::AbstractFeatureLayer) - featuredefn = layerdefn(layer) - fielddefns = (getfielddefn(featuredefn, i) for i in 0:nfield(layer)-1) - names_fields = (getname(fielddefn) for fielddefn in fielddefns) - types_fields = (_FIELDTYPE[gettype(fielddefn)] for fielddefn in fielddefns) - + field_names, geom_names, featuredefn, fielddefns = schema_names(layer) ngeom = ArchGDAL.ngeom(featuredefn) geomdefns = (ArchGDAL.getgeomdefn(featuredefn, i) for i in 0:ngeom-1) - geom_names = (ArchGDAL.getname(geomdefn) for geomdefn in geomdefns) + field_types = (_FIELDTYPE[gettype(fielddefn)] for fielddefn in fielddefns) geom_types = (IGeometry for i in 1:ngeom) - - Tables.Schema((names_fields..., geom_names...), (types_fields..., geom_types...)) + Tables.Schema((field_names..., geom_names...), (field_types..., geom_types...)) end Tables.istable(::Type{<:Table}) = true @@ -34,13 +29,13 @@ function Base.iterate(t::Table, st = 0) if iszero(st) resetreading!(layer) end - return get_row(layer), st + 1 + return nextnamedtuple(layer), st + 1 end function Base.getindex(t::Table, idx::Int) layer = getlayer(t) setnextbyindex!(layer, idx) - return get_row(layer) + return nextnamedtuple(layer) end Base.IteratorSize(::Type{<:Table}) = Base.HasLength() @@ -50,19 +45,28 @@ Base.IteratorEltype(::Type{<:Table}) = Base.HasEltype() Base.propertynames(t::Table) = Tables.schema(getlayer(t)).names Base.getproperty(t::Table, s::Symbol) = [getproperty(row, s) for row in t] -function get_row(layer::IFeatureLayer) - featuredefn = layerdefn(layer) - fielddefns = (getfielddefn(featuredefn, i) for i in 0:nfield(layer)-1) - field_names = Tuple(Symbol(getname(fielddefn)) for fielddefn in fielddefns) - geom_names = (Symbol(getname(getgeomdefn(featuredefn, i-1))) for i in 1:ngeom(layer)) - - nextfeature(layer) do feature +""" +nextnamedtuple(layer::IFeatureLayer) +Returns the feature row of a layer as a `NamedTuple` +Calling it iteratively will work similar to `nextfeature` i.e. give the consecutive feature as `NamedTuple` +""" +function nextnamedtuple(layer::IFeatureLayer) + field_names, geom_names = schema_names(layer) + return nextfeature(layer) do feature prop = (getfield(feature, name) for name in field_names) geom = (getgeom(feature, idx-1) for idx in 1:length(geom_names)) - row = NamedTuple{(field_names..., geom_names...)}((prop..., geom...)) + NamedTuple{(field_names..., geom_names...)}((prop..., geom...)) end end +function schema_names(layer::AbstractFeatureLayer) + featuredefn = layerdefn(layer) + fielddefns = (getfielddefn(featuredefn, i) for i in 0:nfield(layer)-1) + field_names = (Symbol(getname(fielddefn)) for fielddefn in fielddefns) + geom_names = (Symbol(getname(getgeomdefn(featuredefn, i-1))) for i in 1:ngeom(layer)) + return (field_names, geom_names, featuredefn, fielddefns) +end + function Base.show(io::IO, t::Table) println(io, "Table with $(nfeature(getlayer(t))) features") end diff --git a/test/test_tables.jl b/test/test_tables.jl index b139f9d1..6b29e700 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -3,8 +3,8 @@ import ArchGDAL; const AG = ArchGDAL using Tables @testset "Tables Support" begin - dataset = AG.read(joinpath(@__DIR__, "data/point.geojson")) - dataset1 = AG.read(joinpath(@__DIR__, "data/multi_geom.csv"), options = ["GEOM_POSSIBLE_NAMES=point,linestring", "KEEP_GEOM_COLUMNS=NO"]) + dataset = AG.read("point.geojson") + dataset1 = AG.read("multi_geom.csv", options = ["GEOM_POSSIBLE_NAMES=point,linestring", "KEEP_GEOM_COLUMNS=NO"]) @test dataset isa ArchGDAL.IDataset @test dataset1 isa ArchGDAL.IDataset layer = AG.getlayer(dataset, 0) @@ -12,7 +12,6 @@ using Tables gt = AG.Table(layer) gt1 = AG.Table(layer1) - @testset "read layer to table" begin @test AG.getlayer(gt) === layer @test AG.getlayer(gt1) === layer1 @@ -43,5 +42,15 @@ using Tables @test iterate(gt1, 3) === nothing @test typeof([getindex(gt, i) for i in 0:size(gt)-1]) == typeof([iterate(gt, i)[1] for i in 0:size(gt)-1]) @test typeof([getindex(gt1, i) for i in 0:size(gt1)-1]) == typeof([iterate(gt1, i)[1] for i in 0:size(gt1)-1]) + + AG.resetreading!(layer) + AG.resetreading!(layer1) + + @test AG.nextnamedtuple(layer) isa NamedTuple{(:FID, :pointname, Symbol("")),Tuple{Float64,String,ArchGDAL.IGeometry}} + @test AG.nextnamedtuple(layer1) isa NamedTuple{(:id, :zoom, :location, :point, :linestring),Tuple{String,String,String,ArchGDAL.IGeometry,ArchGDAL.IGeometry}} + for i in 1:4 + @test AG.schema_names(layer)[i] isa Base.Generator || AG.schema_names(layer)[i] isa ArchGDAL.IFeatureDefnView + @test AG.schema_names(layer1)[i] isa Base.Generator || AG.schema_names(layer1)[i] isa ArchGDAL.IFeatureDefnView + end end end From 07f210c5acf43168c996e14416fcc3c282b5c49e Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Wed, 26 Aug 2020 14:29:37 +0530 Subject: [PATCH 55/62] Fix path in tests --- test/test_tables.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_tables.jl b/test/test_tables.jl index 6b29e700..6daf3c60 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -3,8 +3,8 @@ import ArchGDAL; const AG = ArchGDAL using Tables @testset "Tables Support" begin - dataset = AG.read("point.geojson") - dataset1 = AG.read("multi_geom.csv", options = ["GEOM_POSSIBLE_NAMES=point,linestring", "KEEP_GEOM_COLUMNS=NO"]) + dataset = AG.read(joinpath(@__DIR__, "data/point.geojson")) + dataset1 = AG.read(joinpath(@__DIR__, "data/multi_geom.csv"), options = ["GEOM_POSSIBLE_NAMES=point,linestring", "KEEP_GEOM_COLUMNS=NO"]) @test dataset isa ArchGDAL.IDataset @test dataset1 isa ArchGDAL.IDataset layer = AG.getlayer(dataset, 0) From eff39c9798ed325edc22579942d8fe92d8440a6b Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Wed, 26 Aug 2020 14:57:39 +0530 Subject: [PATCH 56/62] Add nextnamedtuple to docs --- docs/src/tables.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/src/tables.md b/docs/src/tables.md index 10e3d476..bff8d8d4 100644 --- a/docs/src/tables.md +++ b/docs/src/tables.md @@ -63,3 +63,10 @@ Finally layers can be converted to DataFrames to perform miscellaneous spatial o ```@example tables df = DataFrame(table) ``` +In some cases the `nextfeature` might become a bit tedious to use. In which case the `ArchGDAL.nextnamedtuple()` method comes in handy. Though built upon `nextfeature`, simply calling it, yields the `feature` as a `NamedTuple`. Though one might have to use `ArchGDAL.resetreading!(layer)` method to reset the layer reading to the start. + +```@example tables +ArchGDAL.resetrading!(layer) +feat1 = ArchGDAL.nextnamedtuple(layer) +feat2 = ArchGDAL.nextnamedtuple(layer) +``` From 7677ae798c61837ac2eccf77b77d060cd3c61644 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Fri, 28 Aug 2020 03:37:33 +0530 Subject: [PATCH 57/62] typo fix --- docs/src/tables.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/tables.md b/docs/src/tables.md index bff8d8d4..af8b5b7c 100644 --- a/docs/src/tables.md +++ b/docs/src/tables.md @@ -66,7 +66,7 @@ df = DataFrame(table) In some cases the `nextfeature` might become a bit tedious to use. In which case the `ArchGDAL.nextnamedtuple()` method comes in handy. Though built upon `nextfeature`, simply calling it, yields the `feature` as a `NamedTuple`. Though one might have to use `ArchGDAL.resetreading!(layer)` method to reset the layer reading to the start. ```@example tables -ArchGDAL.resetrading!(layer) +ArchGDAL.resetreading!(layer) feat1 = ArchGDAL.nextnamedtuple(layer) feat2 = ArchGDAL.nextnamedtuple(layer) ``` From 87eb4f7c04d7f056d2643c323de2a0c58b451d72 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Mon, 31 Aug 2020 20:09:58 +0530 Subject: [PATCH 58/62] Test for null/missing --- test/remotefiles.jl | 1 + test/test_tables.jl | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/test/remotefiles.jl b/test/remotefiles.jl index 3c3631d9..77554b53 100644 --- a/test/remotefiles.jl +++ b/test/remotefiles.jl @@ -21,6 +21,7 @@ julia> open(filepath/filename) do f """ remotefiles = [ ("data/multi_geom.csv", "00520017658b66ff21e40cbf553672fa8e280cddae6e7a5d1f8bd36bcd521770"), + ("data/missing_testcase.csv", "d49ba446aae9ef334350b64c876b4de652f28595fdecf78bea4e16af4033f7c6"), ("data/point.geojson", "8744593479054a67c784322e0c198bfa880c9388b39a2ddd4c56726944711bd9"), ("data/utmsmall.tif", "f40dae6e8b5e18f3648e9f095e22a0d7027014bb463418d32f732c3756d8c54f"), ("gdalworkshop/world.tif", "b376dc8af62f9894b5050a6a9273ac0763ae2990b556910d35d4a8f4753278bb"), diff --git a/test/test_tables.jl b/test/test_tables.jl index 6daf3c60..8fb81ea4 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -5,39 +5,53 @@ using Tables @testset "Tables Support" begin dataset = AG.read(joinpath(@__DIR__, "data/point.geojson")) dataset1 = AG.read(joinpath(@__DIR__, "data/multi_geom.csv"), options = ["GEOM_POSSIBLE_NAMES=point,linestring", "KEEP_GEOM_COLUMNS=NO"]) + dataset2 = AG.read(joinpath(@__DIR__, "data/missing_testcase.csv"), options = ["GEOM_POSSIBLE_NAMES=point,linestring", "KEEP_GEOM_COLUMNS=NO"]) @test dataset isa ArchGDAL.IDataset @test dataset1 isa ArchGDAL.IDataset + @test dataset2 isa ArchGDAL.IDataset layer = AG.getlayer(dataset, 0) layer1 = AG.getlayer(dataset1, 0) + layer2 = AG.getlayer(dataset2, 0) gt = AG.Table(layer) gt1 = AG.Table(layer1) - + gt2 = AG.Table(layer2) + @testset "read layer to table" begin @test AG.getlayer(gt) === layer @test AG.getlayer(gt1) === layer1 + @test AG.getlayer(gt2) === layer2 @test sprint(print, gt) == "Table with 4 features\n" @test sprint(print, gt1) == "Table with 2 features\n" + @test sprint(print, gt2) == "Table with 9 features\n" end @testset "Tables methods" begin @test Tables.schema(layer).names == propertynames(gt) @test Tables.schema(layer1).names == propertynames(gt1) + @test Tables.schema(layer2).names == propertynames(gt2) @test Tables.istable(AG.Table) == true @test Tables.rows(gt) == AG.Table(layer) @test Tables.rows(gt1) == AG.Table(layer1) + @test Tables.rows(gt2) == AG.Table(layer2) end @testset "Misc. methods" begin @test Base.size(gt) == 4 @test Base.size(gt1) == 2 + @test Base.size(gt2) == 9 @test Base.length(gt) == 4 @test Base.length(gt1) == 2 + @test Base.length(gt2) == 9 @test Base.IteratorSize(typeof(gt)) == Base.HasLength() @test Base.IteratorEltype(typeof(gt1)) == Base.HasEltype() @test propertynames(gt) == (:FID, :pointname, Symbol("")) @test propertynames(gt1) == (:id, :zoom, :location, :point, :linestring) + @test propertynames(gt2) == (:id, :zoom, :location, :point, :linestring) @test getproperty(gt, :FID) == [iterate(gt, i)[1].FID for i in 0:size(gt)-1] @test getproperty(gt1, :zoom) == [iterate(gt1, i)[1].zoom for i in 0:size(gt1)-1] + @test sprint(print, gt2[4].linestring) == sprint(print, gt2[2].point) + @test sprint(print, gt2[8].linestring) == sprint(print, gt2[6].point) + @test collect(findall(x->x=="missing", df[:,i]) for i in 1:3) == [6, [4, 8], [3, 7, 8]] @test iterate(gt, 5) === nothing @test iterate(gt1, 3) === nothing @test typeof([getindex(gt, i) for i in 0:size(gt)-1]) == typeof([iterate(gt, i)[1] for i in 0:size(gt)-1]) From d5c0ed371351f9ba669185d97d065b81f5f96a60 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Tue, 1 Sep 2020 10:39:35 +0530 Subject: [PATCH 59/62] Refactor tests --- test/test_tables.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_tables.jl b/test/test_tables.jl index 8fb81ea4..b29a2cfb 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -51,7 +51,7 @@ using Tables @test getproperty(gt1, :zoom) == [iterate(gt1, i)[1].zoom for i in 0:size(gt1)-1] @test sprint(print, gt2[4].linestring) == sprint(print, gt2[2].point) @test sprint(print, gt2[8].linestring) == sprint(print, gt2[6].point) - @test collect(findall(x->x=="missing", df[:,i]) for i in 1:3) == [6, [4, 8], [3, 7, 8]] + @test collect(findall(x->x=="missing", getproperty(gt2, i)) for i in [:id, :zoom, :location]) == [[6], [4, 8], [3, 7, 8]] @test iterate(gt, 5) === nothing @test iterate(gt1, 3) === nothing @test typeof([getindex(gt, i) for i in 0:size(gt)-1]) == typeof([iterate(gt, i)[1] for i in 0:size(gt)-1]) From be2df2a81c0c4615ebd5d6d2a6d98c5f0e02f745 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Tue, 1 Sep 2020 13:02:31 +0530 Subject: [PATCH 60/62] nit's --- src/tables.jl | 2 +- test/test_featurelayer.jl | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/tables.jl b/src/tables.jl index 78a01dbb..6694a3ef 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -46,8 +46,8 @@ Base.propertynames(t::Table) = Tables.schema(getlayer(t)).names Base.getproperty(t::Table, s::Symbol) = [getproperty(row, s) for row in t] """ -nextnamedtuple(layer::IFeatureLayer) Returns the feature row of a layer as a `NamedTuple` + Calling it iteratively will work similar to `nextfeature` i.e. give the consecutive feature as `NamedTuple` """ function nextnamedtuple(layer::IFeatureLayer) diff --git a/test/test_featurelayer.jl b/test/test_featurelayer.jl index df1205c0..2e5b5fc4 100644 --- a/test/test_featurelayer.jl +++ b/test/test_featurelayer.jl @@ -147,14 +147,14 @@ import ArchGDAL; const AG = ArchGDAL @test AG.testcapability(layer,"OLCRandomWrite") == false end end - @test AG.findfieldindex(layer,"FID", true) == 0 - @test AG.findfieldindex(layer,:FID, false) == 0 - @test AG.findfieldindex(layer,"pointname", true) == 1 - @test AG.findfieldindex(layer,"pointname", false) == 1 - @test AG.findfieldindex(layer,"geom", true) == -1 - @test AG.findfieldindex(layer,"geom", true) == -1 - @test AG.findfieldindex(layer,"rubbish", true) == -1 - @test AG.findfieldindex(layer,"rubbish", false) == -1 + @test AG.findfieldindex(layer, "FID", true) == 0 + @test AG.findfieldindex(layer, :FID, false) == 0 + @test AG.findfieldindex(layer, "pointname", true) == 1 + @test AG.findfieldindex(layer, "pointname", false) == 1 + @test AG.findfieldindex(layer, "geom", true) == -1 + @test AG.findfieldindex(layer, "geom", true) == -1 + @test AG.findfieldindex(layer, "rubbish", true) == -1 + @test AG.findfieldindex(layer, "rubbish", false) == -1 @test sprint(print, AG.envelope(layer, 0, true)) == "GDAL.OGREnvelope(100.0, 100.2785, 0.0, 0.0893)" @test sprint(print, AG.envelope(layer, true)) == "GDAL.OGREnvelope(100.0, 100.2785, 0.0, 0.0893)" end From 3247789509a10abd34386f3ec9feae8c418c9c76 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Mon, 14 Sep 2020 03:04:14 +0530 Subject: [PATCH 61/62] 1-based indexing --- docs/src/tables.md | 4 ++-- src/tables.jl | 4 ++-- test/test_tables.jl | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/src/tables.md b/docs/src/tables.md index af8b5b7c..0f8ae9f6 100644 --- a/docs/src/tables.md +++ b/docs/src/tables.md @@ -28,7 +28,7 @@ The [`ArchGDAL.Table`](@ref) method accepts an `ArchGDAL.FeatureLayer`. table = ArchGDAL.Table(layer) ``` -Individual rows can be retrieved using the `Base.getindex(t::ArchGDAL.Table, idx::Int)` method. +Individual rows can be retrieved using the `Base.getindex(t::ArchGDAL.Table, idx::Int)` method or simply `table[idx]`. ```@example tables row = table[1] @@ -56,7 +56,7 @@ table = ArchGDAL.Table(layer) Exatracting a row from the table, we see that the row/feature is made up of two geometries viz. `point` and `linestring`. ```@example tables -row = Base.getindex(table, 1) +row = table[1] ``` Finally layers can be converted to DataFrames to perform miscellaneous spatial operations. diff --git a/src/tables.jl b/src/tables.jl index 6694a3ef..ed788df0 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -34,7 +34,7 @@ end function Base.getindex(t::Table, idx::Int) layer = getlayer(t) - setnextbyindex!(layer, idx) + setnextbyindex!(layer, idx-1) return nextnamedtuple(layer) end @@ -44,7 +44,7 @@ Base.length(t::Table) = size(t) Base.IteratorEltype(::Type{<:Table}) = Base.HasEltype() Base.propertynames(t::Table) = Tables.schema(getlayer(t)).names Base.getproperty(t::Table, s::Symbol) = [getproperty(row, s) for row in t] - + """ Returns the feature row of a layer as a `NamedTuple` diff --git a/test/test_tables.jl b/test/test_tables.jl index b29a2cfb..81ca3bd3 100644 --- a/test/test_tables.jl +++ b/test/test_tables.jl @@ -49,13 +49,13 @@ using Tables @test propertynames(gt2) == (:id, :zoom, :location, :point, :linestring) @test getproperty(gt, :FID) == [iterate(gt, i)[1].FID for i in 0:size(gt)-1] @test getproperty(gt1, :zoom) == [iterate(gt1, i)[1].zoom for i in 0:size(gt1)-1] - @test sprint(print, gt2[4].linestring) == sprint(print, gt2[2].point) - @test sprint(print, gt2[8].linestring) == sprint(print, gt2[6].point) + @test sprint(print, gt2[5].linestring) == sprint(print, gt2[3].point) + @test sprint(print, gt2[9].linestring) == sprint(print, gt2[7].point) @test collect(findall(x->x=="missing", getproperty(gt2, i)) for i in [:id, :zoom, :location]) == [[6], [4, 8], [3, 7, 8]] @test iterate(gt, 5) === nothing @test iterate(gt1, 3) === nothing - @test typeof([getindex(gt, i) for i in 0:size(gt)-1]) == typeof([iterate(gt, i)[1] for i in 0:size(gt)-1]) - @test typeof([getindex(gt1, i) for i in 0:size(gt1)-1]) == typeof([iterate(gt1, i)[1] for i in 0:size(gt1)-1]) + @test typeof([getindex(gt, i) for i in 1:size(gt)]) == typeof([iterate(gt, i)[1] for i in 0:size(gt)-1]) + @test typeof([getindex(gt1, i) for i in 1:size(gt1)]) == typeof([iterate(gt1, i)[1] for i in 0:size(gt1)-1]) AG.resetreading!(layer) AG.resetreading!(layer1) @@ -64,7 +64,7 @@ using Tables @test AG.nextnamedtuple(layer1) isa NamedTuple{(:id, :zoom, :location, :point, :linestring),Tuple{String,String,String,ArchGDAL.IGeometry,ArchGDAL.IGeometry}} for i in 1:4 @test AG.schema_names(layer)[i] isa Base.Generator || AG.schema_names(layer)[i] isa ArchGDAL.IFeatureDefnView - @test AG.schema_names(layer1)[i] isa Base.Generator || AG.schema_names(layer1)[i] isa ArchGDAL.IFeatureDefnView + @test AG.schema_names(layer1)[i] isa Base.Generator || AG.schema_names(layer1)[i] isa ArchGDAL.IFeatureDefnView end end end From 84380a5db6799640c3bb63676b484d5891172fd2 Mon Sep 17 00:00:00 2001 From: ARSH SHARMA <184517@nith.ac.in> Date: Tue, 15 Sep 2020 00:48:06 +0530 Subject: [PATCH 62/62] 32bit fix --- src/tables.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tables.jl b/src/tables.jl index ed788df0..0d6d6d13 100644 --- a/src/tables.jl +++ b/src/tables.jl @@ -32,7 +32,7 @@ function Base.iterate(t::Table, st = 0) return nextnamedtuple(layer), st + 1 end -function Base.getindex(t::Table, idx::Int) +function Base.getindex(t::Table, idx::Integer) layer = getlayer(t) setnextbyindex!(layer, idx-1) return nextnamedtuple(layer)