Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve naming in data frame abstraction #857

Merged
merged 4 commits into from
Jun 1, 2024
Merged

Improve naming in data frame abstraction #857

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
dc0703c
Improve naming in the schema package
bchapuis May 30, 2024
c986660
Clean and refactor the baremaps-data module
bchapuis May 30, 2024
88c0d8b
Add some documentation
bchapuis May 30, 2024
d7358c6
Fix comment
bchapuis Jun 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 19 additions & 19 deletions ...rage/flatgeobuf/FlatGeoBufDataSchema.java → ...orage/flatgeobuf/FlatGeoBufDataStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,41 +21,41 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import org.apache.baremaps.data.schema.DataSchema;
import org.apache.baremaps.data.schema.DataTable;
import org.apache.baremaps.data.schema.DataTableException;
import org.apache.baremaps.data.storage.DataStore;
import org.apache.baremaps.data.storage.DataStoreException;
import org.apache.baremaps.data.storage.DataTable;

/**
* A schema corresponding to the flatgeobuf files of a directory.
* A {@link DataStore} corresponding to the flatgeobuf files of a directory.
*/
public class FlatGeoBufDataSchema implements DataSchema {
public class FlatGeoBufDataStore implements DataStore {

private final Path directory;

public FlatGeoBufDataSchema(Path directory) {
public FlatGeoBufDataStore(Path directory) {
this.directory = directory;
}

/**
* {@inheritDoc}
*/
@Override
public List<String> list() throws DataTableException {
public List<String> list() throws DataStoreException {
try (var files = Files.list(directory)) {
return files
.filter(file -> file.toString().toLowerCase().endsWith(".fgb"))
.map(file -> file.getFileName().toString())
.toList();
} catch (IOException e) {
throw new DataTableException(e);
throw new DataStoreException(e);
}
}

/**
* {@inheritDoc}
*/
@Override
public DataTable get(String name) throws DataTableException {
public DataTable get(String name) throws DataStoreException {
var path = directory.resolve(name);
return new FlatGeoBufDataTable(path);
}
Expand All @@ -64,39 +64,39 @@ public DataTable get(String name) throws DataTableException {
* {@inheritDoc}
*/
@Override
public void add(DataTable table) throws DataTableException {
var filename = table.rowType().name();
filename = filename.endsWith(".fgb") ? filename : filename + ".fgb";
add(filename, table);
public void add(DataTable table) throws DataStoreException {
var fileName = table.schema().name();
fileName = fileName.endsWith(".fgb") ? fileName : fileName + ".fgb";
add(fileName, table);
}

@Override
public void add(String name, DataTable table) throws DataTableException {
public void add(String name, DataTable table) throws DataStoreException {
var path = directory.resolve(name);
try {
Files.deleteIfExists(path);
Files.createFile(path);
var flatGeoBufTable = new FlatGeoBufDataTable(path, table.rowType());
var flatGeoBufTable = new FlatGeoBufDataTable(path, table.schema());
flatGeoBufTable.write(table);
} catch (IOException e) {
throw new DataTableException(e);
throw new DataStoreException(e);
}
}

/**
* {@inheritDoc}
*/
@Override
public void remove(String name) throws DataTableException {
public void remove(String name) throws DataStoreException {
var path = directory.resolve(name);
if (name.equals(path.getFileName().toString())) {
try {
Files.delete(path);
} catch (IOException e) {
throw new DataTableException(e);
throw new DataStoreException(e);
}
} else {
throw new DataTableException("Table not found");
throw new DataStoreException("Table not found");
}
}
}
66 changes: 32 additions & 34 deletions baremaps-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufDataTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@
import java.util.Iterator;
import java.util.NoSuchElementException;
import org.apache.baremaps.data.collection.DataCollection;
import org.apache.baremaps.data.schema.DataRow;
import org.apache.baremaps.data.schema.DataRowType;
import org.apache.baremaps.data.schema.DataTable;
import org.apache.baremaps.data.storage.DataRow;
import org.apache.baremaps.data.storage.DataSchema;
import org.apache.baremaps.data.storage.DataTable;
import org.locationtech.jts.geom.*;
import org.wololo.flatgeobuf.Constants;
import org.wololo.flatgeobuf.GeometryConversions;
Expand All @@ -39,13 +39,13 @@
import org.wololo.flatgeobuf.generated.GeometryType;

/**
* A table that stores rows in a flatgeobuf file.
* A {@link DataTable} that stores rows in a flatgeobuf file.
*/
public class FlatGeoBufDataTable implements DataTable {

private final Path file;

private DataRowType rowType;
private DataSchema schema;

/**
* Constructs a table from a flatgeobuf file (used for reading).
Expand All @@ -54,40 +54,38 @@ public class FlatGeoBufDataTable implements DataTable {
*/
public FlatGeoBufDataTable(Path file) {
this.file = file;
this.rowType = readRowType(file);
this.schema = readSchema(file);
}


private static DataSchema readSchema(Path file) {
try (var channel = FileChannel.open(file, StandardOpenOption.READ)) {
// try to read the schema from the file
var buffer = ByteBuffer.allocate(1 << 20).order(ByteOrder.LITTLE_ENDIAN);
HeaderMeta headerMeta = readHeaderMeta(channel, buffer);
return FlatGeoBufTypeConversion.asSchema(headerMeta);
} catch (IOException e) {
return null;
}
}

/**
* Constructs a table from a flatgeobuf file and a row type (used for writing).
* Constructs a table from a flatgeobuf file and a schema (used for writing).
*
* @param file the path to the flatgeobuf file
* @param rowType the row type of the table
* @param schema the schema of the table
*/
public FlatGeoBufDataTable(Path file, DataRowType rowType) {
public FlatGeoBufDataTable(Path file, DataSchema schema) {
this.file = file;
this.rowType = rowType;
this.schema = schema;
}

/**
* {@inheritDoc}
*/
@Override
public DataRowType rowType() {
return rowType;
}

/**
* {@inheritDoc}
*/
public static DataRowType readRowType(Path file) {
try (var channel = FileChannel.open(file, StandardOpenOption.READ)) {
// try to read the row type from the file
var buffer = ByteBuffer.allocate(1 << 20).order(ByteOrder.LITTLE_ENDIAN);
HeaderMeta headerMeta = readHeaderMeta(channel, buffer);
return FlatGeoBufTypeConversion.asRowType(headerMeta);
} catch (IOException e) {
return null;
}
public DataSchema schema() {
return schema;
}

/**
Expand All @@ -111,7 +109,7 @@ public Iterator<DataRow> iterator() {
buffer.clear();

// create the feature stream
return new RowIterator(channel, headerMeta, rowType, buffer);
return new RowIterator(channel, headerMeta, schema, buffer);
} catch (IOException e) {
throw new RuntimeException(e);
}
Expand Down Expand Up @@ -173,8 +171,8 @@ public void write(DataCollection<DataRow> features) throws IOException {
headerMeta.indexNodeSize = 16;
headerMeta.srid = 3857;
headerMeta.featuresCount = features.size();
headerMeta.name = rowType.name();
headerMeta.columns = FlatGeoBufTypeConversion.asColumns(rowType.columns());
headerMeta.name = schema.name();
headerMeta.columns = FlatGeoBufTypeConversion.asColumns(schema.columns());
HeaderMeta.write(headerMeta, outputStream, bufferBuilder);

var indexSize =
Expand Down Expand Up @@ -235,7 +233,7 @@ public static class RowIterator implements Iterator<DataRow> {

private final HeaderMeta headerMeta;

private final DataRowType rowType;
private final DataSchema schema;

private final SeekableByteChannel channel;

Expand All @@ -248,14 +246,14 @@ public static class RowIterator implements Iterator<DataRow> {
*
* @param channel the channel to read from
* @param headerMeta the header meta
* @param rowType the row type of the table
* @param schema the schema of the table
* @param buffer the buffer to use
*/
public RowIterator(SeekableByteChannel channel, HeaderMeta headerMeta,
DataRowType rowType, ByteBuffer buffer) {
DataSchema schema, ByteBuffer buffer) {
this.channel = channel;
this.headerMeta = headerMeta;
this.rowType = rowType;
this.schema = schema;
this.buffer = buffer;
}

Expand All @@ -278,7 +276,7 @@ public DataRow next() {

var featureSize = buffer.getInt();
var row =
FlatGeoBufTypeConversion.asRow(headerMeta, rowType, Feature.getRootAsFeature(buffer));
FlatGeoBufTypeConversion.asRow(headerMeta, schema, Feature.getRootAsFeature(buffer));

buffer.position(Integer.BYTES + featureSize);
buffer.compact();
Expand Down
10 changes: 5 additions & 5 deletions ...s-core/src/main/java/org/apache/baremaps/storage/flatgeobuf/FlatGeoBufTypeConversion.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.stream.Collectors;
import org.apache.baremaps.data.schema.*;
import org.apache.baremaps.data.schema.DataColumn.Type;
import org.apache.baremaps.data.storage.*;
import org.apache.baremaps.data.storage.DataColumn.Type;
import org.wololo.flatgeobuf.ColumnMeta;
import org.wololo.flatgeobuf.GeometryConversions;
import org.wololo.flatgeobuf.HeaderMeta;
Expand All @@ -50,16 +50,16 @@ public class FlatGeoBufTypeConversion {
types.put(Type.STRING, ColumnType.String);
}

public static DataRowType asRowType(HeaderMeta headerMeta) {
public static DataSchema asSchema(HeaderMeta headerMeta) {
var name = headerMeta.name;
var columns = headerMeta.columns.stream()
.map(column -> new DataColumnImpl(column.name, Type.fromBinding(column.getBinding())))
.map(DataColumn.class::cast)
.toList();
return new DataRowTypeImpl(name, columns);
return new DataSchemaImpl(name, columns);
}

public static DataRow asRow(HeaderMeta headerMeta, DataRowType dataType, Feature feature) {
public static DataRow asRow(HeaderMeta headerMeta, DataSchema dataType, Feature feature) {
var values = new ArrayList();

var geometryBuffer = feature.geometry();
Expand Down
26 changes: 13 additions & 13 deletions ...rage/geopackage/GeoPackageDataSchema.java → ...orage/geopackage/GeoPackageDataStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,23 +22,23 @@
import java.util.List;
import mil.nga.geopackage.GeoPackage;
import mil.nga.geopackage.GeoPackageManager;
import org.apache.baremaps.data.schema.DataSchema;
import org.apache.baremaps.data.schema.DataTable;
import org.apache.baremaps.data.schema.DataTableException;
import org.apache.baremaps.data.storage.DataStore;
import org.apache.baremaps.data.storage.DataStoreException;
import org.apache.baremaps.data.storage.DataTable;

/**
* A schema corresponding to a GeoPackage database.
* A {@link DataStore} corresponding to a GeoPackage file.
*/
public class GeoPackageDataSchema implements DataSchema, AutoCloseable {
public class GeoPackageDataStore implements DataStore, AutoCloseable {

private final GeoPackage geoPackage;

/**
* Constructs a schema from a GeoPackage database.
* Constructs a {@link GeoPackageDataStore} from a GeoPackage file.
*
* @param file the path to the GeoPackage database
* @param file the path to the GeoPackage file
*/
public GeoPackageDataSchema(Path file) {
public GeoPackageDataStore(Path file) {
this.geoPackage = GeoPackageManager.open(file.toFile());
}

Expand All @@ -54,36 +54,36 @@ public void close() throws Exception {
* {@inheritDoc}
*/
@Override
public List<String> list() throws DataTableException {
public List<String> list() throws DataStoreException {
return geoPackage.getFeatureTables();
}

/**
* {@inheritDoc}
*/
@Override
public DataTable get(String name) throws DataTableException {
public DataTable get(String name) throws DataStoreException {
return new GeoPackageDataTable(geoPackage.getFeatureDao(name));
}

/**
* {@inheritDoc}
*/
@Override
public void add(DataTable table) throws DataTableException {
public void add(DataTable table) throws DataStoreException {
throw new UnsupportedOperationException();
}

@Override
public void add(String name, DataTable table) throws DataTableException {
public void add(String name, DataTable table) throws DataStoreException {
throw new UnsupportedOperationException();
}

/**
* {@inheritDoc}
*/
@Override
public void remove(String name) throws DataTableException {
public void remove(String name) throws DataStoreException {
throw new UnsupportedOperationException();
}
}
Loading
Loading