From 5195fd08585d4458aee9363a6e5171de83ee875d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Rodrigo?= Date: Wed, 25 May 2022 21:14:14 +0200 Subject: [PATCH] Add option enableKeepPartialLinestring to generate anyway partial geometry when missing/invalid nodes are found --- doc/detailed-usage.adoc | 24 +++++++++++++++++++ .../pgsimple/v0_6/PostgreSqlCopyWriter.java | 10 ++++++-- .../v0_6/PostgreSqlCopyWriterFactory.java | 7 +++++- .../pgsimple/v0_6/PostgreSqlDumpWriter.java | 9 +++++-- .../v0_6/PostgreSqlDumpWriterFactory.java | 8 ++++++- .../pgsimple/v0_6/PostgreSqlWriter.java | 12 +++++++--- .../v0_6/PostgreSqlWriterFactory.java | 8 ++++++- .../v0_6/impl/CopyFilesetBuilder.java | 10 ++++++-- .../v0_6/impl/WayGeometryBuilder.java | 9 +++++-- .../pgsnapshot/v0_6/PostgreSqlCopyWriter.java | 10 ++++++-- .../v0_6/PostgreSqlCopyWriterFactory.java | 6 +++++ .../pgsnapshot/v0_6/PostgreSqlDumpWriter.java | 11 ++++++--- .../v0_6/PostgreSqlDumpWriterFactory.java | 8 ++++++- .../v0_6/impl/CopyFilesetBuilder.java | 11 ++++++--- .../v0_6/impl/WayGeometryBuilder.java | 9 +++++-- 15 files changed, 127 insertions(+), 25 deletions(-) diff --git a/doc/detailed-usage.adoc b/doc/detailed-usage.adoc index 1ab80e6ba..43630c636 100644 --- a/doc/detailed-usage.adoc +++ b/doc/detailed-usage.adoc @@ -1680,6 +1680,12 @@ improvements compared to the query approach. |yes, no |no |enableLinestringBuilder |As per the enableBboxBuilder option but for the linestring geometry column. |yes, no |no +|enableKeepPartialLinestring |This option affects how linestrings are +built from option enableLinestringBuilder. When an invalid or a missing +node location is encountered the linestring is not built by default. +Enabling this option keeps it as a partial linestring. It will result in +a different geometry than the original one. |yes, no |no + |nodeLocationStoreType |This option only takes effect if at least one of the enableBboxBuilder and enableLinestringBuilder options are enabled. Both geometry builder implementations require knowledge of all node @@ -1891,6 +1897,12 @@ much slower but still faster than relying on the default database geometry building implementation, or the "CompactTempFile" option which is more efficient for smaller datasets. |"InMemory", "TempFile", "CompactTempFile" |"CompactTempFile" + +|enableKeepPartialLinestring |This option affects how linestrings are +built. When an invalid or a missing node location is encountered the +linestring is not built by default. Enabling this option keeps it as a +partial linestring. It will result in a different geometry than the +original one. |yes, no |no |======================================================================= ==== --write-pgsimp-dump (--wsd) @@ -1923,6 +1935,12 @@ improvements compared to the query approach. |yes, no |no |enableLinestringBuilder |As per the enableBboxBuilder option but for the linestring geometry column. |yes, no |no +|enableKeepPartialLinestring |This option affects how linestrings are +built from option enableLinestringBuilder. When an invalid or a missing +node location is encountered the linestring is not built by default. +Enabling this option keeps it as a partial linestring. It will result in +a different geometry than the original one. |yes, no |no + |nodeLocationStoreType |This option only takes effect if at least one of the enableBboxBuilder and enableLinestringBuilder options are enabled. Both geometry builder implementations require knowledge of all node @@ -1934,6 +1952,12 @@ you may use the "InMemory" option. Otherwise you must choose between the the default database geometry building implementation, or the "CompactTempFile" option which is more efficient for smaller datasets. |"InMemory", "TempFile", "CompactTempFile" |"CompactTempFile" + +|enableKeepPartialLinestring |This option affects how linestrings are +built. When an invalid or a missing node location is encountered the +linestring is not built by default. Enabling this option keeps it as a +partial linestring. It will result in a different geometry than the +original one. |yes, no |no |======================================================================= ==== --truncate-pgsimp (--ts) diff --git a/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlCopyWriter.java b/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlCopyWriter.java index 0c7ac891c..3347dd857 100644 --- a/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlCopyWriter.java +++ b/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlCopyWriter.java @@ -34,6 +34,7 @@ public class PostgreSqlCopyWriter implements Sink { private NodeLocationStoreType storeType; private boolean populateBbox; private boolean populateLinestring; + private boolean enableKeepPartialLinestring; private boolean initialized; @@ -46,13 +47,17 @@ public class PostgreSqlCopyWriter implements Sink { * Contains preferences configuring database behaviour. * @param storeType * The node location storage type used by the geometry builders. + * @param enableKeepPartialLinestring + * If true, the way linestring is build even on invalid or missing + * nodes. */ public PostgreSqlCopyWriter( DatabaseLoginCredentials loginCredentials, DatabasePreferences preferences, - NodeLocationStoreType storeType) { + NodeLocationStoreType storeType, boolean enableKeepPartialLinestring) { this.loginCredentials = loginCredentials; this.preferences = preferences; this.storeType = storeType; + this.enableKeepPartialLinestring = enableKeepPartialLinestring; copyFileset = new TempCopyFileset(); } @@ -69,7 +74,8 @@ private void initialize() { populateLinestring = capabilityChecker.isWayLinestringSupported(); copyFilesetBuilder = - new CopyFilesetBuilder(copyFileset, populateBbox, populateLinestring, storeType); + new CopyFilesetBuilder(copyFileset, populateBbox, populateLinestring, enableKeepPartialLinestring, + storeType); copyFilesetLoader = new CopyFilesetLoader(loginCredentials, preferences, copyFileset); diff --git a/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlCopyWriterFactory.java b/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlCopyWriterFactory.java index 88341a776..cf1ddccc5 100644 --- a/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlCopyWriterFactory.java +++ b/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlCopyWriterFactory.java @@ -18,6 +18,8 @@ public class PostgreSqlCopyWriterFactory extends DatabaseTaskManagerFactory { private static final String ARG_NODE_LOCATION_STORE_TYPE = "nodeLocationStoreType"; private static final String DEFAULT_NODE_LOCATION_STORE_TYPE = "CompactTempFile"; + private static final String ARG_ENABLE_KEEP_PARTIAL_LIENSTRING = "enableKeepPartialLinestring"; + private static final boolean DEFAULT_ENABLE_KEEP_PARTIAL_LIENSTRING = false; /** * {@inheritDoc} @@ -27,6 +29,7 @@ protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) { DatabaseLoginCredentials loginCredentials; DatabasePreferences preferences; NodeLocationStoreType storeType; + boolean enableKeepPartialLinestring; // Get the task arguments. loginCredentials = getDatabaseLoginCredentials(taskConfig); @@ -34,10 +37,12 @@ protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) { storeType = Enum.valueOf( NodeLocationStoreType.class, getStringArgument(taskConfig, ARG_NODE_LOCATION_STORE_TYPE, DEFAULT_NODE_LOCATION_STORE_TYPE)); + enableKeepPartialLinestring = getBooleanArgument(taskConfig, ARG_ENABLE_KEEP_PARTIAL_LIENSTRING, + DEFAULT_ENABLE_KEEP_PARTIAL_LIENSTRING); return new SinkManager( taskConfig.getId(), - new PostgreSqlCopyWriter(loginCredentials, preferences, storeType), + new PostgreSqlCopyWriter(loginCredentials, preferences, storeType, enableKeepPartialLinestring), taskConfig.getPipeArgs() ); } diff --git a/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlDumpWriter.java b/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlDumpWriter.java index 79b1b41c4..cfab41d83 100644 --- a/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlDumpWriter.java +++ b/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlDumpWriter.java @@ -37,18 +37,23 @@ public class PostgreSqlDumpWriter implements Sink { * processing instead of relying on the database to build them * after import. This increases processing but is faster than * relying on the database. + * @param enableKeepPartialLinestring + * If true, the way linestring is build even on invalid or missing + * nodes. * @param storeType * The node location storage type used by the geometry builders. */ public PostgreSqlDumpWriter( File filePrefix, boolean enableBboxBuilder, - boolean enableLinestringBuilder, NodeLocationStoreType storeType) { + boolean enableLinestringBuilder, boolean enableKeepPartialLinestring, + NodeLocationStoreType storeType) { DirectoryCopyFileset copyFileset; copyFileset = new DirectoryCopyFileset(filePrefix); copyFilesetBuilder = - new CopyFilesetBuilder(copyFileset, enableBboxBuilder, enableLinestringBuilder, storeType); + new CopyFilesetBuilder(copyFileset, enableBboxBuilder, enableLinestringBuilder, + enableKeepPartialLinestring, storeType); } diff --git a/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlDumpWriterFactory.java b/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlDumpWriterFactory.java index 3d65647c1..13fe67914 100644 --- a/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlDumpWriterFactory.java +++ b/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlDumpWriterFactory.java @@ -18,10 +18,12 @@ public class PostgreSqlDumpWriterFactory extends TaskManagerFactory { private static final String ARG_ENABLE_BBOX_BUILDER = "enableBboxBuilder"; private static final String ARG_ENABLE_LINESTRING_BUILDER = "enableLinestringBuilder"; + private static final String ARG_ENABLE_KEEP_PARTIAL_LINESTRING = "enableKeepPartialLinestring"; private static final String ARG_FILE_NAME = "directory"; private static final String ARG_NODE_LOCATION_STORE_TYPE = "nodeLocationStoreType"; private static final boolean DEFAULT_ENABLE_BBOX_BUILDER = false; private static final boolean DEFAULT_ENABLE_LINESTRING_BUILDER = false; + private static final boolean DEFAULT_ENABLE_KEEP_PARTIAL_LINESTRING = false; private static final String DEFAULT_FILE_PREFIX = "pgimport"; private static final String DEFAULT_NODE_LOCATION_STORE_TYPE = "CompactTempFile"; @@ -35,6 +37,7 @@ protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) { File filePrefix; boolean enableBboxBuilder; boolean enableLinestringBuilder; + boolean enableKeepPartialLinestring; NodeLocationStoreType storeType; // Get the task arguments. @@ -44,6 +47,8 @@ protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) { taskConfig, ARG_ENABLE_BBOX_BUILDER, DEFAULT_ENABLE_BBOX_BUILDER); enableLinestringBuilder = getBooleanArgument( taskConfig, ARG_ENABLE_LINESTRING_BUILDER, DEFAULT_ENABLE_LINESTRING_BUILDER); + enableKeepPartialLinestring = getBooleanArgument( + taskConfig, ARG_ENABLE_KEEP_PARTIAL_LINESTRING, DEFAULT_ENABLE_KEEP_PARTIAL_LINESTRING); storeType = Enum.valueOf( NodeLocationStoreType.class, getStringArgument(taskConfig, ARG_NODE_LOCATION_STORE_TYPE, DEFAULT_NODE_LOCATION_STORE_TYPE)); @@ -53,7 +58,8 @@ protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) { return new SinkManager( taskConfig.getId(), - new PostgreSqlDumpWriter(filePrefix, enableBboxBuilder, enableLinestringBuilder, storeType), + new PostgreSqlDumpWriter(filePrefix, enableBboxBuilder, enableLinestringBuilder, + enableKeepPartialLinestring, storeType), taskConfig.getPipeArgs() ); } diff --git a/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlWriter.java b/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlWriter.java index bd688e131..939d36e12 100644 --- a/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlWriter.java +++ b/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlWriter.java @@ -70,6 +70,7 @@ public class PostgreSqlWriter implements Sink, EntityProcessor { private DatabaseContext dbCtx; private boolean enableBboxBuilder; private boolean enableLinestringBuilder; + private boolean enableKeepPartialLinestring; private SchemaVersionValidator schemaVersionValidator; private IndexManager indexManager; private List nodeBuffer; @@ -129,16 +130,21 @@ public class PostgreSqlWriter implements Sink, EntityProcessor { * processing instead of relying on the database to build them * after import. This increases processing but is faster than * relying on the database. + * @param enableKeepPartialLinestring + * If true, the way linestring is build even on invalid or missing + * nodes. * @param storeType * The node location storage type used by the geometry builders. */ public PostgreSqlWriter( DatabaseLoginCredentials loginCredentials, DatabasePreferences preferences, - boolean enableBboxBuilder, boolean enableLinestringBuilder, NodeLocationStoreType storeType) { + boolean enableBboxBuilder, boolean enableLinestringBuilder, + boolean enableKeepPartialLinestring, NodeLocationStoreType storeType) { dbCtx = new DatabaseContext(loginCredentials); this.enableBboxBuilder = enableBboxBuilder; this.enableLinestringBuilder = enableLinestringBuilder; + this.enableKeepPartialLinestring = enableKeepPartialLinestring; schemaVersionValidator = new SchemaVersionValidator(dbCtx, preferences); indexManager = new IndexManager(dbCtx, !enableBboxBuilder, !enableLinestringBuilder); @@ -387,7 +393,7 @@ private void flushWays(boolean complete) { geometries.add(wayGeometryBuilder.createWayBbox(way)); } if (enableLinestringBuilder) { - geometries.add(wayGeometryBuilder.createWayLinestring(way)); + geometries.add(wayGeometryBuilder.createWayLinestring(way, enableKeepPartialLinestring)); } prmIndex = wayBuilder.populateEntityParameters(bulkWayStatement, prmIndex, way, geometries); } @@ -416,7 +422,7 @@ private void flushWays(boolean complete) { geometries.add(wayGeometryBuilder.createWayBbox(way)); } if (enableLinestringBuilder) { - geometries.add(wayGeometryBuilder.createWayLinestring(way)); + geometries.add(wayGeometryBuilder.createWayLinestring(way, enableKeepPartialLinestring)); } wayBuilder.populateEntityParameters(singleWayStatement, 1, way, geometries); diff --git a/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlWriterFactory.java b/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlWriterFactory.java index ad4c102af..cb9ec8974 100644 --- a/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlWriterFactory.java +++ b/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/PostgreSqlWriterFactory.java @@ -18,9 +18,11 @@ public class PostgreSqlWriterFactory extends DatabaseTaskManagerFactory { private static final String ARG_ENABLE_BBOX_BUILDER = "enableBboxBuilder"; private static final String ARG_ENABLE_LINESTRING_BUILDER = "enableLinestringBuilder"; + private static final String ARG_ENABLE_KEEP_PARTIAL_LINESTRING = "enableKeepPartialLinestring"; private static final String ARG_NODE_LOCATION_STORE_TYPE = "nodeLocationStoreType"; private static final boolean DEFAULT_ENABLE_BBOX_BUILDER = false; private static final boolean DEFAULT_ENABLE_LINESTRING_BUILDER = false; + private static final boolean DEFAULT_ENABLE_KEEP_PARTIAL_LINESTRING = false; private static final String DEFAULT_NODE_LOCATION_STORE_TYPE = "CompactTempFile"; /** @@ -32,6 +34,7 @@ protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) { DatabasePreferences preferences; boolean enableBboxBuilder; boolean enableLinestringBuilder; + boolean enableKeepPartialLinestring; NodeLocationStoreType storeType; // Get the task arguments. @@ -40,13 +43,16 @@ protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) { enableBboxBuilder = getBooleanArgument(taskConfig, ARG_ENABLE_BBOX_BUILDER, DEFAULT_ENABLE_BBOX_BUILDER); enableLinestringBuilder = getBooleanArgument( taskConfig, ARG_ENABLE_LINESTRING_BUILDER, DEFAULT_ENABLE_LINESTRING_BUILDER); + enableKeepPartialLinestring = getBooleanArgument( + taskConfig, ARG_ENABLE_KEEP_PARTIAL_LINESTRING, DEFAULT_ENABLE_KEEP_PARTIAL_LINESTRING); storeType = Enum.valueOf( NodeLocationStoreType.class, getStringArgument(taskConfig, ARG_NODE_LOCATION_STORE_TYPE, DEFAULT_NODE_LOCATION_STORE_TYPE)); return new SinkManager( taskConfig.getId(), - new PostgreSqlWriter(loginCredentials, preferences, enableBboxBuilder, enableLinestringBuilder, storeType), + new PostgreSqlWriter(loginCredentials, preferences, enableBboxBuilder, enableLinestringBuilder, + enableKeepPartialLinestring, storeType), taskConfig.getPipeArgs() ); } diff --git a/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/impl/CopyFilesetBuilder.java b/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/impl/CopyFilesetBuilder.java index 6fbdea92c..db41beac4 100644 --- a/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/impl/CopyFilesetBuilder.java +++ b/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/impl/CopyFilesetBuilder.java @@ -35,6 +35,7 @@ public class CopyFilesetBuilder implements Sink, EntityProcessor { private boolean enableBboxBuilder; private boolean enableLinestringBuilder; + private boolean enableKeepPartialLinestring; private WayGeometryBuilder wayGeometryBuilder; private CompletableContainer writerContainer; private MemberTypeValueMapper memberTypeValueMapper; @@ -66,14 +67,19 @@ public class CopyFilesetBuilder implements Sink, EntityProcessor { * processing instead of relying on the database to build them * after import. This increases processing but is faster than * relying on the database. + * @param enableKeepPartialLinestring + * If true, the way linestring is build even on invalid or missing + * nodes. * @param storeType * The node location storage type used by the geometry builders. */ public CopyFilesetBuilder( CopyFileset copyFileset, boolean enableBboxBuilder, - boolean enableLinestringBuilder, NodeLocationStoreType storeType) { + boolean enableLinestringBuilder, boolean enableKeepPartialLinestring, + NodeLocationStoreType storeType) { this.enableBboxBuilder = enableBboxBuilder; this.enableLinestringBuilder = enableLinestringBuilder; + this.enableKeepPartialLinestring = enableKeepPartialLinestring; writerContainer = new CompletableContainer(); @@ -184,7 +190,7 @@ public void process(WayContainer wayContainer) { wayWriter.writeField(wayGeometryBuilder.createWayBbox(way)); } if (enableLinestringBuilder) { - wayWriter.writeField(wayGeometryBuilder.createWayLinestring(way)); + wayWriter.writeField(wayGeometryBuilder.createWayLinestring(way, enableKeepPartialLinestring)); } wayWriter.endRecord(); diff --git a/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/impl/WayGeometryBuilder.java b/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/impl/WayGeometryBuilder.java index 496c2061f..d0099d822 100644 --- a/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/impl/WayGeometryBuilder.java +++ b/osmosis-pgsimple/src/main/java/org/openstreetmap/osmosis/pgsimple/v0_6/impl/WayGeometryBuilder.java @@ -191,9 +191,12 @@ public Polygon createWayBbox(Way way) { * * @param way * The way to create the linestring for. + * @param enableKeepPartialLinestring + * If true, the way linestring is build even on invalid or missing + * nodes. * @return The linestring representing the way. */ - public LineString createWayLinestring(Way way) { + public LineString createWayLinestring(Way way, boolean enableKeepPartialLinestring) { List linePoints; int numValidNodes = 0; @@ -207,7 +210,9 @@ public LineString createWayLinestring(Way way) { numValidNodes++; linePoints.add(new Point(nodeLocation.getLongitude(), nodeLocation.getLatitude())); } else { - return null; + if (!enableKeepPartialLinestring) { + return null; + } } } diff --git a/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlCopyWriter.java b/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlCopyWriter.java index 4c6647a60..0292ea133 100644 --- a/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlCopyWriter.java +++ b/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlCopyWriter.java @@ -35,6 +35,7 @@ public class PostgreSqlCopyWriter implements Sink { private NodeLocationStoreType storeType; private boolean populateBbox; private boolean populateLinestring; + private boolean enableKeepPartialLinestring; private boolean keepInvalidWays; private boolean initialized; private DatabaseContext dbCtx; @@ -49,6 +50,9 @@ public class PostgreSqlCopyWriter implements Sink { * Contains preferences configuring database behaviour. * @param storeType * The node location storage type used by the geometry builders. + * @param enableKeepPartialLinestring + * If true, the way linestring is build even on invalid or missing + * nodes. * @param keepInvalidWays * If true, zero and single node ways are kept. Otherwise they are * silently dropped to avoid putting invalid geometries into the @@ -56,10 +60,11 @@ public class PostgreSqlCopyWriter implements Sink { */ public PostgreSqlCopyWriter( DatabaseLoginCredentials loginCredentials, DatabasePreferences preferences, - NodeLocationStoreType storeType, boolean keepInvalidWays) { + boolean enableKeepPartialLinestring, NodeLocationStoreType storeType, boolean keepInvalidWays) { this.loginCredentials = loginCredentials; this.preferences = preferences; this.storeType = storeType; + this.enableKeepPartialLinestring = enableKeepPartialLinestring; this.keepInvalidWays = keepInvalidWays; this.dbCtx = new DatabaseContext(loginCredentials); this.locker = new DatabaseLocker(dbCtx.getDataSource(), true); @@ -76,7 +81,8 @@ private void initialize() { populateLinestring = capabilityChecker.isWayLinestringSupported(); copyFilesetBuilder = - new CopyFilesetBuilder(copyFileset, populateBbox, populateLinestring, storeType, keepInvalidWays); + new CopyFilesetBuilder(copyFileset, populateBbox, populateLinestring, enableKeepPartialLinestring, + storeType, keepInvalidWays); copyFilesetLoader = new CopyFilesetLoader(loginCredentials, preferences, copyFileset); diff --git a/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlCopyWriterFactory.java b/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlCopyWriterFactory.java index d44593422..585470294 100644 --- a/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlCopyWriterFactory.java +++ b/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlCopyWriterFactory.java @@ -16,6 +16,8 @@ public class PostgreSqlCopyWriterFactory extends DatabaseTaskManagerFactory { private static final String ARG_NODE_LOCATION_STORE_TYPE = "nodeLocationStoreType"; private static final String DEFAULT_NODE_LOCATION_STORE_TYPE = "CompactTempFile"; + private static final String ARG_ENABLE_KEEP_PARTIAL_LIENSTRING = "enableKeepPartialLinestring"; + private static final boolean DEFAULT_ENABLE_KEEP_PARTIAL_LIENSTRING = false; private static final String ARG_KEEP_INVALID_WAYS = "keepInvalidWays"; private static final boolean DEFAULT_KEEP_INVALID_WAYS = true; @@ -24,9 +26,12 @@ public class PostgreSqlCopyWriterFactory extends DatabaseTaskManagerFactory { */ @Override protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) { + boolean enableKeepPartialLinestring; NodeLocationStoreType storeType; boolean keepInvalidWays; + enableKeepPartialLinestring = getBooleanArgument(taskConfig, ARG_ENABLE_KEEP_PARTIAL_LIENSTRING, + DEFAULT_ENABLE_KEEP_PARTIAL_LIENSTRING); storeType = Enum.valueOf( NodeLocationStoreType.class, getStringArgument(taskConfig, ARG_NODE_LOCATION_STORE_TYPE, DEFAULT_NODE_LOCATION_STORE_TYPE)); @@ -37,6 +42,7 @@ protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) { new PostgreSqlCopyWriter( getDatabaseLoginCredentials(taskConfig), getDatabasePreferences(taskConfig), + enableKeepPartialLinestring, storeType, keepInvalidWays), taskConfig.getPipeArgs() diff --git a/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlDumpWriter.java b/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlDumpWriter.java index 4df4bdfb8..36a406b0e 100644 --- a/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlDumpWriter.java +++ b/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlDumpWriter.java @@ -37,6 +37,9 @@ public class PostgreSqlDumpWriter implements Sink { * processing instead of relying on the database to build them * after import. This increases processing but is faster than * relying on the database. + * @param enableKeepPartialLinestring + * If true, the way linestring is build even on invalid or missing + * nodes. * @param storeType * The node location storage type used by the geometry builders. * @param keepInvalidWays @@ -46,14 +49,16 @@ public class PostgreSqlDumpWriter implements Sink { */ public PostgreSqlDumpWriter( File filePrefix, boolean enableBboxBuilder, - boolean enableLinestringBuilder, NodeLocationStoreType storeType, - boolean keepInvalidWays) { + boolean enableLinestringBuilder, + boolean enableKeepPartialLinestring, + NodeLocationStoreType storeType, boolean keepInvalidWays) { DirectoryCopyFileset copyFileset; copyFileset = new DirectoryCopyFileset(filePrefix); copyFilesetBuilder = - new CopyFilesetBuilder(copyFileset, enableBboxBuilder, enableLinestringBuilder, storeType, keepInvalidWays); + new CopyFilesetBuilder(copyFileset, enableBboxBuilder, enableLinestringBuilder, + enableKeepPartialLinestring, storeType, keepInvalidWays); } diff --git a/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlDumpWriterFactory.java b/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlDumpWriterFactory.java index 0955f8d5a..2eb41ea04 100644 --- a/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlDumpWriterFactory.java +++ b/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/PostgreSqlDumpWriterFactory.java @@ -18,11 +18,13 @@ public class PostgreSqlDumpWriterFactory extends TaskManagerFactory { private static final String ARG_ENABLE_BBOX_BUILDER = "enableBboxBuilder"; private static final String ARG_ENABLE_LINESTRING_BUILDER = "enableLinestringBuilder"; + private static final String ARG_ENABLE_KEEP_PARTIAL_LINESTRING = "enableKeepPartialLinestring"; private static final String ARG_KEEP_INVALID_WAYS = "keepInvalidWays"; private static final String ARG_FILE_NAME = "directory"; private static final String ARG_NODE_LOCATION_STORE_TYPE = "nodeLocationStoreType"; private static final boolean DEFAULT_ENABLE_BBOX_BUILDER = false; private static final boolean DEFAULT_ENABLE_LINESTRING_BUILDER = false; + private static final boolean DEFAULT_ENABLE_KEEP_PARTIAL_LINESTRING = false; private static final boolean DEFAULT_KEEP_INVALID_WAYS = true; private static final String DEFAULT_FILE_PREFIX = "pgimport"; private static final String DEFAULT_NODE_LOCATION_STORE_TYPE = "CompactTempFile"; @@ -37,6 +39,7 @@ protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) { File filePrefix; boolean enableBboxBuilder; boolean enableLinestringBuilder; + boolean enableKeepPartialLinestring; boolean keepInvalidWays; NodeLocationStoreType storeType; @@ -47,6 +50,8 @@ protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) { taskConfig, ARG_ENABLE_BBOX_BUILDER, DEFAULT_ENABLE_BBOX_BUILDER); enableLinestringBuilder = getBooleanArgument( taskConfig, ARG_ENABLE_LINESTRING_BUILDER, DEFAULT_ENABLE_LINESTRING_BUILDER); + enableKeepPartialLinestring = getBooleanArgument( + taskConfig, ARG_ENABLE_KEEP_PARTIAL_LINESTRING, DEFAULT_ENABLE_KEEP_PARTIAL_LINESTRING); keepInvalidWays = getBooleanArgument(taskConfig, ARG_KEEP_INVALID_WAYS, DEFAULT_KEEP_INVALID_WAYS); storeType = Enum.valueOf( NodeLocationStoreType.class, @@ -58,7 +63,8 @@ protected TaskManager createTaskManagerImpl(TaskConfiguration taskConfig) { return new SinkManager( taskConfig.getId(), new PostgreSqlDumpWriter( - filePrefix, enableBboxBuilder, enableLinestringBuilder, storeType, keepInvalidWays), + filePrefix, enableBboxBuilder, enableLinestringBuilder, enableKeepPartialLinestring, storeType, + keepInvalidWays), taskConfig.getPipeArgs() ); } diff --git a/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/impl/CopyFilesetBuilder.java b/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/impl/CopyFilesetBuilder.java index 33b60b954..c537a9c43 100644 --- a/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/impl/CopyFilesetBuilder.java +++ b/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/impl/CopyFilesetBuilder.java @@ -39,6 +39,7 @@ public class CopyFilesetBuilder implements Sink, EntityProcessor { private boolean enableBboxBuilder; private boolean enableLinestringBuilder; + private boolean enableKeepPartialLinestring; private boolean keepInvalidWays; private WayGeometryBuilder wayGeometryBuilder; private CompletableContainer writerContainer; @@ -68,6 +69,9 @@ public class CopyFilesetBuilder implements Sink, EntityProcessor { * processing instead of relying on the database to build them * after import. This increases processing but is faster than * relying on the database. + * @param enableKeepPartialLinestring + * If true, the way linestring is build even on invalid or missing + * nodes. * @param storeType * The node location storage type used by the geometry builders. * @param keepInvalidWays @@ -77,10 +81,11 @@ public class CopyFilesetBuilder implements Sink, EntityProcessor { */ public CopyFilesetBuilder( CopyFileset copyFileset, boolean enableBboxBuilder, - boolean enableLinestringBuilder, NodeLocationStoreType storeType, - boolean keepInvalidWays) { + boolean enableLinestringBuilder, boolean enableKeepPartialLinestring, + NodeLocationStoreType storeType, boolean keepInvalidWays) { this.enableBboxBuilder = enableBboxBuilder; this.enableLinestringBuilder = enableLinestringBuilder; + this.enableKeepPartialLinestring = enableKeepPartialLinestring; this.keepInvalidWays = keepInvalidWays; writerContainer = new CompletableContainer(); @@ -203,7 +208,7 @@ public void process(WayContainer wayContainer) { wayWriter.writeField(wayGeometryBuilder.createWayBbox(way)); } if (enableLinestringBuilder) { - wayWriter.writeField(wayGeometryBuilder.createWayLinestring(way)); + wayWriter.writeField(wayGeometryBuilder.createWayLinestring(way, enableKeepPartialLinestring)); } wayWriter.endRecord(); diff --git a/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/impl/WayGeometryBuilder.java b/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/impl/WayGeometryBuilder.java index 2eac918bf..557b8b0ca 100644 --- a/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/impl/WayGeometryBuilder.java +++ b/osmosis-pgsnapshot/src/main/java/org/openstreetmap/osmosis/pgsnapshot/v0_6/impl/WayGeometryBuilder.java @@ -191,9 +191,12 @@ public Polygon createWayBbox(Way way) { * * @param way * The way to create the linestring for. + * @param enableKeepPartialLinestring + * If true, the way linestring is build even on invalid or missing + * nodes. * @return The linestring representing the way. */ - public LineString createWayLinestring(Way way) { + public LineString createWayLinestring(Way way, boolean enableKeepPartialLinestring) { List linePoints; linePoints = new ArrayList(); @@ -205,7 +208,9 @@ public LineString createWayLinestring(Way way) { if (nodeLocation.isValid()) { linePoints.add(new Point(nodeLocation.getLongitude(), nodeLocation.getLatitude())); } else { - return null; + if (!enableKeepPartialLinestring) { + return null; + } } } return createLinestring(linePoints);