diff --git a/docs/modules/storage/pages/import-export.adoc b/docs/modules/storage/pages/import-export.adoc index ea89c4f8f..2929e9810 100644 --- a/docs/modules/storage/pages/import-export.adoc +++ b/docs/modules/storage/pages/import-export.adoc @@ -13,41 +13,21 @@ NOTE: The created binary type data files contain only records of the according t [source, java, title="Export"] ---- +// Setup file system and storage, used in all examples below NioFileSystem fileSystem = NioFileSystem.New(); - EmbeddedStorageManager storage = EmbeddedStorage.start( fileSystem.ensureDirectoryPath("storage") ); -String fileSuffix = "bin"; -StorageConnection connection = storage.createConnection(); -StorageEntityTypeExportStatistics exportResult = connection.exportTypes( - new StorageEntityTypeExportFileProvider.Default( - fileSystem.ensureDirectoryPath("export-dir"), - fileSuffix - ), - typeHandler -> true // export all, customize if necessary +StorageConnection connection = storage.createConnection(); +StorageEntityTypeExportStatistics exportResult = connection.exportTypes( + fileSystem.ensureDirectoryPath("export-dir") ); -XSequence exportFiles = CQL - .from(exportResult.typeStatistics().values()) - .project(s -> Paths.get(s.file().identifier())) - .execute() -; ---- [source, java, title="Import"] ---- -NioFileSystem fileSystem = NioFileSystem.New(); - -EmbeddedStorageManager storage = EmbeddedStorage.start( - fileSystem.ensureDirectoryPath("storage") -); - -StorageConnection connection = storage.createConnection(); -connection.importFiles(X.Enum( - fileSystem.ensureFilePath("type1.bin"), - fileSystem.ensureFilePath("type2.bin") -)); +connection.importFiles(exportResult.files()); ---- == Data Conversion @@ -65,51 +45,19 @@ The file's size is at the possible minimum and the performance of the converter [source, java, title="Binary to CSV"] ---- -NioFileSystem fileSystem = NioFileSystem.New(); - -EmbeddedStorageManager storage = EmbeddedStorage.start( - fileSystem.ensureDirectoryPath("storage") +StorageDataConverterTypeBinaryToCsv converter = StorageDataConverterTypeBinaryToCsv.New( + fileSystem.ensureDirectoryPath("csv-dir"), + storage.typeDictionary() ); - -StorageDataConverterTypeBinaryToCsv converter = - new StorageDataConverterTypeBinaryToCsv.UTF8( - StorageDataConverterCsvConfiguration.defaultConfiguration(), - new StorageEntityTypeConversionFileProvider.Default( - fileSystem.ensureDirectoryPath("csv-dir"), - "csv" - ), - storage.typeDictionary(), - null, // no type name mapping - 4096, // read buffer size - 4096 // write buffer size -); -AReadableFile dataFile = fileSystem.ensureFilePath("type1.bin").useReading(); -try -{ - converter.convertDataFile(dataFile); -} -finally -{ - dataFile.close(); -} +converter.convertDataFiles(exportResult.files()); ---- [source, java, title="CSV to Binary"] ---- -NioFileSystem fileSystem = NioFileSystem.New(); - -EmbeddedStorageManager storage = EmbeddedStorage.start( - fileSystem.ensureDirectoryPath("storage") -); - StorageDataConverterTypeCsvToBinary converter = - StorageDataConverterTypeCsvToBinary.New( - StorageDataConverterCsvConfiguration.defaultConfiguration(), - storage.typeDictionary(), - new StorageEntityTypeConversionFileProvider.Default( - fileSystem.ensureDirectoryPath("bin-dir"), - "dat" - ) + StorageDataConverterTypeCsvToBinary.New( + storage.typeDictionary(), + fileSystem.ensureDirectoryPath("csv-dir") ); converter.convertCsv(fileSystem.ensureFilePath("type1.csv")); ---- diff --git a/storage/storage/src/main/java/one/microstream/storage/types/StorageConnection.java b/storage/storage/src/main/java/one/microstream/storage/types/StorageConnection.java index afe8b2626..b00bcd163 100644 --- a/storage/storage/src/main/java/one/microstream/storage/types/StorageConnection.java +++ b/storage/storage/src/main/java/one/microstream/storage/types/StorageConnection.java @@ -323,6 +323,45 @@ public default StorageEntityTypeExportStatistics exportTypes( { return this.exportTypes(exportFileProvider, null); } + + /** + * Alias for {@code this.exportTypes(StorageEntityTypeExportFileProvider.New(targetDirectory));}, meaning all types are exported. + * + * @param targetDirectory the target directory for the export files + * + * @return a {@link StorageEntityTypeExportStatistics} information instance about the completed export. + * + * @see #exportTypes(StorageEntityTypeExportFileProvider, Predicate) + * @since 08.00.00 + */ + public default StorageEntityTypeExportStatistics exportTypes( + final ADirectory targetDirectory + ) + { + return this.exportTypes(StorageEntityTypeExportFileProvider.New(targetDirectory)); + } + + /** + * Alias for {@code this.exportTypes(StorageEntityTypeExportFileProvider.New(targetDirectory), isExportType);}. + * + * @param targetDirectory the target directory for the export files + * @param isExportType a {@link Predicate} selecting which type's entity data to be exported. + * + * @return a {@link StorageEntityTypeExportStatistics} information instance about the completed export. + * + * @see #exportTypes(StorageEntityTypeExportFileProvider, Predicate) + * @since 08.00.00 + */ + public default StorageEntityTypeExportStatistics exportTypes( + final ADirectory targetDirectory, + final Predicate isExportType + ) + { + return this.exportTypes( + StorageEntityTypeExportFileProvider.New(targetDirectory), + isExportType + ); + } /** * Imports all files specified by the passed Enum (ordered set) of {@link AFile} in order.
diff --git a/storage/storage/src/main/java/one/microstream/storage/types/StorageDataConverterTypeBinaryToCsv.java b/storage/storage/src/main/java/one/microstream/storage/types/StorageDataConverterTypeBinaryToCsv.java index 0fd78514f..4bcc1f188 100644 --- a/storage/storage/src/main/java/one/microstream/storage/types/StorageDataConverterTypeBinaryToCsv.java +++ b/storage/storage/src/main/java/one/microstream/storage/types/StorageDataConverterTypeBinaryToCsv.java @@ -28,6 +28,7 @@ import java.nio.ByteOrder; import one.microstream.X; +import one.microstream.afs.types.ADirectory; import one.microstream.afs.types.AFS; import one.microstream.afs.types.AFile; import one.microstream.afs.types.AReadableFile; @@ -62,6 +63,28 @@ public interface StorageDataConverterTypeBinaryToCsv { public void convertDataFile(AReadableFile file); + + /** + * Batch-converts a list of files. + * + * @param files the binary files to convert to CSV + * @since 08.00.00 + */ + public default > void convertDataFiles(final I files) + { + for(final AFile file : files) + { + final AReadableFile dataFile = file.useReading(); + try + { + this.convertDataFile(dataFile); + } + finally + { + dataFile.close(); + } + } + } @@ -93,6 +116,88 @@ public String mapTypeName(final PersistenceTypeDescriptionMember columnType) } }; } + + + /** + * Pseudo-constructor method to create a new {@link StorageDataConverterTypeBinaryToCsv}. + * + * @param targetDirectory write target directory + * @param typeDictionary the type dictionary to use + * @return a new {@link StorageDataConverterTypeBinaryToCsv} + * @since 08.00.00 + */ + public static StorageDataConverterTypeBinaryToCsv New( + final ADirectory targetDirectory, + final PersistenceTypeDictionary typeDictionary + ) + { + return New( + StorageDataConverterCsvConfiguration.defaultConfiguration(), + new StorageEntityTypeConversionFileProvider.Default( + targetDirectory, + "csv" + ), + typeDictionary + ); + } + + + /** + * Pseudo-constructor method to create a new {@link StorageDataConverterTypeBinaryToCsv}. + * + * @param configuration the CSV configuration to use + * @param fileProvider target file provider + * @param typeDictionary the type dictionary to use + * @return a new {@link StorageDataConverterTypeBinaryToCsv} + * @since 08.00.00 + */ + public static StorageDataConverterTypeBinaryToCsv New( + final StorageDataConverterCsvConfiguration configuration , + final StorageEntityTypeConversionFileProvider fileProvider , + final PersistenceTypeDictionary typeDictionary + ) + { + return New( + configuration , + fileProvider , + typeDictionary, + null , + 0 , + 0 + ); + } + + + /** + * Pseudo-constructor method to create a new {@link StorageDataConverterTypeBinaryToCsv}. + * + * @param configuration the CSV configuration to use + * @param fileProvider target file provider + * @param typeDictionary the type dictionary to use + * @param typeNameMapper optional type name mapper + * @param readBufferSize buffer size for reading + * @param writeBufferSize buffer size for writing + * @return a new {@link StorageDataConverterTypeBinaryToCsv} + * @since 08.00.00 + */ + public static StorageDataConverterTypeBinaryToCsv New( + final StorageDataConverterCsvConfiguration configuration , + final StorageEntityTypeConversionFileProvider fileProvider , + final PersistenceTypeDictionary typeDictionary , + final TypeNameMapper typeNameMapper , + final int readBufferSize , + final int writeBufferSize + ) + { + return new StorageDataConverterTypeBinaryToCsv.UTF8( + configuration , + fileProvider , + typeDictionary , + typeNameMapper , + readBufferSize , + writeBufferSize + ); + } diff --git a/storage/storage/src/main/java/one/microstream/storage/types/StorageDataConverterTypeCsvToBinary.java b/storage/storage/src/main/java/one/microstream/storage/types/StorageDataConverterTypeCsvToBinary.java index 482ee31b7..a55cbab8a 100644 --- a/storage/storage/src/main/java/one/microstream/storage/types/StorageDataConverterTypeCsvToBinary.java +++ b/storage/storage/src/main/java/one/microstream/storage/types/StorageDataConverterTypeCsvToBinary.java @@ -26,6 +26,7 @@ import java.util.Iterator; import one.microstream.X; +import one.microstream.afs.types.ADirectory; import one.microstream.afs.types.AFS; import one.microstream.afs.types.AFile; import one.microstream.afs.types.AWritableFile; @@ -55,6 +56,17 @@ public interface StorageDataConverterTypeCsvToBinary { public void convertCsv(S source); + + /** + * Batch-converts given list of sources. + * @param source type + * @param sources sources to convert + * @since 08.00.00 + */ + public default > void convertCsv(final I sources) + { + sources.forEach(this::convertCsv); + } @@ -73,9 +85,61 @@ public interface ValueHandler */ public int handleValue(char[] data, int offset, int bound, char separator, char terminator); } + + + /** + * Pseudo-constructor method to create a new {@link StorageDataConverterTypeCsvToBinary}. + *

+ * The default file suffix from {@link StorageEntityTypeExportFileProvider} is used. + * @see StorageEntityTypeExportFileProvider.Defaults#defaultFileSuffix() + * + * @param typeDictionary type information + * @param targetDirectory binary file target directory + * @return a new {@link StorageDataConverterTypeCsvToBinary} + * @since 08.00.00 + */ + public static StorageDataConverterTypeCsvToBinary New( + final PersistenceTypeDictionary typeDictionary , + final ADirectory targetDirectory + ) + { + return New( + typeDictionary, + targetDirectory, + StorageEntityTypeExportFileProvider.Defaults.defaultFileSuffix() + ); + } + /** + * Pseudo-constructor method to create a new {@link StorageDataConverterTypeCsvToBinary}. + * + * @param typeDictionary type information + * @param targetDirectory binary file target directory + * @param targetFileSuffix binary file suffix + * @return a new {@link StorageDataConverterTypeCsvToBinary} + * @since 08.00.00 + */ + public static StorageDataConverterTypeCsvToBinary New( + final PersistenceTypeDictionary typeDictionary , + final ADirectory targetDirectory, + final String targetFileSuffix + ) + { + return New( + StorageDataConverterCsvConfiguration.defaultConfiguration(), + typeDictionary, + StorageEntityTypeConversionFileProvider.New(targetDirectory, targetFileSuffix) + ); + } - + /** + * Pseudo-constructor method to create a new {@link StorageDataConverterTypeCsvToBinary}. + * + * @param configuration the csv configuration to use + * @param typeDictionary type information + * @param fileProvider target file provider + * @return a new {@link StorageDataConverterTypeCsvToBinary} + */ public static StorageDataConverterTypeCsvToBinary New( final StorageDataConverterCsvConfiguration configuration , final PersistenceTypeDictionary typeDictionary, @@ -85,6 +149,15 @@ public static StorageDataConverterTypeCsvToBinary New( return New(configuration, typeDictionary, fileProvider, 0); } + /** + * Pseudo-constructor method to create a new {@link StorageDataConverterTypeCsvToBinary}. + * + * @param configuration the csv configuration to use + * @param typeDictionary type information + * @param fileProvider target file provider + * @param bufferSize custom buffer size or 0 + * @return a new {@link StorageDataConverterTypeCsvToBinary} + */ public static StorageDataConverterTypeCsvToBinary New( final StorageDataConverterCsvConfiguration configuration , final PersistenceTypeDictionary typeDictionary, diff --git a/storage/storage/src/main/java/one/microstream/storage/types/StorageEntityTypeConversionFileProvider.java b/storage/storage/src/main/java/one/microstream/storage/types/StorageEntityTypeConversionFileProvider.java index 88ccad5af..57701b9ab 100644 --- a/storage/storage/src/main/java/one/microstream/storage/types/StorageEntityTypeConversionFileProvider.java +++ b/storage/storage/src/main/java/one/microstream/storage/types/StorageEntityTypeConversionFileProvider.java @@ -21,6 +21,7 @@ */ import static one.microstream.X.notNull; +import static one.microstream.chars.XChars.notEmpty; import one.microstream.afs.types.ADirectory; import one.microstream.afs.types.AFile; @@ -32,6 +33,24 @@ public interface StorageEntityTypeConversionFileProvider { public AWritableFile provideConversionFile(PersistenceTypeDefinition typeDescription, AFile sourceFile); + + /** + * Pseudo-constructor method to create a new {@link StorageEntityTypeConversionFileProvider}. + * @param directory the target directory + * @param fileSuffix the suffix to use for the created files + * @return a new {@link StorageEntityTypeConversionFileProvider} + * @since 08.00.00 + */ + public static StorageEntityTypeConversionFileProvider New( + final ADirectory directory , + final String fileSuffix + ) + { + return new StorageEntityTypeConversionFileProvider.Default( + notNull(directory), + notEmpty(fileSuffix) + ); + } public final class Default implements StorageEntityTypeConversionFileProvider diff --git a/storage/storage/src/main/java/one/microstream/storage/types/StorageEntityTypeExportFileProvider.java b/storage/storage/src/main/java/one/microstream/storage/types/StorageEntityTypeExportFileProvider.java index f2dd52959..3c353359b 100644 --- a/storage/storage/src/main/java/one/microstream/storage/types/StorageEntityTypeExportFileProvider.java +++ b/storage/storage/src/main/java/one/microstream/storage/types/StorageEntityTypeExportFileProvider.java @@ -21,6 +21,7 @@ */ import static one.microstream.X.notNull; +import static one.microstream.chars.XChars.notEmpty; import one.microstream.afs.types.ADirectory; import one.microstream.afs.types.AFile; @@ -82,6 +83,56 @@ public static long getTypeIdFromUniqueTypeFileName(final String uniqueTypeFileNa } } + + /** + * + * @since 08.00.00 + */ + public interface Defaults + { + /** + * Default file suffix for binary files. + * + * @return {@code "bin"} + */ + public static String defaultFileSuffix() + { + return "bin"; + } + } + + + /** + * Pseudo-constructor method to create a new {@link StorageEntityTypeExportFileProvider}. + * + * @param directory target directory for the export files + * @return a new {@link StorageEntityTypeExportFileProvider} + * @since 08.00.00 + */ + public static StorageEntityTypeExportFileProvider New(final ADirectory directory) + { + return New( + directory, + Defaults.defaultFileSuffix() + ); + } + + /** + * Pseudo-constructor method to create a new {@link StorageEntityTypeExportFileProvider}. + * + * @param directory target directory for the export files + * @param fileSuffix suffix for created files + * return a new {@link StorageEntityTypeExportFileProvider} + * @since 08.00.00 + */ + public static StorageEntityTypeExportFileProvider New(final ADirectory directory, final String fileSuffix) + { + return new StorageEntityTypeExportFileProvider.Default( + notNull(directory), + notEmpty(fileSuffix) + ); + } + public final class Default implements StorageEntityTypeExportFileProvider { diff --git a/storage/storage/src/main/java/one/microstream/storage/types/StorageEntityTypeExportStatistics.java b/storage/storage/src/main/java/one/microstream/storage/types/StorageEntityTypeExportStatistics.java index 44e0820ff..e10ca2f2b 100644 --- a/storage/storage/src/main/java/one/microstream/storage/types/StorageEntityTypeExportStatistics.java +++ b/storage/storage/src/main/java/one/microstream/storage/types/StorageEntityTypeExportStatistics.java @@ -24,7 +24,9 @@ import one.microstream.chars.VarString; import one.microstream.chars.XChars; import one.microstream.collections.EqConstHashTable; +import one.microstream.collections.EqHashEnum; import one.microstream.collections.EqHashTable; +import one.microstream.collections.types.XGettingEnum; import one.microstream.collections.types.XGettingTable; import one.microstream.collections.types.XImmutableTable; import one.microstream.typing.KeyValue; @@ -44,6 +46,14 @@ public interface StorageEntityTypeExportStatistics public XGettingTable channelStatistics(); + /** + * Collects and returns all exported files. + * + * @return all export files + * @since 08.00.00 + */ + public XGettingEnum files(); + public final class Default extends AbstractStatistic implements StorageEntityTypeExportStatistics @@ -138,6 +148,16 @@ public static final void assembleTableHeader(final VarString vs) return this.viewChannelStatistics; } + @Override + public XGettingEnum files() + { + final EqHashEnum files = EqHashEnum.New(); + this.viewTypeStatistics.values().forEach(s -> + files.add(s.file()) + ); + return files.immure(); + } + final void assembleTableRecord(final VarString vs) { vs.add("Total");