Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add more convenient API #535

Merged
merged 2 commits into from
Mar 1, 2023
Merged

Add more convenient API #535

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a63de43
Add more convenient API
fh-ms Feb 28, 2023
9387e42
Fix typo
fh-ms Mar 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 12 additions & 64 deletions docs/modules/storage/pages/import-export.adoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,41 +13,21 @@ NOTE: The created binary type data files contain only records of the according t

[source, java, title="Export"]
----
// Setup file system and storage, used in all examples below
NioFileSystem fileSystem = NioFileSystem.New();

EmbeddedStorageManager storage = EmbeddedStorage.start(
fileSystem.ensureDirectoryPath("storage")
);

String fileSuffix = "bin";
StorageConnection connection = storage.createConnection();
StorageEntityTypeExportStatistics exportResult = connection.exportTypes(
new StorageEntityTypeExportFileProvider.Default(
fileSystem.ensureDirectoryPath("export-dir"),
fileSuffix
),
typeHandler -> true // export all, customize if necessary
StorageConnection connection = storage.createConnection();
StorageEntityTypeExportStatistics exportResult = connection.exportTypes(
fileSystem.ensureDirectoryPath("export-dir")
);
XSequence<Path> exportFiles = CQL
.from(exportResult.typeStatistics().values())
.project(s -> Paths.get(s.file().identifier()))
.execute()
;
----

[source, java, title="Import"]
----
NioFileSystem fileSystem = NioFileSystem.New();

EmbeddedStorageManager storage = EmbeddedStorage.start(
fileSystem.ensureDirectoryPath("storage")
);

StorageConnection connection = storage.createConnection();
connection.importFiles(X.Enum(
fileSystem.ensureFilePath("type1.bin"),
fileSystem.ensureFilePath("type2.bin")
));
connection.importFiles(exportResult.files());
----

== Data Conversion
Expand All @@ -65,51 +45,19 @@ The file's size is at the possible minimum and the performance of the converter

[source, java, title="Binary to CSV"]
----
NioFileSystem fileSystem = NioFileSystem.New();

EmbeddedStorageManager storage = EmbeddedStorage.start(
fileSystem.ensureDirectoryPath("storage")
StorageDataConverterTypeBinaryToCsv converter = StorageDataConverterTypeBinaryToCsv.New(
fileSystem.ensureDirectoryPath("csv-dir"),
storage.typeDictionary()
);

StorageDataConverterTypeBinaryToCsv converter =
new StorageDataConverterTypeBinaryToCsv.UTF8(
StorageDataConverterCsvConfiguration.defaultConfiguration(),
new StorageEntityTypeConversionFileProvider.Default(
fileSystem.ensureDirectoryPath("csv-dir"),
"csv"
),
storage.typeDictionary(),
null, // no type name mapping
4096, // read buffer size
4096 // write buffer size
);
AReadableFile dataFile = fileSystem.ensureFilePath("type1.bin").useReading();
try
{
converter.convertDataFile(dataFile);
}
finally
{
dataFile.close();
}
converter.convertDataFiles(exportResult.files());
----

[source, java, title="CSV to Binary"]
----
NioFileSystem fileSystem = NioFileSystem.New();

EmbeddedStorageManager storage = EmbeddedStorage.start(
fileSystem.ensureDirectoryPath("storage")
);

StorageDataConverterTypeCsvToBinary<AFile> converter =
StorageDataConverterTypeCsvToBinary.New(
StorageDataConverterCsvConfiguration.defaultConfiguration(),
storage.typeDictionary(),
new StorageEntityTypeConversionFileProvider.Default(
fileSystem.ensureDirectoryPath("bin-dir"),
"dat"
)
StorageDataConverterTypeCsvToBinary.New(
storage.typeDictionary(),
fileSystem.ensureDirectoryPath("csv-dir")
);
converter.convertCsv(fileSystem.ensureFilePath("type1.csv"));
----
39 changes: 39 additions & 0 deletions storage/storage/src/main/java/one/microstream/storage/types/StorageConnection.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -323,6 +323,45 @@ public default StorageEntityTypeExportStatistics exportTypes(
{
return this.exportTypes(exportFileProvider, null);
}

/**
* Alias for {@code this.exportTypes(StorageEntityTypeExportFileProvider.New(targetDirectory));}, meaning all types are exported.
*
* @param targetDirectory the target directory for the export files
*
* @return a {@link StorageEntityTypeExportStatistics} information instance about the completed export.
*
* @see #exportTypes(StorageEntityTypeExportFileProvider, Predicate)
* @since 08.00.00
*/
public default StorageEntityTypeExportStatistics exportTypes(
final ADirectory targetDirectory
)
{
return this.exportTypes(StorageEntityTypeExportFileProvider.New(targetDirectory));
}

/**
* Alias for {@code this.exportTypes(StorageEntityTypeExportFileProvider.New(targetDirectory), isExportType);}.
*
* @param targetDirectory the target directory for the export files
* @param isExportType a {@link Predicate} selecting which type's entity data to be exported.
*
* @return a {@link StorageEntityTypeExportStatistics} information instance about the completed export.
*
* @see #exportTypes(StorageEntityTypeExportFileProvider, Predicate)
* @since 08.00.00
*/
public default StorageEntityTypeExportStatistics exportTypes(
final ADirectory targetDirectory,
final Predicate<? super StorageEntityTypeHandler> isExportType
)
{
return this.exportTypes(
StorageEntityTypeExportFileProvider.New(targetDirectory),
isExportType
);
}

/**
* Imports all files specified by the passed Enum (ordered set) of {@link AFile} in order.<br>
Expand Down
105 changes: 105 additions & 0 deletions ...rage/src/main/java/one/microstream/storage/types/StorageDataConverterTypeBinaryToCsv.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.nio.ByteOrder;

import one.microstream.X;
import one.microstream.afs.types.ADirectory;
import one.microstream.afs.types.AFS;
import one.microstream.afs.types.AFile;
import one.microstream.afs.types.AReadableFile;
Expand Down Expand Up @@ -62,6 +63,28 @@
public interface StorageDataConverterTypeBinaryToCsv
{
public void convertDataFile(AReadableFile file);

/**
* Batch-converts a list of files.
*
* @param files the binary files to convert to CSV
* @since 08.00.00
*/
public default <I extends Iterable<AFile>> void convertDataFiles(final I files)
{
for(final AFile file : files)
{
final AReadableFile dataFile = file.useReading();
try
{
this.convertDataFile(dataFile);
}
finally
{
dataFile.close();
}
}
}



Expand Down Expand Up @@ -93,6 +116,88 @@ public String mapTypeName(final PersistenceTypeDescriptionMember columnType)
}
};
}


/**
* Pseudo-constructor method to create a new {@link StorageDataConverterTypeBinaryToCsv}.
*
* @param targetDirectory write target directory
* @param typeDictionary the type dictionary to use
* @return a new {@link StorageDataConverterTypeBinaryToCsv}
* @since 08.00.00
*/
public static StorageDataConverterTypeBinaryToCsv New(
final ADirectory targetDirectory,
final PersistenceTypeDictionary typeDictionary
)
{
return New(
StorageDataConverterCsvConfiguration.defaultConfiguration(),
new StorageEntityTypeConversionFileProvider.Default(
targetDirectory,
"csv"
),
typeDictionary
);
}


/**
* Pseudo-constructor method to create a new {@link StorageDataConverterTypeBinaryToCsv}.
*
* @param configuration the CSV configuration to use
* @param fileProvider target file provider
* @param typeDictionary the type dictionary to use
* @return a new {@link StorageDataConverterTypeBinaryToCsv}
* @since 08.00.00
*/
public static StorageDataConverterTypeBinaryToCsv New(
final StorageDataConverterCsvConfiguration configuration ,
final StorageEntityTypeConversionFileProvider fileProvider ,
final PersistenceTypeDictionary typeDictionary
)
{
return New(
configuration ,
fileProvider ,
typeDictionary,
null ,
0 ,
0
);
}


/**
* Pseudo-constructor method to create a new {@link StorageDataConverterTypeBinaryToCsv}.
*
* @param configuration the CSV configuration to use
* @param fileProvider target file provider
* @param typeDictionary the type dictionary to use
* @param typeNameMapper optional type name mapper
* @param readBufferSize buffer size for reading
* @param writeBufferSize buffer size for writing
* @return a new {@link StorageDataConverterTypeBinaryToCsv}
* @since 08.00.00
*/
public static StorageDataConverterTypeBinaryToCsv New(
final StorageDataConverterCsvConfiguration configuration ,
final StorageEntityTypeConversionFileProvider fileProvider ,
final PersistenceTypeDictionary typeDictionary ,
final TypeNameMapper typeNameMapper ,
final int readBufferSize ,
final int writeBufferSize
)
{
return new StorageDataConverterTypeBinaryToCsv.UTF8(
configuration ,
fileProvider ,
typeDictionary ,
typeNameMapper ,
readBufferSize ,
writeBufferSize
);
}



Expand Down
75 changes: 74 additions & 1 deletion ...rage/src/main/java/one/microstream/storage/types/StorageDataConverterTypeCsvToBinary.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import java.util.Iterator;

import one.microstream.X;
import one.microstream.afs.types.ADirectory;
import one.microstream.afs.types.AFS;
import one.microstream.afs.types.AFile;
import one.microstream.afs.types.AWritableFile;
Expand Down Expand Up @@ -55,6 +56,17 @@
public interface StorageDataConverterTypeCsvToBinary<S>
{
public void convertCsv(S source);

/**
* Batch-converts given list of sources.
* @param <I> source type
* @param sources sources to convert
* @since 08.00.00
*/
public default <I extends Iterable<S>> void convertCsv(final I sources)
{
sources.forEach(this::convertCsv);
}



Expand All @@ -73,9 +85,61 @@ public interface ValueHandler
*/
public int handleValue(char[] data, int offset, int bound, char separator, char terminator);
}


/**
* Pseudo-constructor method to create a new {@link StorageDataConverterTypeCsvToBinary}.
* <p>
* The default file suffix from {@link StorageEntityTypeExportFileProvider} is used.
* @see StorageEntityTypeExportFileProvider.Defaults#defaultFileSuffix()
*
* @param typeDictionary type information
* @param targetDirectory binary file target directory
* @return a new {@link StorageDataConverterTypeCsvToBinary}
* @since 08.00.00
*/
public static StorageDataConverterTypeCsvToBinary<AFile> New(
final PersistenceTypeDictionary typeDictionary ,
final ADirectory targetDirectory
)
{
return New(
typeDictionary,
targetDirectory,
StorageEntityTypeExportFileProvider.Defaults.defaultFileSuffix()
);
}

/**
* Pseudo-constructor method to create a new {@link StorageDataConverterTypeCsvToBinary}.
*
* @param typeDictionary type information
* @param targetDirectory binary file target directory
* @param targetFileSuffix binary file suffix
* @return a new {@link StorageDataConverterTypeCsvToBinary}
* @since 08.00.00
*/
public static StorageDataConverterTypeCsvToBinary<AFile> New(
final PersistenceTypeDictionary typeDictionary ,
final ADirectory targetDirectory,
final String targetFileSuffix
)
{
return New(
StorageDataConverterCsvConfiguration.defaultConfiguration(),
typeDictionary,
StorageEntityTypeConversionFileProvider.New(targetDirectory, targetFileSuffix)
);
}


/**
* Pseudo-constructor method to create a new {@link StorageDataConverterTypeCsvToBinary}.
*
* @param configuration the csv configuration to use
* @param typeDictionary type information
* @param fileProvider target file provider
* @return a new {@link StorageDataConverterTypeCsvToBinary}
*/
public static StorageDataConverterTypeCsvToBinary<AFile> New(
final StorageDataConverterCsvConfiguration configuration ,
final PersistenceTypeDictionary typeDictionary,
Expand All @@ -85,6 +149,15 @@ public static StorageDataConverterTypeCsvToBinary<AFile> New(
return New(configuration, typeDictionary, fileProvider, 0);
}

/**
* Pseudo-constructor method to create a new {@link StorageDataConverterTypeCsvToBinary}.
*
* @param configuration the csv configuration to use
* @param typeDictionary type information
* @param fileProvider target file provider
* @param bufferSize custom buffer size or 0
* @return a new {@link StorageDataConverterTypeCsvToBinary}
*/
public static StorageDataConverterTypeCsvToBinary<AFile> New(
final StorageDataConverterCsvConfiguration configuration ,
final PersistenceTypeDictionary typeDictionary,
Expand Down
Loading