Skip to content

Commit

Permalink
Merge pull request #10771 from rouault/invisible_vsimem
Browse files Browse the repository at this point in the history
Add VSIMemGenerateHiddenFilename() and use it in drivers
  • Loading branch information
rouault authored Sep 16, 2024
2 parents 79577b1 + 9119a48 commit ce6bae8
Show file tree
Hide file tree
Showing 84 changed files with 765 additions and 397 deletions.
4 changes: 2 additions & 2 deletions apps/gdalwarp_lib.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1175,8 +1175,8 @@ static bool DealWithCOGOptions(CPLStringList &aosCreateOptions, int nSrcCount,

GDALWarpAppOptions oClonedOptions(*psOptions);
oClonedOptions.bQuiet = true;
CPLString osTmpFilename;
osTmpFilename.Printf("/vsimem/gdalwarp/%p.tif", &oClonedOptions);
const CPLString osTmpFilename(
VSIMemGenerateHiddenFilename("gdalwarp_tmp.tif"));
CPLStringList aosTmpGTiffCreateOptions;
aosTmpGTiffCreateOptions.SetNameValue("SPARSE_OK", "YES");
aosTmpGTiffCreateOptions.SetNameValue("TILED", "YES");
Expand Down
190 changes: 190 additions & 0 deletions autotest/cpp/test_cpl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5331,4 +5331,194 @@ TEST_F(test_cpl, CPLSpawn)
}
#endif

static bool ENDS_WITH(const char *pszStr, const char *pszEnd)
{
return strlen(pszStr) >= strlen(pszEnd) &&
strcmp(pszStr + strlen(pszStr) - strlen(pszEnd), pszEnd) == 0;
}

TEST_F(test_cpl, VSIMemGenerateHiddenFilename)
{
{
// Initial cleanup
VSIRmdirRecursive("/vsimem/");
VSIRmdirRecursive("/vsimem/.#!HIDDEN!#.");

// Generate unlisted filename
const std::string osFilename1 = VSIMemGenerateHiddenFilename(nullptr);
const char *pszFilename1 = osFilename1.c_str();
EXPECT_TRUE(STARTS_WITH(pszFilename1, "/vsimem/.#!HIDDEN!#./"));
EXPECT_TRUE(ENDS_WITH(pszFilename1, "/unnamed"));

{
// Check the file doesn't exist yet
VSIStatBufL sStat;
EXPECT_EQ(VSIStatL(pszFilename1, &sStat), -1);
}

// Create the file with some content
GByte abyDummyData[1] = {0};
VSIFCloseL(VSIFileFromMemBuffer(pszFilename1, abyDummyData,
sizeof(abyDummyData), false));

{
// Check the file exists now
VSIStatBufL sStat;
EXPECT_EQ(VSIStatL(pszFilename1, &sStat), 0);
}

// Get's back content
EXPECT_EQ(VSIGetMemFileBuffer(pszFilename1, nullptr, false),
abyDummyData);

{
// Check the hidden file doesn't popup
const CPLStringList aosFiles(VSIReadDir("/vsimem/"));
EXPECT_EQ(aosFiles.size(), 0);
}

{
// Check that we can list the below directory if we know it exists
// and there's just one subdir
const CPLStringList aosFiles(VSIReadDir("/vsimem/.#!HIDDEN!#."));
EXPECT_EQ(aosFiles.size(), 1);
}

{
// but that it is not an explicit directory
VSIStatBufL sStat;
EXPECT_EQ(VSIStatL("/vsimem/.#!HIDDEN!#.", &sStat), -1);
}

// Creates second file
const std::string osFilename2 = VSIMemGenerateHiddenFilename(nullptr);
const char *pszFilename2 = osFilename2.c_str();
EXPECT_TRUE(strcmp(pszFilename1, pszFilename2) != 0);

// Create it
VSIFCloseL(VSIFileFromMemBuffer(pszFilename2, abyDummyData,
sizeof(abyDummyData), false));

{
// Check that we can list the root hidden dir if we know it exists
const CPLStringList aosFiles(VSIReadDir("/vsimem/.#!HIDDEN!#."));
EXPECT_EQ(aosFiles.size(), 2);
}

{
// Create an explicit subdirectory in a hidden directory
const std::string osBaseName =
VSIMemGenerateHiddenFilename(nullptr);
const std::string osSubDir =
CPLFormFilename(osBaseName.c_str(), "mysubdir", nullptr);
EXPECT_EQ(VSIMkdir(osSubDir.c_str(), 0), 0);

// Check the subdirectory exists
{
VSIStatBufL sStat;
EXPECT_EQ(VSIStatL(osSubDir.c_str(), &sStat), 0);
}

// but not its hidden parent
{
VSIStatBufL sStat;
EXPECT_EQ(VSIStatL(osBaseName.c_str(), &sStat), -1);
}

// Create file within the subdirectory
VSIFCloseL(VSIFileFromMemBuffer(
CPLFormFilename(osSubDir.c_str(), "my.bin", nullptr),
abyDummyData, sizeof(abyDummyData), false));

{
// Check that we can list the subdirectory
const CPLStringList aosFiles(VSIReadDir(osSubDir.c_str()));
EXPECT_EQ(aosFiles.size(), 1);
}

{
// Check that we can list the root hidden dir if we know it exists
const CPLStringList aosFiles(
VSIReadDir("/vsimem/.#!HIDDEN!#."));
EXPECT_EQ(aosFiles.size(), 3);
}
}

// Directly create a directory with the return of VSIMemGenerateHiddenFilename()
{
const std::string osDirname = VSIMemGenerateHiddenFilename(nullptr);
EXPECT_EQ(VSIMkdir(osDirname.c_str(), 0), 0);

// Check the subdirectory exists
{
VSIStatBufL sStat;
EXPECT_EQ(VSIStatL(osDirname.c_str(), &sStat), 0);
}

// Create file within the subdirectory
VSIFCloseL(VSIFileFromMemBuffer(
CPLFormFilename(osDirname.c_str(), "my.bin", nullptr),
abyDummyData, sizeof(abyDummyData), false));

{
// Check there's a file in this subdirectory
const CPLStringList aosFiles(VSIReadDir(osDirname.c_str()));
EXPECT_EQ(aosFiles.size(), 1);
}

EXPECT_EQ(VSIRmdirRecursive(osDirname.c_str()), 0);

{
// Check there's no longer any file in this subdirectory
const CPLStringList aosFiles(VSIReadDir(osDirname.c_str()));
EXPECT_EQ(aosFiles.size(), 0);
}

{
// Check that it no longer exists
VSIStatBufL sStat;
EXPECT_EQ(VSIStatL(osDirname.c_str(), &sStat), -1);
}
}

// Check that operations on "/vsimem/" do not interfere with hidden files
{
// Create regular file
VSIFCloseL(VSIFileFromMemBuffer("/vsimem/regular_file",
abyDummyData, sizeof(abyDummyData),
false));

// Check it is visible
EXPECT_EQ(CPLStringList(VSIReadDir("/vsimem/")).size(), 1);

// Clean root /vsimem/
VSIRmdirRecursive("/vsimem/");

// No more user files
EXPECT_TRUE(CPLStringList(VSIReadDir("/vsimem/")).empty());

// But still hidden files
EXPECT_TRUE(
!CPLStringList(VSIReadDir("/vsimem/.#!HIDDEN!#.")).empty());
}

// Clean-up hidden files
EXPECT_EQ(VSIRmdirRecursive("/vsimem/.#!HIDDEN!#."), 0);

{
// Check the root hidden dir is empty
const CPLStringList aosFiles(VSIReadDir("/vsimem/.#!HIDDEN!#."));
EXPECT_TRUE(aosFiles.empty());
}

EXPECT_EQ(VSIRmdirRecursive("/vsimem/.#!HIDDEN!#."), 0);
}

{
const std::string osFilename = VSIMemGenerateHiddenFilename("foo.bar");
const char *pszFilename = osFilename.c_str();
EXPECT_TRUE(STARTS_WITH(pszFilename, "/vsimem/.#!HIDDEN!#./"));
EXPECT_TRUE(ENDS_WITH(pszFilename, "/foo.bar"));
}
}
} // namespace
20 changes: 20 additions & 0 deletions autotest/gcore/vsifile.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,26 @@ def vsifile_generic(filename, options=[]):

gdal.Unlink(filename)

if not filename.startswith("/vsicrypt/"):
assert gdal.RmdirRecursive(filename + "/i_dont_exist") == -1

subdir = filename + "/subdir"
assert gdal.MkdirRecursive(subdir + "/subsubdir", 0o755) == 0

assert gdal.VSIStatL(subdir) is not None
assert gdal.VSIStatL(subdir + "/subsubdir") is not None

if not filename.startswith("/vsimem/"):
assert gdal.Rmdir(subdir) == -1
assert gdal.VSIStatL(subdir) is not None

# Safety belt...
assert filename.startswith("tmp/") or filename.startswith("/vsimem/")
assert gdal.RmdirRecursive(filename) == 0

assert gdal.VSIStatL(subdir) is None
assert gdal.VSIStatL(subdir + "/subsubdir") is None


###############################################################################
# Test /vsimem
Expand Down
4 changes: 2 additions & 2 deletions autotest/gdrivers/jp2openjpeg.py
Original file line number Diff line number Diff line change
Expand Up @@ -2744,10 +2744,10 @@ def test_jp2openjpeg_45():
)
del out_ds

dircontent = gdal.ReadDir("/vsimem/")
dircontent = gdal.ReadDir("/vsimem/.#!HIDDEN!#.")
if dircontent:
for filename in dircontent:
assert not filename.startswith("gmljp2")
assert "gmljp2" not in filename

ds = ogr.Open("/vsimem/jp2openjpeg_45.jp2")
assert ds.GetLayerCount() == 1
Expand Down
5 changes: 5 additions & 0 deletions autotest/ogr/ogr_csw.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,14 @@ def module_disable_exceptions():
###############################################################################
@pytest.fixture(autouse=True, scope="module")
def setup_and_cleanup():

vsimem_hidden_before = gdal.ReadDirRecursive("/vsimem/.#!HIDDEN!#.")

with gdal.config_option("CPL_CURL_ENABLE_VSIMEM", "YES"):
yield

assert gdal.ReadDirRecursive("/vsimem/.#!HIDDEN!#.") == vsimem_hidden_before


###############################################################################
# Test underlying OGR drivers
Expand Down
4 changes: 4 additions & 0 deletions autotest/ogr/ogr_wfs.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,13 @@ def ogr_wfs_init():
if gml_ds is None:
pytest.skip("cannot read GML files")

vsimem_hidden_before = gdal.ReadDirRecursive("/vsimem/.#!HIDDEN!#.")

with gdal.config_option("CPL_CURL_ENABLE_VSIMEM", "YES"):
yield

assert gdal.ReadDirRecursive("/vsimem/.#!HIDDEN!#.") == vsimem_hidden_before


@pytest.fixture(
params=["NO", None], scope="module", ids=["without-streaming", "with-streaming"]
Expand Down
7 changes: 4 additions & 3 deletions frmts/cals/calsdataset.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,7 @@ GDALDataset *CALSDataset::Open(GDALOpenInfo *poOpenInfo)

// Create a TIFF header for a single-strip CCITTFAX4 file.
poDS->osTIFFHeaderFilename =
CPLSPrintf("/vsimem/cals/header_%p.tiff", poDS);
VSIMemGenerateHiddenFilename("cals_header.tiff");
VSILFILE *fp = VSIFOpenL(poDS->osTIFFHeaderFilename, "wb");
const int nTagCount = 10;
const int nHeaderSize = 4 + 4 + 2 + nTagCount * 12 + 4;
Expand Down Expand Up @@ -359,7 +359,7 @@ GDALDataset *CALSDataset::Open(GDALOpenInfo *poOpenInfo)

// Create a /vsisparse/ description file assembling the TIFF header
// with the FAX4 codestream that starts at offset 2048 of the CALS file.
poDS->osSparseFilename = CPLSPrintf("/vsimem/cals/sparse_%p.xml", poDS);
poDS->osSparseFilename = VSIMemGenerateHiddenFilename("cals_sparse.xml");
fp = VSIFOpenL(poDS->osSparseFilename, "wb");
CPLAssert(fp);
VSIFPrintfL(fp,
Expand Down Expand Up @@ -473,7 +473,8 @@ GDALDataset *CALSDataset::CreateCopy(const char *pszFilename,

// Write a in-memory TIFF with just the TIFF header to figure out
// how large it will be.
CPLString osTmpFilename(CPLSPrintf("/vsimem/cals/tmp_%p", poSrcDS));
const CPLString osTmpFilename(
VSIMemGenerateHiddenFilename("tmp_tif_header"));
char **papszOptions = nullptr;
papszOptions = CSLSetNameValue(papszOptions, "COMPRESS", "CCITTFAX4");
papszOptions = CSLSetNameValue(papszOptions, "NBITS", "1");
Expand Down
2 changes: 1 addition & 1 deletion frmts/daas/daasdataset.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2420,7 +2420,7 @@ CPLErr GDALDAASRasterBand::GetBlocks(int nBlockXOff, int nBlockYOff,
}
else
{
CPLString osTmpMemFile = CPLSPrintf("/vsimem/daas_%p", this);
const CPLString osTmpMemFile = VSIMemGenerateHiddenFilename("daas");
VSIFCloseL(VSIFileFromMemBuffer(
osTmpMemFile, psResult->pasMimePart[iDataPart].pabyData,
psResult->pasMimePart[iDataPart].nDataLen, false));
Expand Down
7 changes: 4 additions & 3 deletions frmts/ecw/ecwdataset.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2850,10 +2850,11 @@ GDALDataset *ECWDataset::Open(GDALOpenInfo *poOpenInfo, int bIsJPEG2000)
/* There are issues at least in the 5.x series. */
/* -------------------------------------------------------------------- */
#if ECWSDK_VERSION >= 40
constexpr const char *szDETECT_BUG_FILENAME =
"__detect_ecw_uint32_bug__.j2k";
if (bIsJPEG2000 && poDS->eNCSRequestDataType == NCSCT_UINT32 &&
CPLTestBool(CPLGetConfigOption("ECW_CHECK_CORRECT_DECODING", "TRUE")) &&
!STARTS_WITH_CI(poOpenInfo->pszFilename,
"/vsimem/detect_ecw_uint32_bug"))
strstr(poOpenInfo->pszFilename, szDETECT_BUG_FILENAME) == nullptr)
{
static bool bUINT32_Ok = false;
{
Expand All @@ -2878,7 +2879,7 @@ GDALDataset *ECWDataset::Open(GDALOpenInfo *poOpenInfo, int bIsJPEG2000)
0xDF, 0xFF, 0x7F, 0x5F, 0xFF, 0xD9};

const std::string osTmpFilename =
CPLSPrintf("/vsimem/detect_ecw_uint32_bug_%p.j2k", poDS);
VSIMemGenerateHiddenFilename(szDETECT_BUG_FILENAME);
VSIFCloseL(VSIFileFromMemBuffer(
osTmpFilename.c_str(),
const_cast<GByte *>(abyTestUInt32ImageData),
Expand Down
2 changes: 1 addition & 1 deletion frmts/eeda/eedaidataset.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -433,7 +433,7 @@ bool GDALEEDAIRasterBand::DecodeGDALDataset(const GByte *pabyData, int nDataLen,
{
GDALEEDAIDataset *poGDS = reinterpret_cast<GDALEEDAIDataset *>(poDS);

CPLString osTmpFilename(CPLSPrintf("/vsimem/eeai/%p", this));
const CPLString osTmpFilename(VSIMemGenerateHiddenFilename("eedai"));
VSIFCloseL(VSIFileFromMemBuffer(
osTmpFilename, const_cast<GByte *>(pabyData), nDataLen, false));
const char *const apszDrivers[] = {"PNG", "JPEG", "GTIFF", nullptr};
Expand Down
4 changes: 1 addition & 3 deletions frmts/esric/esric_dataset.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -896,9 +896,7 @@ CPLErr ECBand::IReadBlock(int nBlockXOff, int nBlockYOff, void *pData)
GUInt64(size), GUInt64(offset));
return CE_Failure;
}
CPLString magic;
// Should use some sort of unique
magic.Printf("/vsimem/esric_%p.tmp", this);
const CPLString magic(VSIMemGenerateHiddenFilename("esric.tmp"));
auto mfh = VSIFileFromMemBuffer(magic.c_str(), fbuffer.data(), size, false);
VSIFCloseL(mfh);
// Can't open a raster by handle?
Expand Down
Loading

0 comments on commit ce6bae8

Please sign in to comment.