Skip to content

Commit

Permalink
Merge pull request #9151 from rouault/win32_extended_filename_tunings
Browse files Browse the repository at this point in the history 
Win32 extended filenames (starting with "\\?\"): various fixes; add support for network UNC paths
  • Loading branch information
@rouault
rouault authored Jan 29, 2024
2 parents 4ec99d2 + db5169d commit 271d01d
Show file tree
Hide file tree
Showing 5 changed files with 237 additions and 55 deletions.
54 changes: 54 additions & 0 deletions autotest/gcore/vsifile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1236,3 +1236,57 @@ def progress(pct, msg, user_data):

def test_vsimem_illegal_filename():
assert gdal.FileFromMemBuffer("/vsimem/\\\\", "foo") == -1


###############################################################################
# Test operations with Windows special filenames (prefix with "\\?\")


@pytest.mark.skipif(sys.platform != "win32", reason="Windows specific test")
def test_vsifile_win32_special_filenames(tmp_path):

# Try prefix filenames
tmp_path_str = str(tmp_path)
if "/" not in tmp_path_str:
prefix_path = "\\\\?\\" + tmp_path_str
assert gdal.VSIStatL(prefix_path) is not None

assert gdal.MkdirRecursive(prefix_path + "\\foo\\bar", 0o755) == 0
assert gdal.VSIStatL(prefix_path + "\\foo\\bar") is not None

assert gdal.Mkdir(prefix_path + "\\foo\\baz", 0o755) == 0

f = gdal.VSIFOpenL(prefix_path + "\\foo\\file.bin", "wb")
assert f
gdal.VSIFCloseL(f)

assert set(gdal.ReadDir(prefix_path)) == set([".", "..", "foo"])
assert set(gdal.ReadDirRecursive(prefix_path)) == set(
["foo\\", "foo\\file.bin", "foo\\bar\\", "foo\\baz\\"]
)

assert gdal.Sync(prefix_path + "\\foo\\", prefix_path + "\\foo2")
assert set(gdal.ReadDirRecursive(prefix_path + "\\foo2")) == set(
["file.bin", "bar\\", "baz\\"]
)

assert gdal.Rmdir(prefix_path + "\\foo\\bar") == 0
assert gdal.RmdirRecursive(prefix_path + "\\foo") == 0
assert gdal.VSIStatL(prefix_path + "\\foo") is None


###############################################################################
# Test operations with Windows network path


@pytest.mark.skipif(sys.platform != "win32", reason="Windows specific test")
@pytest.mark.skipif(
gdaltest.is_travis_branch("mingw64"), reason="does not work with mingw64"
)
def test_vsifile_win32_network_path():

# Try the code path that converts network paths "\\foo\bar" to prefixed ones
# "\\?\foo\bar"
drive_letter = os.getcwd()[0]
dirname = f"\\\\localhost\\{drive_letter}$"
assert gdal.VSIStatL(dirname) is not None
72 changes: 72 additions & 0 deletions autotest/gcore/vsis3.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3719,6 +3719,78 @@ def test_vsis3_sync_source_target_in_vsis3(aws_test_config, webserver_port):
)


###############################################################################
# Test VSISync() with Windows special filenames (prefix with "\\?\")


@pytest.mark.skipif(sys.platform != "win32", reason="Windows specific test")
def test_vsis3_sync_win32_special_filenames(aws_test_config, webserver_port, tmp_path):

options = ["SYNC_STRATEGY=OVERWRITE"]

tmp_path_str = str(tmp_path)
if "/" in tmp_path_str:
pytest.skip("Found forward slash in tmp_path")

prefix_path = "\\\\?\\" + tmp_path_str

f = gdal.VSIFOpenL(prefix_path + "\\testsync.txt", "wb")
assert f
gdal.VSIFCloseL(f)

# S3 to local: S3 file is newer
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()

handler.add(
"GET",
"/out/testsync.txt",
206,
{
"Content-Length": "3",
"Content-Range": "bytes 0-2/3",
"Last-Modified": "Mon, 01 Jan 2037 00:00:01 GMT",
},
"foo",
)
handler.add(
"GET",
"/out/testsync.txt",
200,
{"Content-Length": "3", "Last-Modified": "Mon, 01 Jan 2037 00:00:01 GMT"},
"foo",
)
with webserver.install_http_handler(handler):
assert gdal.Sync("/vsis3/out/testsync.txt", prefix_path, options=options)

# Local to S3: S3 file is newer
gdal.VSICurlClearCache()
handler = webserver.SequentialHandler()
handler.add("GET", "/out/", 404)
handler.add("GET", "/out/", 404)
handler.add(
"GET",
"/out/?delimiter=%2F&max-keys=100",
200,
{},
"""<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult>
<Prefix></Prefix>
<Marker/>
<IsTruncated>false</IsTruncated>
<Contents>
<Key>testsync.txt</Key>
<LastModified>2037-01-01T00:00:01.000Z</LastModified>
<Size>3</Size>
</Contents>
</ListBucketResult>
""",
)
handler.add("PUT", "/out/testsync.txt", 200)
with webserver.install_http_handler(handler):
assert gdal.Sync(prefix_path + "\\", "/vsis3/out/", options=options)


###############################################################################
# Test rename

Expand Down
50 changes: 39 additions & 11 deletions port/cpl_vsil.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,12 @@ char **VSISiblingFiles(const char *pszFilename)

char **VSIReadDirRecursive(const char *pszPathIn)
{
#if defined(_WIN32)
const char SEP = pszPathIn[0] == '\\' ? '\\' : '/';
#else
constexpr char SEP = '/';
#endif

const char *const apszOptions[] = {"NAME_AND_TYPE_ONLY=YES", nullptr};
VSIDIR *psDir = VSIOpenDir(pszPathIn, -1, apszOptions);
if (!psDir)
Expand All @@ -191,9 +197,9 @@ char **VSIReadDirRecursive(const char *pszPathIn)
while (auto psEntry = VSIGetNextDirEntry(psDir))
{
if (VSI_ISDIR(psEntry->nMode) && psEntry->pszName[0] &&
psEntry->pszName[strlen(psEntry->pszName) - 1] != '/')
psEntry->pszName[strlen(psEntry->pszName) - 1] != SEP)
{
oFiles.AddString((std::string(psEntry->pszName) + '/').c_str());
oFiles.AddString((std::string(psEntry->pszName) + SEP).c_str());
}
else
{
Expand Down Expand Up @@ -1369,6 +1375,12 @@ bool VSIFilesystemHandler::Sync(const char *pszSource, const char *pszTarget,
GDALProgressFunc pProgressFunc,
void *pProgressData, char ***ppapszOutputs)
{
#if defined(_WIN32)
const char SOURCE_SEP = pszSource[0] == '\\' ? '\\' : '/';
#else
constexpr char SOURCE_SEP = '/';
#endif

if (ppapszOutputs)
{
*ppapszOutputs = nullptr;
Expand All @@ -1377,7 +1389,8 @@ bool VSIFilesystemHandler::Sync(const char *pszSource, const char *pszTarget,
VSIStatBufL sSource;
CPLString osSource(pszSource);
CPLString osSourceWithoutSlash(pszSource);
if (osSourceWithoutSlash.back() == '/')
if (osSourceWithoutSlash.back() == '/' ||
osSourceWithoutSlash.back() == '\\')
{
osSourceWithoutSlash.resize(osSourceWithoutSlash.size() - 1);
}
Expand All @@ -1390,7 +1403,7 @@ bool VSIFilesystemHandler::Sync(const char *pszSource, const char *pszTarget,
if (VSI_ISDIR(sSource.st_mode))
{
CPLString osTargetDir(pszTarget);
if (osSource.back() != '/')
if (osSource.back() != '/' && osSource.back() != '\\')
{
osTargetDir = CPLFormFilename(osTargetDir,
CPLGetFilename(pszSource), nullptr);
Expand Down Expand Up @@ -1441,7 +1454,7 @@ bool VSIFilesystemHandler::Sync(const char *pszSource, const char *pszTarget,
void *pScaledProgress = GDALCreateScaledProgress(
double(iFile) / nFileCount, double(iFile + 1) / nFileCount,
pProgressFunc, pProgressData);
ret = Sync((osSubSource + '/').c_str(), osSubTarget,
ret = Sync((osSubSource + SOURCE_SEP).c_str(), osSubTarget,
aosChildOptions.List(), GDALScaledProgress,
pScaledProgress, nullptr);
GDALDestroyScaledProgress(pScaledProgress);
Expand Down Expand Up @@ -1651,12 +1664,18 @@ VSIDIR *VSIFilesystemHandler::OpenDir(const char *pszPath, int nRecurseDepth,

const VSIDIREntry *VSIDIRGeneric::NextDirEntry()
{
#if defined(_WIN32)
const char SEP = osRootPath[0] == '\\' ? '\\' : '/';
#else
constexpr char SEP = '/';
#endif

begin:
if (VSI_ISDIR(entry.nMode) && nRecurseDepth != 0)
{
CPLString osCurFile(osRootPath);
if (!osCurFile.empty())
osCurFile += '/';
osCurFile += SEP;
osCurFile += entry.pszName;
auto subdir =
static_cast<VSIDIRGeneric *>(poFS->VSIFilesystemHandler::OpenDir(
Expand Down Expand Up @@ -1705,15 +1724,15 @@ const VSIDIREntry *VSIDIRGeneric::NextDirEntry()
CPLFree(entry.pszName);
CPLString osName(osBasePath);
if (!osName.empty())
osName += '/';
osName += SEP;
osName += papszContent[nPos];
nPos++;

entry.pszName = CPLStrdup(osName);
entry.nMode = 0;
CPLString osCurFile(osRootPath);
if (!osCurFile.empty())
osCurFile += '/';
osCurFile += SEP;
osCurFile += entry.pszName;

const auto StatFile = [&osCurFile, this]()
Expand Down Expand Up @@ -1743,7 +1762,7 @@ const VSIDIREntry *VSIDIRGeneric::NextDirEntry()
m_osFilterPrefix.size() > osName.size())
{
if (STARTS_WITH(m_osFilterPrefix.c_str(), osName.c_str()) &&
m_osFilterPrefix[osName.size()] == '/')
m_osFilterPrefix[osName.size()] == SEP)
{
StatFile();
if (VSI_ISDIR(entry.nMode))
Expand Down Expand Up @@ -1791,8 +1810,17 @@ int VSIFilesystemHandler::RmdirRecursive(const char *pszDirname)
{
CPLString osDirnameWithoutEndSlash(pszDirname);
if (!osDirnameWithoutEndSlash.empty() &&
osDirnameWithoutEndSlash.back() == '/')
(osDirnameWithoutEndSlash.back() == '/' ||
osDirnameWithoutEndSlash.back() == '\\'))
{
osDirnameWithoutEndSlash.resize(osDirnameWithoutEndSlash.size() - 1);
}

#if defined(_WIN32)
const char SEP = pszDirname[0] == '\\' ? '\\' : '/';
#else
constexpr char SEP = '/';
#endif

CPLStringList aosOptions;
auto poDir =
Expand All @@ -1806,7 +1834,7 @@ int VSIFilesystemHandler::RmdirRecursive(const char *pszDirname)
if (!entry)
break;

const CPLString osFilename(osDirnameWithoutEndSlash + '/' +
const CPLString osFilename(osDirnameWithoutEndSlash + SEP +
entry->pszName);
if ((entry->nMode & S_IFDIR))
{
Expand Down
8 changes: 5 additions & 3 deletions port/cpl_vsil_s3.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3796,7 +3796,8 @@ bool IVSIS3LikeFSHandler::Sync(const char *pszSource, const char *pszTarget,

std::string osSource(pszSource);
std::string osSourceWithoutSlash(pszSource);
if (osSourceWithoutSlash.back() == '/')
if (osSourceWithoutSlash.back() == '/' ||
osSourceWithoutSlash.back() == '\\')
{
osSourceWithoutSlash.resize(osSourceWithoutSlash.size() - 1);
}
Expand Down Expand Up @@ -3845,7 +3846,8 @@ bool IVSIS3LikeFSHandler::Sync(const char *pszSource, const char *pszTarget,
// If the source is likely to be a directory, try to issue a ReadDir()
// if we haven't stat'ed it yet
std::unique_ptr<VSIDIR> poSourceDir;
if (STARTS_WITH(pszSource, GetFSPrefix().c_str()) && osSource.back() == '/')
if (STARTS_WITH(pszSource, GetFSPrefix().c_str()) &&
(osSource.back() == '/' || osSource.back() == '\\'))
{
const char *const apszOptions[] = {"SYNTHETIZE_MISSING_DIRECTORIES=YES",
nullptr};
Expand Down Expand Up @@ -4079,7 +4081,7 @@ bool IVSIS3LikeFSHandler::Sync(const char *pszSource, const char *pszTarget,
if (VSI_ISDIR(sSource.st_mode))
{
osTargetDir = pszTarget;
if (osSource.back() != '/')
if (osSource.back() != '/' && osSource.back() != '\\')
{
osTargetDir = CPLFormFilename(osTargetDir.c_str(),
CPLGetFilename(pszSource), nullptr);
Expand Down
Loading

0 comments on commit 271d01d

Please sign in to comment.