From 4f2970e5ca462c2706da2803ff5436adea24914c Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 18 Jan 2014 00:48:23 +0100 Subject: [PATCH] Win32: support long paths Windows paths are typically limited to MAX_PATH = 260 characters, even though the underlying NTFS file system supports paths up to 32,767 chars. This limitation is also evident in Windows Explorer, cmd.exe and many other applications (including IDEs). Particularly annoying is that most Windows APIs return bogus error codes if a relative path only barely exceeds MAX_PATH in conjunction with the current directory, e.g. ERROR_PATH_NOT_FOUND / ENOENT instead of the infinitely more helpful ERROR_FILENAME_EXCED_RANGE / ENAMETOOLONG. Many Windows wide char APIs support longer than MAX_PATH paths through the file namespace prefix ('\\?\' or '\\?\UNC\') followed by an absolute path. Notable exceptions include functions dealing with executables and the current directory (CreateProcess, LoadLibrary, Get/SetCurrentDirectory) as well as the entire shell API (ShellExecute, SHGetSpecialFolderPath...). Introduce a handle_long_path function to check the length of a specified path properly (and fail with ENAMETOOLONG), and to optionally expand long paths using the '\\?\' file namespace prefix. Short paths will not be modified, so we don't need to worry about device names (NUL, CON, AUX). Contrary to MSDN docs, the GetFullPathNameW function doesn't seem to be limited to MAX_PATH (at least not on Win7), so we can use it to do the heavy lifting of the conversion (translate '/' to '\', eliminate '.' and '..', and make an absolute path). Add long path error checking to xutftowcs_path for APIs with hard MAX_PATH limit. Add a new MAX_LONG_PATH constant and xutftowcs_long_path function for APIs that support long paths. While improved error checking is always active, long paths support must be explicitly enabled via 'core.longpaths' option. This is to prevent end users to shoot themselves in the foot by checking out files that Windows Explorer, cmd/bash or their favorite IDE cannot handle. Thanks-to: Martin W. Kirst Thanks-to: Doug Kelly Signed-off-by: Karsten Blees --- Documentation/config.txt | 7 +++ cache.h | 2 + compat/mingw.c | 129 ++++++++++++++++++++++++++++++--------- compat/mingw.h | 74 ++++++++++++++++++++-- compat/win32/dirent.c | 13 ++-- compat/win32/fscache.c | 17 +++--- config.c | 5 ++ environment.c | 1 + 8 files changed, 201 insertions(+), 47 deletions(-) diff --git a/Documentation/config.txt b/Documentation/config.txt index 15c340bd4cafdc..8810839f494e6e 100644 --- a/Documentation/config.txt +++ b/Documentation/config.txt @@ -626,6 +626,13 @@ core.fscache:: Git for Windows uses this to bulk-read and cache lstat data of entire directories (instead of doing lstat file by file). +core.longpaths:: + Enable long path (> 260) support for builtin commands in Git for + Windows. This is disabled by default, as long paths are not supported + by Windows Explorer, cmd.exe and the Git for Windows tool chain + (msys, bash, tcl, perl...). Only enable this if you know what you're + doing and are prepared to live with a few quirks. + core.createObject:: You can set this to 'link', in which case a hardlink followed by a delete of the source are used to make sure that object creation diff --git a/cache.h b/cache.h index c01a84c2a4761a..53f1c29e2da081 100644 --- a/cache.h +++ b/cache.h @@ -600,6 +600,8 @@ extern enum hide_dotfiles_type hide_dotfiles; extern int core_fscache; +extern int core_long_paths; + enum branch_track { BRANCH_TRACK_UNSPECIFIED = -1, BRANCH_TRACK_NEVER = 0, diff --git a/compat/mingw.c b/compat/mingw.c index 6627139d3df341..4e44e488579d94 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -204,8 +204,8 @@ static int ask_yes_no_if_possible(const char *format, ...) int mingw_unlink(const char *pathname) { int ret, tries = 0; - wchar_t wpathname[MAX_PATH]; - if (xutftowcs_path(wpathname, pathname) < 0) + wchar_t wpathname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; /* read-only files cannot be removed */ @@ -234,7 +234,7 @@ static int is_dir_empty(const wchar_t *wpath) { WIN32_FIND_DATAW findbuf; HANDLE handle; - wchar_t wbuf[MAX_PATH + 2]; + wchar_t wbuf[MAX_LONG_PATH + 2]; wcscpy(wbuf, wpath); wcscat(wbuf, L"\\*"); handle = FindFirstFileW(wbuf, &findbuf); @@ -255,8 +255,8 @@ static int is_dir_empty(const wchar_t *wpath) int mingw_rmdir(const char *pathname) { int ret, tries = 0; - wchar_t wpathname[MAX_PATH]; - if (xutftowcs_path(wpathname, pathname) < 0) + wchar_t wpathname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; while ((ret = _wrmdir(wpathname)) == -1 && tries < ARRAY_SIZE(delay)) { @@ -296,9 +296,9 @@ static int make_hidden(const wchar_t *path) void mingw_mark_as_git_dir(const char *dir) { - wchar_t wdir[MAX_PATH]; + wchar_t wdir[MAX_LONG_PATH]; if (hide_dotfiles != HIDE_DOTFILES_FALSE && !is_bare_repository()) - if (xutftowcs_path(wdir, dir) < 0 || make_hidden(wdir)) + if (xutftowcs_long_path(wdir, dir) < 0 || make_hidden(wdir)) warning("Failed to make '%s' hidden", dir); git_config_set("core.hideDotFiles", hide_dotfiles == HIDE_DOTFILES_FALSE ? "false" : @@ -309,9 +309,12 @@ void mingw_mark_as_git_dir(const char *dir) int mingw_mkdir(const char *path, int mode) { int ret; - wchar_t wpath[MAX_PATH]; - if (xutftowcs_path(wpath, path) < 0) + wchar_t wpath[MAX_LONG_PATH]; + /* CreateDirectoryW path limit is 248 (MAX_PATH - 8.3 file name) */ + if (xutftowcs_path_ex(wpath, path, MAX_LONG_PATH, -1, 248, + core_long_paths) < 0) return -1; + ret = _wmkdir(wpath); if (!ret && hide_dotfiles == HIDE_DOTFILES_TRUE) { /* @@ -331,7 +334,7 @@ int mingw_open (const char *filename, int oflags, ...) va_list args; unsigned mode; int fd; - wchar_t wfilename[MAX_PATH]; + wchar_t wfilename[MAX_LONG_PATH]; va_start(args, oflags); mode = va_arg(args, int); @@ -340,7 +343,7 @@ int mingw_open (const char *filename, int oflags, ...) if (filename && !strcmp(filename, "/dev/null")) filename = "nul"; - if (xutftowcs_path(wfilename, filename) < 0) + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; fd = _wopen(wfilename, oflags, mode); @@ -410,13 +413,13 @@ FILE *mingw_fopen (const char *filename, const char *otype) { int hide = 0; FILE *file; - wchar_t wfilename[MAX_PATH], wotype[4]; + wchar_t wfilename[MAX_LONG_PATH], wotype[4]; if (hide_dotfiles == HIDE_DOTFILES_TRUE && basename((char*)filename)[0] == '.') hide = access(filename, F_OK); if (filename && !strcmp(filename, "/dev/null")) filename = "nul"; - if (xutftowcs_path(wfilename, filename) < 0 || + if (xutftowcs_long_path(wfilename, filename) < 0 || xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0) return NULL; file = _wfopen(wfilename, wotype); @@ -429,13 +432,13 @@ FILE *mingw_freopen (const char *filename, const char *otype, FILE *stream) { int hide = 0; FILE *file; - wchar_t wfilename[MAX_PATH], wotype[4]; + wchar_t wfilename[MAX_LONG_PATH], wotype[4]; if (hide_dotfiles == HIDE_DOTFILES_TRUE && basename((char*)filename)[0] == '.') hide = access(filename, F_OK); if (filename && !strcmp(filename, "/dev/null")) filename = "nul"; - if (xutftowcs_path(wfilename, filename) < 0 || + if (xutftowcs_long_path(wfilename, filename) < 0 || xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0) return NULL; file = _wfreopen(wfilename, wotype, stream); @@ -468,25 +471,32 @@ int mingw_fflush(FILE *stream) int mingw_access(const char *filename, int mode) { - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, filename) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; /* X_OK is not supported by the MSVCRT version */ return _waccess(wfilename, mode & ~X_OK); } +/* cached length of current directory for handle_long_path */ +static int current_directory_len = 0; + int mingw_chdir(const char *dirname) { + int result; wchar_t wdirname[MAX_PATH]; + /* SetCurrentDirectoryW doesn't support long paths */ if (xutftowcs_path(wdirname, dirname) < 0) return -1; - return _wchdir(wdirname); + result = _wchdir(wdirname); + current_directory_len = GetCurrentDirectoryW(0, NULL); + return result; } int mingw_chmod(const char *filename, int mode) { - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, filename) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; return _wchmod(wfilename, mode); } @@ -501,8 +511,8 @@ int mingw_chmod(const char *filename, int mode) static int do_lstat(int follow, const char *file_name, struct stat *buf) { WIN32_FILE_ATTRIBUTE_DATA fdata; - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, file_name) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, file_name) < 0) return -1; if (GetFileAttributesExW(wfilename, GetFileExInfoStandard, &fdata)) { @@ -645,8 +655,8 @@ int mingw_utime (const char *file_name, const struct utimbuf *times) FILETIME mft, aft; int fh, rc; DWORD attrs; - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, file_name) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, file_name) < 0) return -1; /* must have write permission */ @@ -694,6 +704,7 @@ unsigned int sleep (unsigned int seconds) char *mingw_mktemp(char *template) { wchar_t wtemplate[MAX_PATH]; + /* we need to return the path, thus no long paths here! */ if (xutftowcs_path(wtemplate, template) < 0) return NULL; if (!_wmktemp(wtemplate)) @@ -1136,6 +1147,7 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen si.hStdOutput = winansi_get_osfhandle(fhout); si.hStdError = winansi_get_osfhandle(fherr); + /* executables and the current directory don't support long paths */ if (xutftowcs_path(wcmd, cmd) < 0) return -1; if (dir && xutftowcs_path(wdir, dir) < 0) @@ -1628,8 +1640,9 @@ int mingw_rename(const char *pold, const char *pnew) { DWORD attrs, gle; int tries = 0; - wchar_t wpold[MAX_PATH], wpnew[MAX_PATH]; - if (xutftowcs_path(wpold, pold) < 0 || xutftowcs_path(wpnew, pnew) < 0) + wchar_t wpold[MAX_LONG_PATH], wpnew[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpold, pold) < 0 || + xutftowcs_long_path(wpnew, pnew) < 0) return -1; /* @@ -1905,9 +1918,9 @@ int link(const char *oldpath, const char *newpath) { typedef BOOL (WINAPI *T)(LPCWSTR, LPCWSTR, LPSECURITY_ATTRIBUTES); static T create_hard_link = NULL; - wchar_t woldpath[MAX_PATH], wnewpath[MAX_PATH]; - if (xutftowcs_path(woldpath, oldpath) < 0 || - xutftowcs_path(wnewpath, newpath) < 0) + wchar_t woldpath[MAX_LONG_PATH], wnewpath[MAX_LONG_PATH]; + if (xutftowcs_long_path(woldpath, oldpath) < 0 || + xutftowcs_long_path(wnewpath, newpath) < 0) return -1; if (!create_hard_link) { @@ -2106,6 +2119,61 @@ int xwcstoutf(char *utf, const wchar_t *wcs, size_t utflen) return -1; } +int handle_long_path(wchar_t *path, int len, int max_path, int expand) +{ + int result; + wchar_t buf[MAX_LONG_PATH]; + + /* + * we don't need special handling if path is relative to the current + * directory, and current directory + path don't exceed the desired + * max_path limit. This should cover > 99 % of cases with minimal + * performance impact (git almost always uses relative paths). + */ + if ((len < 2 || (!is_dir_sep(path[0]) && path[1] != ':')) && + (current_directory_len + len < max_path)) + return len; + + /* + * handle everything else: + * - absolute paths: "C:\dir\file" + * - absolute UNC paths: "\\server\share\dir\file" + * - absolute paths on current drive: "\dir\file" + * - relative paths on other drive: "X:file" + * - prefixed paths: "\\?\...", "\\.\..." + */ + + /* convert to absolute path using GetFullPathNameW */ + result = GetFullPathNameW(path, MAX_LONG_PATH, buf, NULL); + if (!result) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + /* keep original path if its full path doesn't exceed max_path */ + if (result < max_path) + return len; + /* error out if we shouldn't expand the path or buf is too small */ + if (!expand || result >= MAX_LONG_PATH - 6) { + errno = ENAMETOOLONG; + return -1; + } + + /* prefix full path with "\\?\" or "\\?\UNC\" */ + if (buf[0] == '\\') { + /* ...unless already prefixed */ + if (buf[1] == '\\' && (buf[2] == '?' || buf[2] == '.')) + return len; + + wcscpy(path, L"\\\\?\\UNC\\"); + wcscpy(path + 8, buf + 2); + return result + 6; + } else { + wcscpy(path, L"\\\\?\\"); + wcscpy(path + 4, buf); + return result + 4; + } +} + /* * Disable MSVCRT command line wildcard expansion (__getmainargs called from * mingw startup code, see init.c in mingw runtime). @@ -2209,4 +2277,7 @@ void mingw_startup() /* initialize Unicode console */ winansi_init(); + + /* init length of current directory for handle_long_path */ + current_directory_len = GetCurrentDirectoryW(0, NULL); } diff --git a/compat/mingw.h b/compat/mingw.h index 28f6eef5b11764..b06861ed4c16b0 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -379,6 +379,39 @@ void mingw_open_html(const char *path); void mingw_mark_as_git_dir(const char *dir); #define mark_as_git_dir mingw_mark_as_git_dir +/** + * Max length of long paths (exceeding MAX_PATH). The actual maximum supported + * by NTFS is 32,767 (* sizeof(wchar_t)), but we choose an arbitrary smaller + * value to limit required stack memory. + */ +#define MAX_LONG_PATH 4096 + +/** + * Handles paths that would exceed the MAX_PATH limit of Windows Unicode APIs. + * + * With expand == false, the function only checks for over-long path and fails + * with ENAMETOOLONG. The path parameter is not modified. + * + * With expand == true, an over-long path is automatically converted in place + * to an absolute path prefixed with '\\?\', and the new length is returned. + * The path parameter must point to a buffer of MAX_LONG_PATH wide characters. + * + * Parameters: + * path: path to check and / or convert + * len: size of path on input (number of wide chars without \0) + * max_path: max short path length to check (usually MAX_PATH = 260, but just + * 248 for CreateDirectoryW) + * expand: false to only check the length, true to expand the path to a + * '\\?\'-prefixed absolute path + * + * Return: + * length of the resulting path, or -1 on failure + * + * Errors: + * ENAMETOOLONG if path is too long + */ +int handle_long_path(wchar_t *path, int len, int max_path, int expand); + /** * Converts UTF-8 encoded string to UTF-16LE. * @@ -437,18 +470,49 @@ static inline int xutftowcs(wchar_t *wcs, const char *utf, size_t wcslen) } /** - * Simplified file system specific variant of xutftowcsn, assumes output - * buffer size is MAX_PATH wide chars and input string is \0-terminated, - * fails with ENAMETOOLONG if input string is too long. + * Simplified file system specific wrapper of xutftowcsn and handle_long_path. + * Converts ERANGE to ENAMETOOLONG. If expand is true, wcs must be at least + * MAX_LONG_PATH wide chars (see handle_long_path). */ -static inline int xutftowcs_path(wchar_t *wcs, const char *utf) +static inline int xutftowcs_path_ex(wchar_t *wcs, const char *utf, + size_t wcslen, int utflen, int max_path, int expand) { - int result = xutftowcsn(wcs, utf, MAX_PATH, -1); + int result = xutftowcsn(wcs, utf, wcslen, utflen); if (result < 0 && errno == ERANGE) errno = ENAMETOOLONG; + if (result >= 0) + result = handle_long_path(wcs, result, max_path, expand); return result; } +/** + * Simplified file system specific variant of xutftowcsn, assumes output + * buffer size is MAX_PATH wide chars and input string is \0-terminated, + * fails with ENAMETOOLONG if input string is too long. Typically used for + * Windows APIs that don't support long paths, e.g. SetCurrentDirectory, + * LoadLibrary, CreateProcess... + */ +static inline int xutftowcs_path(wchar_t *wcs, const char *utf) +{ + return xutftowcs_path_ex(wcs, utf, MAX_PATH, -1, MAX_PATH, 0); +} + +/* need to re-declare that here as mingw.h is included before cache.h */ +extern int core_long_paths; + +/** + * Simplified file system specific variant of xutftowcsn for Windows APIs + * that support long paths via '\\?\'-prefix, assumes output buffer size is + * MAX_LONG_PATH wide chars, fails with ENAMETOOLONG if input string is too + * long. The 'core.longpaths' git-config option controls whether the path + * is only checked or expanded to a long path. + */ +static inline int xutftowcs_long_path(wchar_t *wcs, const char *utf) +{ + return xutftowcs_path_ex(wcs, utf, MAX_LONG_PATH, -1, MAX_PATH, + core_long_paths); +} + /** * Converts UTF-16LE encoded string to UTF-8. * diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 6b87042182d950..c18fd0024cf316 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -63,19 +63,22 @@ static int dirent_closedir(dirent_DIR *dir) DIR *dirent_opendir(const char *name) { - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + wchar_t pattern[MAX_LONG_PATH + 2]; /* + 2 for "\*" */ WIN32_FIND_DATAW fdata; HANDLE h; int len; dirent_DIR *dir; - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((len = xutftowcs_path(pattern, name)) < 0) + /* convert name to UTF-16 and check length */ + if ((len = xutftowcs_long_path(pattern, name)) < 0) return NULL; - /* append optional '/' and wildcard '*' */ + /* + * append optional '\' and wildcard '*'. Note: we need to use '\' as + * Windows doesn't translate '/' to '\' for "\\?\"-prefixed paths. + */ if (len && !is_dir_sep(pattern[len - 1])) - pattern[len++] = '/'; + pattern[len++] = '\\'; pattern[len++] = '*'; pattern[len] = 0; diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 5388d64377b700..da3bb2cae1d05e 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -162,23 +162,24 @@ static struct fsentry *fseentry_create_entry(struct fsentry *list, */ static struct fsentry *fsentry_create_list(const struct fsentry *dir) { - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + wchar_t pattern[MAX_LONG_PATH + 2]; /* + 2 for "\*" */ WIN32_FIND_DATAW fdata; HANDLE h; int wlen; struct fsentry *list, **phead; DWORD err; - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((wlen = xutftowcsn(pattern, dir->name, MAX_PATH, dir->len)) < 0) { - if (errno == ERANGE) - errno = ENAMETOOLONG; + /* convert name to UTF-16 and check length */ + if ((wlen = xutftowcs_path_ex(pattern, dir->name, MAX_LONG_PATH, + dir->len, MAX_PATH - 2, core_long_paths)) < 0) return NULL; - } - /* append optional '/' and wildcard '*' */ + /* + * append optional '\' and wildcard '*'. Note: we need to use '\' as + * Windows doesn't translate '/' to '\' for "\\?\"-prefixed paths. + */ if (wlen) - pattern[wlen++] = '/'; + pattern[wlen++] = '\\'; pattern[wlen++] = '*'; pattern[wlen] = 0; diff --git a/config.c b/config.c index 2edf1e7b0f93a3..fb6ad8cedce944 100644 --- a/config.c +++ b/config.c @@ -895,6 +895,11 @@ static int git_default_core_config(const char *var, const char *value) return 0; } + if (!strcmp(var, "core.longpaths")) { + core_long_paths = git_config_bool(var, value); + return 0; + } + /* Add other config variables here and to Documentation/config.txt. */ return 0; } diff --git a/environment.c b/environment.c index 66d67acda37c4a..697093d3185d8a 100644 --- a/environment.c +++ b/environment.c @@ -64,6 +64,7 @@ struct startup_info *startup_info; unsigned long pack_size_limit_cfg; enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; int core_fscache; +int core_long_paths; /* * The character that begins a commented line in user-editable file