Commit 231aa9d6 authored by Quentin Smith's avatar Quentin Smith

os: use extended-length paths on Windows when possible

Windows has a limit of 260 characters on normal paths, but it's possible
to use longer paths by using "extended-length paths" that begin with
`\\?\`. This commit attempts to transparently convert an absolute path
to an extended-length path, following the subtly different rules those
paths require. It does not attempt to handle relative paths, which
continue to be passed to the operating system unmodified.

This adds a new test, TestLongPath, to the os package. This test makes
sure that it is possible to write a path at least 400 characters long
and runs on every platform. It also tests symlinks and hardlinks, though
symlinks are not testable with our builder configuration.

HasLink is moved to internal/testenv so it can be used by multiple tests.

https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx
has Microsoft's documentation on extended-length paths.

Fixes #3358.
Fixes #10577.
Fixes #17500.

Change-Id: I4ff6bb2ef9c9a4468d383d98379f65cf9c448218
Reviewed-on: https://go-review.googlesource.com/32451
Run-TryBot: Quentin Smith <quentin@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: default avatarRuss Cox <rsc@golang.org>
parent 2058511e
......@@ -153,6 +153,22 @@ func MustHaveSymlink(t *testing.T) {
}
}
// HasLink reports whether the current system can use os.Link.
func HasLink() bool {
// From Android release M (Marshmallow), hard linking files is blocked
// and an attempt to call link() on a file will return EACCES.
// - https://code.google.com/p/android-developer-preview/issues/detail?id=3150
return runtime.GOOS != "plan9" && runtime.GOOS != "android"
}
// MustHaveLink reports whether the current system can use os.Link.
// If not, MustHaveLink calls t.Skip with an explanation.
func MustHaveLink(t *testing.T) {
if !HasLink() {
t.Skipf("skipping test: hardlinks are not supported on %s/%s", runtime.GOOS, runtime.GOARCH)
}
}
var flaky = flag.Bool("flaky", false, "run known-flaky tests too")
func SkipFlaky(t *testing.T, issue int) {
......
......@@ -11,4 +11,5 @@ var (
GetCPP = &getCP
ReadFileP = &readFile
ResetGetConsoleCPAndReadFileFuncs = resetGetConsoleCPAndReadFileFuncs
FixLongPath = fixLongPath
)
......@@ -203,7 +203,7 @@ func (f *File) WriteString(s string) (n int, err error) {
// Mkdir creates a new directory with the specified name and permission bits.
// If there is an error, it will be of type *PathError.
func Mkdir(name string, perm FileMode) error {
e := syscall.Mkdir(name, syscallMode(perm))
e := syscall.Mkdir(fixLongPath(name), syscallMode(perm))
if e != nil {
return &PathError{"mkdir", name, e}
......
......@@ -11,6 +11,11 @@ import (
"time"
)
// fixLongPath is a noop on non-Windows platforms.
func fixLongPath(path string) string {
return path
}
// file is the real representation of *File.
// The extra level of indirection ensures that no clients of os
// can overwrite this data, which could cause the finalizer
......
......@@ -18,7 +18,7 @@ func sigpipe() // implemented in package runtime
func Readlink(name string) (string, error) {
for len := 128; ; len *= 2 {
b := make([]byte, len)
n, e := fixCount(syscall.Readlink(name, b))
n, e := fixCount(syscall.Readlink(fixLongPath(name), b))
if e != nil {
return "", &PathError{"readlink", name, e}
}
......@@ -134,7 +134,7 @@ func Chtimes(name string, atime time.Time, mtime time.Time) error {
var utimes [2]syscall.Timespec
utimes[0] = syscall.NsecToTimespec(atime.UnixNano())
utimes[1] = syscall.NsecToTimespec(mtime.UnixNano())
if e := syscall.UtimesNano(name, utimes[0:]); e != nil {
if e := syscall.UtimesNano(fixLongPath(name), utimes[0:]); e != nil {
return &PathError{"chtimes", name, e}
}
return nil
......
......@@ -11,6 +11,11 @@ import (
"syscall"
)
// fixLongPath is a noop on non-Windows platforms.
func fixLongPath(path string) string {
return path
}
func rename(oldname, newname string) error {
fi, err := Lstat(newname)
if err == nil && fi.IsDir() {
......
......@@ -86,7 +86,7 @@ const DevNull = "NUL"
func (f *file) isdir() bool { return f != nil && f.dirinfo != nil }
func openFile(name string, flag int, perm FileMode) (file *File, err error) {
r, e := syscall.Open(name, flag|syscall.O_CLOEXEC, syscallMode(perm))
r, e := syscall.Open(fixLongPath(name), flag|syscall.O_CLOEXEC, syscallMode(perm))
if e != nil {
return nil, e
}
......@@ -95,10 +95,13 @@ func openFile(name string, flag int, perm FileMode) (file *File, err error) {
func openDir(name string) (file *File, err error) {
var mask string
if len(name) == 2 && name[1] == ':' { // it is a drive letter, like C:
mask = name + `*`
path := fixLongPath(name)
if len(path) == 2 && path[1] == ':' || (len(path) > 0 && path[len(path)-1] == '\\') { // it is a drive letter, like C:
mask = path + `*`
} else {
mask = name + `\*`
mask = path + `\*`
}
maskp, e := syscall.UTF16PtrFromString(mask)
if e != nil {
......@@ -114,11 +117,11 @@ func openDir(name string) (file *File, err error) {
return nil, e
}
var fa syscall.Win32FileAttributeData
namep, e := syscall.UTF16PtrFromString(name)
pathp, e := syscall.UTF16PtrFromString(path)
if e != nil {
return nil, e
}
e = syscall.GetFileAttributesEx(namep, syscall.GetFileExInfoStandard, (*byte)(unsafe.Pointer(&fa)))
e = syscall.GetFileAttributesEx(pathp, syscall.GetFileExInfoStandard, (*byte)(unsafe.Pointer(&fa)))
if e != nil {
return nil, e
}
......@@ -127,7 +130,7 @@ func openDir(name string) (file *File, err error) {
}
d.isempty = true
}
d.path = name
d.path = path
if !isAbs(d.path) {
d.path, e = syscall.FullPath(d.path)
if e != nil {
......@@ -439,7 +442,7 @@ func Truncate(name string, size int64) error {
// Remove removes the named file or directory.
// If there is an error, it will be of type *PathError.
func Remove(name string) error {
p, e := syscall.UTF16PtrFromString(name)
p, e := syscall.UTF16PtrFromString(fixLongPath(name))
if e != nil {
return &PathError{"remove", name, e}
}
......@@ -476,7 +479,7 @@ func Remove(name string) error {
}
func rename(oldname, newname string) error {
e := windows.Rename(oldname, newname)
e := windows.Rename(fixLongPath(oldname), fixLongPath(newname))
if e != nil {
return &LinkError{"rename", oldname, newname, e}
}
......@@ -521,11 +524,11 @@ func TempDir() string {
// Link creates newname as a hard link to the oldname file.
// If there is an error, it will be of type *LinkError.
func Link(oldname, newname string) error {
n, err := syscall.UTF16PtrFromString(newname)
n, err := syscall.UTF16PtrFromString(fixLongPath(newname))
if err != nil {
return &LinkError{"link", oldname, newname, err}
}
o, err := syscall.UTF16PtrFromString(oldname)
o, err := syscall.UTF16PtrFromString(fixLongPath(oldname))
if err != nil {
return &LinkError{"link", oldname, newname, err}
}
......@@ -556,11 +559,11 @@ func Symlink(oldname, newname string) error {
fi, err := Lstat(destpath)
isdir := err == nil && fi.IsDir()
n, err := syscall.UTF16PtrFromString(newname)
n, err := syscall.UTF16PtrFromString(fixLongPath(newname))
if err != nil {
return &LinkError{"symlink", oldname, newname, err}
}
o, err := syscall.UTF16PtrFromString(oldname)
o, err := syscall.UTF16PtrFromString(fixLongPath(oldname))
if err != nil {
return &LinkError{"symlink", oldname, newname, err}
}
......
......@@ -600,15 +600,8 @@ func TestReaddirOfFile(t *testing.T) {
}
func TestHardLink(t *testing.T) {
if runtime.GOOS == "plan9" {
t.Skip("skipping on plan9, hardlinks not supported")
}
// From Android release M (Marshmallow), hard linking files is blocked
// and an attempt to call link() on a file will return EACCES.
// - https://code.google.com/p/android-developer-preview/issues/detail?id=3150
if runtime.GOOS == "android" {
t.Skip("skipping on android, hardlinks not supported")
}
testenv.MustHaveLink(t)
defer chtmpdir(t)()
from, to := "hardlinktestfrom", "hardlinktestto"
Remove(from) // Just in case.
......@@ -1708,6 +1701,61 @@ func TestReadAtEOF(t *testing.T) {
}
}
func TestLongPath(t *testing.T) {
tmpdir := newDir("TestLongPath", t)
defer func() {
if err := RemoveAll(tmpdir); err != nil {
t.Fatalf("RemoveAll failed: %v", err)
}
}()
for len(tmpdir) < 400 {
tmpdir += "/dir3456789"
}
if err := MkdirAll(tmpdir, 0755); err != nil {
t.Fatalf("MkdirAll failed: %v", err)
}
data := []byte("hello world\n")
if err := ioutil.WriteFile(tmpdir+"/foo.txt", data, 0644); err != nil {
t.Fatalf("ioutil.WriteFile() failed: %v", err)
}
if err := Rename(tmpdir+"/foo.txt", tmpdir+"/bar.txt"); err != nil {
t.Fatalf("Rename failed: %v", err)
}
mtime := time.Now().Truncate(time.Minute)
if err := Chtimes(tmpdir+"/bar.txt", mtime, mtime); err != nil {
t.Fatalf("Chtimes failed: %v", err)
}
names := []string{"bar.txt"}
if testenv.HasSymlink() {
if err := Symlink(tmpdir+"/bar.txt", tmpdir+"/symlink.txt"); err != nil {
t.Fatalf("Symlink failed: %v", err)
}
names = append(names, "symlink.txt")
}
if testenv.HasLink() {
if err := Link(tmpdir+"/bar.txt", tmpdir+"/link.txt"); err != nil {
t.Fatalf("Link failed: %v", err)
}
names = append(names, "link.txt")
}
for _, wantSize := range []int64{int64(len(data)), 0} {
for _, name := range names {
path := tmpdir + "/" + name
dir, err := Stat(path)
if err != nil {
t.Fatalf("Stat(%q) failed: %v", path, err)
}
filesize := size(path, t)
if dir.Size() != filesize || filesize != wantSize {
t.Errorf("Size(%q) is %d, len(ReadFile()) is %d, want %d", path, dir.Size(), filesize, wantSize)
}
}
if err := Truncate(tmpdir+"/bar.txt", 0); err != nil {
t.Fatalf("Truncate failed: %v")
}
}
}
func testKillProcess(t *testing.T, processKiller func(p *Process)) {
testenv.MustHaveExec(t)
......
......@@ -127,3 +127,66 @@ func dirname(path string) string {
}
return vol + dir
}
// fixLongPath returns the extended-length (\\?\-prefixed) form of
// path if possible, in order to avoid the default 260 character file
// path limit imposed by Windows. If path is not easily converted to
// the extended-length form (for example, if path is a relative path
// or contains .. elements), fixLongPath returns path unmodified.
func fixLongPath(path string) string {
// The extended form begins with \\?\, as in
// \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt.
// The extended form disables evaluation of . and .. path
// elements and disables the interpretation of / as equivalent
// to \. The conversion here rewrites / to \ and elides
// . elements as well as trailing or duplicate separators. For
// simplicity it avoids the conversion entirely for relative
// paths or paths containing .. elements. For now,
// \\server\share paths are not converted to
// \\?\UNC\server\share paths because the rules for doing so
// are less well-specified.
//
// For details of \\?\ paths, see:
// https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath
if len(path) == 0 || (len(path) >= 2 && path[:2] == `\\`) {
// Don't canonicalize UNC paths.
return path
}
if !isAbs(path) {
// Relative path
return path
}
const prefix = `\\?`
pathbuf := make([]byte, len(prefix)+len(path)+len(`\`))
copy(pathbuf, prefix)
n := len(path)
r, w := 0, len(prefix)
for r < n {
switch {
case IsPathSeparator(path[r]):
// empty block
r++
case path[r] == '.' && (r+1 == n || IsPathSeparator(path[r+1])):
// /./
r++
case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || IsPathSeparator(path[r+2])):
// /../ is currently unhandled
return path
default:
pathbuf[w] = '\\'
w++
for ; r < n && !IsPathSeparator(path[r]); r++ {
pathbuf[w] = path[r]
w++
}
}
}
// A drive's root directory needs a trailing \
if w == len(`\\?\c:`) {
pathbuf[w] = '\\'
w++
}
return string(pathbuf[:w])
}
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package os_test
import (
"os"
"testing"
)
func TestFixLongPath(t *testing.T) {
for _, test := range []struct{ in, want string }{
{`C:\foo.txt`, `\\?\C:\foo.txt`},
{`C:/foo.txt`, `\\?\C:\foo.txt`},
{`C:\foo\\bar\.\baz\\`, `\\?\C:\foo\bar\baz`},
{`C:\`, `\\?\C:\`}, // drives must have a trailing slash
{`\\unc\path`, `\\unc\path`},
{`foo.txt`, `foo.txt`},
{`C:foo.txt`, `C:foo.txt`},
{`c:\foo\..\bar\baz`, `c:\foo\..\bar\baz`},
{`\\?\c:\windows\foo.txt`, `\\?\c:\windows\foo.txt`},
{`\\?\c:\windows/foo.txt`, `\\?\c:\windows/foo.txt`},
} {
if got := os.FixLongPath(test.in); got != test.want {
t.Errorf("fixLongPath(%q) = %q; want %q", test.in, got, test.want)
}
}
}
......@@ -90,7 +90,7 @@ func Lstat(name string) (FileInfo, error) {
return &devNullStat, nil
}
fs := &fileStat{name: basename(name)}
namep, e := syscall.UTF16PtrFromString(name)
namep, e := syscall.UTF16PtrFromString(fixLongPath(name))
if e != nil {
return nil, &PathError{"Lstat", name, e}
}
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment