Index: src/pkg/archive/tar/common.go
===================================================================
--- a/src/pkg/archive/tar/common.go
+++ b/src/pkg/archive/tar/common.go
@@ -13,6 +13,7 @@
package tar
import (
+ "bytes"
"errors"
"fmt"
"os"
@@ -174,6 +175,23 @@
c_ISSOCK = 0140000 // Socket
)
+// Keywords for the PAX Extended Header
+const (
+ paxAtime = "atime"
+ paxCharset = "charset"
+ paxComment = "comment"
+ paxCtime = "ctime" // please note that ctime is not a valid pax header.
+ paxGid = "gid"
+ paxGname = "gname"
+ paxLinkpath = "linkpath"
+ paxMtime = "mtime"
+ paxPath = "path"
+ paxSize = "size"
+ paxUid = "uid"
+ paxUname = "uname"
+ paxNone = ""
+)
+
// FileInfoHeader creates a partially-populated Header from fi.
// If fi describes a symlink, FileInfoHeader records link as the link target.
// If fi describes a directory, a slash is appended to the name.
@@ -257,3 +275,25 @@
b, *sp = s[0:n], s[n:]
return
}
+
+func isASCII(s string) bool {
+ for _, c := range s {
+ if c >= 0x80 {
+ return false
+ }
+ }
+ return true
+}
+
+func toASCII(s string) string {
+ if isASCII(s) {
+ return s
+ }
+ var buf bytes.Buffer
+ for _, c := range s {
+ if c < 0x80 {
+ buf.WriteByte(byte(c))
+ }
+ }
+ return buf.String()
+}
Index: src/pkg/archive/tar/reader.go
===================================================================
--- a/src/pkg/archive/tar/reader.go
+++ b/src/pkg/archive/tar/reader.go
@@ -95,45 +95,45 @@
func mergePAX(hdr *Header, headers map[string]string) error {
for k, v := range headers {
switch k {
- case "path":
+ case paxPath:
hdr.Name = v
- case "linkpath":
+ case paxLinkpath:
hdr.Linkname = v
- case "gname":
+ case paxGname:
hdr.Gname = v
- case "uname":
+ case paxUname:
hdr.Uname = v
- case "uid":
+ case paxUid:
uid, err := strconv.ParseInt(v, 10, 0)
if err != nil {
return err
}
hdr.Uid = int(uid)
- case "gid":
+ case paxGid:
gid, err := strconv.ParseInt(v, 10, 0)
if err != nil {
return err
}
hdr.Gid = int(gid)
- case "atime":
+ case paxAtime:
t, err := parsePAXTime(v)
if err != nil {
return err
}
hdr.AccessTime = t
- case "mtime":
+ case paxMtime:
t, err := parsePAXTime(v)
if err != nil {
return err
}
hdr.ModTime = t
- case "ctime":
+ case paxCtime:
t, err := parsePAXTime(v)
if err != nil {
return err
}
hdr.ChangeTime = t
- case "size":
+ case paxSize:
size, err := strconv.ParseInt(v, 10, 0)
if err != nil {
return err
Index: src/pkg/archive/tar/writer.go
===================================================================
--- a/src/pkg/archive/tar/writer.go
+++ b/src/pkg/archive/tar/writer.go
@@ -24,6 +24,7 @@
ErrFieldTooLong = errors.New("archive/tar: header field too long")
ErrWriteAfterClose = errors.New("archive/tar: write after close")
errNameTooLong = errors.New("archive/tar: name too long")
+ errInvalidHeader = errors.New("archive/tar: header field too long or contains invalid values")
)
// A Writer provides sequential writing of a tar archive in POSIX.1 format.
@@ -37,6 +38,7 @@
pad int64 // amount of padding to write after current file entry
closed bool
usedBinary bool // whether the binary numeric field extension was used
+ preferPax bool // use pax header instead of binary numeric header
}
// NewWriter creates a new Writer writing to w.
@@ -65,16 +67,23 @@
}
// Write s into b, terminating it with a NUL if there is room.
-func (tw *Writer) cString(b []byte, s string) {
+// If the value is too long for the field and allowPax is true add a paxheader record instead
+func (tw *Writer) cString(b []byte, s string, allowPax bool, paxKeyword string, paxHeaders map[string]string) {
+ needsPaxHeader := allowPax && len(s) > len(b) || !isASCII(s)
+ if needsPaxHeader {
+ paxHeaders[paxKeyword] = s
+ return
+ }
if len(s) > len(b) {
if tw.err == nil {
tw.err = ErrFieldTooLong
}
return
}
- copy(b, s)
- if len(s) < len(b) {
- b[len(s)] = 0
+ ascii := toASCII(s)
+ copy(b, ascii)
+ if len(ascii) < len(b) {
+ b[len(ascii)] = 0
}
}
@@ -85,17 +94,27 @@
for len(s)+1 < len(b) {
s = "0" + s
}
- tw.cString(b, s)
+ tw.cString(b, s, false, paxNone, nil)
}
// Write x into b, either as octal or as binary (GNUtar/star extension).
-func (tw *Writer) numeric(b []byte, x int64) {
+// If the value is too long for the field and writingPax is enabled both for the field and the add a paxheader record instead
+func (tw *Writer) numeric(b []byte, x int64, allowPax bool, paxKeyword string, paxHeaders map[string]string) {
// Try octal first.
s := strconv.FormatInt(x, 8)
if len(s) < len(b) {
tw.octal(b, x)
return
}
+
+ // If it is too long for octal, and pax is preferred, use a pax header
+ if allowPax && tw.preferPax {
+ tw.octal(b, 0)
+ s := strconv.FormatInt(x, 10)
+ paxHeaders[paxKeyword] = s
+ return
+ }
+
// Too big: use binary (big-endian).
tw.usedBinary = true
for i := len(b) - 1; x > 0 && i >= 0; i-- {
@@ -115,6 +134,15 @@
// WriteHeader calls Flush if it is not the first header.
// Calling after a Close will return ErrWriteAfterClose.
func (tw *Writer) WriteHeader(hdr *Header) error {
+ return tw.writeHeader(hdr, true)
+}
+
+// WriteHeader writes hdr and prepares to accept the file's contents.
+// WriteHeader calls Flush if it is not the first header.
+// Calling after a Close will return ErrWriteAfterClose.
+// As this method is called internally by writePax header to allow it to
+// suppress writing the pax header.
+func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error {
if tw.closed {
return ErrWriteAfterClose
}
@@ -124,31 +152,21 @@
if tw.err != nil {
return tw.err
}
- // Decide whether or not to use PAX extensions
+
+ // a map to hold pax header records, if any are needed
+ paxHeaders := make(map[string]string)
+
// TODO(shanemhansen): we might want to use PAX headers for
// subsecond time resolution, but for now let's just capture
- // the long name/long symlink use case.
- suffix := hdr.Name
- prefix := ""
- if len(hdr.Name) > fileNameSize || len(hdr.Linkname) > fileNameSize {
- var err error
- prefix, suffix, err = tw.splitUSTARLongName(hdr.Name)
- // Either we were unable to pack the long name into ustar format
- // or the link name is too long; use PAX headers.
- if err == errNameTooLong || len(hdr.Linkname) > fileNameSize {
- if err := tw.writePAXHeader(hdr); err != nil {
- return err
- }
- } else if err != nil {
- return err
- }
- }
- tw.nb = int64(hdr.Size)
- tw.pad = -tw.nb & (blockSize - 1) // blockSize is a power of two
+ // too long fields or non ascii characters
header := make([]byte, blockSize)
s := slicer(header)
- tw.cString(s.next(fileNameSize), suffix)
+
+ // keep a reference to the filename to allow to overwrite it later if we detect that we can use ustar longnames instead of pax
+ pathHeaderBytes := s.next(fileNameSize)
+
+ tw.cString(pathHeaderBytes, hdr.Name, true, paxPath, paxHeaders)
// Handle out of range ModTime carefully.
var modTime int64
@@ -156,27 +174,55 @@
modTime = hdr.ModTime.Unix()
}
- tw.octal(s.next(8), hdr.Mode) // 100:108
- tw.numeric(s.next(8), int64(hdr.Uid)) // 108:116
- tw.numeric(s.next(8), int64(hdr.Gid)) // 116:124
- tw.numeric(s.next(12), hdr.Size) // 124:136
- tw.numeric(s.next(12), modTime) // 136:148
- s.next(8) // chksum (148:156)
- s.next(1)[0] = hdr.Typeflag // 156:157
- tw.cString(s.next(100), hdr.Linkname) // linkname (157:257)
- copy(s.next(8), []byte("ustar\x0000")) // 257:265
- tw.cString(s.next(32), hdr.Uname) // 265:297
- tw.cString(s.next(32), hdr.Gname) // 297:329
- tw.numeric(s.next(8), hdr.Devmajor) // 329:337
- tw.numeric(s.next(8), hdr.Devminor) // 337:345
- tw.cString(s.next(155), prefix) // 345:500
+ tw.octal(s.next(8), hdr.Mode) // 100:108
+ tw.numeric(s.next(8), int64(hdr.Uid), true, paxUid, paxHeaders) // 108:116
+ tw.numeric(s.next(8), int64(hdr.Gid), true, paxGid, paxHeaders) // 116:124
+ tw.numeric(s.next(12), hdr.Size, true, paxSize, paxHeaders) // 124:136
+ tw.numeric(s.next(12), modTime, false, paxNone, nil) // 136:148 --- consider using pax for finer granularity
+ s.next(8) // chksum (148:156)
+ s.next(1)[0] = hdr.Typeflag // 156:157
+
+ tw.cString(s.next(100), hdr.Linkname, true, paxLinkpath, paxHeaders)
+
+ copy(s.next(8), []byte("ustar\x0000")) // 257:265
+ tw.cString(s.next(32), hdr.Uname, true, paxUname, paxHeaders) // 265:297
+ tw.cString(s.next(32), hdr.Gname, true, paxGname, paxHeaders) // 297:329
+ tw.numeric(s.next(8), hdr.Devmajor, false, paxNone, nil) // 329:337
+ tw.numeric(s.next(8), hdr.Devminor, false, paxNone, nil) // 337:345
+
+ // keep a reference to the prefix to allow to overwrite it later if we detect that we can use ustar longnames instead of pax
+ prefixHeaderBytes := s.next(155)
+ tw.cString(prefixHeaderBytes, "", false, paxNone, nil) // 345:500 prefix
+
// Use the GNU magic instead of POSIX magic if we used any GNU extensions.
if tw.usedBinary {
copy(header[257:265], []byte("ustar \x00"))
}
- // Use the ustar magic if we used ustar long names.
- if len(prefix) > 0 {
- copy(header[257:265], []byte("ustar\000"))
+
+ _, paxPathUsed := paxHeaders[paxPath]
+ // try to use a ustar header when only the name is too long
+ if !tw.preferPax && len(paxHeaders) == 1 && paxPathUsed {
+ suffix := hdr.Name
+ prefix := ""
+ if len(hdr.Name) > fileNameSize && isASCII(hdr.Name) {
+ var err error
+ prefix, suffix, err = tw.splitUSTARLongName(hdr.Name)
+ if err == nil {
+ // ok we can use a ustar long name instead of pax, now correct the fields
+
+ // remove the path field from the pax header. this will suppress the pax header
+ delete(paxHeaders, paxPath)
+
+ // update the path fields
+ tw.cString(pathHeaderBytes, suffix, false, paxNone, nil)
+ tw.cString(prefixHeaderBytes, prefix, false, paxNone, nil)
+
+ // Use the ustar magic if we used ustar long names.
+ if len(prefix) > 0 {
+ copy(header[257:265], []byte("ustar\000"))
+ }
+ }
+ }
}
// The chksum field is terminated by a NUL and a space.
@@ -190,8 +236,18 @@
return tw.err
}
+ if len(paxHeaders) > 0 {
+ if !allowPax {
+ return errInvalidHeader
+ }
+ if err := tw.writePAXHeader(hdr, paxHeaders); err != nil {
+ return err
+ }
+ }
+ tw.nb = int64(hdr.Size)
+ tw.pad = (blockSize - (tw.nb % blockSize)) % blockSize
+
_, tw.err = tw.w.Write(header)
-
return tw.err
}
@@ -218,7 +274,7 @@
// writePaxHeader writes an extended pax header to the
// archive.
-func (tw *Writer) writePAXHeader(hdr *Header) error {
+func (tw *Writer) writePAXHeader(hdr *Header, paxHeaders map[string]string) error {
// Prepare extended header
ext := new(Header)
ext.Typeflag = TypeXHeader
@@ -229,18 +285,23 @@
// with the current pid.
pid := os.Getpid()
dir, file := path.Split(hdr.Name)
- ext.Name = path.Join(dir,
- fmt.Sprintf("PaxHeaders.%d", pid), file)[0:100]
+ fullName := path.Join(dir,
+ fmt.Sprintf("PaxHeaders.%d", pid), file)
+
+ ascii := toASCII(fullName)
+ if len(ascii) > 100 {
+ ascii = ascii[:100]
+ }
+ ext.Name = ascii
// Construct the body
var buf bytes.Buffer
- if len(hdr.Name) > fileNameSize {
- fmt.Fprint(&buf, paxHeader("path="+hdr.Name))
+
+ for k, v := range paxHeaders {
+ fmt.Fprint(&buf, paxHeader(k+"="+v))
}
- if len(hdr.Linkname) > fileNameSize {
- fmt.Fprint(&buf, paxHeader("linkpath="+hdr.Linkname))
- }
+
ext.Size = int64(len(buf.Bytes()))
- if err := tw.WriteHeader(ext); err != nil {
+ if err := tw.writeHeader(ext, false); err != nil {
return err
}
if _, err := tw.Write(buf.Bytes()); err != nil {
Index: src/pkg/archive/tar/writer_test.go
===================================================================
--- a/src/pkg/archive/tar/writer_test.go
+++ b/src/pkg/archive/tar/writer_test.go
@@ -243,15 +243,110 @@
}
}
+func TestPaxSymlink(t *testing.T) {
+ // Create an archive with a large linkname
+ fileinfo, err := os.Stat("testdata/small.txt")
+ if err != nil {
+ t.Fatal(err)
+ }
+ hdr, err := FileInfoHeader(fileinfo, "")
+ hdr.Typeflag = TypeSymlink
+ if err != nil {
+ t.Fatalf("os.Stat:1 %v", err)
+ }
+ // Force a PAX long linkname to be written
+ longLinkname := strings.Repeat("1234567890/1234567890", 10)
+ hdr.Linkname = longLinkname
+
+ hdr.Size = 0
+ var buf bytes.Buffer
+ writer := NewWriter(&buf)
+ if err := writer.WriteHeader(hdr); err != nil {
+ t.Fatal(err)
+ }
+ if err := writer.Close(); err != nil {
+ t.Fatal(err)
+ }
+ // Simple test to make sure PAX extensions are in effect
+ if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.")) {
+ t.Fatal("Expected at least one PAX header to be written.")
+ }
+ // Test that we can get a long name back out of the archive.
+ reader := NewReader(&buf)
+ hdr, err = reader.Next()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if hdr.Linkname != longLinkname {
+ t.Fatal("Couldn't recover long link name")
+ }
+}
+
+func TestPaxNonAscii(t *testing.T) {
+ // Create an archive with non ascii. These should trigger a pax header
+ // because pax headers have a defined utf-8 encoding.
+ fileinfo, err := os.Stat("testdata/small.txt")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ hdr, err := FileInfoHeader(fileinfo, "")
+ if err != nil {
+ t.Fatalf("os.Stat:1 %v", err)
+ }
+
+ // some sample data
+ chineseFilename := "文件名"
+ chineseGroupname := "組"
+ chineseUsername := "用戶名"
+
+ hdr.Name = chineseFilename
+ hdr.Gname = chineseGroupname
+ hdr.Uname = chineseUsername
+
+ contents := strings.Repeat(" ", int(hdr.Size))
+
+ var buf bytes.Buffer
+ writer := NewWriter(&buf)
+ if err := writer.WriteHeader(hdr); err != nil {
+ t.Fatal(err)
+ }
+ if _, err = writer.Write([]byte(contents)); err != nil {
+ t.Fatal(err)
+ }
+ if err := writer.Close(); err != nil {
+ t.Fatal(err)
+ }
+ // Simple test to make sure PAX extensions are in effect
+ if !bytes.Contains(buf.Bytes(), []byte("PaxHeaders.")) {
+ t.Fatal("Expected at least one PAX header to be written.")
+ }
+ // Test that we can get a long name back out of the archive.
+ reader := NewReader(&buf)
+ hdr, err = reader.Next()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if hdr.Name != chineseFilename {
+ t.Fatal("Couldn't recover unicode name")
+ }
+ if hdr.Gname != chineseGroupname {
+ t.Fatal("Couldn't recover unicode group")
+ }
+ if hdr.Uname != chineseUsername {
+ t.Fatal("Couldn't recover unicode user")
+ }
+}
+
func TestPAXHeader(t *testing.T) {
medName := strings.Repeat("CD", 50)
longName := strings.Repeat("AB", 100)
paxTests := [][2]string{
- {"name=/etc/hosts", "19 name=/etc/hosts\n"},
+ {paxPath + "=/etc/hosts", "19 path=/etc/hosts\n"},
{"a=b", "6 a=b\n"}, // Single digit length
{"a=names", "11 a=names\n"}, // Test case involving carries
- {"name=" + longName, fmt.Sprintf("210 name=%s\n", longName)},
- {"name=" + medName, fmt.Sprintf("110 name=%s\n", medName)}}
+ {paxPath + "=" + longName, fmt.Sprintf("210 path=%s\n", longName)},
+ {paxPath + "=" + medName, fmt.Sprintf("110 path=%s\n", medName)}}
for _, test := range paxTests {
key, expected := test[0], test[1]