diff options
Diffstat (limited to 'libgo/go/encoding')
28 files changed, 411 insertions, 224 deletions
diff --git a/libgo/go/encoding/asn1/marshal.go b/libgo/go/encoding/asn1/marshal.go index c181e43f979..774bee74baa 100644 --- a/libgo/go/encoding/asn1/marshal.go +++ b/libgo/go/encoding/asn1/marshal.go @@ -24,7 +24,7 @@ type forkableWriter struct { } func newForkableWriter() *forkableWriter { - return &forkableWriter{bytes.NewBuffer(nil), nil, nil} + return &forkableWriter{new(bytes.Buffer), nil, nil} } func (f *forkableWriter) fork() (pre, post *forkableWriter) { diff --git a/libgo/go/encoding/base32/base32.go b/libgo/go/encoding/base32/base32.go index c75c7c19d15..71da6e22b12 100644 --- a/libgo/go/encoding/base32/base32.go +++ b/libgo/go/encoding/base32/base32.go @@ -125,6 +125,13 @@ func (enc *Encoding) Encode(dst, src []byte) { } } +// EncodeToString returns the base32 encoding of src. +func (enc *Encoding) EncodeToString(src []byte) string { + buf := make([]byte, enc.EncodedLen(len(src))) + enc.Encode(buf, src) + return string(buf) +} + type encoder struct { err error enc *Encoding @@ -221,24 +228,32 @@ func (e CorruptInputError) Error() string { // decode is like Decode but returns an additional 'end' value, which // indicates if end-of-message padding was encountered and thus any -// additional data is an error. decode also assumes len(src)%8==0, -// since it is meant for internal use. +// additional data is an error. func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { - for i := 0; i < len(src)/8 && !end; i++ { + osrc := src + for len(src) > 0 && !end { // Decode quantum using the base32 alphabet var dbuf [8]byte dlen := 8 // do the top bytes contain any data? dbufloop: - for j := 0; j < 8; j++ { - in := src[i*8+j] - if in == '=' && j >= 2 && i == len(src)/8-1 { + for j := 0; j < 8; { + if len(src) == 0 { + return n, false, CorruptInputError(len(osrc) - len(src) - j) + } + in := src[0] + src = src[1:] + if in == '\r' || in == '\n' { + // Ignore this character. + continue + } + if in == '=' && j >= 2 && len(src) < 8 { // We've reached the end and there's // padding, the rest should be padded - for k := j; k < 8; k++ { - if src[i*8+k] != '=' { - return n, false, CorruptInputError(i*8 + j) + for k := 0; k < 8-j-1; k++ { + if len(src) > k && src[k] != '=' { + return n, false, CorruptInputError(len(osrc) - len(src) + k - 1) } } dlen = j @@ -247,28 +262,30 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { } dbuf[j] = enc.decodeMap[in] if dbuf[j] == 0xFF { - return n, false, CorruptInputError(i*8 + j) + return n, false, CorruptInputError(len(osrc) - len(src) - 1) } + j++ } // Pack 8x 5-bit source blocks into 5 byte destination // quantum switch dlen { case 7, 8: - dst[i*5+4] = dbuf[6]<<5 | dbuf[7] + dst[4] = dbuf[6]<<5 | dbuf[7] fallthrough case 6, 5: - dst[i*5+3] = dbuf[4]<<7 | dbuf[5]<<2 | dbuf[6]>>3 + dst[3] = dbuf[4]<<7 | dbuf[5]<<2 | dbuf[6]>>3 fallthrough case 4: - dst[i*5+2] = dbuf[3]<<4 | dbuf[4]>>1 + dst[2] = dbuf[3]<<4 | dbuf[4]>>1 fallthrough case 3: - dst[i*5+1] = dbuf[1]<<6 | dbuf[2]<<1 | dbuf[3]>>4 + dst[1] = dbuf[1]<<6 | dbuf[2]<<1 | dbuf[3]>>4 fallthrough case 2: - dst[i*5+0] = dbuf[0]<<3 | dbuf[1]>>2 + dst[0] = dbuf[0]<<3 | dbuf[1]>>2 } + dst = dst[5:] switch dlen { case 2: n += 1 @@ -289,15 +306,19 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { // DecodedLen(len(src)) bytes to dst and returns the number of bytes // written. If src contains invalid base32 data, it will return the // number of bytes successfully written and CorruptInputError. +// New line characters (\r and \n) are ignored. func (enc *Encoding) Decode(dst, src []byte) (n int, err error) { - if len(src)%8 != 0 { - return 0, CorruptInputError(len(src) / 8 * 8) - } - n, _, err = enc.decode(dst, src) return } +// DecodeString returns the bytes represented by the base32 string s. +func (enc *Encoding) DecodeString(s string) ([]byte, error) { + dbuf := make([]byte, enc.DecodedLen(len(s))) + n, err := enc.Decode(dbuf, []byte(s)) + return dbuf[:n], err +} + type decoder struct { err error enc *Encoding diff --git a/libgo/go/encoding/base32/base32_test.go b/libgo/go/encoding/base32/base32_test.go index facf5d04eeb..98365e18cfc 100644 --- a/libgo/go/encoding/base32/base32_test.go +++ b/libgo/go/encoding/base32/base32_test.go @@ -51,9 +51,8 @@ func testEqual(t *testing.T, msg string, args ...interface{}) bool { func TestEncode(t *testing.T) { for _, p := range pairs { - buf := make([]byte, StdEncoding.EncodedLen(len(p.decoded))) - StdEncoding.Encode(buf, []byte(p.decoded)) - testEqual(t, "Encode(%q) = %q, want %q", p.decoded, string(buf), p.encoded) + got := StdEncoding.EncodeToString([]byte(p.decoded)) + testEqual(t, "Encode(%q) = %q, want %q", p.decoded, got, p.encoded) } } @@ -99,6 +98,10 @@ func TestDecode(t *testing.T) { testEqual(t, "Decode(%q) = %q, want %q", p.encoded, string(dbuf[0:count]), p.decoded) + + dbuf, err = StdEncoding.DecodeString(p.encoded) + testEqual(t, "DecodeString(%q) = error %v, want %v", p.encoded, err, error(nil)) + testEqual(t, "DecodeString(%q) = %q, want %q", p.encoded, string(dbuf), p.decoded) } } @@ -191,3 +194,29 @@ func TestBig(t *testing.T) { t.Errorf("Decode(Encode(%d-byte string)) failed at offset %d", n, i) } } + +func TestNewLineCharacters(t *testing.T) { + // Each of these should decode to the string "sure", without errors. + const expected = "sure" + examples := []string{ + "ON2XEZI=", + "ON2XEZI=\r", + "ON2XEZI=\n", + "ON2XEZI=\r\n", + "ON2XEZ\r\nI=", + "ON2X\rEZ\nI=", + "ON2X\nEZ\rI=", + "ON2XEZ\nI=", + "ON2XEZI\n=", + } + for _, e := range examples { + buf, err := StdEncoding.DecodeString(e) + if err != nil { + t.Errorf("Decode(%q) failed: %v", e, err) + continue + } + if s := string(buf); s != expected { + t.Errorf("Decode(%q) = %q, want %q", e, s, expected) + } + } +} diff --git a/libgo/go/encoding/base64/base64.go b/libgo/go/encoding/base64/base64.go index 889b565e3f5..55f9f67a43a 100644 --- a/libgo/go/encoding/base64/base64.go +++ b/libgo/go/encoding/base64/base64.go @@ -208,22 +208,30 @@ func (e CorruptInputError) Error() string { // decode is like Decode but returns an additional 'end' value, which // indicates if end-of-message padding was encountered and thus any -// additional data is an error. decode also assumes len(src)%4==0, -// since it is meant for internal use. +// additional data is an error. func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { - for i := 0; i < len(src)/4 && !end; i++ { + osrc := src + for len(src) > 0 && !end { // Decode quantum using the base64 alphabet var dbuf [4]byte dlen := 4 dbufloop: - for j := 0; j < 4; j++ { - in := src[i*4+j] - if in == '=' && j >= 2 && i == len(src)/4-1 { + for j := 0; j < 4; { + if len(src) == 0 { + return n, false, CorruptInputError(len(osrc) - len(src) - j) + } + in := src[0] + src = src[1:] + if in == '\r' || in == '\n' { + // Ignore this character. + continue + } + if in == '=' && j >= 2 && len(src) < 4 { // We've reached the end and there's // padding - if src[i*4+3] != '=' { - return n, false, CorruptInputError(i*4 + 2) + if len(src) > 0 && src[0] != '=' { + return n, false, CorruptInputError(len(osrc) - len(src) - 1) } dlen = j end = true @@ -231,22 +239,24 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { } dbuf[j] = enc.decodeMap[in] if dbuf[j] == 0xFF { - return n, false, CorruptInputError(i*4 + j) + return n, false, CorruptInputError(len(osrc) - len(src) - 1) } + j++ } // Pack 4x 6-bit source blocks into 3 byte destination // quantum switch dlen { case 4: - dst[i*3+2] = dbuf[2]<<6 | dbuf[3] + dst[2] = dbuf[2]<<6 | dbuf[3] fallthrough case 3: - dst[i*3+1] = dbuf[1]<<4 | dbuf[2]>>2 + dst[1] = dbuf[1]<<4 | dbuf[2]>>2 fallthrough case 2: - dst[i*3+0] = dbuf[0]<<2 | dbuf[1]>>4 + dst[0] = dbuf[0]<<2 | dbuf[1]>>4 } + dst = dst[3:] n += dlen - 1 } @@ -257,11 +267,8 @@ func (enc *Encoding) decode(dst, src []byte) (n int, end bool, err error) { // DecodedLen(len(src)) bytes to dst and returns the number of bytes // written. If src contains invalid base64 data, it will return the // number of bytes successfully written and CorruptInputError. +// New line characters (\r and \n) are ignored. func (enc *Encoding) Decode(dst, src []byte) (n int, err error) { - if len(src)%4 != 0 { - return 0, CorruptInputError(len(src) / 4 * 4) - } - n, _, err = enc.decode(dst, src) return } diff --git a/libgo/go/encoding/base64/base64_test.go b/libgo/go/encoding/base64/base64_test.go index 8310d8ab9e3..3e9a84393b6 100644 --- a/libgo/go/encoding/base64/base64_test.go +++ b/libgo/go/encoding/base64/base64_test.go @@ -197,3 +197,29 @@ func TestBig(t *testing.T) { t.Errorf("Decode(Encode(%d-byte string)) failed at offset %d", n, i) } } + +func TestNewLineCharacters(t *testing.T) { + // Each of these should decode to the string "sure", without errors. + const expected = "sure" + examples := []string{ + "c3VyZQ==", + "c3VyZQ==\r", + "c3VyZQ==\n", + "c3VyZQ==\r\n", + "c3VyZ\r\nQ==", + "c3V\ryZ\nQ==", + "c3V\nyZ\rQ==", + "c3VyZ\nQ==", + "c3VyZQ\n==", + } + for _, e := range examples { + buf, err := StdEncoding.DecodeString(e) + if err != nil { + t.Errorf("Decode(%q) failed: %v", e, err) + continue + } + if s := string(buf); s != expected { + t.Errorf("Decode(%q) = %q, want %q", e, s, expected) + } + } +} diff --git a/libgo/go/encoding/binary/binary.go b/libgo/go/encoding/binary/binary.go index d2f8b1e6248..4be83f53bd4 100644 --- a/libgo/go/encoding/binary/binary.go +++ b/libgo/go/encoding/binary/binary.go @@ -163,7 +163,7 @@ func Read(r io.Reader, order ByteOrder, data interface{}) error { default: return errors.New("binary.Read: invalid type " + d.Type().String()) } - size := TotalSize(v) + size := dataSize(v) if size < 0 { return errors.New("binary.Read: invalid type " + v.Type().String()) } @@ -242,7 +242,7 @@ func Write(w io.Writer, order ByteOrder, data interface{}) error { return err } v := reflect.Indirect(reflect.ValueOf(data)) - size := TotalSize(v) + size := dataSize(v) if size < 0 { return errors.New("binary.Write: invalid type " + v.Type().String()) } @@ -253,7 +253,11 @@ func Write(w io.Writer, order ByteOrder, data interface{}) error { return err } -func TotalSize(v reflect.Value) int { +// dataSize returns the number of bytes the actual data represented by v occupies in memory. +// For compound structures, it sums the sizes of the elements. Thus, for instance, for a slice +// it returns the length of the slice times the element size and does not count the memory +// occupied by the header. +func dataSize(v reflect.Value) int { if v.Kind() == reflect.Slice { elem := sizeof(v.Type().Elem()) if elem < 0 { diff --git a/libgo/go/encoding/binary/binary_test.go b/libgo/go/encoding/binary/binary_test.go index 3e7057ea227..ff361b7e379 100644 --- a/libgo/go/encoding/binary/binary_test.go +++ b/libgo/go/encoding/binary/binary_test.go @@ -187,7 +187,7 @@ func BenchmarkReadStruct(b *testing.B) { bsr := &byteSliceReader{} var buf bytes.Buffer Write(&buf, BigEndian, &s) - n := TotalSize(reflect.ValueOf(s)) + n := dataSize(reflect.ValueOf(s)) b.SetBytes(int64(n)) t := s b.ResetTimer() diff --git a/libgo/go/encoding/csv/reader.go b/libgo/go/encoding/csv/reader.go index ae0f567b9df..9aa398e58b2 100644 --- a/libgo/go/encoding/csv/reader.go +++ b/libgo/go/encoding/csv/reader.go @@ -156,6 +156,9 @@ func (r *Reader) Read() (record []string, err error) { // ReadAll reads all the remaining records from r. // Each record is a slice of fields. +// A successful call returns err == nil, not err == EOF. Because ReadAll is +// defined to read until EOF, it does not treat end of file as an error to be +// reported. func (r *Reader) ReadAll() (records [][]string, err error) { for { record, err := r.Read() diff --git a/libgo/go/encoding/gob/codec_test.go b/libgo/go/encoding/gob/codec_test.go index 73844b920c1..d365f826345 100644 --- a/libgo/go/encoding/gob/codec_test.go +++ b/libgo/go/encoding/gob/codec_test.go @@ -8,9 +8,11 @@ import ( "bytes" "errors" "math" + "math/rand" "reflect" "strings" "testing" + "time" "unsafe" ) @@ -1407,3 +1409,60 @@ func TestDebugStruct(t *testing.T) { } debugFunc(debugBuffer) } + +func encFuzzDec(rng *rand.Rand, in interface{}) error { + buf := new(bytes.Buffer) + enc := NewEncoder(buf) + if err := enc.Encode(&in); err != nil { + return err + } + + b := buf.Bytes() + for i, bi := range b { + if rng.Intn(10) < 3 { + b[i] = bi + uint8(rng.Intn(256)) + } + } + + dec := NewDecoder(buf) + var e interface{} + if err := dec.Decode(&e); err != nil { + return err + } + return nil +} + +// This does some "fuzz testing" by attempting to decode a sequence of random bytes. +func TestFuzz(t *testing.T) { + if testing.Short() { + return + } + + // all possible inputs + input := []interface{}{ + new(int), + new(float32), + new(float64), + new(complex128), + &ByteStruct{255}, + &ArrayStruct{}, + &StringStruct{"hello"}, + &GobTest1{0, &StringStruct{"hello"}}, + } + testFuzz(t, time.Now().UnixNano(), 100, input...) +} + +func TestFuzzRegressions(t *testing.T) { + // An instance triggering a type name of length ~102 GB. + testFuzz(t, 1328492090837718000, 100, new(float32)) +} + +func testFuzz(t *testing.T, seed int64, n int, input ...interface{}) { + t.Logf("seed=%d n=%d\n", seed, n) + for _, e := range input { + rng := rand.New(rand.NewSource(seed)) + for i := 0; i < n; i++ { + encFuzzDec(rng, e) + } + } +} diff --git a/libgo/go/encoding/gob/debug.go b/libgo/go/encoding/gob/debug.go index 6dc7fc9aca0..b54ef46f52c 100644 --- a/libgo/go/encoding/gob/debug.go +++ b/libgo/go/encoding/gob/debug.go @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// Delete the next line to include this file in the gob package. -// +build ignore +// Delete the next line to include in the gob package. +// +build gob-debug package gob diff --git a/libgo/go/encoding/gob/decode.go b/libgo/go/encoding/gob/decode.go index 4d1325d176c..8191062d309 100644 --- a/libgo/go/encoding/gob/decode.go +++ b/libgo/go/encoding/gob/decode.go @@ -690,7 +690,11 @@ func (dec *Decoder) decodeInterface(ityp reflect.Type, state *decoderState, p ui // Create a writable interface reflect.Value. We need one even for the nil case. ivalue := allocValue(ityp) // Read the name of the concrete type. - b := make([]byte, state.decodeUint()) + nr := state.decodeUint() + if nr < 0 || nr > 1<<31 { // zero is permissible for anonymous types + errorf("invalid type name length %d", nr) + } + b := make([]byte, nr) state.b.Read(b) name := string(b) if name == "" { diff --git a/libgo/go/encoding/gob/decoder.go b/libgo/go/encoding/gob/decoder.go index fb28c8caf53..c5c7d3fdb10 100644 --- a/libgo/go/encoding/gob/decoder.go +++ b/libgo/go/encoding/gob/decoder.go @@ -135,7 +135,7 @@ func (dec *Decoder) nextUint() uint64 { // and returns the type id of the next value. It returns -1 at // EOF. Upon return, the remainder of dec.buf is the value to be // decoded. If this is an interface value, it can be ignored by -// simply resetting that buffer. +// resetting that buffer. func (dec *Decoder) decodeTypeSequence(isInterface bool) typeId { for dec.err == nil { if dec.buf.Len() == 0 { diff --git a/libgo/go/encoding/gob/doc.go b/libgo/go/encoding/gob/doc.go index 05ebef19593..c9ad18e7641 100644 --- a/libgo/go/encoding/gob/doc.go +++ b/libgo/go/encoding/gob/doc.go @@ -70,7 +70,7 @@ operation will fail. Structs, arrays and slices are also supported. Strings and arrays of bytes are supported with a special, efficient representation (see below). When a slice is decoded, if the existing slice has capacity the slice will be extended in place; -if not, a new array is allocated. Regardless, the length of the resuling slice +if not, a new array is allocated. Regardless, the length of the resulting slice reports the number of elements decoded. Functions and channels cannot be sent in a gob. Attempting @@ -162,7 +162,7 @@ description, constructed from these types: StructT *StructType MapT *MapType } - type ArrayType struct { + type arrayType struct { CommonType Elem typeId Len int @@ -171,19 +171,19 @@ description, constructed from these types: Name string // the name of the struct type Id int // the id of the type, repeated so it's inside the type } - type SliceType struct { + type sliceType struct { CommonType Elem typeId } - type StructType struct { + type structType struct { CommonType Field []*fieldType // the fields of the struct. } - type FieldType struct { + type fieldType struct { Name string // the name of the field. Id int // the type id of the field, which must be already defined } - type MapType struct { + type mapType struct { CommonType Key typeId Elem typeId @@ -308,15 +308,15 @@ reserved). // Set the field number implicitly to -1; this is done at the beginning // of every struct, including nested structs. 03 // Add 3 to field number; now 2 (wireType.structType; this is a struct). - // structType starts with an embedded commonType, which appears + // structType starts with an embedded CommonType, which appears // as a regular structure here too. - 01 // add 1 to field number (now 0); start of embedded commonType. + 01 // add 1 to field number (now 0); start of embedded CommonType. 01 // add 1 to field number (now 0, the name of the type) 05 // string is (unsigned) 5 bytes long - 50 6f 69 6e 74 // wireType.structType.commonType.name = "Point" + 50 6f 69 6e 74 // wireType.structType.CommonType.name = "Point" 01 // add 1 to field number (now 1, the id of the type) - ff 82 // wireType.structType.commonType._id = 65 - 00 // end of embedded wiretype.structType.commonType struct + ff 82 // wireType.structType.CommonType._id = 65 + 00 // end of embedded wiretype.structType.CommonType struct 01 // add 1 to field number (now 1, the field array in wireType.structType) 02 // There are two fields in the type (len(structType.field)) 01 // Start of first field structure; add 1 to get field number 0: field[0].name diff --git a/libgo/go/encoding/gob/encoder_test.go b/libgo/go/encoding/gob/encoder_test.go index 7a30f9107e6..9a62cf9c2ad 100644 --- a/libgo/go/encoding/gob/encoder_test.go +++ b/libgo/go/encoding/gob/encoder_test.go @@ -570,8 +570,7 @@ func TestGobMapInterfaceEncode(t *testing.T) { "bo": []bool{false}, "st": []string{"s"}, } - buf := bytes.NewBuffer(nil) - enc := NewEncoder(buf) + enc := NewEncoder(new(bytes.Buffer)) err := enc.Encode(m) if err != nil { t.Errorf("encode map: %s", err) @@ -579,7 +578,7 @@ func TestGobMapInterfaceEncode(t *testing.T) { } func TestSliceReusesMemory(t *testing.T) { - buf := bytes.NewBuffer(nil) + buf := new(bytes.Buffer) // Bytes { x := []byte("abcd") diff --git a/libgo/go/encoding/gob/error.go b/libgo/go/encoding/gob/error.go index fbae8b683da..92cc0c615e3 100644 --- a/libgo/go/encoding/gob/error.go +++ b/libgo/go/encoding/gob/error.go @@ -33,7 +33,11 @@ func error_(err error) { // plain error. It overwrites the error return of the function that deferred its call. func catchError(err *error) { if e := recover(); e != nil { - *err = e.(gobError).err // Will re-panic if not one of our errors, such as a runtime error. + ge, ok := e.(gobError) + if !ok { + panic(e) + } + *err = ge.err } return } diff --git a/libgo/go/encoding/gob/timing_test.go b/libgo/go/encoding/gob/timing_test.go index 1017eb7f51d..b9371c42309 100644 --- a/libgo/go/encoding/gob/timing_test.go +++ b/libgo/go/encoding/gob/timing_test.go @@ -53,8 +53,9 @@ func TestCountEncodeMallocs(t *testing.T) { var buf bytes.Buffer enc := NewEncoder(&buf) bench := &Bench{7, 3.2, "now is the time", []byte("for all good men")} - runtime.UpdateMemStats() - mallocs := 0 - runtime.MemStats.Mallocs + memstats := new(runtime.MemStats) + runtime.ReadMemStats(memstats) + mallocs := 0 - memstats.Mallocs const count = 1000 for i := 0; i < count; i++ { err := enc.Encode(bench) @@ -62,8 +63,8 @@ func TestCountEncodeMallocs(t *testing.T) { t.Fatal("encode:", err) } } - runtime.UpdateMemStats() - mallocs += runtime.MemStats.Mallocs + runtime.ReadMemStats(memstats) + mallocs += memstats.Mallocs fmt.Printf("mallocs per encode of type Bench: %d\n", mallocs/count) } @@ -79,8 +80,9 @@ func TestCountDecodeMallocs(t *testing.T) { } } dec := NewDecoder(&buf) - runtime.UpdateMemStats() - mallocs := 0 - runtime.MemStats.Mallocs + memstats := new(runtime.MemStats) + runtime.ReadMemStats(memstats) + mallocs := 0 - memstats.Mallocs for i := 0; i < count; i++ { *bench = Bench{} err := dec.Decode(&bench) @@ -88,7 +90,7 @@ func TestCountDecodeMallocs(t *testing.T) { t.Fatal("decode:", err) } } - runtime.UpdateMemStats() - mallocs += runtime.MemStats.Mallocs + runtime.ReadMemStats(memstats) + mallocs += memstats.Mallocs fmt.Printf("mallocs per decode of type Bench: %d\n", mallocs/count) } diff --git a/libgo/go/encoding/gob/type.go b/libgo/go/encoding/gob/type.go index 71a28be7cab..39006efdb2d 100644 --- a/libgo/go/encoding/gob/type.go +++ b/libgo/go/encoding/gob/type.go @@ -180,7 +180,10 @@ func (t typeId) name() string { return t.gobType().name() } -// Common elements of all types. +// CommonType holds elements of all types. +// It is a historical artifact, kept for binary compatibility and exported +// only for the benefit of the package's encoding of type descriptors. It is +// not intended for direct use by clients. type CommonType struct { Name string Id typeId diff --git a/libgo/go/encoding/hex/hex.go b/libgo/go/encoding/hex/hex.go index eb7e7ca8d38..167d00e032a 100644 --- a/libgo/go/encoding/hex/hex.go +++ b/libgo/go/encoding/hex/hex.go @@ -7,8 +7,9 @@ package hex import ( "bytes" + "errors" + "fmt" "io" - "strconv" ) const hextable = "0123456789abcdef" @@ -29,16 +30,14 @@ func Encode(dst, src []byte) int { return len(src) * 2 } -// OddLengthInputError results from decoding an odd length slice. -type OddLengthInputError struct{} +// ErrLength results from decoding an odd length slice. +var ErrLength = errors.New("encoding/hex: odd length hex string") -func (OddLengthInputError) Error() string { return "odd length hex string" } +// InvalidByteError values describe errors resulting from an invalid byte in a hex string. +type InvalidByteError byte -// InvalidHexCharError results from finding an invalid character in a hex string. -type InvalidHexCharError byte - -func (e InvalidHexCharError) Error() string { - return "invalid hex char: " + strconv.Itoa(int(e)) +func (e InvalidByteError) Error() string { + return fmt.Sprintf("encoding/hex: invalid byte: %#U", rune(e)) } func DecodedLen(x int) int { return x / 2 } @@ -46,21 +45,20 @@ func DecodedLen(x int) int { return x / 2 } // Decode decodes src into DecodedLen(len(src)) bytes, returning the actual // number of bytes written to dst. // -// If Decode encounters invalid input, it returns an OddLengthInputError or an -// InvalidHexCharError. +// If Decode encounters invalid input, it returns an error describing the failure. func Decode(dst, src []byte) (int, error) { if len(src)%2 == 1 { - return 0, OddLengthInputError{} + return 0, ErrLength } for i := 0; i < len(src)/2; i++ { a, ok := fromHexChar(src[i*2]) if !ok { - return 0, InvalidHexCharError(src[i*2]) + return 0, InvalidByteError(src[i*2]) } b, ok := fromHexChar(src[i*2+1]) if !ok { - return 0, InvalidHexCharError(src[i*2+1]) + return 0, InvalidByteError(src[i*2+1]) } dst[i] = (a << 4) | b } @@ -103,8 +101,8 @@ func DecodeString(s string) ([]byte, error) { // Dump returns a string that contains a hex dump of the given data. The format // of the hex dump matches the output of `hexdump -C` on the command line. func Dump(data []byte) string { - buf := bytes.NewBuffer(nil) - dumper := Dumper(buf) + var buf bytes.Buffer + dumper := Dumper(&buf) dumper.Write(data) dumper.Close() return string(buf.Bytes()) diff --git a/libgo/go/encoding/hex/hex_test.go b/libgo/go/encoding/hex/hex_test.go index 8e1838e51e6..2d24fd0a146 100644 --- a/libgo/go/encoding/hex/hex_test.go +++ b/libgo/go/encoding/hex/hex_test.go @@ -9,141 +9,98 @@ import ( "testing" ) -type encodeTest struct { - in, out []byte +type encDecTest struct { + enc string + dec []byte } -var encodeTests = []encodeTest{ - {[]byte{}, []byte{}}, - {[]byte{0x01}, []byte{'0', '1'}}, - {[]byte{0xff}, []byte{'f', 'f'}}, - {[]byte{0xff, 00}, []byte{'f', 'f', '0', '0'}}, - {[]byte{0}, []byte{'0', '0'}}, - {[]byte{1}, []byte{'0', '1'}}, - {[]byte{2}, []byte{'0', '2'}}, - {[]byte{3}, []byte{'0', '3'}}, - {[]byte{4}, []byte{'0', '4'}}, - {[]byte{5}, []byte{'0', '5'}}, - {[]byte{6}, []byte{'0', '6'}}, - {[]byte{7}, []byte{'0', '7'}}, - {[]byte{8}, []byte{'0', '8'}}, - {[]byte{9}, []byte{'0', '9'}}, - {[]byte{10}, []byte{'0', 'a'}}, - {[]byte{11}, []byte{'0', 'b'}}, - {[]byte{12}, []byte{'0', 'c'}}, - {[]byte{13}, []byte{'0', 'd'}}, - {[]byte{14}, []byte{'0', 'e'}}, - {[]byte{15}, []byte{'0', 'f'}}, +var encDecTests = []encDecTest{ + {"", []byte{}}, + {"0001020304050607", []byte{0, 1, 2, 3, 4, 5, 6, 7}}, + {"08090a0b0c0d0e0f", []byte{8, 9, 10, 11, 12, 13, 14, 15}}, + {"f0f1f2f3f4f5f6f7", []byte{0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7}}, + {"f8f9fafbfcfdfeff", []byte{0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff}}, + {"67", []byte{'g'}}, + {"e3a1", []byte{0xe3, 0xa1}}, } func TestEncode(t *testing.T) { - for i, test := range encodeTests { - dst := make([]byte, EncodedLen(len(test.in))) - n := Encode(dst, test.in) + for i, test := range encDecTests { + dst := make([]byte, EncodedLen(len(test.dec))) + n := Encode(dst, test.dec) if n != len(dst) { t.Errorf("#%d: bad return value: got: %d want: %d", i, n, len(dst)) } - if bytes.Compare(dst, test.out) != 0 { - t.Errorf("#%d: got: %#v want: %#v", i, dst, test.out) + if string(dst) != test.enc { + t.Errorf("#%d: got: %#v want: %#v", i, dst, test.enc) } } } -type decodeTest struct { - in, out []byte - ok bool -} - -var decodeTests = []decodeTest{ - {[]byte{}, []byte{}, true}, - {[]byte{'0'}, []byte{}, false}, - {[]byte{'0', 'g'}, []byte{}, false}, - {[]byte{'0', '\x01'}, []byte{}, false}, - {[]byte{'0', '0'}, []byte{0}, true}, - {[]byte{'0', '1'}, []byte{1}, true}, - {[]byte{'0', '2'}, []byte{2}, true}, - {[]byte{'0', '3'}, []byte{3}, true}, - {[]byte{'0', '4'}, []byte{4}, true}, - {[]byte{'0', '5'}, []byte{5}, true}, - {[]byte{'0', '6'}, []byte{6}, true}, - {[]byte{'0', '7'}, []byte{7}, true}, - {[]byte{'0', '8'}, []byte{8}, true}, - {[]byte{'0', '9'}, []byte{9}, true}, - {[]byte{'0', 'a'}, []byte{10}, true}, - {[]byte{'0', 'b'}, []byte{11}, true}, - {[]byte{'0', 'c'}, []byte{12}, true}, - {[]byte{'0', 'd'}, []byte{13}, true}, - {[]byte{'0', 'e'}, []byte{14}, true}, - {[]byte{'0', 'f'}, []byte{15}, true}, - {[]byte{'0', 'A'}, []byte{10}, true}, - {[]byte{'0', 'B'}, []byte{11}, true}, - {[]byte{'0', 'C'}, []byte{12}, true}, - {[]byte{'0', 'D'}, []byte{13}, true}, - {[]byte{'0', 'E'}, []byte{14}, true}, - {[]byte{'0', 'F'}, []byte{15}, true}, -} - func TestDecode(t *testing.T) { - for i, test := range decodeTests { - dst := make([]byte, DecodedLen(len(test.in))) - n, err := Decode(dst, test.in) - if err == nil && n != len(dst) { + for i, test := range encDecTests { + dst := make([]byte, DecodedLen(len(test.enc))) + n, err := Decode(dst, []byte(test.enc)) + if err != nil { t.Errorf("#%d: bad return value: got:%d want:%d", i, n, len(dst)) - } - if test.ok != (err == nil) { - t.Errorf("#%d: unexpected err value: %s", i, err) - } - if err == nil && bytes.Compare(dst, test.out) != 0 { - t.Errorf("#%d: got: %#v want: %#v", i, dst, test.out) + } else if !bytes.Equal(dst, test.dec) { + t.Errorf("#%d: got: %#v want: %#v", i, dst, test.dec) } } } -type encodeStringTest struct { - in []byte - out string -} - -var encodeStringTests = []encodeStringTest{ - {[]byte{}, ""}, - {[]byte{0}, "00"}, - {[]byte{0, 1}, "0001"}, - {[]byte{0, 1, 255}, "0001ff"}, +func TestEncodeToString(t *testing.T) { + for i, test := range encDecTests { + s := EncodeToString(test.dec) + if s != test.enc { + t.Errorf("#%d got:%s want:%s", i, s, test.enc) + } + } } -func TestEncodeToString(t *testing.T) { - for i, test := range encodeStringTests { - s := EncodeToString(test.in) - if s != test.out { - t.Errorf("#%d got:%s want:%s", i, s, test.out) +func TestDecodeString(t *testing.T) { + for i, test := range encDecTests { + dst, err := DecodeString(test.enc) + if err != nil { + t.Errorf("#%d: unexpected err value: %s", i, err) + continue + } + if bytes.Compare(dst, test.dec) != 0 { + t.Errorf("#%d: got: %#v want: #%v", i, dst, test.dec) } } } -type decodeStringTest struct { +type errTest struct { in string - out []byte - ok bool + err string } -var decodeStringTests = []decodeStringTest{ - {"", []byte{}, true}, - {"0", []byte{}, false}, - {"00", []byte{0}, true}, - {"0\x01", []byte{}, false}, - {"0g", []byte{}, false}, - {"00ff00", []byte{0, 255, 0}, true}, - {"0000ff", []byte{0, 0, 255}, true}, +var errTests = []errTest{ + {"0", "encoding/hex: odd length hex string"}, + {"0g", "encoding/hex: invalid byte: U+0067 'g'"}, + {"0\x01", "encoding/hex: invalid byte: U+0001"}, } -func TestDecodeString(t *testing.T) { - for i, test := range decodeStringTests { - dst, err := DecodeString(test.in) - if test.ok != (err == nil) { - t.Errorf("#%d: unexpected err value: %s", i, err) +func TestInvalidErr(t *testing.T) { + for i, test := range errTests { + dst := make([]byte, DecodedLen(len(test.in))) + _, err := Decode(dst, []byte(test.in)) + if err == nil { + t.Errorf("#%d: expected error; got none") + } else if err.Error() != test.err { + t.Errorf("#%d: got: %v want: %v", i, err, test.err) } - if err == nil && bytes.Compare(dst, test.out) != 0 { - t.Errorf("#%d: got: %#v want: #%v", i, dst, test.out) + } +} + +func TestInvalidStringErr(t *testing.T) { + for i, test := range errTests { + _, err := DecodeString(test.in) + if err == nil { + t.Errorf("#%d: expected error; got none") + } else if err.Error() != test.err { + t.Errorf("#%d: got: %v want: %v", i, err, test.err) } } } @@ -155,8 +112,8 @@ func TestDumper(t *testing.T) { } for stride := 1; stride < len(in); stride++ { - out := bytes.NewBuffer(nil) - dumper := Dumper(out) + var out bytes.Buffer + dumper := Dumper(&out) done := 0 for done < len(in) { todo := done + stride diff --git a/libgo/go/encoding/json/decode_test.go b/libgo/go/encoding/json/decode_test.go index cc3103f032f..775becfa7c9 100644 --- a/libgo/go/encoding/json/decode_test.go +++ b/libgo/go/encoding/json/decode_test.go @@ -598,3 +598,24 @@ var pallValueIndent = `{ }` var pallValueCompact = strings.Map(noSpace, pallValueIndent) + +func TestRefUnmarshal(t *testing.T) { + type S struct { + // Ref is defined in encode_test.go. + R0 Ref + R1 *Ref + } + want := S{ + R0: 12, + R1: new(Ref), + } + *want.R1 = 12 + + var got S + if err := Unmarshal([]byte(`{"R0":"ref","R1":"ref"}`), &got); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("got %+v, want %+v", got, want) + } +} diff --git a/libgo/go/encoding/json/encode.go b/libgo/go/encoding/json/encode.go index eac14a47ed7..83e73c09cb4 100644 --- a/libgo/go/encoding/json/encode.go +++ b/libgo/go/encoding/json/encode.go @@ -262,8 +262,18 @@ func (e *encodeState) reflectValueQuoted(v reflect.Value, quoted bool) { return } - if j, ok := v.Interface().(Marshaler); ok && (v.Kind() != reflect.Ptr || !v.IsNil()) { - b, err := j.MarshalJSON() + m, ok := v.Interface().(Marshaler) + if !ok { + // T doesn't match the interface. Check against *T too. + if v.Kind() != reflect.Ptr && v.CanAddr() { + m, ok = v.Addr().Interface().(Marshaler) + if ok { + v = v.Addr() + } + } + } + if ok && (v.Kind() != reflect.Ptr || !v.IsNil()) { + b, err := m.MarshalJSON() if err == nil { // copy JSON into buffer, checking validity. err = Compact(&e.Buffer, b) diff --git a/libgo/go/encoding/json/encode_test.go b/libgo/go/encoding/json/encode_test.go index 0e39559a463..7a726a91c47 100644 --- a/libgo/go/encoding/json/encode_test.go +++ b/libgo/go/encoding/json/encode_test.go @@ -126,3 +126,44 @@ func TestUnsupportedValues(t *testing.T) { } } } + +// Ref has Marshaler and Unmarshaler methods with pointer receiver. +type Ref int + +func (*Ref) MarshalJSON() ([]byte, error) { + return []byte(`"ref"`), nil +} + +func (r *Ref) UnmarshalJSON([]byte) error { + *r = 12 + return nil +} + +// Val has Marshaler methods with value receiver. +type Val int + +func (Val) MarshalJSON() ([]byte, error) { + return []byte(`"val"`), nil +} + +func TestRefValMarshal(t *testing.T) { + var s = struct { + R0 Ref + R1 *Ref + V0 Val + V1 *Val + }{ + R0: 12, + R1: new(Ref), + V0: 13, + V1: new(Val), + } + const want = `{"R0":"ref","R1":"ref","V0":"val","V1":"val"}` + b, err := Marshal(&s) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + if got := string(b); got != want { + t.Errorf("got %q, want %q", got, want) + } +} diff --git a/libgo/go/encoding/json/scanner.go b/libgo/go/encoding/json/scanner.go index 2661f410e01..054b6b3d564 100644 --- a/libgo/go/encoding/json/scanner.go +++ b/libgo/go/encoding/json/scanner.go @@ -185,18 +185,9 @@ func isSpace(c rune) bool { return c == ' ' || c == '\t' || c == '\r' || c == '\n' } -// NOTE(rsc): The various instances of -// -// if c <= ' ' && (c == ' ' || c == '\t' || c == '\r' || c == '\n') -// -// below should all be if c <= ' ' && isSpace(c), but inlining -// the checks makes a significant difference (>10%) in tight loops -// such as nextValue. These should be rewritten with the clearer -// function call once 6g knows to inline the call. - // stateBeginValueOrEmpty is the state after reading `[`. func stateBeginValueOrEmpty(s *scanner, c int) int { - if c <= ' ' && (c == ' ' || c == '\t' || c == '\r' || c == '\n') { + if c <= ' ' && isSpace(rune(c)) { return scanSkipSpace } if c == ']' { @@ -207,7 +198,7 @@ func stateBeginValueOrEmpty(s *scanner, c int) int { // stateBeginValue is the state at the beginning of the input. func stateBeginValue(s *scanner, c int) int { - if c <= ' ' && (c == ' ' || c == '\t' || c == '\r' || c == '\n') { + if c <= ' ' && isSpace(rune(c)) { return scanSkipSpace } switch c { @@ -247,7 +238,7 @@ func stateBeginValue(s *scanner, c int) int { // stateBeginStringOrEmpty is the state after reading `{`. func stateBeginStringOrEmpty(s *scanner, c int) int { - if c <= ' ' && (c == ' ' || c == '\t' || c == '\r' || c == '\n') { + if c <= ' ' && isSpace(rune(c)) { return scanSkipSpace } if c == '}' { @@ -260,7 +251,7 @@ func stateBeginStringOrEmpty(s *scanner, c int) int { // stateBeginString is the state after reading `{"key": value,`. func stateBeginString(s *scanner, c int) int { - if c <= ' ' && (c == ' ' || c == '\t' || c == '\r' || c == '\n') { + if c <= ' ' && isSpace(rune(c)) { return scanSkipSpace } if c == '"' { @@ -280,7 +271,7 @@ func stateEndValue(s *scanner, c int) int { s.endTop = true return stateEndTop(s, c) } - if c <= ' ' && (c == ' ' || c == '\t' || c == '\r' || c == '\n') { + if c <= ' ' && isSpace(rune(c)) { s.step = stateEndValue return scanSkipSpace } diff --git a/libgo/go/encoding/pem/pem.go b/libgo/go/encoding/pem/pem.go index 3eb7c9fa0d9..38afbb42aff 100644 --- a/libgo/go/encoding/pem/pem.go +++ b/libgo/go/encoding/pem/pem.go @@ -251,7 +251,7 @@ func Encode(out io.Writer, b *Block) (err error) { } func EncodeToMemory(b *Block) []byte { - buf := bytes.NewBuffer(nil) - Encode(buf, b) + var buf bytes.Buffer + Encode(&buf, b) return buf.Bytes() } diff --git a/libgo/go/encoding/pem/pem_test.go b/libgo/go/encoding/pem/pem_test.go index 11efe554487..9ae1578a501 100644 --- a/libgo/go/encoding/pem/pem_test.go +++ b/libgo/go/encoding/pem/pem_test.go @@ -73,7 +73,7 @@ var lineBreakerTests = []lineBreakerTest{ func TestLineBreaker(t *testing.T) { for i, test := range lineBreakerTests { - buf := bytes.NewBuffer(nil) + buf := new(bytes.Buffer) var breaker lineBreaker breaker.out = buf _, err := breaker.Write([]byte(test.in)) @@ -93,7 +93,7 @@ func TestLineBreaker(t *testing.T) { } for i, test := range lineBreakerTests { - buf := bytes.NewBuffer(nil) + buf := new(bytes.Buffer) var breaker lineBreaker breaker.out = buf diff --git a/libgo/go/encoding/xml/marshal_test.go b/libgo/go/encoding/xml/marshal_test.go index e0be3320086..0f6c0f0795d 100644 --- a/libgo/go/encoding/xml/marshal_test.go +++ b/libgo/go/encoding/xml/marshal_test.go @@ -532,6 +532,11 @@ var marshalTests = []struct { Value: &NameInField{Name{Space: "ns", Local: "foo"}}, ExpectXML: `<NameInField><foo xmlns="ns"></foo></NameInField>`, }, + { + Value: &NameInField{Name{Space: "ns", Local: "foo"}}, + ExpectXML: `<NameInField><foo xmlns="ns"><ignore></ignore></foo></NameInField>`, + UnmarshalOnly: true, + }, // Marshaling zero xml.Name uses the tag or field name. { diff --git a/libgo/go/encoding/xml/read.go b/libgo/go/encoding/xml/read.go index 871fe059cfa..bde875a0123 100644 --- a/libgo/go/encoding/xml/read.go +++ b/libgo/go/encoding/xml/read.go @@ -265,12 +265,13 @@ func (p *Decoder) unmarshal(val reflect.Value, start *StartElement) error { saveData = v case reflect.Struct: - sv = v - typ := sv.Type() + typ := v.Type() if typ == nameType { v.Set(reflect.ValueOf(start.Name)) break } + + sv = v tinfo, err = getTypeInfo(typ) if err != nil { return err @@ -541,19 +542,21 @@ Loop: panic("unreachable") } -// Have already read a start element. -// Read tokens until we find the end element. -// Token is taking care of making sure the -// end element matches the start element we saw. -func (p *Decoder) Skip() error { +// Skip reads tokens until it has consumed the end element +// matching the most recent start element already consumed. +// It recurs if it encounters a start element, so it can be used to +// skip nested structures. +// It returns nil if it finds an end element matching the start +// element; otherwise it returns an error describing the problem. +func (d *Decoder) Skip() error { for { - tok, err := p.Token() + tok, err := d.Token() if err != nil { return err } switch tok.(type) { case StartElement: - if err := p.Skip(); err != nil { + if err := d.Skip(); err != nil { return err } case EndElement: diff --git a/libgo/go/encoding/xml/typeinfo.go b/libgo/go/encoding/xml/typeinfo.go index 2bf2c6b3032..5475f290d18 100644 --- a/libgo/go/encoding/xml/typeinfo.go +++ b/libgo/go/encoding/xml/typeinfo.go @@ -193,7 +193,7 @@ func structFieldInfo(typ reflect.Type, f *reflect.StructField) (*fieldInfo, erro // If the field type has an XMLName field, the names must match // so that the behavior of both marshalling and unmarshalling - // is straighforward and unambiguous. + // is straightforward and unambiguous. if finfo.flags&fElement != 0 { ftyp := f.Type xmlname := lookupXMLName(ftyp) |
