Skip to content

G
go
Switch branch/tag
Find file Normal viewHistoryPermalink
xml_test.go 22 KB
Newer Older
Brad Fitzpatrick's avatar
all: make copyright headers consistent with one space after period  
Brad Fitzpatrick committed
1 
// Copyright 2009 The Go Authors. All rights reserved.
Russ Cox's avatar
XML lexing  
Russ Cox committed
2 3 4 5 6 7 
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package xml

import (
Olivier Saingre's avatar
encoding/xml: rewrite invalid code points to U+FFFD in Marshal, Escape  
Olivier Saingre committed
8 
	"bytes"
Gustavo Niemeyer's avatar
encoding/xml: fix decoding of unknown entities in non-strict mode  
Gustavo Niemeyer committed
9 
	"fmt"
Robert Griesemer's avatar
1) Change default gofmt default settings for  
Robert Griesemer committed
10 11 
	"io"
	"reflect"
Brad Fitzpatrick's avatar
xml: Parser hook for non-UTF-8 charset converters  
Brad Fitzpatrick committed
12 
	"strings"
Robert Griesemer's avatar
1) Change default gofmt default settings for  
Robert Griesemer committed
13 
	"testing"
Alex A Skinner's avatar
encoding/xml: Do not pass through invalid utf8 bytes  
Alex A Skinner committed
14 
	"unicode/utf8"
Russ Cox's avatar
XML lexing  
Russ Cox committed
15 16 17 18 19 20 
)

const testInput = `
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
Robert Griesemer's avatar
remove uses of string concatenation from src and misc directory  
Robert Griesemer committed
21 22 
<body xmlns:foo="ns1" xmlns="ns2" xmlns:tag="ns3" ` +
	"\r\n\t" + `  >
Russ Cox's avatar
XML lexing  
Russ Cox committed
23 
  <hello lang="en">World &lt;&gt;&apos;&quot; &#x767d;&#40300;翔</hello>
Patrick Smith's avatar
encoding/xml: expand allowed entity names  
Patrick Smith committed
24 
  <query>&何; &is-it;</query>
Russ Cox's avatar
XML lexing  
Russ Cox committed
25 26 27 28 29 
  <goodbye />
  <outer foo:attr="value" xmlns:tag="ns4">
    <inner/>
  </outer>
  <tag:name>
Abhinav Gupta's avatar
xml: Fixed CDATA parsing.  
Abhinav Gupta committed
30 
    <![CDATA[Some text here.]]>
Russ Cox's avatar
XML lexing  
Russ Cox committed
31 32 33 
  </tag:name>
</body><!-- missing final newline -->`
Patrick Smith's avatar
encoding/xml: expand allowed entity names  
Patrick Smith committed
34 35 
var testEntity = map[string]string{"何": "What", "is-it": "is it?"}
Russ Cox's avatar
XML lexing  
Russ Cox committed
36 
var rawTokens = []Token{
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
37 
	CharData("\n"),
Russ Cox's avatar
strings: delete Runes, Bytes  
Russ Cox committed
38 
	ProcInst{"xml", []byte(`version="1.0" encoding="UTF-8"`)},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
39 40 
	CharData("\n"),
	Directive(`DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
Russ Cox's avatar
strings: delete Runes, Bytes  
Russ Cox committed
41 
  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"`),
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
42 
	CharData("\n"),
Robert Griesemer's avatar
gofmt -s -w src misc  
Robert Griesemer committed
43 
	StartElement{Name{"", "body"}, []Attr{{Name{"xmlns", "foo"}, "ns1"}, {Name{"", "xmlns"}, "ns2"}, {Name{"xmlns", "tag"}, "ns3"}}},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
44 
	CharData("\n  "),
Robert Griesemer's avatar
gofmt -s -w src misc  
Robert Griesemer committed
45 
	StartElement{Name{"", "hello"}, []Attr{{Name{"", "lang"}, "en"}}},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
46 
	CharData("World <>'\" 白鵬翔"),
Russ Cox's avatar
XML lexing  
Russ Cox committed
47 
	EndElement{Name{"", "hello"}},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
48 
	CharData("\n  "),
Patrick Smith's avatar
encoding/xml: expand allowed entity names  
Patrick Smith committed
49 50 51 52 
	StartElement{Name{"", "query"}, []Attr{}},
	CharData("What is it?"),
	EndElement{Name{"", "query"}},
	CharData("\n  "),
Mikio Hara's avatar
xml: fix build  
Mikio Hara committed
53 
	StartElement{Name{"", "goodbye"}, []Attr{}},
Russ Cox's avatar
XML lexing  
Russ Cox committed
54 
	EndElement{Name{"", "goodbye"}},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
55 
	CharData("\n  "),
Robert Griesemer's avatar
gofmt -s -w src misc  
Robert Griesemer committed
56 
	StartElement{Name{"", "outer"}, []Attr{{Name{"foo", "attr"}, "value"}, {Name{"xmlns", "tag"}, "ns4"}}},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
57 
	CharData("\n    "),
Mikio Hara's avatar
xml: fix build  
Mikio Hara committed
58 
	StartElement{Name{"", "inner"}, []Attr{}},
Russ Cox's avatar
XML lexing  
Russ Cox committed
59 
	EndElement{Name{"", "inner"}},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
60 
	CharData("\n  "),
Russ Cox's avatar
XML lexing  
Russ Cox committed
61 
	EndElement{Name{"", "outer"}},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
62 
	CharData("\n  "),
Mikio Hara's avatar
xml: fix build  
Mikio Hara committed
63 
	StartElement{Name{"tag", "name"}, []Attr{}},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
64 65 66 
	CharData("\n    "),
	CharData("Some text here."),
	CharData("\n  "),
Russ Cox's avatar
XML lexing  
Russ Cox committed
67 
	EndElement{Name{"tag", "name"}},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
68 
	CharData("\n"),
Russ Cox's avatar
XML lexing  
Russ Cox committed
69 
	EndElement{Name{"", "body"}},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
70 
	Comment(" missing final newline "),
Russ Cox's avatar
XML lexing  
Russ Cox committed
71 72 73 
}

var cookedTokens = []Token{
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
74 
	CharData("\n"),
Russ Cox's avatar
strings: delete Runes, Bytes  
Russ Cox committed
75 
	ProcInst{"xml", []byte(`version="1.0" encoding="UTF-8"`)},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
76 77 
	CharData("\n"),
	Directive(`DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
Russ Cox's avatar
strings: delete Runes, Bytes  
Russ Cox committed
78 
  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"`),
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
79 
	CharData("\n"),
Robert Griesemer's avatar
gofmt -s -w src misc  
Robert Griesemer committed
80 
	StartElement{Name{"ns2", "body"}, []Attr{{Name{"xmlns", "foo"}, "ns1"}, {Name{"", "xmlns"}, "ns2"}, {Name{"xmlns", "tag"}, "ns3"}}},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
81 
	CharData("\n  "),
Robert Griesemer's avatar
gofmt -s -w src misc  
Robert Griesemer committed
82 
	StartElement{Name{"ns2", "hello"}, []Attr{{Name{"", "lang"}, "en"}}},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
83 
	CharData("World <>'\" 白鵬翔"),
Russ Cox's avatar
XML lexing  
Russ Cox committed
84 
	EndElement{Name{"ns2", "hello"}},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
85 
	CharData("\n  "),
Patrick Smith's avatar
encoding/xml: expand allowed entity names  
Patrick Smith committed
86 87 88 89 
	StartElement{Name{"ns2", "query"}, []Attr{}},
	CharData("What is it?"),
	EndElement{Name{"ns2", "query"}},
	CharData("\n  "),
Mikio Hara's avatar
xml: fix build  
Mikio Hara committed
90 
	StartElement{Name{"ns2", "goodbye"}, []Attr{}},
Russ Cox's avatar
XML lexing  
Russ Cox committed
91 
	EndElement{Name{"ns2", "goodbye"}},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
92 
	CharData("\n  "),
Robert Griesemer's avatar
gofmt -s -w src misc  
Robert Griesemer committed
93 
	StartElement{Name{"ns2", "outer"}, []Attr{{Name{"ns1", "attr"}, "value"}, {Name{"xmlns", "tag"}, "ns4"}}},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
94 
	CharData("\n    "),
Mikio Hara's avatar
xml: fix build  
Mikio Hara committed
95 
	StartElement{Name{"ns2", "inner"}, []Attr{}},
Russ Cox's avatar
XML lexing  
Russ Cox committed
96 
	EndElement{Name{"ns2", "inner"}},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
97 
	CharData("\n  "),
Russ Cox's avatar
XML lexing  
Russ Cox committed
98 
	EndElement{Name{"ns2", "outer"}},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
99 
	CharData("\n  "),
Mikio Hara's avatar
xml: fix build  
Mikio Hara committed
100 
	StartElement{Name{"ns3", "name"}, []Attr{}},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
101 102 103 
	CharData("\n    "),
	CharData("Some text here."),
	CharData("\n  "),
Russ Cox's avatar
XML lexing  
Russ Cox committed
104 
	EndElement{Name{"ns3", "name"}},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
105 
	CharData("\n"),
Russ Cox's avatar
XML lexing  
Russ Cox committed
106 
	EndElement{Name{"ns2", "body"}},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
107 
	Comment(" missing final newline "),
Russ Cox's avatar
XML lexing  
Russ Cox committed
108 109 
}
Brad Fitzpatrick's avatar
xml: Parser hook for non-UTF-8 charset converters  
Brad Fitzpatrick committed
110 111 112 113 114 
const testInputAltEncoding = `
<?xml version="1.0" encoding="x-testing-uppercase"?>
<TAG>VALUE</TAG>`

var rawTokensAltEncoding = []Token{
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
115 
	CharData("\n"),
Brad Fitzpatrick's avatar
xml: Parser hook for non-UTF-8 charset converters  
Brad Fitzpatrick committed
116 
	ProcInst{"xml", []byte(`version="1.0" encoding="x-testing-uppercase"`)},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
117 
	CharData("\n"),
Mikio Hara's avatar
xml: fix build  
Mikio Hara committed
118 
	StartElement{Name{"", "tag"}, []Attr{}},
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
119 
	CharData("value"),
Brad Fitzpatrick's avatar
xml: Parser hook for non-UTF-8 charset converters  
Brad Fitzpatrick committed
120 121 122 
	EndElement{Name{"", "tag"}},
}
Arvindh Rajesh Tamilmani's avatar
xml: handle unexpected EOF while parsing and fix a bug in name  
Arvindh Rajesh Tamilmani committed
123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 
var xmlInput = []string{
	// unexpected EOF cases
	"<",
	"<t",
	"<t ",
	"<t/",
	"<!",
	"<!-",
	"<!--",
	"<!--c-",
	"<!--c--",
	"<!d",
	"<t></",
	"<t></t",
	"<?",
	"<?p",
	"<t a",
	"<t a=",
	"<t a='",
	"<t a=''",
	"<t/><![",
	"<t/><![C",
	"<t/><![CDATA[d",
	"<t/><![CDATA[d]",
	"<t/><![CDATA[d]]",

	// other Syntax errors
	"<>",
	"<t/a",
	"<0 />",
	"<?0 >",
	//	"<!0 >",	// let the Token() caller handle
	"</0>",
	"<t 0=''>",
	"<t a='&'>",
	"<t a='<'>",
	"<t>&nbspc;</t>",
	"<t a>",
	"<t a=>",
	"<t a=v>",
	//	"<![CDATA[d]]>",	// let the Token() caller handle
	"<t></e>",
	"<t></>",
	"<t></t!",
	"<t>cdata]]></t>",
}
Russ Cox's avatar
XML lexing  
Russ Cox committed
170 
func TestRawToken(t *testing.T) {
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
171 
	d := NewDecoder(strings.NewReader(testInput))
Patrick Smith's avatar
encoding/xml: expand allowed entity names  
Patrick Smith committed
172 
	d.Entity = testEntity
Russ Cox's avatar
encoding/xml: add InputOffset method to Decoder  
Russ Cox committed
173 
	testRawToken(t, d, testInput, rawTokens)
Brad Fitzpatrick's avatar
xml: Parser hook for non-UTF-8 charset converters  
Brad Fitzpatrick committed
174 175 
}
Gustavo Niemeyer's avatar
encoding/xml: fix decoding of unknown entities in non-strict mode  
Gustavo Niemeyer committed
176 177 178 179 180 
const nonStrictInput = `
<tag>non&entity</tag>
<tag>&unknown;entity</tag>
<tag>&#123</tag>
<tag>&#zzz;</tag>
Patrick Smith's avatar
encoding/xml: expand allowed entity names  
Patrick Smith committed
181 182 183 184 
<tag>&なまえ3;</tag>
<tag>&lt-gt;</tag>
<tag>&;</tag>
<tag>&0a;</tag>
Gustavo Niemeyer's avatar
encoding/xml: fix decoding of unknown entities in non-strict mode  
Gustavo Niemeyer committed
185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 
`

var nonStrictTokens = []Token{
	CharData("\n"),
	StartElement{Name{"", "tag"}, []Attr{}},
	CharData("non&entity"),
	EndElement{Name{"", "tag"}},
	CharData("\n"),
	StartElement{Name{"", "tag"}, []Attr{}},
	CharData("&unknown;entity"),
	EndElement{Name{"", "tag"}},
	CharData("\n"),
	StartElement{Name{"", "tag"}, []Attr{}},
	CharData("&#123"),
	EndElement{Name{"", "tag"}},
	CharData("\n"),
	StartElement{Name{"", "tag"}, []Attr{}},
	CharData("&#zzz;"),
	EndElement{Name{"", "tag"}},
	CharData("\n"),
Patrick Smith's avatar
encoding/xml: expand allowed entity names  
Patrick Smith committed
205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 
	StartElement{Name{"", "tag"}, []Attr{}},
	CharData("&なまえ3;"),
	EndElement{Name{"", "tag"}},
	CharData("\n"),
	StartElement{Name{"", "tag"}, []Attr{}},
	CharData("&lt-gt;"),
	EndElement{Name{"", "tag"}},
	CharData("\n"),
	StartElement{Name{"", "tag"}, []Attr{}},
	CharData("&;"),
	EndElement{Name{"", "tag"}},
	CharData("\n"),
	StartElement{Name{"", "tag"}, []Attr{}},
	CharData("&0a;"),
	EndElement{Name{"", "tag"}},
	CharData("\n"),
Gustavo Niemeyer's avatar
encoding/xml: fix decoding of unknown entities in non-strict mode  
Gustavo Niemeyer committed
221 222 223 224 225 
}

func TestNonStrictRawToken(t *testing.T) {
	d := NewDecoder(strings.NewReader(nonStrictInput))
	d.Strict = false
Russ Cox's avatar
encoding/xml: add InputOffset method to Decoder  
Russ Cox committed
226 
	testRawToken(t, d, nonStrictInput, nonStrictTokens)
Gustavo Niemeyer's avatar
encoding/xml: fix decoding of unknown entities in non-strict mode  
Gustavo Niemeyer committed
227 228 
}
Brad Fitzpatrick's avatar
xml: Parser hook for non-UTF-8 charset converters  
Brad Fitzpatrick committed
229 230 231 232 233 
type downCaser struct {
	t *testing.T
	r io.ByteReader
}
Russ Cox's avatar
src/pkg/[n-z]*: gofix -r error -force=error  
Russ Cox committed
234 
func (d *downCaser) ReadByte() (c byte, err error) {
Brad Fitzpatrick's avatar
xml: Parser hook for non-UTF-8 charset converters  
Brad Fitzpatrick committed
235 236 237 238 239 240 241 
	c, err = d.r.ReadByte()
	if c >= 'A' && c <= 'Z' {
		c += 'a' - 'A'
	}
	return
}
Russ Cox's avatar
src/pkg/[n-z]*: gofix -r error -force=error  
Russ Cox committed
242 
func (d *downCaser) Read(p []byte) (int, error) {
Brad Fitzpatrick's avatar
xml: Parser hook for non-UTF-8 charset converters  
Brad Fitzpatrick committed
243 
	d.t.Fatalf("unexpected Read call on downCaser reader")
Russ Cox's avatar
various: reduce overuse of os.EINVAL + others  
Russ Cox committed
244 
	panic("unreachable")
Brad Fitzpatrick's avatar
xml: Parser hook for non-UTF-8 charset converters  
Brad Fitzpatrick committed
245 246 247 
}

func TestRawTokenAltEncoding(t *testing.T) {
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
248 249 
	d := NewDecoder(strings.NewReader(testInputAltEncoding))
	d.CharsetReader = func(charset string, input io.Reader) (io.Reader, error) {
Brad Fitzpatrick's avatar
xml: Parser hook for non-UTF-8 charset converters  
Brad Fitzpatrick committed
250 251 252 253 254 
		if charset != "x-testing-uppercase" {
			t.Fatalf("unexpected charset %q", charset)
		}
		return &downCaser{t, input.(io.ByteReader)}, nil
	}
Russ Cox's avatar
encoding/xml: add InputOffset method to Decoder  
Russ Cox committed
255 
	testRawToken(t, d, testInputAltEncoding, rawTokensAltEncoding)
Brad Fitzpatrick's avatar
xml: Parser hook for non-UTF-8 charset converters  
Brad Fitzpatrick committed
256 
}
Russ Cox's avatar
XML lexing  
Russ Cox committed
257
Brad Fitzpatrick's avatar
xml: Parser hook for non-UTF-8 charset converters  
Brad Fitzpatrick committed
258 
func TestRawTokenAltEncodingNoConverter(t *testing.T) {
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
259 260 
	d := NewDecoder(strings.NewReader(testInputAltEncoding))
	token, err := d.RawToken()
Brad Fitzpatrick's avatar
xml: Parser hook for non-UTF-8 charset converters  
Brad Fitzpatrick committed
261 262 263 264 265 266 
	if token == nil {
		t.Fatalf("expected a token on first RawToken call")
	}
	if err != nil {
		t.Fatal(err)
	}
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
267 
	token, err = d.RawToken()
Brad Fitzpatrick's avatar
xml: Parser hook for non-UTF-8 charset converters  
Brad Fitzpatrick committed
268 269 270 271 272 273 274 
	if token != nil {
		t.Errorf("expected a nil token; got %#v", token)
	}
	if err == nil {
		t.Fatalf("expected an error on second RawToken call")
	}
	const encoding = "x-testing-uppercase"
Russ Cox's avatar
src/pkg/[n-z]*: gofix -r error -force=error  
Russ Cox committed
275 
	if !strings.Contains(err.Error(), encoding) {
Brad Fitzpatrick's avatar
xml: Parser hook for non-UTF-8 charset converters  
Brad Fitzpatrick committed
276 277 278 279 280 
		t.Errorf("expected error to contain %q; got error: %v",
			encoding, err)
	}
}
Russ Cox's avatar
encoding/xml: add InputOffset method to Decoder  
Russ Cox committed
281 282 
func testRawToken(t *testing.T, d *Decoder, raw string, rawTokens []Token) {
	lastEnd := int64(0)
Russ Cox's avatar
XML lexing  
Russ Cox committed
283 
	for i, want := range rawTokens {
Russ Cox's avatar
encoding/xml: add InputOffset method to Decoder  
Russ Cox committed
284 
		start := d.InputOffset()
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
285 
		have, err := d.RawToken()
Russ Cox's avatar
encoding/xml: add InputOffset method to Decoder  
Russ Cox committed
286 
		end := d.InputOffset()
Russ Cox's avatar
XML lexing  
Russ Cox committed
287 
		if err != nil {
Robert Griesemer's avatar
remove semis after statements in one-statement statement lists  
Robert Griesemer committed
288 
			t.Fatalf("token %d: unexpected error: %s", i, err)
Russ Cox's avatar
XML lexing  
Russ Cox committed
289 290 
		}
		if !reflect.DeepEqual(have, want) {
Gustavo Niemeyer's avatar
encoding/xml: fix decoding of unknown entities in non-strict mode  
Gustavo Niemeyer committed
291 292 293 294 295 296 297 298 299 300 301 302 
			var shave, swant string
			if _, ok := have.(CharData); ok {
				shave = fmt.Sprintf("CharData(%q)", have)
			} else {
				shave = fmt.Sprintf("%#v", have)
			}
			if _, ok := want.(CharData); ok {
				swant = fmt.Sprintf("CharData(%q)", want)
			} else {
				swant = fmt.Sprintf("%#v", want)
			}
			t.Errorf("token %d = %s, want %s", i, shave, swant)
Chris Dollin's avatar
xml: permit nested directives  
Chris Dollin committed
303 
		}
Russ Cox's avatar
encoding/xml: add InputOffset method to Decoder  
Russ Cox committed
304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 

		// Check that InputOffset returned actual token.
		switch {
		case start < lastEnd:
			t.Errorf("token %d: position [%d,%d) for %T is before previous token", i, start, end, have)
		case start >= end:
			// Special case: EndElement can be synthesized.
			if start == end && end == lastEnd {
				break
			}
			t.Errorf("token %d: position [%d,%d) for %T is empty", i, start, end, have)
		case end > int64(len(raw)):
			t.Errorf("token %d: position [%d,%d) for %T extends beyond input", i, start, end, have)
		default:
			text := raw[start:end]
			if strings.ContainsAny(text, "<>") && (!strings.HasPrefix(text, "<") || !strings.HasSuffix(text, ">")) {
				t.Errorf("token %d: misaligned raw token %#q for %T", i, text, have)
			}
		}
		lastEnd = end
Chris Dollin's avatar
xml: permit nested directives  
Chris Dollin committed
324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 
	}
}

// Ensure that directives (specifically !DOCTYPE) include the complete
// text of any nested directives, noting that < and > do not change
// nesting depth if they are in single or double quotes.

var nestedDirectivesInput = `
<!DOCTYPE [<!ENTITY rdf "http://www.w3.org/1999/02/22-rdf-syntax-ns#">]>
<!DOCTYPE [<!ENTITY xlt ">">]>
<!DOCTYPE [<!ENTITY xlt "<">]>
<!DOCTYPE [<!ENTITY xlt '>'>]>
<!DOCTYPE [<!ENTITY xlt '<'>]>
<!DOCTYPE [<!ENTITY xlt '">'>]>
<!DOCTYPE [<!ENTITY xlt "'<">]>
`

var nestedDirectivesTokens = []Token{
Russ Cox's avatar
allow direct conversion between string and named []byte, []rune  
Russ Cox committed
342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 
	CharData("\n"),
	Directive(`DOCTYPE [<!ENTITY rdf "http://www.w3.org/1999/02/22-rdf-syntax-ns#">]`),
	CharData("\n"),
	Directive(`DOCTYPE [<!ENTITY xlt ">">]`),
	CharData("\n"),
	Directive(`DOCTYPE [<!ENTITY xlt "<">]`),
	CharData("\n"),
	Directive(`DOCTYPE [<!ENTITY xlt '>'>]`),
	CharData("\n"),
	Directive(`DOCTYPE [<!ENTITY xlt '<'>]`),
	CharData("\n"),
	Directive(`DOCTYPE [<!ENTITY xlt '">'>]`),
	CharData("\n"),
	Directive(`DOCTYPE [<!ENTITY xlt "'<">]`),
	CharData("\n"),
Chris Dollin's avatar
xml: permit nested directives  
Chris Dollin committed
357 358 359 
}

func TestNestedDirectives(t *testing.T) {
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
360 
	d := NewDecoder(strings.NewReader(nestedDirectivesInput))
Chris Dollin's avatar
xml: permit nested directives  
Chris Dollin committed
361 362 

	for i, want := range nestedDirectivesTokens {
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
363 
		have, err := d.Token()
Chris Dollin's avatar
xml: permit nested directives  
Chris Dollin committed
364 365 366 367 
		if err != nil {
			t.Fatalf("token %d: unexpected error: %s", i, err)
		}
		if !reflect.DeepEqual(have, want) {
Robert Griesemer's avatar
remove semis after statements in one-statement statement lists  
Robert Griesemer committed
368 
			t.Errorf("token %d = %#v want %#v", i, have, want)
Russ Cox's avatar
XML lexing  
Russ Cox committed
369 370 371 372 373 
		}
	}
}

func TestToken(t *testing.T) {
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
374 
	d := NewDecoder(strings.NewReader(testInput))
Patrick Smith's avatar
encoding/xml: expand allowed entity names  
Patrick Smith committed
375 
	d.Entity = testEntity
Russ Cox's avatar
XML lexing  
Russ Cox committed
376 377 

	for i, want := range cookedTokens {
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
378 
		have, err := d.Token()
Russ Cox's avatar
XML lexing  
Russ Cox committed
379 
		if err != nil {
Robert Griesemer's avatar
remove semis after statements in one-statement statement lists  
Robert Griesemer committed
380 
			t.Fatalf("token %d: unexpected error: %s", i, err)
Russ Cox's avatar
XML lexing  
Russ Cox committed
381 382 
		}
		if !reflect.DeepEqual(have, want) {
Robert Griesemer's avatar
remove semis after statements in one-statement statement lists  
Robert Griesemer committed
383 
			t.Errorf("token %d = %#v want %#v", i, have, want)
Russ Cox's avatar
XML lexing  
Russ Cox committed
384 385 386 
		}
	}
}
Arvindh Rajesh Tamilmani's avatar
xml: handle unexpected EOF while parsing and fix a bug in name  
Arvindh Rajesh Tamilmani committed
387 388 389 

func TestSyntax(t *testing.T) {
	for i := range xmlInput {
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
390 
		d := NewDecoder(strings.NewReader(xmlInput[i]))
Russ Cox's avatar
src/pkg/[n-z]*: gofix -r error -force=error  
Russ Cox committed
391 
		var err error
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
392 
		for _, err = d.Token(); err == nil; _, err = d.Token() {
Arvindh Rajesh Tamilmani's avatar
xml: handle unexpected EOF while parsing and fix a bug in name  
Arvindh Rajesh Tamilmani committed
393 
		}
Kyle Consalus's avatar
xml: add line numbers to syntax errors.  
Kyle Consalus committed
394 
		if _, ok := err.(*SyntaxError); !ok {
Arvindh Rajesh Tamilmani's avatar
xml: handle unexpected EOF while parsing and fix a bug in name  
Arvindh Rajesh Tamilmani committed
395 396 397 398 
			t.Fatalf(`xmlInput "%s": expected SyntaxError not received`, xmlInput[i])
		}
	}
}
Michael Hoisie's avatar
Allow underscores in XML element names (except for leading characters)  
Michael Hoisie committed
399
Rob Pike's avatar
allow any scalar type in xml.Unmarshal.  
Rob Pike committed
400 
type allScalars struct {
Russ Cox's avatar
xml: fix reflect error  
Russ Cox committed
401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 
	True1     bool
	True2     bool
	False1    bool
	False2    bool
	Int       int
	Int8      int8
	Int16     int16
	Int32     int32
	Int64     int64
	Uint      int
	Uint8     uint8
	Uint16    uint16
	Uint32    uint32
	Uint64    uint64
	Uintptr   uintptr
	Float32   float32
	Float64   float64
	String    string
	PtrString *string
Rob Pike's avatar
allow any scalar type in xml.Unmarshal.  
Rob Pike committed
420 421 422 
}

var all = allScalars{
Russ Cox's avatar
xml: fix reflect error  
Russ Cox committed
423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 
	True1:     true,
	True2:     true,
	False1:    false,
	False2:    false,
	Int:       1,
	Int8:      -2,
	Int16:     3,
	Int32:     -4,
	Int64:     5,
	Uint:      6,
	Uint8:     7,
	Uint16:    8,
	Uint32:    9,
	Uint64:    10,
	Uintptr:   11,
	Float32:   13.0,
	Float64:   14.0,
	String:    "15",
	PtrString: &sixteen,
Rob Pike's avatar
allow any scalar type in xml.Unmarshal.  
Rob Pike committed
442 443 
}
Russ Cox's avatar
xml: fix reflect error  
Russ Cox committed
444 445 
var sixteen = "16"
Rob Pike's avatar
allow any scalar type in xml.Unmarshal.  
Rob Pike committed
446 
const testScalarsInput = `<allscalars>
Gustavo Niemeyer's avatar
xml: major Go 1 fixup  
Gustavo Niemeyer committed
447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 
	<True1>true</True1>
	<True2>1</True2>
	<False1>false</False1>
	<False2>0</False2>
	<Int>1</Int>
	<Int8>-2</Int8>
	<Int16>3</Int16>
	<Int32>-4</Int32>
	<Int64>5</Int64>
	<Uint>6</Uint>
	<Uint8>7</Uint8>
	<Uint16>8</Uint16>
	<Uint32>9</Uint32>
	<Uint64>10</Uint64>
	<Uintptr>11</Uintptr>
	<Float>12.0</Float>
	<Float32>13.0</Float32>
	<Float64>14.0</Float64>
	<String>15</String>
	<PtrString>16</PtrString>
Rob Pike's avatar
allow any scalar type in xml.Unmarshal.  
Rob Pike committed
467 468 469 470 
</allscalars>`

func TestAllScalars(t *testing.T) {
	var a allScalars
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
471 
	err := Unmarshal([]byte(testScalarsInput), &a)
Rob Pike's avatar
allow any scalar type in xml.Unmarshal.  
Rob Pike committed
472 473 474 475 476 

	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(a, all) {
Russ Cox's avatar
xml: fix reflect error  
Russ Cox committed
477 
		t.Errorf("have %+v want %+v", a, all)
Rob Pike's avatar
allow any scalar type in xml.Unmarshal.  
Rob Pike committed
478 479 480 
	}
}
Michael Hoisie's avatar
Allow underscores in XML element names (except for leading characters)  
Michael Hoisie committed
481 
type item struct {
Karel Pazdera's avatar
encoding/xml: improve package based on the suggestions from metalinter  
Karel Pazdera committed
482 
	FieldA string
Michael Hoisie's avatar
Allow underscores in XML element names (except for leading characters)  
Michael Hoisie committed
483 484 485 
}

func TestIssue569(t *testing.T) {
Karel Pazdera's avatar
encoding/xml: improve package based on the suggestions from metalinter  
Karel Pazdera committed
486 
	data := `<item><FieldA>abcd</FieldA></item>`
Michael Hoisie's avatar
Allow underscores in XML element names (except for leading characters)  
Michael Hoisie committed
487 
	var i item
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
488 
	err := Unmarshal([]byte(data), &i)
Michael Hoisie's avatar
Allow underscores in XML element names (except for leading characters)  
Michael Hoisie committed
489
Karel Pazdera's avatar
encoding/xml: improve package based on the suggestions from metalinter  
Karel Pazdera committed
490 
	if err != nil || i.FieldA != "abcd" {
Rob Pike's avatar
throughout: fix broken calls to Printf etc.  
Rob Pike committed
491 
		t.Fatal("Expecting abcd")
Michael Hoisie's avatar
Allow underscores in XML element names (except for leading characters)  
Michael Hoisie committed
492 493 
	}
}
Amrut Joshi's avatar
xml: allow unquoted attribute values in non-Strict mode  
Amrut Joshi committed
494 495 496 

func TestUnquotedAttrs(t *testing.T) {
	data := "<tag attr=azAZ09:-_\t>"
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
497 498 499 
	d := NewDecoder(strings.NewReader(data))
	d.Strict = false
	token, err := d.Token()
Kyle Consalus's avatar
xml: add line numbers to syntax errors.  
Kyle Consalus committed
500 
	if _, ok := err.(*SyntaxError); ok {
Amrut Joshi's avatar
xml: allow unquoted attribute values in non-Strict mode  
Amrut Joshi committed
501 502 503 504 505 506 507 508 509 510 511 512 513 
		t.Errorf("Unexpected error: %v", err)
	}
	if token.(StartElement).Name.Local != "tag" {
		t.Errorf("Unexpected tag name: %v", token.(StartElement).Name.Local)
	}
	attr := token.(StartElement).Attr[0]
	if attr.Value != "azAZ09:-_" {
		t.Errorf("Unexpected attribute value: %v", attr.Value)
	}
	if attr.Name.Local != "attr" {
		t.Errorf("Unexpected attribute name: %v", attr.Name.Local)
	}
}
Kyle Consalus's avatar
xml: add CopyToken  
Kyle Consalus committed
514
Volker Dobler's avatar
xml: allow attributes without value in non-strict mode.  
Volker Dobler committed
515 516 517 518 519 520 521 522 
func TestValuelessAttrs(t *testing.T) {
	tests := [][3]string{
		{"<p nowrap>", "p", "nowrap"},
		{"<p nowrap >", "p", "nowrap"},
		{"<input checked/>", "input", "checked"},
		{"<input checked />", "input", "checked"},
	}
	for _, test := range tests {
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
523 524 525 
		d := NewDecoder(strings.NewReader(test[0]))
		d.Strict = false
		token, err := d.Token()
Volker Dobler's avatar
xml: allow attributes without value in non-strict mode.  
Volker Dobler committed
526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 
		if _, ok := err.(*SyntaxError); ok {
			t.Errorf("Unexpected error: %v", err)
		}
		if token.(StartElement).Name.Local != test[1] {
			t.Errorf("Unexpected tag name: %v", token.(StartElement).Name.Local)
		}
		attr := token.(StartElement).Attr[0]
		if attr.Value != test[2] {
			t.Errorf("Unexpected attribute value: %v", attr.Value)
		}
		if attr.Name.Local != test[2] {
			t.Errorf("Unexpected attribute name: %v", attr.Name.Local)
		}
	}
}
Kyle Consalus's avatar
xml: add CopyToken  
Kyle Consalus committed
542 543 544 545 546 547 548 549 550 551 552 553 554 555 
func TestCopyTokenCharData(t *testing.T) {
	data := []byte("same data")
	var tok1 Token = CharData(data)
	tok2 := CopyToken(tok1)
	if !reflect.DeepEqual(tok1, tok2) {
		t.Error("CopyToken(CharData) != CharData")
	}
	data[1] = 'o'
	if reflect.DeepEqual(tok1, tok2) {
		t.Error("CopyToken(CharData) uses same buffer.")
	}
}

func TestCopyTokenStartElement(t *testing.T) {
Robert Griesemer's avatar
gofmt -s -w src misc  
Robert Griesemer committed
556 
	elt := StartElement{Name{"", "hello"}, []Attr{{Name{"", "lang"}, "en"}}}
Kyle Consalus's avatar
xml: add CopyToken  
Kyle Consalus committed
557 558 
	var tok1 Token = elt
	tok2 := CopyToken(tok1)
Russ Cox's avatar
encoding/xml: fix copy bug  
Russ Cox committed
559 560 561 
	if tok1.(StartElement).Attr[0].Value != "en" {
		t.Error("CopyToken overwrote Attr[0]")
	}
Kyle Consalus's avatar
xml: add CopyToken  
Kyle Consalus committed
562 563 564 
	if !reflect.DeepEqual(tok1, tok2) {
		t.Error("CopyToken(StartElement) != StartElement")
	}
Russ Cox's avatar
encoding/xml: fix copy bug  
Russ Cox committed
565 
	tok1.(StartElement).Attr[0] = Attr{Name{"", "lang"}, "de"}
Kyle Consalus's avatar
xml: add CopyToken  
Kyle Consalus committed
566 567 568 569 
	if reflect.DeepEqual(tok1, tok2) {
		t.Error("CopyToken(CharData) uses same buffer.")
	}
}
Kyle Consalus's avatar
xml: add line numbers to syntax errors.  
Kyle Consalus committed
570 571 572 

func TestSyntaxErrorLineNum(t *testing.T) {
	testInput := "<P>Foo<P>\n\n<P>Bar</>\n"
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
573 
	d := NewDecoder(strings.NewReader(testInput))
Russ Cox's avatar
src/pkg/[n-z]*: gofix -r error -force=error  
Russ Cox committed
574 
	var err error
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
575 
	for _, err = d.Token(); err == nil; _, err = d.Token() {
Kyle Consalus's avatar
xml: add line numbers to syntax errors.  
Kyle Consalus committed
576 577 578 579 580 581 582 583 584 
	}
	synerr, ok := err.(*SyntaxError)
	if !ok {
		t.Error("Expected SyntaxError.")
	}
	if synerr.Line != 3 {
		t.Error("SyntaxError didn't have correct line number.")
	}
}
Russ Cox's avatar
xml: allow text segments to end at EOF  
Russ Cox committed
585 586 587 

func TestTrailingRawToken(t *testing.T) {
	input := `<FOO></FOO>  `
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
588 
	d := NewDecoder(strings.NewReader(input))
Russ Cox's avatar
src/pkg/[n-z]*: gofix -r error -force=error  
Russ Cox committed
589 
	var err error
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
590 
	for _, err = d.RawToken(); err == nil; _, err = d.RawToken() {
Russ Cox's avatar
xml: allow text segments to end at EOF  
Russ Cox committed
591 
	}
Russ Cox's avatar
src/pkg/[n-z]*: gofix -r error -force=error  
Russ Cox committed
592 
	if err != io.EOF {
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
593 
		t.Fatalf("d.RawToken() = _, %v, want _, io.EOF", err)
Russ Cox's avatar
xml: allow text segments to end at EOF  
Russ Cox committed
594 595 596 597 598 
	}
}

func TestTrailingToken(t *testing.T) {
	input := `<FOO></FOO>  `
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
599 
	d := NewDecoder(strings.NewReader(input))
Russ Cox's avatar
src/pkg/[n-z]*: gofix -r error -force=error  
Russ Cox committed
600 
	var err error
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
601 
	for _, err = d.Token(); err == nil; _, err = d.Token() {
Russ Cox's avatar
xml: allow text segments to end at EOF  
Russ Cox committed
602 
	}
Russ Cox's avatar
src/pkg/[n-z]*: gofix -r error -force=error  
Russ Cox committed
603 
	if err != io.EOF {
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
604 
		t.Fatalf("d.Token() = _, %v, want _, io.EOF", err)
Russ Cox's avatar
xml: allow text segments to end at EOF  
Russ Cox committed
605 606 
	}
}
Dan Sinclair's avatar
xml: Allow entities inside CDATA tags  
Dan Sinclair committed
607 608 609 

func TestEntityInsideCDATA(t *testing.T) {
	input := `<test><![CDATA[ &val=foo ]]></test>`
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
610 
	d := NewDecoder(strings.NewReader(input))
Russ Cox's avatar
src/pkg/[n-z]*: gofix -r error -force=error  
Russ Cox committed
611 
	var err error
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
612 
	for _, err = d.Token(); err == nil; _, err = d.Token() {
Dan Sinclair's avatar
xml: Allow entities inside CDATA tags  
Dan Sinclair committed
613 
	}
Russ Cox's avatar
src/pkg/[n-z]*: gofix -r error -force=error  
Russ Cox committed
614 
	if err != io.EOF {
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
615 
		t.Fatalf("d.Token() = _, %v, want _, io.EOF", err)
Dan Sinclair's avatar
xml: Allow entities inside CDATA tags  
Dan Sinclair committed
616 617 
	}
}
Nigel Kerr's avatar
xml: disallow invalid Unicode code points  
Nigel Kerr committed
618 619 620 621 622 623 624 625 626 627 

var characterTests = []struct {
	in  string
	err string
}{
	{"\x12<doc/>", "illegal character code U+0012"},
	{"<?xml version=\"1.0\"?>\x0b<doc/>", "illegal character code U+000B"},
	{"\xef\xbf\xbe<doc/>", "illegal character code U+FFFE"},
	{"<?xml version=\"1.0\"?><doc>\r\n<hiya/>\x07<toots/></doc>", "illegal character code U+0007"},
	{"<?xml version=\"1.0\"?><doc \x12='value'>what's up</doc>", "expected attribute name in element"},
Russ Cox's avatar
encoding/xml: fix spurious "no semicolon" in error  
Russ Cox committed
628 
	{"<doc>&abc\x01;</doc>", "invalid character entity &abc (no semicolon)"},
Gustavo Niemeyer's avatar
encoding/xml: fix decoding of unknown entities in non-strict mode  
Gustavo Niemeyer committed
629 
	{"<doc>&\x01;</doc>", "invalid character entity & (no semicolon)"},
Russ Cox's avatar
encoding/xml: fix spurious "no semicolon" in error  
Russ Cox committed
630 631 
	{"<doc>&\xef\xbf\xbe;</doc>", "invalid character entity &\uFFFE;"},
	{"<doc>&hello;</doc>", "invalid character entity &hello;"},
Nigel Kerr's avatar
xml: disallow invalid Unicode code points  
Nigel Kerr committed
632 633 634 635 636 
}

func TestDisallowedCharacters(t *testing.T) {

	for i, tt := range characterTests {
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
637 
		d := NewDecoder(strings.NewReader(tt.in))
Russ Cox's avatar
src/pkg/[n-z]*: gofix -r error -force=error  
Russ Cox committed
638 
		var err error
Nigel Kerr's avatar
xml: disallow invalid Unicode code points  
Nigel Kerr committed
639 640 

		for err == nil {
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
641 
			_, err = d.Token()
Nigel Kerr's avatar
xml: disallow invalid Unicode code points  
Nigel Kerr committed
642 643 644 
		}
		synerr, ok := err.(*SyntaxError)
		if !ok {
Gustavo Niemeyer's avatar
encoding/xml: bring API closer to other packages  
Gustavo Niemeyer committed
645 
			t.Fatalf("input %d d.Token() = _, %v, want _, *SyntaxError", i, err)
Nigel Kerr's avatar
xml: disallow invalid Unicode code points  
Nigel Kerr committed
646 647 
		}
		if synerr.Msg != tt.err {
Russ Cox's avatar
encoding/xml: fix spurious "no semicolon" in error  
Russ Cox committed
648 
			t.Fatalf("input %d synerr.Msg wrong: want %q, got %q", i, tt.err, synerr.Msg)
Nigel Kerr's avatar
xml: disallow invalid Unicode code points  
Nigel Kerr committed
649 650 651 
		}
	}
}
Brad Fitzpatrick's avatar
xml: Parser hook for non-UTF-8 charset converters  
Brad Fitzpatrick committed
652 653 

var procInstTests = []struct {
Giulio Iotti's avatar
xml: add check of version in document declaration  
Giulio Iotti committed
654 655 
	input  string
	expect [2]string
Brad Fitzpatrick's avatar
xml: Parser hook for non-UTF-8 charset converters  
Brad Fitzpatrick committed
656 
}{
Giulio Iotti's avatar
xml: add check of version in document declaration  
Giulio Iotti committed
657 658 659 660 661 
	{`version="1.0" encoding="utf-8"`, [2]string{"1.0", "utf-8"}},
	{`version="1.0" encoding='utf-8'`, [2]string{"1.0", "utf-8"}},
	{`version="1.0" encoding='utf-8' `, [2]string{"1.0", "utf-8"}},
	{`version="1.0" encoding=utf-8`, [2]string{"1.0", ""}},
	{`encoding="FOO" `, [2]string{"", "FOO"}},
Brad Fitzpatrick's avatar
xml: Parser hook for non-UTF-8 charset converters  
Brad Fitzpatrick committed
662 663 664 665 
}

func TestProcInstEncoding(t *testing.T) {
	for _, test := range procInstTests {
Giulio Iotti's avatar
xml: add check of version in document declaration  
Giulio Iotti committed
666 667 668 669 670 
		if got := procInst("version", test.input); got != test.expect[0] {
			t.Errorf("procInst(version, %q) = %q; want %q", test.input, got, test.expect[0])
		}
		if got := procInst("encoding", test.input); got != test.expect[1] {
			t.Errorf("procInst(encoding, %q) = %q; want %q", test.input, got, test.expect[1])
Brad Fitzpatrick's avatar
xml: Parser hook for non-UTF-8 charset converters  
Brad Fitzpatrick committed
671 672 673 
		}
	}
}
Shawn Smith's avatar
encoding/xml: parse comments in DOCTYPE  
Shawn Smith committed
674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 

// Ensure that directives with comments include the complete
// text of any nested directives.

var directivesWithCommentsInput = `
<!DOCTYPE [<!-- a comment --><!ENTITY rdf "http://www.w3.org/1999/02/22-rdf-syntax-ns#">]>
<!DOCTYPE [<!ENTITY go "Golang"><!-- a comment-->]>
<!DOCTYPE <!-> <!> <!----> <!-->--> <!--->--> [<!ENTITY go "Golang"><!-- a comment-->]>
`

var directivesWithCommentsTokens = []Token{
	CharData("\n"),
	Directive(`DOCTYPE [<!ENTITY rdf "http://www.w3.org/1999/02/22-rdf-syntax-ns#">]`),
	CharData("\n"),
	Directive(`DOCTYPE [<!ENTITY go "Golang">]`),
	CharData("\n"),
	Directive(`DOCTYPE <!-> <!>    [<!ENTITY go "Golang">]`),
	CharData("\n"),
}

func TestDirectivesWithComments(t *testing.T) {
	d := NewDecoder(strings.NewReader(directivesWithCommentsInput))

	for i, want := range directivesWithCommentsTokens {
		have, err := d.Token()
		if err != nil {
			t.Fatalf("token %d: unexpected error: %s", i, err)
		}
		if !reflect.DeepEqual(have, want) {
			t.Errorf("token %d = %#v want %#v", i, have, want)
		}
	}
}
Olivier Saingre's avatar
encoding/xml: make sure Encoder.Encode reports Write errors.  
Olivier Saingre committed
707 708 709 710 711 712 713 714 715 716 717 

// Writer whose Write method always returns an error.
type errWriter struct{}

func (errWriter) Write(p []byte) (n int, err error) { return 0, fmt.Errorf("unwritable") }

func TestEscapeTextIOErrors(t *testing.T) {
	expectErr := "unwritable"
	err := EscapeText(errWriter{}, []byte{'A'})

	if err == nil || err.Error() != expectErr {
Olivier Saingre's avatar
encoding/xml: rewrite invalid code points to U+FFFD in Marshal, Escape  
Olivier Saingre committed
718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 
		t.Errorf("have %v, want %v", err, expectErr)
	}
}

func TestEscapeTextInvalidChar(t *testing.T) {
	input := []byte("A \x00 terminated string.")
	expected := "A \uFFFD terminated string."

	buff := new(bytes.Buffer)
	if err := EscapeText(buff, input); err != nil {
		t.Fatalf("have %v, want nil", err)
	}
	text := buff.String()

	if text != expected {
		t.Errorf("have %v, want %v", text, expected)
Olivier Saingre's avatar
encoding/xml: make sure Encoder.Encode reports Write errors.  
Olivier Saingre committed
734 735 
	}
}
Alex A Skinner's avatar
encoding/xml: Do not pass through invalid utf8 bytes  
Alex A Skinner committed
736 737 738 739 740 741 742 743 744 745 746 

func TestIssue5880(t *testing.T) {
	type T []byte
	data, err := Marshal(T{192, 168, 0, 1})
	if err != nil {
		t.Errorf("Marshal error: %v", err)
	}
	if !utf8.Valid(data) {
		t.Errorf("Marshal generated invalid UTF-8: %x", data)
	}
}
Robert Stepanek's avatar
encoding/xml: Return SyntaxError for unmatched root start elements.  
Robert Stepanek committed
747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 

func TestIssue11405(t *testing.T) {
	testCases := []string{
		"<root>",
		"<root><foo>",
		"<root><foo></foo>",
	}
	for _, tc := range testCases {
		d := NewDecoder(strings.NewReader(tc))
		var err error
		for {
			_, err = d.Token()
			if err != nil {
				break
			}
		}
		if _, ok := err.(*SyntaxError); !ok {
			t.Errorf("%s: Token: Got error %v, want SyntaxError", tc, err)
		}
	}
}
Giulio Iotti's avatar
encoding/xml: case-insensitive encoding recognition  
Giulio Iotti committed
768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 

func TestIssue12417(t *testing.T) {
	testCases := []struct {
		s  string
		ok bool
	}{
		{`<?xml encoding="UtF-8" version="1.0"?><root/>`, true},
		{`<?xml encoding="UTF-8" version="1.0"?><root/>`, true},
		{`<?xml encoding="utf-8" version="1.0"?><root/>`, true},
		{`<?xml encoding="uuu-9" version="1.0"?><root/>`, false},
	}
	for _, tc := range testCases {
		d := NewDecoder(strings.NewReader(tc.s))
		var err error
		for {
			_, err = d.Token()
			if err != nil {
				if err == io.EOF {
					err = nil
				}
				break
			}
		}
		if err != nil && tc.ok {
			t.Errorf("%q: Encoding charset: expected no error, got %s", tc.s, err)
			continue
		}
		if err == nil && !tc.ok {
			t.Errorf("%q: Encoding charset: expected error, got nil", tc.s)
		}
	}
}
Sam Whited's avatar
encoding/xml: add decode wrapper  
Sam Whited committed
800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 

func tokenMap(mapping func(t Token) Token) func(TokenReader) TokenReader {
	return func(src TokenReader) TokenReader {
		return mapper{
			t: src,
			f: mapping,
		}
	}
}

type mapper struct {
	t TokenReader
	f func(Token) Token
}

func (m mapper) Token() (Token, error) {
	tok, err := m.t.Token()
	if err != nil {
		return nil, err
	}
	return m.f(tok), nil
}

func TestNewTokenDecoderIdempotent(t *testing.T) {
	d := NewDecoder(strings.NewReader(`<br/>`))
	d2 := NewTokenDecoder(d)
	if d != d2 {
		t.Error("NewTokenDecoder did not detect underlying Decoder")
	}
}

func TestWrapDecoder(t *testing.T) {
	d := NewDecoder(strings.NewReader(`<quote>[Re-enter Clown with a letter, and FABIAN]</quote>`))
	m := tokenMap(func(t Token) Token {
		switch tok := t.(type) {
		case StartElement:
			if tok.Name.Local == "quote" {
				tok.Name.Local = "blocking"
				return tok
			}
		case EndElement:
			if tok.Name.Local == "quote" {
				tok.Name.Local = "blocking"
				return tok
			}
		}
		return t
	})

	d = NewTokenDecoder(m(d))

	o := struct {
		XMLName  Name   `xml:"blocking"`
		Chardata string `xml:",chardata"`
	}{}

	if err := d.Decode(&o); err != nil {
		t.Fatal("Got unexpected error while decoding:", err)
	}

	if o.Chardata != "[Re-enter Clown with a letter, and FABIAN]" {
		t.Fatalf("Got unexpected chardata: `%s`\n", o.Chardata)
	}
}
Sam Whited's avatar
encoding/xml: don't panic when custom Unmarshaler sees StartElement  
Sam Whited committed
864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 

type tokReader struct{}

func (tokReader) Token() (Token, error) {
	return StartElement{}, nil
}

type Failure struct{}

func (Failure) UnmarshalXML(*Decoder, StartElement) error {
	return nil
}

func TestTokenUnmarshaler(t *testing.T) {
	defer func() {
		if r := recover(); r != nil {
			t.Error("Unexpected panic using custom token unmarshaler")
		}
	}()

	d := NewTokenDecoder(tokReader{})
	d.Decode(&Failure{})
}
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7