1// Copyright 2010 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5/*
6Package html implements an HTML5-compliant tokenizer and parser.
7
8Tokenization is done by creating a Tokenizer for an io.Reader r. It is the
9caller's responsibility to ensure that r provides UTF-8 encoded HTML.
10
11	z := html.NewTokenizer(r)
12
13Given a Tokenizer z, the HTML is tokenized by repeatedly calling z.Next(),
14which parses the next token and returns its type, or an error:
15
16	for {
17		tt := z.Next()
18		if tt == html.ErrorToken {
19			// ...
20			return ...
21		}
22		// Process the current token.
23	}
24
25There are two APIs for retrieving the current token. The high-level API is to
26call Token; the low-level API is to call Text or TagName / TagAttr. Both APIs
27allow optionally calling Raw after Next but before Token, Text, TagName, or
28TagAttr. In EBNF notation, the valid call sequence per token is:
29
30	Next {Raw} [ Token | Text | TagName {TagAttr} ]
31
32Token returns an independent data structure that completely describes a token.
33Entities (such as "<") are unescaped, tag names and attribute keys are
34lower-cased, and attributes are collected into a []Attribute. For example:
35
36	for {
37		if z.Next() == html.ErrorToken {
38			// Returning io.EOF indicates success.
39			return z.Err()
40		}
41		emitToken(z.Token())
42	}
43
44The low-level API performs fewer allocations and copies, but the contents of
45the []byte values returned by Text, TagName and TagAttr may change on the next
46call to Next. For example, to extract an HTML page's anchor text:
47
48	depth := 0
49	for {
50		tt := z.Next()
51		switch tt {
52		case html.ErrorToken:
53			return z.Err()
54		case html.TextToken:
55			if depth > 0 {
56				// emitBytes should copy the []byte it receives,
57				// if it doesn't process it immediately.
58				emitBytes(z.Text())
59			}
60		case html.StartTagToken, html.EndTagToken:
61			tn, _ := z.TagName()
62			if len(tn) == 1 && tn[0] == 'a' {
63				if tt == html.StartTagToken {
64					depth++
65				} else {
66					depth--
67				}
68			}
69		}
70	}
71
72Parsing is done by calling Parse with an io.Reader, which returns the root of
73the parse tree (the document element) as a *Node. It is the caller's
74responsibility to ensure that the Reader provides UTF-8 encoded HTML. For
75example, to process each anchor node in depth-first order:
76
77	doc, err := html.Parse(r)
78	if err != nil {
79		// ...
80	}
81	var f func(*html.Node)
82	f = func(n *html.Node) {
83		if n.Type == html.ElementNode && n.Data == "a" {
84			// Do something with n...
85		}
86		for c := n.FirstChild; c != nil; c = c.NextSibling {
87			f(c)
88		}
89	}
90	f(doc)
91
92The relevant specifications include:
93https://html.spec.whatwg.org/multipage/syntax.html and
94https://html.spec.whatwg.org/multipage/syntax.html#tokenization
95*/
96package html // import "golang.org/x/net/html"
97
98// The tokenization algorithm implemented by this package is not a line-by-line
99// transliteration of the relatively verbose state-machine in the WHATWG
100// specification. A more direct approach is used instead, where the program
101// counter implies the state, such as whether it is tokenizing a tag or a text
102// node. Specification compliance is verified by checking expected and actual
103// outputs over a test suite rather than aiming for algorithmic fidelity.
104
105// TODO(nigeltao): Does a DOM API belong in this package or a separate one?
106// TODO(nigeltao): How does parsing interact with a JavaScript engine?
107