1// Copyright 2011 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5// Extract example functions from file ASTs. 6 7package doc 8 9import ( 10 "go/ast" 11 "go/token" 12 "path" 13 "regexp" 14 "sort" 15 "strconv" 16 "strings" 17 "unicode" 18 "unicode/utf8" 19) 20 21// An Example represents an example function found in a source files. 22type Example struct { 23 Name string // name of the item being exemplified 24 Doc string // example function doc string 25 Code ast.Node 26 Play *ast.File // a whole program version of the example 27 Comments []*ast.CommentGroup 28 Output string // expected output 29 Unordered bool 30 EmptyOutput bool // expect empty output 31 Order int // original source code order 32} 33 34// Examples returns the examples found in the files, sorted by Name field. 35// The Order fields record the order in which the examples were encountered. 36// 37// Playable Examples must be in a package whose name ends in "_test". 38// An Example is "playable" (the Play field is non-nil) in either of these 39// circumstances: 40// - The example function is self-contained: the function references only 41// identifiers from other packages (or predeclared identifiers, such as 42// "int") and the test file does not include a dot import. 43// - The entire test file is the example: the file contains exactly one 44// example function, zero test or benchmark functions, and at least one 45// top-level function, type, variable, or constant declaration other 46// than the example function. 47func Examples(files ...*ast.File) []*Example { 48 var list []*Example 49 for _, file := range files { 50 hasTests := false // file contains tests or benchmarks 51 numDecl := 0 // number of non-import declarations in the file 52 var flist []*Example 53 for _, decl := range file.Decls { 54 if g, ok := decl.(*ast.GenDecl); ok && g.Tok != token.IMPORT { 55 numDecl++ 56 continue 57 } 58 f, ok := decl.(*ast.FuncDecl) 59 if !ok || f.Recv != nil { 60 continue 61 } 62 numDecl++ 63 name := f.Name.Name 64 if isTest(name, "Test") || isTest(name, "Benchmark") { 65 hasTests = true 66 continue 67 } 68 if !isTest(name, "Example") { 69 continue 70 } 71 if f.Body == nil { // ast.File.Body nil dereference (see issue 28044) 72 continue 73 } 74 var doc string 75 if f.Doc != nil { 76 doc = f.Doc.Text() 77 } 78 output, unordered, hasOutput := exampleOutput(f.Body, file.Comments) 79 flist = append(flist, &Example{ 80 Name: name[len("Example"):], 81 Doc: doc, 82 Code: f.Body, 83 Play: playExample(file, f), 84 Comments: file.Comments, 85 Output: output, 86 Unordered: unordered, 87 EmptyOutput: output == "" && hasOutput, 88 Order: len(flist), 89 }) 90 } 91 if !hasTests && numDecl > 1 && len(flist) == 1 { 92 // If this file only has one example function, some 93 // other top-level declarations, and no tests or 94 // benchmarks, use the whole file as the example. 95 flist[0].Code = file 96 flist[0].Play = playExampleFile(file) 97 } 98 list = append(list, flist...) 99 } 100 // sort by name 101 sort.Slice(list, func(i, j int) bool { 102 return list[i].Name < list[j].Name 103 }) 104 return list 105} 106 107var outputPrefix = regexp.MustCompile(`(?i)^[[:space:]]*(unordered )?output:`) 108 109// Extracts the expected output and whether there was a valid output comment 110func exampleOutput(b *ast.BlockStmt, comments []*ast.CommentGroup) (output string, unordered, ok bool) { 111 if _, last := lastComment(b, comments); last != nil { 112 // test that it begins with the correct prefix 113 text := last.Text() 114 if loc := outputPrefix.FindStringSubmatchIndex(text); loc != nil { 115 if loc[2] != -1 { 116 unordered = true 117 } 118 text = text[loc[1]:] 119 // Strip zero or more spaces followed by \n or a single space. 120 text = strings.TrimLeft(text, " ") 121 if len(text) > 0 && text[0] == '\n' { 122 text = text[1:] 123 } 124 return text, unordered, true 125 } 126 } 127 return "", false, false // no suitable comment found 128} 129 130// isTest tells whether name looks like a test, example, or benchmark. 131// It is a Test (say) if there is a character after Test that is not a 132// lower-case letter. (We don't want Testiness.) 133func isTest(name, prefix string) bool { 134 if !strings.HasPrefix(name, prefix) { 135 return false 136 } 137 if len(name) == len(prefix) { // "Test" is ok 138 return true 139 } 140 rune, _ := utf8.DecodeRuneInString(name[len(prefix):]) 141 return !unicode.IsLower(rune) 142} 143 144// playExample synthesizes a new *ast.File based on the provided 145// file with the provided function body as the body of main. 146func playExample(file *ast.File, f *ast.FuncDecl) *ast.File { 147 body := f.Body 148 149 if !strings.HasSuffix(file.Name.Name, "_test") { 150 // We don't support examples that are part of the 151 // greater package (yet). 152 return nil 153 } 154 155 // Collect top-level declarations in the file. 156 topDecls := make(map[*ast.Object]ast.Decl) 157 typMethods := make(map[string][]ast.Decl) 158 159 for _, decl := range file.Decls { 160 switch d := decl.(type) { 161 case *ast.FuncDecl: 162 if d.Recv == nil { 163 topDecls[d.Name.Obj] = d 164 } else { 165 if len(d.Recv.List) == 1 { 166 t := d.Recv.List[0].Type 167 tname, _ := baseTypeName(t) 168 typMethods[tname] = append(typMethods[tname], d) 169 } 170 } 171 case *ast.GenDecl: 172 for _, spec := range d.Specs { 173 switch s := spec.(type) { 174 case *ast.TypeSpec: 175 topDecls[s.Name.Obj] = d 176 case *ast.ValueSpec: 177 for _, name := range s.Names { 178 topDecls[name.Obj] = d 179 } 180 } 181 } 182 } 183 } 184 185 // Find unresolved identifiers and uses of top-level declarations. 186 unresolved := make(map[string]bool) 187 var depDecls []ast.Decl 188 hasDepDecls := make(map[ast.Decl]bool) 189 190 var inspectFunc func(ast.Node) bool 191 inspectFunc = func(n ast.Node) bool { 192 switch e := n.(type) { 193 case *ast.Ident: 194 if e.Obj == nil && e.Name != "_" { 195 unresolved[e.Name] = true 196 } else if d := topDecls[e.Obj]; d != nil { 197 if !hasDepDecls[d] { 198 hasDepDecls[d] = true 199 depDecls = append(depDecls, d) 200 } 201 } 202 return true 203 case *ast.SelectorExpr: 204 // For selector expressions, only inspect the left hand side. 205 // (For an expression like fmt.Println, only add "fmt" to the 206 // set of unresolved names, not "Println".) 207 ast.Inspect(e.X, inspectFunc) 208 return false 209 case *ast.KeyValueExpr: 210 // For key value expressions, only inspect the value 211 // as the key should be resolved by the type of the 212 // composite literal. 213 ast.Inspect(e.Value, inspectFunc) 214 return false 215 } 216 return true 217 } 218 ast.Inspect(body, inspectFunc) 219 for i := 0; i < len(depDecls); i++ { 220 switch d := depDecls[i].(type) { 221 case *ast.FuncDecl: 222 // Inspect types of parameters and results. See #28492. 223 if d.Type.Params != nil { 224 for _, p := range d.Type.Params.List { 225 ast.Inspect(p.Type, inspectFunc) 226 } 227 } 228 if d.Type.Results != nil { 229 for _, r := range d.Type.Results.List { 230 ast.Inspect(r.Type, inspectFunc) 231 } 232 } 233 234 ast.Inspect(d.Body, inspectFunc) 235 case *ast.GenDecl: 236 for _, spec := range d.Specs { 237 switch s := spec.(type) { 238 case *ast.TypeSpec: 239 ast.Inspect(s.Type, inspectFunc) 240 241 depDecls = append(depDecls, typMethods[s.Name.Name]...) 242 case *ast.ValueSpec: 243 if s.Type != nil { 244 ast.Inspect(s.Type, inspectFunc) 245 } 246 for _, val := range s.Values { 247 ast.Inspect(val, inspectFunc) 248 } 249 } 250 } 251 } 252 } 253 254 // Remove predeclared identifiers from unresolved list. 255 for n := range unresolved { 256 if predeclaredTypes[n] || predeclaredConstants[n] || predeclaredFuncs[n] { 257 delete(unresolved, n) 258 } 259 } 260 261 // Use unresolved identifiers to determine the imports used by this 262 // example. The heuristic assumes package names match base import 263 // paths for imports w/o renames (should be good enough most of the time). 264 namedImports := make(map[string]string) // [name]path 265 var blankImports []ast.Spec // _ imports 266 for _, s := range file.Imports { 267 p, err := strconv.Unquote(s.Path.Value) 268 if err != nil { 269 continue 270 } 271 if p == "syscall/js" { 272 // We don't support examples that import syscall/js, 273 // because the package syscall/js is not available in the playground. 274 return nil 275 } 276 n := path.Base(p) 277 if s.Name != nil { 278 n = s.Name.Name 279 switch n { 280 case "_": 281 blankImports = append(blankImports, s) 282 continue 283 case ".": 284 // We can't resolve dot imports (yet). 285 return nil 286 } 287 } 288 if unresolved[n] { 289 namedImports[n] = p 290 delete(unresolved, n) 291 } 292 } 293 294 // If there are other unresolved identifiers, give up because this 295 // synthesized file is not going to build. 296 if len(unresolved) > 0 { 297 return nil 298 } 299 300 // Include documentation belonging to blank imports. 301 var comments []*ast.CommentGroup 302 for _, s := range blankImports { 303 if c := s.(*ast.ImportSpec).Doc; c != nil { 304 comments = append(comments, c) 305 } 306 } 307 308 // Include comments that are inside the function body. 309 for _, c := range file.Comments { 310 if body.Pos() <= c.Pos() && c.End() <= body.End() { 311 comments = append(comments, c) 312 } 313 } 314 315 // Strip the "Output:" or "Unordered output:" comment and adjust body 316 // end position. 317 body, comments = stripOutputComment(body, comments) 318 319 // Include documentation belonging to dependent declarations. 320 for _, d := range depDecls { 321 switch d := d.(type) { 322 case *ast.GenDecl: 323 if d.Doc != nil { 324 comments = append(comments, d.Doc) 325 } 326 case *ast.FuncDecl: 327 if d.Doc != nil { 328 comments = append(comments, d.Doc) 329 } 330 } 331 } 332 333 // Synthesize import declaration. 334 importDecl := &ast.GenDecl{ 335 Tok: token.IMPORT, 336 Lparen: 1, // Need non-zero Lparen and Rparen so that printer 337 Rparen: 1, // treats this as a factored import. 338 } 339 for n, p := range namedImports { 340 s := &ast.ImportSpec{Path: &ast.BasicLit{Value: strconv.Quote(p)}} 341 if path.Base(p) != n { 342 s.Name = ast.NewIdent(n) 343 } 344 importDecl.Specs = append(importDecl.Specs, s) 345 } 346 importDecl.Specs = append(importDecl.Specs, blankImports...) 347 348 // Synthesize main function. 349 funcDecl := &ast.FuncDecl{ 350 Name: ast.NewIdent("main"), 351 Type: f.Type, 352 Body: body, 353 } 354 355 decls := make([]ast.Decl, 0, 2+len(depDecls)) 356 decls = append(decls, importDecl) 357 decls = append(decls, depDecls...) 358 decls = append(decls, funcDecl) 359 360 sort.Slice(decls, func(i, j int) bool { 361 return decls[i].Pos() < decls[j].Pos() 362 }) 363 364 sort.Slice(comments, func(i, j int) bool { 365 return comments[i].Pos() < comments[j].Pos() 366 }) 367 368 // Synthesize file. 369 return &ast.File{ 370 Name: ast.NewIdent("main"), 371 Decls: decls, 372 Comments: comments, 373 } 374} 375 376// playExampleFile takes a whole file example and synthesizes a new *ast.File 377// such that the example is function main in package main. 378func playExampleFile(file *ast.File) *ast.File { 379 // Strip copyright comment if present. 380 comments := file.Comments 381 if len(comments) > 0 && strings.HasPrefix(comments[0].Text(), "Copyright") { 382 comments = comments[1:] 383 } 384 385 // Copy declaration slice, rewriting the ExampleX function to main. 386 var decls []ast.Decl 387 for _, d := range file.Decls { 388 if f, ok := d.(*ast.FuncDecl); ok && isTest(f.Name.Name, "Example") { 389 // Copy the FuncDecl, as it may be used elsewhere. 390 newF := *f 391 newF.Name = ast.NewIdent("main") 392 newF.Body, comments = stripOutputComment(f.Body, comments) 393 d = &newF 394 } 395 decls = append(decls, d) 396 } 397 398 // Copy the File, as it may be used elsewhere. 399 f := *file 400 f.Name = ast.NewIdent("main") 401 f.Decls = decls 402 f.Comments = comments 403 return &f 404} 405 406// stripOutputComment finds and removes the "Output:" or "Unordered output:" 407// comment from body and comments, and adjusts the body block's end position. 408func stripOutputComment(body *ast.BlockStmt, comments []*ast.CommentGroup) (*ast.BlockStmt, []*ast.CommentGroup) { 409 // Do nothing if there is no "Output:" or "Unordered output:" comment. 410 i, last := lastComment(body, comments) 411 if last == nil || !outputPrefix.MatchString(last.Text()) { 412 return body, comments 413 } 414 415 // Copy body and comments, as the originals may be used elsewhere. 416 newBody := &ast.BlockStmt{ 417 Lbrace: body.Lbrace, 418 List: body.List, 419 Rbrace: last.Pos(), 420 } 421 newComments := make([]*ast.CommentGroup, len(comments)-1) 422 copy(newComments, comments[:i]) 423 copy(newComments[i:], comments[i+1:]) 424 return newBody, newComments 425} 426 427// lastComment returns the last comment inside the provided block. 428func lastComment(b *ast.BlockStmt, c []*ast.CommentGroup) (i int, last *ast.CommentGroup) { 429 if b == nil { 430 return 431 } 432 pos, end := b.Pos(), b.End() 433 for j, cg := range c { 434 if cg.Pos() < pos { 435 continue 436 } 437 if cg.End() > end { 438 break 439 } 440 i, last = j, cg 441 } 442 return 443} 444