1/*
2Copyright 2019 The Kubernetes Authors.
3
4Licensed under the Apache License, Version 2.0 (the "License");
5you may not use this file except in compliance with the License.
6You may obtain a copy of the License at
7
8    http://www.apache.org/licenses/LICENSE-2.0
9
10Unless required by applicable law or agreed to in writing, software
11distributed under the License is distributed on an "AS IS" BASIS,
12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13See the License for the specific language governing permissions and
14limitations under the License.
15*/
16
17package main
18
19import (
20	"encoding/json"
21	"flag"
22	"fmt"
23	"go/ast"
24	"go/parser"
25	"go/token"
26	"io"
27	"log"
28	"os"
29	"regexp"
30	"sort"
31	"strconv"
32	"strings"
33	"text/template"
34
35	"gopkg.in/yaml.v2"
36
37	"github.com/onsi/ginkgo/types"
38)
39
40// ConformanceData describes the structure of the conformance.yaml file
41type ConformanceData struct {
42	// A URL to the line of code in the kube src repo for the test. Omitted from the YAML to avoid exposing line number.
43	URL string `yaml:"-"`
44	// Extracted from the "Testname:" comment before the test
45	TestName string
46	// CodeName is taken from the actual ginkgo descriptions, e.g. `[sig-apps] Foo should bar [Conformance]`
47	CodeName string
48	// Extracted from the "Description:" comment before the test
49	Description string
50	// Version when this test is added or modified ex: v1.12, v1.13
51	Release string
52	// File is the filename where the test is defined. We intentionally don't save the line here to avoid meaningless changes.
53	File string
54}
55
56var (
57	baseURL = flag.String("url", "https://github.com/kubernetes/kubernetes/tree/master/", "location of the current source")
58	k8sPath = flag.String("source", "", "location of the current source on the current machine")
59	confDoc = flag.Bool("docs", false, "write a conformance document")
60	version = flag.String("version", "v1.9", "version of this conformance document")
61
62	// If a test name contains any of these tags, it is ineligble for promotion to conformance
63	regexIneligibleTags = regexp.MustCompile(`\[(Alpha|Feature:[^\]]+|Flaky)\]`)
64
65	// Conformance comments should be within this number of lines to the call itself.
66	// Allowing for more than one in case a spare comment or two is below it.
67	conformanceCommentsLineWindow = 5
68
69	seenLines map[string]struct{}
70)
71
72type frame struct {
73	Function string
74
75	// File and Line are the file name and line number of the
76	// location in this frame. For non-leaf frames, this will be
77	// the location of a call. These may be the empty string and
78	// zero, respectively, if not known.
79	File string
80	Line int
81}
82
83func main() {
84	flag.Parse()
85
86	if len(flag.Args()) < 1 {
87		log.Fatalln("Requires the name of the test details file as first and only argument.")
88	}
89	testDetailsFile := flag.Args()[0]
90	f, err := os.Open(testDetailsFile)
91	if err != nil {
92		log.Fatalf("Failed to open file %v: %v", testDetailsFile, err)
93	}
94	defer f.Close()
95
96	seenLines = map[string]struct{}{}
97	dec := json.NewDecoder(f)
98	testInfos := []*ConformanceData{}
99	for {
100		var spec *types.SpecSummary
101		if err := dec.Decode(&spec); err == io.EOF {
102			break
103		} else if err != nil {
104			log.Fatal(err)
105		}
106
107		if isConformance(spec) {
108			testInfo := getTestInfo(spec)
109			if testInfo != nil {
110				testInfos = append(testInfos, testInfo)
111			}
112
113			if err := validateTestName(testInfo.CodeName); err != nil {
114				log.Fatal(err)
115			}
116		}
117	}
118
119	sort.Slice(testInfos, func(i, j int) bool { return testInfos[i].CodeName < testInfos[j].CodeName })
120	saveAllTestInfo(testInfos)
121}
122
123func isConformance(spec *types.SpecSummary) bool {
124	return strings.Contains(getTestName(spec), "[Conformance]")
125}
126
127func getTestInfo(spec *types.SpecSummary) *ConformanceData {
128	var c *ConformanceData
129	var err error
130	// The key to this working is that we don't need to parse every file or walk
131	// every componentCodeLocation. The last componentCodeLocation is going to typically start
132	// with the ConformanceIt(...) call and the next call in that callstack will be the
133	// ast.Node which is attached to the comment that we want.
134	for i := len(spec.ComponentCodeLocations) - 1; i > 0; i-- {
135		fullstacktrace := spec.ComponentCodeLocations[i].FullStackTrace
136		c, err = getConformanceDataFromStackTrace(fullstacktrace)
137		if err != nil {
138			log.Printf("Error looking for conformance data: %v", err)
139		}
140		if c != nil {
141			break
142		}
143	}
144
145	if c == nil {
146		log.Printf("Did not find test info for spec: %#v\n", getTestName(spec))
147		return nil
148	}
149
150	c.CodeName = getTestName(spec)
151	return c
152}
153
154func getTestName(spec *types.SpecSummary) string {
155	return strings.Join(spec.ComponentTexts[1:], " ")
156}
157
158func saveAllTestInfo(dataSet []*ConformanceData) {
159	if *confDoc {
160		// Note: this assumes that you're running from the root of the kube src repo
161		templ, err := template.ParseFiles("./test/conformance/cf_header.md")
162		if err != nil {
163			fmt.Printf("Error reading the Header file information: %s\n\n", err)
164		}
165		data := struct {
166			Version string
167		}{
168			Version: *version,
169		}
170		templ.Execute(os.Stdout, data)
171
172		for _, data := range dataSet {
173			fmt.Printf("## [%s](%s)\n\n", data.TestName, data.URL)
174			fmt.Printf("- Added to conformance in release %s\n", data.Release)
175			fmt.Printf("- Defined in code as: %s\n\n", data.CodeName)
176			fmt.Printf("%s\n\n", data.Description)
177		}
178		return
179	}
180
181	// Serialize the list as a whole. Generally meant to end up as conformance.txt which tracks the set of tests.
182	b, err := yaml.Marshal(dataSet)
183	if err != nil {
184		log.Printf("Error marshalling data into YAML: %v", err)
185	}
186	fmt.Println(string(b))
187}
188
189func getConformanceDataFromStackTrace(fullstackstrace string) (*ConformanceData, error) {
190	// The full stacktrace to parse from ginkgo is of the form:
191	// k8s.io/kubernetes/test/e2e/storage/utils.SIGDescribe(0x51f4c4f, 0xf, 0x53a0dd8, 0xc000ab6e01)\n\ttest/e2e/storage/utils/framework.go:23 +0x75\n ... ...
192	// So we need to split it into lines, remove whitespace, and then grab the files/lines.
193	stack := strings.Replace(fullstackstrace, "\t", "", -1)
194	calls := strings.Split(stack, "\n")
195	frames := []frame{}
196	i := 0
197	for i < len(calls) {
198		fileLine := strings.Split(calls[i+1], " ")
199		lineinfo := strings.Split(fileLine[0], ":")
200		line, err := strconv.Atoi(lineinfo[1])
201		if err != nil {
202			panic(err)
203		}
204		frames = append(frames, frame{
205			Function: calls[i],
206			File:     lineinfo[0],
207			Line:     line,
208		})
209		i += 2
210	}
211
212	// filenames are in one of two special GOPATHs depending on if they were
213	// built dockerized or with the host go
214	// we want to trim this prefix to produce portable relative paths
215	k8sSRC := *k8sPath + "/_output/local/go/src/k8s.io/kubernetes/"
216	for i := range frames {
217		trimmedFile := strings.TrimPrefix(frames[i].File, k8sSRC)
218		trimmedFile = strings.TrimPrefix(trimmedFile, "/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/")
219		frames[i].File = trimmedFile
220	}
221
222	for _, curFrame := range frames {
223		if _, seen := seenLines[fmt.Sprintf("%v:%v", curFrame.File, curFrame.Line)]; seen {
224			continue
225		}
226
227		freader, err := os.Open(curFrame.File)
228		if err != nil {
229			return nil, err
230		}
231		defer freader.Close()
232		cd, err := scanFileForFrame(curFrame.File, freader, curFrame)
233		if err != nil {
234			return nil, err
235		}
236		if cd != nil {
237			return cd, nil
238		}
239	}
240
241	return nil, nil
242}
243
244// scanFileForFrame will scan the target and look for a conformance comment attached to the function
245// described by the target frame. If the comment can't be found then nil, nil is returned.
246func scanFileForFrame(filename string, src interface{}, targetFrame frame) (*ConformanceData, error) {
247	fset := token.NewFileSet() // positions are relative to fset
248	f, err := parser.ParseFile(fset, filename, src, parser.ParseComments)
249	if err != nil {
250		return nil, err
251	}
252
253	cmap := ast.NewCommentMap(fset, f, f.Comments)
254	for _, cs := range cmap {
255		for _, c := range cs {
256			if cd := tryCommentGroupAndFrame(fset, c, targetFrame); cd != nil {
257				return cd, nil
258			}
259		}
260	}
261	return nil, nil
262}
263
264func validateTestName(s string) error {
265	matches := regexIneligibleTags.FindAllString(s, -1)
266	if matches != nil {
267		return fmt.Errorf("'%s' cannot have invalid tags %v", s, strings.Join(matches, ","))
268	}
269	return nil
270}
271
272func tryCommentGroupAndFrame(fset *token.FileSet, cg *ast.CommentGroup, f frame) *ConformanceData {
273	if !shouldProcessCommentGroup(fset, cg, f) {
274		return nil
275	}
276
277	// Each file/line will either be some helper function (not a conformance comment) or apply to just a single test. Don't revisit.
278	if seenLines != nil {
279		seenLines[fmt.Sprintf("%v:%v", f.File, f.Line)] = struct{}{}
280	}
281	cd := commentToConformanceData(cg.Text())
282	if cd == nil {
283		return nil
284	}
285
286	cd.URL = fmt.Sprintf("%s%s#L%d", *baseURL, f.File, f.Line)
287	cd.File = f.File
288	return cd
289}
290
291func shouldProcessCommentGroup(fset *token.FileSet, cg *ast.CommentGroup, f frame) bool {
292	lineDiff := f.Line - fset.Position(cg.End()).Line
293	return lineDiff > 0 && lineDiff <= conformanceCommentsLineWindow
294}
295
296func commentToConformanceData(comment string) *ConformanceData {
297	lines := strings.Split(comment, "\n")
298	descLines := []string{}
299	cd := &ConformanceData{}
300	var curLine string
301	for _, line := range lines {
302		line = strings.TrimSpace(line)
303		if len(line) == 0 {
304			continue
305		}
306		if sline := regexp.MustCompile("^Testname\\s*:\\s*").Split(line, -1); len(sline) == 2 {
307			curLine = "Testname"
308			cd.TestName = sline[1]
309			continue
310		}
311		if sline := regexp.MustCompile("^Release\\s*:\\s*").Split(line, -1); len(sline) == 2 {
312			curLine = "Release"
313			cd.Release = sline[1]
314			continue
315		}
316		if sline := regexp.MustCompile("^Description\\s*:\\s*").Split(line, -1); len(sline) == 2 {
317			curLine = "Description"
318			descLines = append(descLines, sline[1])
319			continue
320		}
321
322		// Line has no header
323		if curLine == "Description" {
324			descLines = append(descLines, line)
325		}
326	}
327	if cd.TestName == "" {
328		return nil
329	}
330
331	cd.Description = strings.Join(descLines, " ")
332	return cd
333}
334