1/* 2Copyright 2019 The Kubernetes Authors. 3 4Licensed under the Apache License, Version 2.0 (the "License"); 5you may not use this file except in compliance with the License. 6You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10Unless required by applicable law or agreed to in writing, software 11distributed under the License is distributed on an "AS IS" BASIS, 12WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13See the License for the specific language governing permissions and 14limitations under the License. 15*/ 16 17package main 18 19import ( 20 "encoding/json" 21 "flag" 22 "fmt" 23 "go/ast" 24 "go/parser" 25 "go/token" 26 "io" 27 "log" 28 "os" 29 "regexp" 30 "sort" 31 "strconv" 32 "strings" 33 "text/template" 34 35 "gopkg.in/yaml.v2" 36 37 "github.com/onsi/ginkgo/types" 38) 39 40// ConformanceData describes the structure of the conformance.yaml file 41type ConformanceData struct { 42 // A URL to the line of code in the kube src repo for the test. Omitted from the YAML to avoid exposing line number. 43 URL string `yaml:"-"` 44 // Extracted from the "Testname:" comment before the test 45 TestName string 46 // CodeName is taken from the actual ginkgo descriptions, e.g. `[sig-apps] Foo should bar [Conformance]` 47 CodeName string 48 // Extracted from the "Description:" comment before the test 49 Description string 50 // Version when this test is added or modified ex: v1.12, v1.13 51 Release string 52 // File is the filename where the test is defined. We intentionally don't save the line here to avoid meaningless changes. 53 File string 54} 55 56var ( 57 baseURL = flag.String("url", "https://github.com/kubernetes/kubernetes/tree/master/", "location of the current source") 58 k8sPath = flag.String("source", "", "location of the current source on the current machine") 59 confDoc = flag.Bool("docs", false, "write a conformance document") 60 version = flag.String("version", "v1.9", "version of this conformance document") 61 62 // If a test name contains any of these tags, it is ineligble for promotion to conformance 63 regexIneligibleTags = regexp.MustCompile(`\[(Alpha|Feature:[^\]]+|Flaky)\]`) 64 65 // Conformance comments should be within this number of lines to the call itself. 66 // Allowing for more than one in case a spare comment or two is below it. 67 conformanceCommentsLineWindow = 5 68 69 seenLines map[string]struct{} 70) 71 72type frame struct { 73 Function string 74 75 // File and Line are the file name and line number of the 76 // location in this frame. For non-leaf frames, this will be 77 // the location of a call. These may be the empty string and 78 // zero, respectively, if not known. 79 File string 80 Line int 81} 82 83func main() { 84 flag.Parse() 85 86 if len(flag.Args()) < 1 { 87 log.Fatalln("Requires the name of the test details file as first and only argument.") 88 } 89 testDetailsFile := flag.Args()[0] 90 f, err := os.Open(testDetailsFile) 91 if err != nil { 92 log.Fatalf("Failed to open file %v: %v", testDetailsFile, err) 93 } 94 defer f.Close() 95 96 seenLines = map[string]struct{}{} 97 dec := json.NewDecoder(f) 98 testInfos := []*ConformanceData{} 99 for { 100 var spec *types.SpecSummary 101 if err := dec.Decode(&spec); err == io.EOF { 102 break 103 } else if err != nil { 104 log.Fatal(err) 105 } 106 107 if isConformance(spec) { 108 testInfo := getTestInfo(spec) 109 if testInfo != nil { 110 testInfos = append(testInfos, testInfo) 111 } 112 113 if err := validateTestName(testInfo.CodeName); err != nil { 114 log.Fatal(err) 115 } 116 } 117 } 118 119 sort.Slice(testInfos, func(i, j int) bool { return testInfos[i].CodeName < testInfos[j].CodeName }) 120 saveAllTestInfo(testInfos) 121} 122 123func isConformance(spec *types.SpecSummary) bool { 124 return strings.Contains(getTestName(spec), "[Conformance]") 125} 126 127func getTestInfo(spec *types.SpecSummary) *ConformanceData { 128 var c *ConformanceData 129 var err error 130 // The key to this working is that we don't need to parse every file or walk 131 // every componentCodeLocation. The last componentCodeLocation is going to typically start 132 // with the ConformanceIt(...) call and the next call in that callstack will be the 133 // ast.Node which is attached to the comment that we want. 134 for i := len(spec.ComponentCodeLocations) - 1; i > 0; i-- { 135 fullstacktrace := spec.ComponentCodeLocations[i].FullStackTrace 136 c, err = getConformanceDataFromStackTrace(fullstacktrace) 137 if err != nil { 138 log.Printf("Error looking for conformance data: %v", err) 139 } 140 if c != nil { 141 break 142 } 143 } 144 145 if c == nil { 146 log.Printf("Did not find test info for spec: %#v\n", getTestName(spec)) 147 return nil 148 } 149 150 c.CodeName = getTestName(spec) 151 return c 152} 153 154func getTestName(spec *types.SpecSummary) string { 155 return strings.Join(spec.ComponentTexts[1:], " ") 156} 157 158func saveAllTestInfo(dataSet []*ConformanceData) { 159 if *confDoc { 160 // Note: this assumes that you're running from the root of the kube src repo 161 templ, err := template.ParseFiles("./test/conformance/cf_header.md") 162 if err != nil { 163 fmt.Printf("Error reading the Header file information: %s\n\n", err) 164 } 165 data := struct { 166 Version string 167 }{ 168 Version: *version, 169 } 170 templ.Execute(os.Stdout, data) 171 172 for _, data := range dataSet { 173 fmt.Printf("## [%s](%s)\n\n", data.TestName, data.URL) 174 fmt.Printf("- Added to conformance in release %s\n", data.Release) 175 fmt.Printf("- Defined in code as: %s\n\n", data.CodeName) 176 fmt.Printf("%s\n\n", data.Description) 177 } 178 return 179 } 180 181 // Serialize the list as a whole. Generally meant to end up as conformance.txt which tracks the set of tests. 182 b, err := yaml.Marshal(dataSet) 183 if err != nil { 184 log.Printf("Error marshalling data into YAML: %v", err) 185 } 186 fmt.Println(string(b)) 187} 188 189func getConformanceDataFromStackTrace(fullstackstrace string) (*ConformanceData, error) { 190 // The full stacktrace to parse from ginkgo is of the form: 191 // k8s.io/kubernetes/test/e2e/storage/utils.SIGDescribe(0x51f4c4f, 0xf, 0x53a0dd8, 0xc000ab6e01)\n\ttest/e2e/storage/utils/framework.go:23 +0x75\n ... ... 192 // So we need to split it into lines, remove whitespace, and then grab the files/lines. 193 stack := strings.Replace(fullstackstrace, "\t", "", -1) 194 calls := strings.Split(stack, "\n") 195 frames := []frame{} 196 i := 0 197 for i < len(calls) { 198 fileLine := strings.Split(calls[i+1], " ") 199 lineinfo := strings.Split(fileLine[0], ":") 200 line, err := strconv.Atoi(lineinfo[1]) 201 if err != nil { 202 panic(err) 203 } 204 frames = append(frames, frame{ 205 Function: calls[i], 206 File: lineinfo[0], 207 Line: line, 208 }) 209 i += 2 210 } 211 212 // filenames are in one of two special GOPATHs depending on if they were 213 // built dockerized or with the host go 214 // we want to trim this prefix to produce portable relative paths 215 k8sSRC := *k8sPath + "/_output/local/go/src/k8s.io/kubernetes/" 216 for i := range frames { 217 trimmedFile := strings.TrimPrefix(frames[i].File, k8sSRC) 218 trimmedFile = strings.TrimPrefix(trimmedFile, "/go/src/k8s.io/kubernetes/_output/dockerized/go/src/k8s.io/kubernetes/") 219 frames[i].File = trimmedFile 220 } 221 222 for _, curFrame := range frames { 223 if _, seen := seenLines[fmt.Sprintf("%v:%v", curFrame.File, curFrame.Line)]; seen { 224 continue 225 } 226 227 freader, err := os.Open(curFrame.File) 228 if err != nil { 229 return nil, err 230 } 231 defer freader.Close() 232 cd, err := scanFileForFrame(curFrame.File, freader, curFrame) 233 if err != nil { 234 return nil, err 235 } 236 if cd != nil { 237 return cd, nil 238 } 239 } 240 241 return nil, nil 242} 243 244// scanFileForFrame will scan the target and look for a conformance comment attached to the function 245// described by the target frame. If the comment can't be found then nil, nil is returned. 246func scanFileForFrame(filename string, src interface{}, targetFrame frame) (*ConformanceData, error) { 247 fset := token.NewFileSet() // positions are relative to fset 248 f, err := parser.ParseFile(fset, filename, src, parser.ParseComments) 249 if err != nil { 250 return nil, err 251 } 252 253 cmap := ast.NewCommentMap(fset, f, f.Comments) 254 for _, cs := range cmap { 255 for _, c := range cs { 256 if cd := tryCommentGroupAndFrame(fset, c, targetFrame); cd != nil { 257 return cd, nil 258 } 259 } 260 } 261 return nil, nil 262} 263 264func validateTestName(s string) error { 265 matches := regexIneligibleTags.FindAllString(s, -1) 266 if matches != nil { 267 return fmt.Errorf("'%s' cannot have invalid tags %v", s, strings.Join(matches, ",")) 268 } 269 return nil 270} 271 272func tryCommentGroupAndFrame(fset *token.FileSet, cg *ast.CommentGroup, f frame) *ConformanceData { 273 if !shouldProcessCommentGroup(fset, cg, f) { 274 return nil 275 } 276 277 // Each file/line will either be some helper function (not a conformance comment) or apply to just a single test. Don't revisit. 278 if seenLines != nil { 279 seenLines[fmt.Sprintf("%v:%v", f.File, f.Line)] = struct{}{} 280 } 281 cd := commentToConformanceData(cg.Text()) 282 if cd == nil { 283 return nil 284 } 285 286 cd.URL = fmt.Sprintf("%s%s#L%d", *baseURL, f.File, f.Line) 287 cd.File = f.File 288 return cd 289} 290 291func shouldProcessCommentGroup(fset *token.FileSet, cg *ast.CommentGroup, f frame) bool { 292 lineDiff := f.Line - fset.Position(cg.End()).Line 293 return lineDiff > 0 && lineDiff <= conformanceCommentsLineWindow 294} 295 296func commentToConformanceData(comment string) *ConformanceData { 297 lines := strings.Split(comment, "\n") 298 descLines := []string{} 299 cd := &ConformanceData{} 300 var curLine string 301 for _, line := range lines { 302 line = strings.TrimSpace(line) 303 if len(line) == 0 { 304 continue 305 } 306 if sline := regexp.MustCompile("^Testname\\s*:\\s*").Split(line, -1); len(sline) == 2 { 307 curLine = "Testname" 308 cd.TestName = sline[1] 309 continue 310 } 311 if sline := regexp.MustCompile("^Release\\s*:\\s*").Split(line, -1); len(sline) == 2 { 312 curLine = "Release" 313 cd.Release = sline[1] 314 continue 315 } 316 if sline := regexp.MustCompile("^Description\\s*:\\s*").Split(line, -1); len(sline) == 2 { 317 curLine = "Description" 318 descLines = append(descLines, sline[1]) 319 continue 320 } 321 322 // Line has no header 323 if curLine == "Description" { 324 descLines = append(descLines, line) 325 } 326 } 327 if cd.TestName == "" { 328 return nil 329 } 330 331 cd.Description = strings.Join(descLines, " ") 332 return cd 333} 334