1// Copyright 2017 Google LLC. All Rights Reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14 15package compiler 16 17import ( 18 "fmt" 19 "io/ioutil" 20 "log" 21 "net/http" 22 "net/url" 23 "path/filepath" 24 "strings" 25 "sync" 26 27 yaml "gopkg.in/yaml.v3" 28) 29 30var verboseReader = false 31 32var fileCache map[string][]byte 33var infoCache map[string]*yaml.Node 34 35var fileCacheEnable = true 36var infoCacheEnable = true 37 38// These locks are used to synchronize accesses to the fileCache and infoCache 39// maps (above). They are global state and can throw thread-related errors 40// when modified from separate goroutines. The general strategy is to protect 41// all public functions in this file with mutex Lock() calls. As a result, to 42// avoid deadlock, these public functions should not call other public 43// functions, so some public functions have private equivalents. 44// In the future, we might consider replacing the maps with sync.Map and 45// eliminating these mutexes. 46var fileCacheMutex sync.Mutex 47var infoCacheMutex sync.Mutex 48 49func initializeFileCache() { 50 if fileCache == nil { 51 fileCache = make(map[string][]byte, 0) 52 } 53} 54 55func initializeInfoCache() { 56 if infoCache == nil { 57 infoCache = make(map[string]*yaml.Node, 0) 58 } 59} 60 61// EnableFileCache turns on file caching. 62func EnableFileCache() { 63 fileCacheMutex.Lock() 64 defer fileCacheMutex.Unlock() 65 fileCacheEnable = true 66} 67 68// EnableInfoCache turns on parsed info caching. 69func EnableInfoCache() { 70 infoCacheMutex.Lock() 71 defer infoCacheMutex.Unlock() 72 infoCacheEnable = true 73} 74 75// DisableFileCache turns off file caching. 76func DisableFileCache() { 77 fileCacheMutex.Lock() 78 defer fileCacheMutex.Unlock() 79 fileCacheEnable = false 80} 81 82// DisableInfoCache turns off parsed info caching. 83func DisableInfoCache() { 84 infoCacheMutex.Lock() 85 defer infoCacheMutex.Unlock() 86 infoCacheEnable = false 87} 88 89// RemoveFromFileCache removes an entry from the file cache. 90func RemoveFromFileCache(fileurl string) { 91 fileCacheMutex.Lock() 92 defer fileCacheMutex.Unlock() 93 if !fileCacheEnable { 94 return 95 } 96 initializeFileCache() 97 delete(fileCache, fileurl) 98} 99 100// RemoveFromInfoCache removes an entry from the info cache. 101func RemoveFromInfoCache(filename string) { 102 infoCacheMutex.Lock() 103 defer infoCacheMutex.Unlock() 104 if !infoCacheEnable { 105 return 106 } 107 initializeInfoCache() 108 delete(infoCache, filename) 109} 110 111// GetInfoCache returns the info cache map. 112func GetInfoCache() map[string]*yaml.Node { 113 infoCacheMutex.Lock() 114 defer infoCacheMutex.Unlock() 115 if infoCache == nil { 116 initializeInfoCache() 117 } 118 return infoCache 119} 120 121// ClearFileCache clears the file cache. 122func ClearFileCache() { 123 fileCacheMutex.Lock() 124 defer fileCacheMutex.Unlock() 125 fileCache = make(map[string][]byte, 0) 126} 127 128// ClearInfoCache clears the info cache. 129func ClearInfoCache() { 130 infoCacheMutex.Lock() 131 defer infoCacheMutex.Unlock() 132 infoCache = make(map[string]*yaml.Node) 133} 134 135// ClearCaches clears all caches. 136func ClearCaches() { 137 ClearFileCache() 138 ClearInfoCache() 139} 140 141// FetchFile gets a specified file from the local filesystem or a remote location. 142func FetchFile(fileurl string) ([]byte, error) { 143 fileCacheMutex.Lock() 144 defer fileCacheMutex.Unlock() 145 return fetchFile(fileurl) 146} 147 148func fetchFile(fileurl string) ([]byte, error) { 149 var bytes []byte 150 initializeFileCache() 151 if fileCacheEnable { 152 bytes, ok := fileCache[fileurl] 153 if ok { 154 if verboseReader { 155 log.Printf("Cache hit %s", fileurl) 156 } 157 return bytes, nil 158 } 159 if verboseReader { 160 log.Printf("Fetching %s", fileurl) 161 } 162 } 163 response, err := http.Get(fileurl) 164 if err != nil { 165 return nil, err 166 } 167 defer response.Body.Close() 168 if response.StatusCode != 200 { 169 return nil, fmt.Errorf("Error downloading %s: %s", fileurl, response.Status) 170 } 171 bytes, err = ioutil.ReadAll(response.Body) 172 if fileCacheEnable && err == nil { 173 fileCache[fileurl] = bytes 174 } 175 return bytes, err 176} 177 178// ReadBytesForFile reads the bytes of a file. 179func ReadBytesForFile(filename string) ([]byte, error) { 180 fileCacheMutex.Lock() 181 defer fileCacheMutex.Unlock() 182 return readBytesForFile(filename) 183} 184 185func readBytesForFile(filename string) ([]byte, error) { 186 // is the filename a url? 187 fileurl, _ := url.Parse(filename) 188 if fileurl.Scheme != "" { 189 // yes, fetch it 190 bytes, err := fetchFile(filename) 191 if err != nil { 192 return nil, err 193 } 194 return bytes, nil 195 } 196 // no, it's a local filename 197 bytes, err := ioutil.ReadFile(filename) 198 if err != nil { 199 return nil, err 200 } 201 return bytes, nil 202} 203 204// ReadInfoFromBytes unmarshals a file as a *yaml.Node. 205func ReadInfoFromBytes(filename string, bytes []byte) (*yaml.Node, error) { 206 infoCacheMutex.Lock() 207 defer infoCacheMutex.Unlock() 208 return readInfoFromBytes(filename, bytes) 209} 210 211func readInfoFromBytes(filename string, bytes []byte) (*yaml.Node, error) { 212 initializeInfoCache() 213 if infoCacheEnable { 214 cachedInfo, ok := infoCache[filename] 215 if ok { 216 if verboseReader { 217 log.Printf("Cache hit info for file %s", filename) 218 } 219 return cachedInfo, nil 220 } 221 if verboseReader { 222 log.Printf("Reading info for file %s", filename) 223 } 224 } 225 var info yaml.Node 226 err := yaml.Unmarshal(bytes, &info) 227 if err != nil { 228 return nil, err 229 } 230 if infoCacheEnable && len(filename) > 0 { 231 infoCache[filename] = &info 232 } 233 return &info, nil 234} 235 236// ReadInfoForRef reads a file and return the fragment needed to resolve a $ref. 237func ReadInfoForRef(basefile string, ref string) (*yaml.Node, error) { 238 fileCacheMutex.Lock() 239 defer fileCacheMutex.Unlock() 240 infoCacheMutex.Lock() 241 defer infoCacheMutex.Unlock() 242 initializeInfoCache() 243 if infoCacheEnable { 244 info, ok := infoCache[ref] 245 if ok { 246 if verboseReader { 247 log.Printf("Cache hit for ref %s#%s", basefile, ref) 248 } 249 return info, nil 250 } 251 if verboseReader { 252 log.Printf("Reading info for ref %s#%s", basefile, ref) 253 } 254 } 255 basedir, _ := filepath.Split(basefile) 256 parts := strings.Split(ref, "#") 257 var filename string 258 if parts[0] != "" { 259 filename = parts[0] 260 if _, err := url.ParseRequestURI(parts[0]); err != nil { 261 // It is not an URL, so the file is local 262 filename = basedir + parts[0] 263 } 264 } else { 265 filename = basefile 266 } 267 bytes, err := readBytesForFile(filename) 268 if err != nil { 269 return nil, err 270 } 271 info, err := readInfoFromBytes(filename, bytes) 272 if info != nil && info.Kind == yaml.DocumentNode { 273 info = info.Content[0] 274 } 275 if err != nil { 276 log.Printf("File error: %v\n", err) 277 } else { 278 if info == nil { 279 return nil, NewError(nil, fmt.Sprintf("could not resolve %s", ref)) 280 } 281 if len(parts) > 1 { 282 path := strings.Split(parts[1], "/") 283 for i, key := range path { 284 if i > 0 { 285 m := info 286 if true { 287 found := false 288 for i := 0; i < len(m.Content); i += 2 { 289 if m.Content[i].Value == key { 290 info = m.Content[i+1] 291 found = true 292 } 293 } 294 if !found { 295 infoCache[ref] = nil 296 return nil, NewError(nil, fmt.Sprintf("could not resolve %s", ref)) 297 } 298 } 299 } 300 } 301 } 302 } 303 if infoCacheEnable { 304 infoCache[ref] = info 305 } 306 return info, nil 307} 308