1package lexer 2 3import ( 4 "io" 5 6 "github.com/goccy/go-yaml/scanner" 7 "github.com/goccy/go-yaml/token" 8) 9 10// Tokenize split to token instances from string 11func Tokenize(src string) token.Tokens { 12 var s scanner.Scanner 13 s.Init(src) 14 var tokens token.Tokens 15 for { 16 subTokens, err := s.Scan() 17 if err == io.EOF { 18 break 19 } 20 tokens.Add(subTokens...) 21 } 22 return tokens 23} 24