1package lexer
2
3import (
4	"io"
5
6	"github.com/goccy/go-yaml/scanner"
7	"github.com/goccy/go-yaml/token"
8)
9
10// Tokenize split to token instances from string
11func Tokenize(src string) token.Tokens {
12	var s scanner.Scanner
13	s.Init(src)
14	var tokens token.Tokens
15	for {
16		subTokens, err := s.Scan()
17		if err == io.EOF {
18			break
19		}
20		tokens.Add(subTokens...)
21	}
22	return tokens
23}
24