1package chunkenc 2 3import ( 4 "context" 5 "errors" 6 "fmt" 7 "io" 8 "strings" 9 "time" 10 11 "github.com/grafana/loki/pkg/iter" 12 "github.com/grafana/loki/pkg/logproto" 13 "github.com/grafana/loki/pkg/logql/log" 14) 15 16// Errors returned by the chunk interface. 17var ( 18 ErrChunkFull = errors.New("chunk full") 19 ErrOutOfOrder = errors.New("entry out of order") 20 ErrInvalidSize = errors.New("invalid size") 21 ErrInvalidFlag = errors.New("invalid flag") 22 ErrInvalidChecksum = errors.New("invalid chunk checksum") 23) 24 25// Encoding is the identifier for a chunk encoding. 26type Encoding byte 27 28// The different available encodings. 29// Make sure to preserve the order, as these numeric values are written to the chunks! 30const ( 31 EncNone Encoding = iota 32 EncGZIP 33 EncDumb 34 EncLZ4_64k 35 EncSnappy 36 EncLZ4_256k 37 EncLZ4_1M 38 EncLZ4_4M 39 EncFlate 40 EncZstd 41) 42 43var supportedEncoding = []Encoding{ 44 EncNone, 45 EncGZIP, 46 EncLZ4_64k, 47 EncSnappy, 48 EncLZ4_256k, 49 EncLZ4_1M, 50 EncLZ4_4M, 51 EncFlate, 52 EncZstd, 53} 54 55func (e Encoding) String() string { 56 switch e { 57 case EncGZIP: 58 return "gzip" 59 case EncNone: 60 return "none" 61 case EncDumb: 62 return "dumb" 63 case EncLZ4_64k: 64 return "lz4-64k" 65 case EncLZ4_256k: 66 return "lz4-256k" 67 case EncLZ4_1M: 68 return "lz4-1M" 69 case EncLZ4_4M: 70 return "lz4" 71 case EncSnappy: 72 return "snappy" 73 case EncFlate: 74 return "flate" 75 case EncZstd: 76 return "zstd" 77 default: 78 return "unknown" 79 } 80} 81 82// ParseEncoding parses an chunk encoding (compression algorithm) by its name. 83func ParseEncoding(enc string) (Encoding, error) { 84 for _, e := range supportedEncoding { 85 if strings.EqualFold(e.String(), enc) { 86 return e, nil 87 } 88 } 89 return 0, fmt.Errorf("invalid encoding: %s, supported: %s", enc, SupportedEncoding()) 90 91} 92 93// SupportedEncoding returns the list of supported Encoding. 94func SupportedEncoding() string { 95 var sb strings.Builder 96 for i := range supportedEncoding { 97 sb.WriteString(supportedEncoding[i].String()) 98 if i != len(supportedEncoding)-1 { 99 sb.WriteString(", ") 100 } 101 } 102 return sb.String() 103} 104 105// Chunk is the interface for the compressed logs chunk format. 106type Chunk interface { 107 Bounds() (time.Time, time.Time) 108 SpaceFor(*logproto.Entry) bool 109 Append(*logproto.Entry) error 110 Iterator(ctx context.Context, mintT, maxtT time.Time, direction logproto.Direction, pipeline log.StreamPipeline) (iter.EntryIterator, error) 111 SampleIterator(ctx context.Context, from, through time.Time, extractor log.StreamSampleExtractor) iter.SampleIterator 112 // Returns the list of blocks in the chunks. 113 Blocks(mintT, maxtT time.Time) []Block 114 // Size returns the number of entries in a chunk 115 Size() int 116 Bytes() ([]byte, error) 117 BytesWith([]byte) ([]byte, error) // uses provided []byte for buffer instantiation 118 io.WriterTo 119 BlockCount() int 120 Utilization() float64 121 UncompressedSize() int 122 CompressedSize() int 123 Close() error 124 Encoding() Encoding 125 Rebound(start, end time.Time) (Chunk, error) 126} 127 128// Block is a chunk block. 129type Block interface { 130 // MinTime is the minimum time of entries in the block 131 MinTime() int64 132 // MaxTime is the maximum time of entries in the block 133 MaxTime() int64 134 // Offset is the offset/position of the block in the chunk. Offset is unique for a given block per chunk. 135 Offset() int 136 // Entries is the amount of entries in the block. 137 Entries() int 138 // Iterator returns an entry iterator for the block. 139 Iterator(ctx context.Context, pipeline log.StreamPipeline) iter.EntryIterator 140 // SampleIterator returns a sample iterator for the block. 141 SampleIterator(ctx context.Context, extractor log.StreamSampleExtractor) iter.SampleIterator 142} 143