1// Package lz4stream provides the types that support reading and writing LZ4 data streams. 2package lz4stream 3 4import ( 5 "encoding/binary" 6 "fmt" 7 "io" 8 "io/ioutil" 9 10 "github.com/pierrec/lz4/v4/internal/lz4block" 11 "github.com/pierrec/lz4/v4/internal/lz4errors" 12 "github.com/pierrec/lz4/v4/internal/xxh32" 13) 14 15//go:generate go run gen.go 16 17const ( 18 frameMagic uint32 = 0x184D2204 19 frameSkipMagic uint32 = 0x184D2A50 20 frameMagicLegacy uint32 = 0x184C2102 21) 22 23func NewFrame() *Frame { 24 return &Frame{} 25} 26 27type Frame struct { 28 buf [15]byte // frame descriptor needs at most 4(magic)+4+8+1=11 bytes 29 Magic uint32 30 Descriptor FrameDescriptor 31 Blocks Blocks 32 Checksum uint32 33 checksum xxh32.XXHZero 34} 35 36// Reset allows reusing the Frame. 37// The Descriptor configuration is not modified. 38func (f *Frame) Reset(num int) { 39 f.Magic = 0 40 f.Descriptor.Checksum = 0 41 f.Descriptor.ContentSize = 0 42 _ = f.Blocks.close(f, num) 43 f.Checksum = 0 44} 45 46func (f *Frame) InitW(dst io.Writer, num int, legacy bool) { 47 if legacy { 48 f.Magic = frameMagicLegacy 49 idx := lz4block.Index(lz4block.Block8Mb) 50 f.Descriptor.Flags.BlockSizeIndexSet(idx) 51 } else { 52 f.Magic = frameMagic 53 f.Descriptor.initW() 54 } 55 f.Blocks.initW(f, dst, num) 56 f.checksum.Reset() 57} 58 59func (f *Frame) CloseW(dst io.Writer, num int) error { 60 if err := f.Blocks.close(f, num); err != nil { 61 return err 62 } 63 if f.isLegacy() { 64 return nil 65 } 66 buf := f.buf[:0] 67 // End mark (data block size of uint32(0)). 68 buf = append(buf, 0, 0, 0, 0) 69 if f.Descriptor.Flags.ContentChecksum() { 70 buf = f.checksum.Sum(buf) 71 } 72 _, err := dst.Write(buf) 73 return err 74} 75 76func (f *Frame) isLegacy() bool { 77 return f.Magic == frameMagicLegacy 78} 79 80func (f *Frame) ParseHeaders(src io.Reader) error { 81 if f.Magic > 0 { 82 // Header already read. 83 return nil 84 } 85 86newFrame: 87 var err error 88 if f.Magic, err = f.readUint32(src); err != nil { 89 return err 90 } 91 switch m := f.Magic; { 92 case m == frameMagic || m == frameMagicLegacy: 93 // All 16 values of frameSkipMagic are valid. 94 case m>>8 == frameSkipMagic>>8: 95 skip, err := f.readUint32(src) 96 if err != nil { 97 return err 98 } 99 if _, err := io.CopyN(ioutil.Discard, src, int64(skip)); err != nil { 100 return err 101 } 102 goto newFrame 103 default: 104 return lz4errors.ErrInvalidFrame 105 } 106 if err := f.Descriptor.initR(f, src); err != nil { 107 return err 108 } 109 f.checksum.Reset() 110 return nil 111} 112 113func (f *Frame) InitR(src io.Reader, num int) (chan []byte, error) { 114 return f.Blocks.initR(f, num, src) 115} 116 117func (f *Frame) CloseR(src io.Reader) (err error) { 118 if f.isLegacy() { 119 return nil 120 } 121 if !f.Descriptor.Flags.ContentChecksum() { 122 return nil 123 } 124 if f.Checksum, err = f.readUint32(src); err != nil { 125 return err 126 } 127 if c := f.checksum.Sum32(); c != f.Checksum { 128 return fmt.Errorf("%w: got %x; expected %x", lz4errors.ErrInvalidFrameChecksum, c, f.Checksum) 129 } 130 return nil 131} 132 133type FrameDescriptor struct { 134 Flags DescriptorFlags 135 ContentSize uint64 136 Checksum uint8 137} 138 139func (fd *FrameDescriptor) initW() { 140 fd.Flags.VersionSet(1) 141 fd.Flags.BlockIndependenceSet(true) 142} 143 144func (fd *FrameDescriptor) Write(f *Frame, dst io.Writer) error { 145 if fd.Checksum > 0 { 146 // Header already written. 147 return nil 148 } 149 150 buf := f.buf[:4] 151 // Write the magic number here even though it belongs to the Frame. 152 binary.LittleEndian.PutUint32(buf, f.Magic) 153 if !f.isLegacy() { 154 buf = buf[:4+2] 155 binary.LittleEndian.PutUint16(buf[4:], uint16(fd.Flags)) 156 157 if fd.Flags.Size() { 158 buf = buf[:4+2+8] 159 binary.LittleEndian.PutUint64(buf[4+2:], fd.ContentSize) 160 } 161 fd.Checksum = descriptorChecksum(buf[4:]) 162 buf = append(buf, fd.Checksum) 163 } 164 165 _, err := dst.Write(buf) 166 return err 167} 168 169func (fd *FrameDescriptor) initR(f *Frame, src io.Reader) error { 170 if f.isLegacy() { 171 idx := lz4block.Index(lz4block.Block8Mb) 172 f.Descriptor.Flags.BlockSizeIndexSet(idx) 173 return nil 174 } 175 // Read the flags and the checksum, hoping that there is not content size. 176 buf := f.buf[:3] 177 if _, err := io.ReadFull(src, buf); err != nil { 178 return err 179 } 180 descr := binary.LittleEndian.Uint16(buf) 181 fd.Flags = DescriptorFlags(descr) 182 if fd.Flags.Size() { 183 // Append the 8 missing bytes. 184 buf = buf[:3+8] 185 if _, err := io.ReadFull(src, buf[3:]); err != nil { 186 return err 187 } 188 fd.ContentSize = binary.LittleEndian.Uint64(buf[2:]) 189 } 190 fd.Checksum = buf[len(buf)-1] // the checksum is the last byte 191 buf = buf[:len(buf)-1] // all descriptor fields except checksum 192 if c := descriptorChecksum(buf); fd.Checksum != c { 193 return fmt.Errorf("%w: got %x; expected %x", lz4errors.ErrInvalidHeaderChecksum, c, fd.Checksum) 194 } 195 // Validate the elements that can be. 196 if idx := fd.Flags.BlockSizeIndex(); !idx.IsValid() { 197 return lz4errors.ErrOptionInvalidBlockSize 198 } 199 return nil 200} 201 202func descriptorChecksum(buf []byte) byte { 203 return byte(xxh32.ChecksumZero(buf) >> 8) 204} 205