[VOL-5486] Upgrade library versions
Change-Id: I8b4e88699e03f44ee13e467867f45ae3f0a63c4b
Signed-off-by: Abhay Kumar <abhay.kumar@radisys.com>
diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go
index 4871dd0..8f8223c 100644
--- a/vendor/github.com/klauspost/compress/zstd/encoder.go
+++ b/vendor/github.com/klauspost/compress/zstd/encoder.go
@@ -6,8 +6,10 @@
import (
"crypto/rand"
+ "errors"
"fmt"
"io"
+ "math"
rdebug "runtime/debug"
"sync"
@@ -33,7 +35,7 @@
Block() *blockEnc
CRC() *xxhash.Digest
AppendCRC([]byte) []byte
- WindowSize(size int) int32
+ WindowSize(size int64) int32
UseBlock(*blockEnc)
Reset(d *dict, singleBlock bool)
}
@@ -48,6 +50,8 @@
err error
writeErr error
nWritten int64
+ nInput int64
+ frameContentSize int64
headerWritten bool
eofWritten bool
fullFrameWritten bool
@@ -96,23 +100,25 @@
if cap(s.filling) == 0 {
s.filling = make([]byte, 0, e.o.blockSize)
}
- if cap(s.current) == 0 {
- s.current = make([]byte, 0, e.o.blockSize)
- }
- if cap(s.previous) == 0 {
- s.previous = make([]byte, 0, e.o.blockSize)
+ if e.o.concurrent > 1 {
+ if cap(s.current) == 0 {
+ s.current = make([]byte, 0, e.o.blockSize)
+ }
+ if cap(s.previous) == 0 {
+ s.previous = make([]byte, 0, e.o.blockSize)
+ }
+ s.current = s.current[:0]
+ s.previous = s.previous[:0]
+ if s.writing == nil {
+ s.writing = &blockEnc{lowMem: e.o.lowMem}
+ s.writing.init()
+ }
+ s.writing.initNewEncode()
}
if s.encoder == nil {
s.encoder = e.o.encoder()
}
- if s.writing == nil {
- s.writing = &blockEnc{lowMem: e.o.lowMem}
- s.writing.init()
- }
- s.writing.initNewEncode()
s.filling = s.filling[:0]
- s.current = s.current[:0]
- s.previous = s.previous[:0]
s.encoder.Reset(e.o.dict, false)
s.headerWritten = false
s.eofWritten = false
@@ -120,7 +126,21 @@
s.w = w
s.err = nil
s.nWritten = 0
+ s.nInput = 0
s.writeErr = nil
+ s.frameContentSize = 0
+}
+
+// ResetContentSize will reset and set a content size for the next stream.
+// If the bytes written does not match the size given an error will be returned
+// when calling Close().
+// This is removed when Reset is called.
+// Sizes <= 0 results in no content size set.
+func (e *Encoder) ResetContentSize(w io.Writer, size int64) {
+ e.Reset(w)
+ if size >= 0 {
+ e.state.frameContentSize = size
+ }
}
// Write data to the encoder.
@@ -130,6 +150,9 @@
// and write CRC if requested.
func (e *Encoder) Write(p []byte) (n int, err error) {
s := &e.state
+ if s.eofWritten {
+ return 0, ErrEncoderClosed
+ }
for len(p) > 0 {
if len(p)+len(s.filling) < e.o.blockSize {
if e.o.crc {
@@ -183,13 +206,14 @@
return nil
}
if final && len(s.filling) > 0 {
- s.current = e.EncodeAll(s.filling, s.current[:0])
+ s.current = e.encodeAll(s.encoder, s.filling, s.current[:0])
var n2 int
n2, s.err = s.w.Write(s.current)
if s.err != nil {
return s.err
}
s.nWritten += int64(n2)
+ s.nInput += int64(len(s.filling))
s.current = s.current[:0]
s.filling = s.filling[:0]
s.headerWritten = true
@@ -200,17 +224,14 @@
var tmp [maxHeaderSize]byte
fh := frameHeader{
- ContentSize: 0,
- WindowSize: uint32(s.encoder.WindowSize(0)),
+ ContentSize: uint64(s.frameContentSize),
+ WindowSize: uint32(s.encoder.WindowSize(s.frameContentSize)),
SingleSegment: false,
Checksum: e.o.crc,
DictID: e.o.dict.ID(),
}
- dst, err := fh.appendTo(tmp[:0])
- if err != nil {
- return err
- }
+ dst := fh.appendTo(tmp[:0])
s.headerWritten = true
s.wWg.Wait()
var n2 int
@@ -241,11 +262,41 @@
return s.err
}
+ // SYNC:
+ if e.o.concurrent == 1 {
+ src := s.filling
+ s.nInput += int64(len(s.filling))
+ if debugEncoder {
+ println("Adding sync block,", len(src), "bytes, final:", final)
+ }
+ enc := s.encoder
+ blk := enc.Block()
+ blk.reset(nil)
+ enc.Encode(blk, src)
+ blk.last = final
+ if final {
+ s.eofWritten = true
+ }
+
+ s.err = blk.encode(src, e.o.noEntropy, !e.o.allLitEntropy)
+ if s.err != nil {
+ return s.err
+ }
+ _, s.err = s.w.Write(blk.output)
+ s.nWritten += int64(len(blk.output))
+ s.filling = s.filling[:0]
+ return s.err
+ }
+
// Move blocks forward.
s.filling, s.current, s.previous = s.previous[:0], s.filling, s.current
+ s.nInput += int64(len(s.current))
s.wg.Add(1)
+ if final {
+ s.eofWritten = true
+ }
go func(src []byte) {
- if debug {
+ if debugEncoder {
println("Adding block,", len(src), "bytes, final:", final)
}
defer func() {
@@ -259,9 +310,6 @@
blk := enc.Block()
enc.Encode(blk, src)
blk.last = final
- if final {
- s.eofWritten = true
- }
// Wait for pending writes.
s.wWg.Wait()
if s.writeErr != nil {
@@ -282,22 +330,8 @@
}
s.wWg.Done()
}()
- err := errIncompressible
- // If we got the exact same number of literals as input,
- // assume the literals cannot be compressed.
- if len(src) != len(blk.literals) || len(src) != e.o.blockSize {
- err = blk.encode(src, e.o.noEntropy, !e.o.allLitEntropy)
- }
- switch err {
- case errIncompressible:
- if debug {
- println("Storing incompressible block as raw")
- }
- blk.encodeRaw(src)
- // In fast mode, we do not transfer offsets, so we don't have to deal with changing the.
- case nil:
- default:
- s.writeErr = err
+ s.writeErr = blk.encode(src, e.o.noEntropy, !e.o.allLitEntropy)
+ if s.writeErr != nil {
return
}
_, s.writeErr = s.w.Write(blk.output)
@@ -313,7 +347,7 @@
//
// The Copy function uses ReaderFrom if available.
func (e *Encoder) ReadFrom(r io.Reader) (n int64, err error) {
- if debug {
+ if debugEncoder {
println("Using ReadFrom")
}
@@ -336,20 +370,20 @@
switch err {
case io.EOF:
e.state.filling = e.state.filling[:len(e.state.filling)-len(src)]
- if debug {
+ if debugEncoder {
println("ReadFrom: got EOF final block:", len(e.state.filling))
}
return n, nil
case nil:
default:
- if debug {
+ if debugEncoder {
println("ReadFrom: got error:", err)
}
e.state.err = err
return n, err
}
if len(src) > 0 {
- if debug {
+ if debugEncoder {
println("ReadFrom: got space left in source:", len(src))
}
continue
@@ -371,12 +405,20 @@
if len(s.filling) > 0 {
err := e.nextBlock(false)
if err != nil {
+ // Ignore Flush after Close.
+ if errors.Is(s.err, ErrEncoderClosed) {
+ return nil
+ }
return err
}
}
s.wg.Wait()
s.wWg.Wait()
if s.err != nil {
+ // Ignore Flush after Close.
+ if errors.Is(s.err, ErrEncoderClosed) {
+ return nil
+ }
return s.err
}
return s.writeErr
@@ -392,8 +434,16 @@
}
err := e.nextBlock(true)
if err != nil {
+ if errors.Is(s.err, ErrEncoderClosed) {
+ return nil
+ }
return err
}
+ if s.frameContentSize > 0 {
+ if s.nInput != s.frameContentSize {
+ return fmt.Errorf("frame content size %d given, but %d bytes was written", s.frameContentSize, s.nInput)
+ }
+ }
if e.state.fullFrameWritten {
return s.err
}
@@ -424,6 +474,11 @@
}
_, s.err = s.w.Write(frame)
}
+ if s.err == nil {
+ s.err = ErrEncoderClosed
+ return nil
+ }
+
return s.err
}
@@ -434,6 +489,15 @@
// Data compressed with EncodeAll can be decoded with the Decoder,
// using either a stream or DecodeAll.
func (e *Encoder) EncodeAll(src, dst []byte) []byte {
+ e.init.Do(e.initialize)
+ enc := <-e.encoders
+ defer func() {
+ e.encoders <- enc
+ }()
+ return e.encodeAll(enc, src, dst)
+}
+
+func (e *Encoder) encodeAll(enc encoder, src, dst []byte) []byte {
if len(src) == 0 {
if e.o.fullZero {
// Add frame header.
@@ -445,7 +509,7 @@
Checksum: false,
DictID: 0,
}
- dst, _ = fh.appendTo(dst)
+ dst = fh.appendTo(dst)
// Write raw block as last one only.
var blk blockHeader
@@ -456,21 +520,15 @@
}
return dst
}
- e.init.Do(e.initialize)
- enc := <-e.encoders
- defer func() {
- // Release encoder reference to last block.
- // If a non-single block is needed the encoder will reset again.
- e.encoders <- enc
- }()
- // Use single segments when above minimum window and below 1MB.
- single := len(src) < 1<<20 && len(src) > MinWindowSize
+
+ // Use single segments when above minimum window and below window size.
+ single := len(src) <= e.o.windowSize && len(src) > MinWindowSize
if e.o.single != nil {
single = *e.o.single
}
fh := frameHeader{
ContentSize: uint64(len(src)),
- WindowSize: uint32(enc.WindowSize(len(src))),
+ WindowSize: uint32(enc.WindowSize(int64(len(src)))),
SingleSegment: single,
Checksum: e.o.crc,
DictID: e.o.dict.ID(),
@@ -480,13 +538,10 @@
if len(dst) == 0 && cap(dst) == 0 && len(src) < 1<<20 && !e.o.lowMem {
dst = make([]byte, 0, len(src))
}
- dst, err := fh.appendTo(dst)
- if err != nil {
- panic(err)
- }
+ dst = fh.appendTo(dst)
// If we can do everything in one block, prefer that.
- if len(src) <= maxCompressedBlockSize {
+ if len(src) <= e.o.blockSize {
enc.Reset(e.o.dict, true)
// Slightly faster with no history and everything in one block.
if e.o.crc {
@@ -502,25 +557,15 @@
// If we got the exact same number of literals as input,
// assume the literals cannot be compressed.
- err := errIncompressible
oldout := blk.output
- if len(blk.literals) != len(src) || len(src) != e.o.blockSize {
- // Output directly to dst
- blk.output = dst
- err = blk.encode(src, e.o.noEntropy, !e.o.allLitEntropy)
- }
+ // Output directly to dst
+ blk.output = dst
- switch err {
- case errIncompressible:
- if debug {
- println("Storing incompressible block as raw")
- }
- dst = blk.encodeRawTo(dst, src)
- case nil:
- dst = blk.output
- default:
+ err := blk.encode(src, e.o.noEntropy, !e.o.allLitEntropy)
+ if err != nil {
panic(err)
}
+ dst = blk.output
blk.output = oldout
} else {
enc.Reset(e.o.dict, false)
@@ -539,25 +584,11 @@
if len(src) == 0 {
blk.last = true
}
- err := errIncompressible
- // If we got the exact same number of literals as input,
- // assume the literals cannot be compressed.
- if len(blk.literals) != len(todo) || len(todo) != e.o.blockSize {
- err = blk.encode(todo, e.o.noEntropy, !e.o.allLitEntropy)
- }
-
- switch err {
- case errIncompressible:
- if debug {
- println("Storing incompressible block as raw")
- }
- dst = blk.encodeRawTo(dst, todo)
- blk.popOffsets()
- case nil:
- dst = append(dst, blk.output...)
- default:
+ err := blk.encode(todo, e.o.noEntropy, !e.o.allLitEntropy)
+ if err != nil {
panic(err)
}
+ dst = append(dst, blk.output...)
blk.reset(nil)
}
}
@@ -567,6 +598,7 @@
// Add padding with content from crypto/rand.Reader
if e.o.pad > 0 {
add := calcSkippableFrame(int64(len(dst)), int64(e.o.pad))
+ var err error
dst, err = skippableFrame(dst, add, rand.Reader)
if err != nil {
panic(err)
@@ -574,3 +606,37 @@
}
return dst
}
+
+// MaxEncodedSize returns the expected maximum
+// size of an encoded block or stream.
+func (e *Encoder) MaxEncodedSize(size int) int {
+ frameHeader := 4 + 2 // magic + frame header & window descriptor
+ if e.o.dict != nil {
+ frameHeader += 4
+ }
+ // Frame content size:
+ if size < 256 {
+ frameHeader++
+ } else if size < 65536+256 {
+ frameHeader += 2
+ } else if size < math.MaxInt32 {
+ frameHeader += 4
+ } else {
+ frameHeader += 8
+ }
+ // Final crc
+ if e.o.crc {
+ frameHeader += 4
+ }
+
+ // Max overhead is 3 bytes/block.
+ // There cannot be 0 blocks.
+ blocks := (size + e.o.blockSize) / e.o.blockSize
+
+ // Combine, add padding.
+ maxSz := frameHeader + 3*blocks + size
+ if e.o.pad > 1 {
+ maxSz += calcSkippableFrame(int64(maxSz), int64(e.o.pad))
+ }
+ return maxSz
+}