2015-06-20 14:33:14 +00:00
|
|
|
package storage
|
|
|
|
|
|
|
|
import (
|
|
|
|
"encoding/json"
|
|
|
|
"errors"
|
|
|
|
"io"
|
|
|
|
"path/filepath"
|
2015-09-23 19:50:23 +00:00
|
|
|
"unicode/utf8"
|
2015-06-20 14:33:14 +00:00
|
|
|
)
|
|
|
|
|
2015-08-11 20:27:05 +00:00
|
|
|
// ErrDuplicatePath occurs when a tar archive has more than one entry for the
|
|
|
|
// same file path
|
2015-06-20 14:33:14 +00:00
|
|
|
var ErrDuplicatePath = errors.New("duplicates of file paths not supported")
|
|
|
|
|
|
|
|
// Packer describes the methods to pack Entries to a storage destination
|
|
|
|
type Packer interface {
|
|
|
|
// AddEntry packs the Entry and returns its position
|
|
|
|
AddEntry(e Entry) (int, error)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Unpacker describes the methods to read Entries from a source
|
|
|
|
type Unpacker interface {
|
|
|
|
// Next returns the next Entry being unpacked, or error, until io.EOF
|
|
|
|
Next() (*Entry, error)
|
|
|
|
}
|
|
|
|
|
|
|
|
/* TODO(vbatts) figure out a good model for this
|
|
|
|
type PackUnpacker interface {
|
|
|
|
Packer
|
|
|
|
Unpacker
|
|
|
|
}
|
|
|
|
*/
|
|
|
|
|
|
|
|
type jsonUnpacker struct {
|
2015-12-02 20:34:12 +00:00
|
|
|
seen seenNames
|
|
|
|
dec *json.Decoder
|
2015-06-20 14:33:14 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
func (jup *jsonUnpacker) Next() (*Entry, error) {
|
|
|
|
var e Entry
|
2015-12-02 20:34:12 +00:00
|
|
|
err := jup.dec.Decode(&e)
|
|
|
|
if err != nil {
|
2015-06-20 14:33:14 +00:00
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// check for dup name
|
|
|
|
if e.Type == FileType {
|
2015-09-23 19:50:23 +00:00
|
|
|
cName := filepath.Clean(e.GetName())
|
2015-06-20 14:33:14 +00:00
|
|
|
if _, ok := jup.seen[cName]; ok {
|
|
|
|
return nil, ErrDuplicatePath
|
|
|
|
}
|
2015-08-11 20:27:05 +00:00
|
|
|
jup.seen[cName] = struct{}{}
|
2015-06-20 14:33:14 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return &e, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewJSONUnpacker provides an Unpacker that reads Entries (SegmentType and
|
|
|
|
// FileType) as a json document.
|
|
|
|
//
|
|
|
|
// Each Entry read are expected to be delimited by new line.
|
|
|
|
func NewJSONUnpacker(r io.Reader) Unpacker {
|
|
|
|
return &jsonUnpacker{
|
2015-12-02 20:34:12 +00:00
|
|
|
dec: json.NewDecoder(r),
|
2015-06-20 14:33:14 +00:00
|
|
|
seen: seenNames{},
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
type jsonPacker struct {
|
|
|
|
w io.Writer
|
|
|
|
e *json.Encoder
|
|
|
|
pos int
|
|
|
|
seen seenNames
|
|
|
|
}
|
|
|
|
|
2015-08-11 20:27:05 +00:00
|
|
|
type seenNames map[string]struct{}
|
2015-06-20 14:33:14 +00:00
|
|
|
|
|
|
|
func (jp *jsonPacker) AddEntry(e Entry) (int, error) {
|
2015-09-23 19:50:23 +00:00
|
|
|
// if Name is not valid utf8, switch it to raw first.
|
|
|
|
if e.Name != "" {
|
|
|
|
if !utf8.ValidString(e.Name) {
|
|
|
|
e.NameRaw = []byte(e.Name)
|
|
|
|
e.Name = ""
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-06-20 14:33:14 +00:00
|
|
|
// check early for dup name
|
|
|
|
if e.Type == FileType {
|
2015-09-23 19:50:23 +00:00
|
|
|
cName := filepath.Clean(e.GetName())
|
2015-06-20 14:33:14 +00:00
|
|
|
if _, ok := jp.seen[cName]; ok {
|
|
|
|
return -1, ErrDuplicatePath
|
|
|
|
}
|
2015-08-11 20:27:05 +00:00
|
|
|
jp.seen[cName] = struct{}{}
|
2015-06-20 14:33:14 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
e.Position = jp.pos
|
|
|
|
err := jp.e.Encode(e)
|
|
|
|
if err != nil {
|
|
|
|
return -1, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// made it this far, increment now
|
|
|
|
jp.pos++
|
|
|
|
return e.Position, nil
|
|
|
|
}
|
|
|
|
|
2015-08-11 20:27:05 +00:00
|
|
|
// NewJSONPacker provides a Packer that writes each Entry (SegmentType and
|
2015-06-20 14:33:14 +00:00
|
|
|
// FileType) as a json document.
|
|
|
|
//
|
|
|
|
// The Entries are delimited by new line.
|
|
|
|
func NewJSONPacker(w io.Writer) Packer {
|
|
|
|
return &jsonPacker{
|
|
|
|
w: w,
|
|
|
|
e: json.NewEncoder(w),
|
|
|
|
seen: seenNames{},
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
TODO(vbatts) perhaps have a more compact packer/unpacker, maybe using msgapck
|
|
|
|
(https://github.com/ugorji/go)
|
|
|
|
|
|
|
|
|
|
|
|
Even though, since our jsonUnpacker and jsonPacker just take
|
|
|
|
io.Reader/io.Writer, then we can get away with passing them a
|
|
|
|
gzip.Reader/gzip.Writer
|
|
|
|
*/
|