package storage import ( "encoding/json" "errors" "io" "path/filepath" "unicode/utf8" ) // ErrDuplicatePath occurs when a tar archive has more than one entry for the // same file path var ErrDuplicatePath = errors.New("duplicates of file paths not supported") // Packer describes the methods to pack Entries to a storage destination type Packer interface { // AddEntry packs the Entry and returns its position AddEntry(e Entry) (int, error) } // Unpacker describes the methods to read Entries from a source type Unpacker interface { // Next returns the next Entry being unpacked, or error, until io.EOF Next() (*Entry, error) } /* TODO(vbatts) figure out a good model for this type PackUnpacker interface { Packer Unpacker } */ type jsonUnpacker struct { seen seenNames dec *json.Decoder } func (jup *jsonUnpacker) Next() (*Entry, error) { var e Entry err := jup.dec.Decode(&e) if err != nil { return nil, err } // check for dup name if e.Type == FileType { cName := filepath.Clean(e.GetName()) if _, ok := jup.seen[cName]; ok { return nil, ErrDuplicatePath } jup.seen[cName] = struct{}{} } return &e, err } // NewJSONUnpacker provides an Unpacker that reads Entries (SegmentType and // FileType) as a json document. // // Each Entry read are expected to be delimited by new line. func NewJSONUnpacker(r io.Reader) Unpacker { return &jsonUnpacker{ dec: json.NewDecoder(r), seen: seenNames{}, } } type jsonPacker struct { w io.Writer e *json.Encoder pos int seen seenNames } type seenNames map[string]struct{} func (jp *jsonPacker) AddEntry(e Entry) (int, error) { // if Name is not valid utf8, switch it to raw first. if e.Name != "" { if !utf8.ValidString(e.Name) { e.NameRaw = []byte(e.Name) e.Name = "" } } // check early for dup name if e.Type == FileType { cName := filepath.Clean(e.GetName()) if _, ok := jp.seen[cName]; ok { return -1, ErrDuplicatePath } jp.seen[cName] = struct{}{} } e.Position = jp.pos err := jp.e.Encode(e) if err != nil { return -1, err } // made it this far, increment now jp.pos++ return e.Position, nil } // NewJSONPacker provides a Packer that writes each Entry (SegmentType and // FileType) as a json document. // // The Entries are delimited by new line. func NewJSONPacker(w io.Writer) Packer { return &jsonPacker{ w: w, e: json.NewEncoder(w), seen: seenNames{}, } } /* TODO(vbatts) perhaps have a more compact packer/unpacker, maybe using msgapck (https://github.com/ugorji/go) Even though, since our jsonUnpacker and jsonPacker just take io.Reader/io.Writer, then we can get away with passing them a gzip.Reader/gzip.Writer */