Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix incorrect tar size calculation #270

Merged
merged 1 commit into from
Jul 31, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion pkg/shp/bundle/bundle.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,10 @@ func Push(ctx context.Context, io *genericclioptions.IOStreams, localDirectory s
defer progress.Close()
}

progress.ChangeMax64(update.Total)
if update.Total != progress.GetMax64() {
progress.ChangeMax64(update.Total)
}

_ = progress.Set64(update.Complete)
}
}
Expand Down
4 changes: 2 additions & 2 deletions pkg/shp/cmd/build/upload.go
Original file line number Diff line number Diff line change
Expand Up @@ -210,14 +210,14 @@ func (u *UploadCommand) performDataStreaming(target *streamer.Target) error {
return err
}

tarsize, err := streamer.GetTarSize(u.sourceDir)
size, err := tarball.Size()
if err != nil {
return err
}

// start writing the data using the tarball format, and streaming it via STDIN, which is
// redirected to the correct container
if err = u.dataStreamer.Stream(target, tarball.Create, tarsize.Size); err != nil {
if err = u.dataStreamer.Stream(target, tarball.Create, size); err != nil {
return err
}

Expand Down
8 changes: 4 additions & 4 deletions pkg/shp/streamer/streamer.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,10 @@ import (
"k8s.io/cli-runtime/pkg/genericclioptions"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"

progressbar "github.com/schollz/progressbar/v3"
"k8s.io/kubectl/pkg/cmd/exec"
"k8s.io/kubectl/pkg/util/interrupt"

progressbar "github.com/schollz/progressbar/v3"
)

// Streamer represents the actor that streams data onto a POD, running on Kubernetes. It does so via
Expand Down Expand Up @@ -43,7 +43,7 @@ func (s *Streamer) execute(opts *exec.ExecOptions) error {

// Stream the data onto the informed target, and it uses the BaseDir as the path to store the data on
// the running POD. The writerFn is employed to expose the writer interface to callers.
func (s *Streamer) Stream(target *Target, writerFn WriterFn, size int64) error {
func (s *Streamer) Stream(target *Target, writerFn WriterFn, size int) error {
var wg sync.WaitGroup
wg.Add(1)

Expand All @@ -60,7 +60,7 @@ func (s *Streamer) Stream(target *Target, writerFn WriterFn, size int64) error {
wg.Done()
}()

progress := progressbar.NewOptions(int(size),
progress := progressbar.NewOptions(size,
progressbar.OptionSetWriter(os.Stderr),
progressbar.OptionEnableColorCodes(true),
progressbar.OptionShowBytes(true),
Expand Down
3 changes: 1 addition & 2 deletions pkg/shp/streamer/streamer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,10 @@ func Test_Streamer(t *testing.T) {
BaseDir: "/",
}

var size int64 = 1000

// streaming mocked standard input data, and asserting both command informed is expected, and
// stdin is preserved
stdin := "standard input"
size := len(stdin)
err := s.Stream(targetPod, func(w io.Writer) error {
_, err := w.Write([]byte(stdin))
return err
Expand Down
43 changes: 9 additions & 34 deletions pkg/shp/streamer/tar.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import (
type Tar struct {
src string // base directory
gitIgnore *ignore.GitIgnore // matcher for git ignored files
Size int64
}

// skipPath inspect each path and makes sure it skips files the tar helper can't handle.
Expand Down Expand Up @@ -69,38 +68,14 @@ func NewTar(src string) (*Tar, error) {
return t, t.bootstrap()
}

func GetTarSize(src string) (*Tar, error) {
t := &Tar{src: src}
return t, t.tarSize()
}

func (t *Tar) tarSize() error {
var size int64

err := filepath.WalkDir(t.src, func(fpath string, d fs.DirEntry, err error) error {
if err != nil {
return err
}

stat, err := d.Info()
if err != nil {
return err
}

if t.skipPath(fpath, stat) {
return nil
}

header, err := tar.FileInfoHeader(stat, stat.Name())
if err != nil {
return err
}

header.Name = trimPrefix(t.src, fpath)
size += header.Size
return nil
})
// Size returns the size in bytes that a call of Create would write into the io.Writer
//
// Note: This performs a whote walk through of the source directory and can be an expensive operation.
func (t *Tar) Size() (int, error) {
wc := &writeCounter{}
if err := t.Create(wc); err != nil {
return -1, err
}

t.Size = size + size*1/100
return err
return wc.total, nil
}
8 changes: 8 additions & 0 deletions pkg/shp/streamer/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,14 @@ import (
"strings"
)

type writeCounter struct{ total int }

func (wc *writeCounter) Write(p []byte) (int, error) {
n := len(p)
wc.total += n
return n, nil
}

func trimPrefix(prefix, fpath string) string {
return strings.TrimPrefix(strings.Replace(fpath, prefix, "", -1), string(filepath.Separator))
}
Expand Down
Loading