Audiobookshelf upload progress calculation implemented

.gitignore

@@ -1,6 +1,6 @@
 bin/
 mock/
-output/
+tmp/
 dist/
 __debug_bin*
 abb_ia.log

internal/audiobookshelf/client.go

@@ -141,7 +141,7 @@ func (c *AudiobookshelfClient) ScanLibrary(libraryID string) error {
 }
 
 // Upload audiobook to The Audibookshelf server
-func (c *AudiobookshelfClient) UploadBook(ab *dto.Audiobook, libraryID string, folderID string, callback func(int, int) ) error {
+func (c *AudiobookshelfClient) UploadBook(ab *dto.Audiobook, libraryID string, folderID string, callback Fn ) error {
 	// Open each file for upload
 	var filesList []*os.File
 	for _, part := range ab.Parts {
@@ -175,7 +175,9 @@ func (c *AudiobookshelfClient) UploadBook(ab *dto.Audiobook, libraryID string, f
 		if err != nil {
 			return err
 		}
-		pr := &progressReader{
+		pr := &ProgressReader{
+			FileId:   i,
+			FileName: filepath.Base(file.Name()),
 			Reader:   file,
 			Size:     fileStat.Size(),
 			Callback: callback,
@@ -214,16 +216,24 @@ func (c *AudiobookshelfClient) UploadBook(ab *dto.Audiobook, libraryID string, f
 	return nil
 }
 
-type progressReader struct {
+// Progress Reader for file upload progress
+type Fn func(fileId int, fileName string, size int64, pos int64, percent int)
+type ProgressReader struct {
+	FileId   int
+	FileName string
 	Reader   io.Reader
 	Size     int64
-	Callback func(int, int)
+	Pos      int64
+	Percent  int
+	Callback Fn
 }
 
-func (pr *progressReader) Read(p []byte) (int, error) {
+func (pr *ProgressReader) Read(p []byte) (int, error) {
 	n, err := pr.Reader.Read(p)
 	if err == nil {
-		pr.Callback(n, int(pr.Size))
+		pr.Pos += int64(n)
+		pr.Percent = int(float64(pr.Pos) / float64(pr.Size) * 100)
+		pr.Callback(pr.FileId, pr.FileName, pr.Size, pr.Pos, pr.Percent)
 	}
 	return n, err
 }

internal/config/config.go

@@ -42,6 +42,7 @@ type Config struct {
 	SampleRateHz           int           `yaml:"SampleRateHz"`
 	MaxFileSizeMb          int           `yaml:"MaxFileSizeMb"`
 	UploadToAudiobookshef  bool          `yaml:"UploadToAudiobookshelf"`
+	ScanAudiobookshef      bool          `yaml:"ScanAudiobookshelf"`
 	AudiobookshelfUrl      string        `yaml:"AudiobookshelfUrl"`
 	AudiobookshelfUser     string        `yaml:"AudiobookshelfUser"`
 	AudiobookshelfPassword string        `yaml:"AudiobookshelfPassword"`
@@ -84,6 +85,7 @@ func Load() {
 	config.SampleRateHz = 44100
 	config.MaxFileSizeMb = 250
 	config.UploadToAudiobookshef = true
+	config.ScanAudiobookshef = true
 	config.AudiobookshelfUser = "admin"
 	config.AudiobookshelfPassword = ""
 	config.AudiobookshelfLibrary = "Internet Archive"
@@ -275,6 +277,14 @@ func (c *Config) IsUploadToAudiobookshef() bool {
 	return c.UploadToAudiobookshef
 }
 
+func (c *Config) SetScanAudiobookshelf(b bool) {
+	c.ScanAudiobookshef = b
+}
+
+func (c *Config) IsScanAudiobookshef() bool {
+	return c.ScanAudiobookshef
+}
+
 func (c *Config) GetAudiobookshelfUrl() string {
 	return c.AudiobookshelfUrl
 }

internal/controller/audiobookShelfController.go

@@ -1,16 +1,31 @@
 package controller
 
 import (
-	"strconv"
+	"fmt"
+	"time"
 
 	"github.com/vpoluyaktov/abb_ia/internal/audiobookshelf"
 	"github.com/vpoluyaktov/abb_ia/internal/dto"
 	"github.com/vpoluyaktov/abb_ia/internal/logger"
 	"github.com/vpoluyaktov/abb_ia/internal/mq"
+	"github.com/vpoluyaktov/abb_ia/internal/utils"
 )
 
 type AudiobookshelfController struct {
-	mq *mq.Dispatcher
+	mq        *mq.Dispatcher
+	ab        *dto.Audiobook
+	startTime time.Time
+	stopFlag  bool
+
+	// progress tracking arrays
+	filesUpload []fileUpload
+}
+
+type fileUpload struct {
+	fileId      int
+	fileSize    int64
+	bytesCopied int64
+	progress    int
 }
 
 func NewAudiobookshelfController(dispatcher *mq.Dispatcher) *AudiobookshelfController {
@@ -74,15 +89,16 @@ func (c *AudiobookshelfController) audiobookshelfScan(cmd *dto.AudiobookshelfSca
 		}
 		logger.Info("A scan launched for library " + libraryName + " on audiobookshlf server")
 	}
+	c.mq.SendMessage(mq.AudiobookshelfController, mq.BuildPage, &dto.ScanComplete{Audiobook: cmd.Audiobook}, true)
 }
 
 func (c *AudiobookshelfController) uploadAudiobook(cmd *dto.AudiobookshelfUploadCommand) {
 	logger.Info(mq.AudiobookshelfController + " received " + cmd.String())
-	ab := cmd.Audiobook
-	url := ab.Config.GetAudiobookshelfUrl()
-	username := ab.Config.GetAudiobookshelfUser()
-	password := ab.Config.GetAudiobookshelfPassword()
-	libraryName := ab.Config.GetAudiobookshelfLibrary()
+	c.ab = cmd.Audiobook
+	url := c.ab.Config.GetAudiobookshelfUrl()
+	username := c.ab.Config.GetAudiobookshelfUser()
+	password := c.ab.Config.GetAudiobookshelfPassword()
+	libraryName := c.ab.Config.GetAudiobookshelfLibrary()
 
 	if url != "" && username != "" && password != "" && libraryName != "" {
 		absClient := audiobookshelf.NewClient(url)
@@ -106,20 +122,92 @@ func (c *AudiobookshelfController) uploadAudiobook(cmd *dto.AudiobookshelfUpload
 			logger.Error("Can't get a folder for library: " + err.Error())
 			return
 		}
-
 		// TODO: Check if a folder selector is needed here. Let's use first folder in a library for upload
-		err = absClient.UploadBook(ab, libraryID, folders[0].ID, c.updateFileUplodProgress)
+		folderID := folders[0].ID
+
+		c.stopFlag = false
+		c.filesUpload = make([]fileUpload, len(c.ab.Parts))
+		go c.updateTotalUploadProgress()
+		err = absClient.UploadBook(c.ab, libraryID, folderID, c.updateFileUplodProgress)
+
 		if err != nil {
 			logger.Error("Can't upload the audiobook to audiobookshelf server: " + err.Error())
-			return
 		}
+		c.stopFlag = true
 	}
+	c.mq.SendMessage(mq.AudiobookshelfController, mq.BuildPage, &dto.UploadComplete{Audiobook: cmd.Audiobook}, true)
 }
 
-func (c *AudiobookshelfController) updateFileUplodProgress(bytesUploaded int, totalBytes int) {
+func (c *AudiobookshelfController) updateFileUplodProgress(fileId int, fileName string, size int64, pos int64, percent int) {
 
-	if totalBytes != 0 {
-		percent := bytesUploaded / totalBytes * 100
-		logger.Debug("Upload percent: " + strconv.Itoa(percent))
+	if c.filesUpload[fileId].progress != percent {
+		// wrong calculation protection
+		if percent < 0 {
+			percent = 0
+		} else if percent > 100 {
+			percent = 100
+		}
+
+		// sent a message only if progress changed
+		c.mq.SendMessage(mq.AudiobookshelfController, mq.BuildPage, &dto.UploadFileProgress{FileId: fileId, FileName: fileName, Percent: percent}, false)
+	}
+	c.filesUpload[fileId].fileId = fileId
+	c.filesUpload[fileId].fileSize = size
+	c.filesUpload[fileId].bytesCopied = pos
+	c.filesUpload[fileId].progress = percent
+}
+
+func (c *AudiobookshelfController) updateTotalUploadProgress() {
+	var percent int = -1
+
+	for !c.stopFlag && percent <= 100 {
+		var totalSize = c.ab.TotalSize
+		var totalBytesCopied int64 = 0
+		filesCopied := 0
+		for _, f := range c.filesUpload {
+			totalBytesCopied += f.bytesCopied
+			if f.progress == 100 {
+				filesCopied++
+			}
+		}
+
+		var p int = 0
+		if totalSize > 0 {
+			p = int(float64(totalBytesCopied) / float64(totalSize) * 100)
+		}
+
+		// fix wrong incorrect calculation
+		if filesCopied == len(c.filesUpload) {
+			p = 100
+			totalBytesCopied = c.ab.TotalSize
+		}
+
+		if percent != p {
+			// sent a message only if progress changed
+			percent = p
+
+			// wrong calculation protection
+			if percent < 0 {
+				percent = 0
+			} else if percent > 100 {
+				percent = 100
+			}
+
+			elapsed := time.Since(c.startTime).Seconds()
+			speed := int64(float64(totalBytesCopied) / elapsed)
+			eta := (100 / (float64(percent) / elapsed)) - elapsed
+			if eta < 0 || eta > (60*60*24*365) {
+				eta = 0
+			}
+
+			elapsedH := utils.SecondsToTime(elapsed)
+			bytesH := utils.BytesToHuman(totalBytesCopied)
+			filesH := fmt.Sprintf("%d/%d", filesCopied, len(c.ab.Parts))
+			speedH := utils.SpeedToHuman(speed)
+			etaH := utils.SecondsToTime(eta)
+
+			c.mq.SendMessage(mq.AudiobookshelfController, mq.BuildPage, &dto.UploadProgress{Elapsed: elapsedH, Percent: percent, Files: filesH, Bytes: bytesH, Speed: speedH, ETA: etaH}, false)
+		}
+		time.Sleep(mq.PullFrequency)
 	}
 }

internal/controller/buildController.go

@@ -85,6 +85,7 @@ func (c *BuildController) startBuild(cmd *dto.BuildCommand) {
 		}
 		part.AACFile = filePath + ".aac"
 		part.M4BFile = filePath + ".m4b"
+		c.files[i].fileName = part.M4BFile
 		c.files[i].totalDuration = part.Duration
 	}
 

internal/controller/cleanupController.go

@@ -52,4 +52,6 @@ func (c *CleanupController) cleanUp(cmd *dto.CleanupCommand) {
 		}
 	}
 	os.Remove(c.ab.CoverFile)
+
+	c.mq.SendMessage(mq.CleanupController, mq.BuildPage, &dto.CleanupComplete{Audiobook: cmd.Audiobook}, true)
 }

internal/controller/copyController.go

@@ -193,7 +193,7 @@ func (c *CopyController) updateFileCopyProgress(fileId int, fileName string, siz
 		}
 
 		// sent a message only if progress changed
-		c.mq.SendMessage(mq.CopyController, mq.BuildPage, &dto.CopyFileProgress{FileId: fileId, FileName: fileName, Percent: percent}, false)
+		c.mq.SendMessage(mq.CopyController, mq.BuildPage, &dto.UploadFileProgress{FileId: fileId, FileName: fileName, Percent: percent}, false)
 	}
 	c.filesCopy[fileId].fileId = fileId
 	c.filesCopy[fileId].fileSize = size

internal/dto/audiobookshelf.go

@@ -10,6 +10,14 @@ func (c *AudiobookshelfScanCommand) String() string {
 	return fmt.Sprintf("AudiobookshelfScanCommand: %s", c.Audiobook.String())
 }
 
+type ScanComplete struct {
+	Audiobook *Audiobook
+}
+
+func (c *ScanComplete) String() string {
+	return fmt.Sprintf("ScanComplete: %s", c.Audiobook.String())
+}
+
 type AudiobookshelfUploadCommand struct {
 	Audiobook *Audiobook
 }
@@ -18,10 +26,33 @@ func (c *AudiobookshelfUploadCommand) String() string {
 	return fmt.Sprintf("AudiobookshelfUploadCommand: %s", c.Audiobook.String())
 }
 
+type UploadFileProgress struct {
+	FileId   int
+	FileName string
+	Percent  int
+}
+
+func (c *UploadFileProgress) String() string {
+	return fmt.Sprintf("UploadFileProgress: %d, %s, %d", c.FileId, c.FileName, c.Percent)
+}
+
+type UploadProgress struct {
+	Elapsed string // time since started
+	Percent int
+	Files   string // files encoded
+	Bytes   string // total bytes copied
+	Speed   string // encode speed bytes/s
+	ETA     string // ETA in seconds
+}
+
+func (c *UploadProgress) String() string {
+	return fmt.Sprintf("UploadProgress: %d", c.Percent)
+}
+
 type UploadComplete struct {
 	Audiobook *Audiobook
 }
 
 func (c *UploadComplete) String() string {
 	return fmt.Sprintf("UploadComplete: %s", c.Audiobook.String())
-}
+}

internal/dto/cleanup.go

@@ -8,4 +8,12 @@ type CleanupCommand struct {
 
 func (c *CleanupCommand) String() string {
 	return fmt.Sprintf("CleanupCommand: %s", c.Audiobook.String())
-}
+}
+
+type CleanupComplete struct {
+	Audiobook *Audiobook
+}
+
+func (c *CleanupComplete) String() string {
+	return fmt.Sprintf("CleanupComplete: %s", c.Audiobook.String())
+}

internal/dto/copy.go

@@ -39,4 +39,4 @@ type CopyComplete struct {
 
 func (c *CopyComplete) String() string {
 	return fmt.Sprintf("CopyComplete: %s", c.Audiobook.String())
-}
+}

internal/mq/recepients.go

@@ -21,7 +21,6 @@ const (
 	ChaptersController       = "ChaptersController"
 	BuildController          = "BuildController"
 	CopyController           = "CopyController"
-	UploadController         = "UploadController"
 	CleanupController        = "CleanupController"
 	AudiobookshelfController = "AudiobookshelfController"
 )