Merge pull request #350 from pixlise/feature/archive-optimiser-tool

Feature/archive optimiser tool
pixlise · Nov 11, 2024 · f1476b9 · f1476b9
2 parents 496f49d + 4b9663a
commit f1476b9
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 8 deletions.
diff --git a/api/dataimport/sdfToRSI/scanSDF_test.go b/api/dataimport/sdfToRSI/scanSDF_test.go
@@ -6,7 +6,7 @@ func Example_scanSDF() {
 	ensureSDFRawExists()
 
 	refs, err := scanSDF("./test-data/BadPath.txt")
-	fmt.Printf("%v|%v\n", len(refs), err)
+	fmt.Printf("%v|%v\n", len(refs), err != nil)
 
 	refs, err = scanSDF("./test-data/sdf_raw.txt")
 	fmt.Printf("err: %v\n", err)
@@ -15,7 +15,7 @@ func Example_scanSDF() {
 	}
 
 	// Output:
-	// 0|open ./test-data/BadPath.txt: The system cannot find the file specified.
+	// 0|true
 	// err: <nil>
 	// 439: start: ''
 	// 464: first-time: '2022-301T13:50:28'

diff --git a/api/dataimport/sdfToRSI/sdfToRSI_test.go b/api/dataimport/sdfToRSI/sdfToRSI_test.go
@@ -31,7 +31,7 @@ func ensureSDFRawExists() {
 
 		for _, f := range archive.File {
 			if len(archive.File) != 1 || f.Name != "sdf_raw.txt" {
-				log.Fatalln("Expected zdf_raw.zip to only contain one file: sdf_raw.txt")
+				log.Fatalln("Expected sdf_raw.zip to only contain one file: sdf_raw.txt")
 			}
 
 			dstFile, err := os.OpenFile("./test-data/sdf_raw.txt", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
@@ -51,4 +51,13 @@ func ensureSDFRawExists() {
 			}
 		}
 	}
+
+	err = os.MkdirAll("./output", 0644)
+	fmt.Printf("mkdir worked: %v\n", err == nil)
+	files, rtts, err := ConvertSDFtoRSIs("./test-data/sdf_raw.txt", "./output/")
+	fmt.Printf("%v, %v: %v\n", files, rtts, err)
+
+	// Output:
+	// mkdir worked: true
+	// [RSI-208536069.csv RSI-208601602.csv], [208536069 208601602]: <nil>
 }
diff --git a/core/utils/zip-file.go b/core/utils/zip-file.go
@@ -38,7 +38,6 @@ func AddFilesToZip(w *zip.Writer, basePath, baseInZip string) error {
 
 	for _, file := range files {
 		readPath := filepath.Join(basePath, file.Name())
-		fmt.Println(readPath)
 		if !file.IsDir() {
 			dat, err := os.ReadFile(readPath)
 			if err != nil {

diff --git a/internal/cmd-line-tools/dataset-archive-optimiser/main.go b/internal/cmd-line-tools/dataset-archive-optimiser/main.go
@@ -159,27 +159,33 @@ func optimise(rtts map[string]string, remoteFS fileaccess.FileAccess, iLog logge
 		fatalError(err)
 	}
 
+	iLog.Infof("Found %v scans to optimise archive for:", len(rttToArchiveFiles))
+	for rtt, files := range rttToArchiveFiles {
+		iLog.Infof(" %v: %v zip files", rtt, len(files))
+	}
+
 	// Handle each file
 	for rtt, _ := range rttToArchiveFiles {
 		localArchivePath, zipFilesOptimised, err := makeOptimisedArchive(rtt, rtts[rtt], remoteFS, workingDir, iLog)
 		if err != nil {
-			fmt.Printf("Error creating optimised archive for %v: %v\n", rtt, err)
+			iLog.Errorf("Error creating optimised archive for %v: %v\n", rtt, err)
 		} else {
 			// Upload the optimised file (should be overwriting the latest one)
 			err = upload(localArchivePath, "Archive", remoteFS, iLog)
 			if err != nil {
-				fmt.Printf("FAILED TO UPLOAD archive file %v: %v\n", localArchivePath, err)
+				iLog.Errorf("FAILED TO UPLOAD archive file %v: %v\n", localArchivePath, err)
 			}
 
 			// Delete the zips that we are replacing
 			for _, zipFile := range zipFilesOptimised {
-
 				// Don't delete what we just uploaded!
 				if !strings.HasSuffix(localArchivePath, zipFile) {
 					zipPath := path.Join("Archive", zipFile)
+
+					iLog.Infof("Deleting from S3: %v", zip.ErrInsecurePath)
 					err = remoteFS.DeleteObject(dataBucket, zipPath)
 					if err != nil {
-						fmt.Printf("Error deleting archive file %v: %v\n", zipPath, err)
+						iLog.Errorf("Error deleting archive file %v: %v\n", zipPath, err)
 					}
 				}
 			}