diff --git a/api/filepaths/filepaths.go b/api/filepaths/filepaths.go index f18304e0..e92a98cd 100644 --- a/api/filepaths/filepaths.go +++ b/api/filepaths/filepaths.go @@ -21,9 +21,6 @@ package filepaths import ( "path" - "strings" - - "github.com/pixlise/core/v4/core/fileaccess" ) // This package contains all file paths that the PIXLISE API should ever need to access. All been centralised @@ -104,11 +101,6 @@ const RootDetectorConfig = "DetectorConfig" // TODO: remove that const PiquantConfigSubDir = "PiquantConfigs" -// Get a PIXLISE detector config file path given the config name -func GetDetectorConfigFilePath(configName string) string { - return path.Join(RootDetectorConfig, configName, "pixlise-config.json") -} - // Get a detector config path (used by PIQUANT) given the config name, version and optionally a file name. If file name is blank // then the directory path above it is returned func GetDetectorConfigPath(configName string, version string, fileName string) string { @@ -138,82 +130,12 @@ const RootPixliseConfigPath = "PixliseConfig" // Auth0 PEM file which API uses to verify JWTs const Auth0PemFileName = "auth0.pem" -// Gets dataset list file (which UI shows as tiles). -// NOTE: This is regenerated every time Datasets/ changes by lambda function: dataset-tile-updater -func GetDatasetListPath() string { - return GetConfigFilePath("datasets.json") -} - -func GetDatasetsAuthPath() string { - return GetConfigFilePath("datasets-auth.json") -} - -func GetPublicObjectsPath() string { - return GetConfigFilePath("public-objects.json") -} - -// Config contains the docker container to use for PIQUANT. Separate from config.json because users can configure this in UI -const PiquantVersionFileName = "piquant-version.json" - -// Contains a list of Dataset IDs to ignore when generating dataset tiles. This is hand -// maintained, only used when we have a bad dataset is downloaded that will never be usable -// This way we can prevent it from being written to /PixliseConfig/datasets.json -// Just a work-around for having OCS fetcher put files there without our control. If a bad -// dataset download happens, there's no point in showing a broken tile for it forever! -const BadDatasetIDsFile = "bad-dataset-ids.json" - // Getting a config file path relative to the root of the bucket func GetConfigFilePath(fileName string) string { return path.Join(RootPixliseConfigPath, fileName) } -//////////////////////////////////////////////////////////////////////////////////// -// User Content Bucket -//////////////////////////////////////////////////////////////////////////////////// - -/* -Root directory for bucket containing all user-created content - - UserContent/ - - ----/ - - --------ElementSets.json - User created element sets - - --------DataExpressions.json - User created expressions - - --------RGBMixes.json - User created RGB mixes - - --------/ - - ------------ROI.json - User created ROIs - - ------------Tags.json - Dataset tags - - ------------SpectrumAnnotation.json - - ------------multi-quant-z-stack.json - The current z-stack on multi-quant panel - - ------------Quantifications/ - - ----------------.bin - The combined.csv file converted to protobuf binary format by quant-converter - - ----------------.csv - Copied from Job Bucket/JobData//output/combined.csv - - -----------------logs/ - Copied from Job Bucket/JobData//piquant-logs/ - - ----------------summary-.json - Quant summary file - - ------------LastPiquantOutput/ - Last output of fit command - - ------------ViewState/ - User view states for each dataset. This stores UI info about how the view was configured - - ----------------quantification.json - The quantification loaded on UI top toolbar - - ----------------roi.json - Colours assigned to ROIs on the UI - - ----------------selection.json - The users current selection of PMCs and/or pixels on UI - - ----------------analysisLayout.json - What widgets go where, top row/bottom row - - ------------------.json - States of various UI panels and where they are - - ----------------- See: GetViewStatePath() - - ----------------- Workspaces/ - - --------------------.json - View state files (like up one directory) flattened to a file and given a workspace name. Note the file also contains the workspace name, the file name may have been modified for saving, eg removal of / - - -------------------- See: GetWorkspacePath() - - -------------------- WorkspaceCollections/ - - --------------------.json - - -------------------- See: GetCollectionPath() - - ----shared/ - All shared objects go here. Kind of like if they belong to a user called "shared". NOTE: User diffraction/roughness files are shared by default! -*/ const RootUserContent = "UserContent" -const elementSetFile = "ElementSets.json" -const rgbMixFile = "RGBMixes.json" -const roiFile = "ROI.json" -const tagFile = "Tags.json" - -// Multi-quant z-stack file name -const MultiQuantZStackFile = "multi-quant-z-stack.json" - -const annotationFile = "SpectrumAnnotation.json" const RootQuantificationPath = "Quantifications" @@ -224,8 +146,6 @@ func GetQuantPath(userId string, scanId string, fileName string) string { return path.Join(RootQuantificationPath, scanId, userId) } -const quantLastOutputSubPath = "LastPiquantOutput" - // File name of last piquant output (used with fit command). Extension added as needed const QuantLastOutputFileName = "output_data" @@ -233,32 +153,13 @@ const QuantLastOutputFileName = "output_data" const QuantLastOutputLogName = "output.log" // Retrieve path for last outputs, eg last run fit command (command is actually "quant"), sits in here with its log file +const rootLastQuantOutput = "LastQuantOutput" + func GetUserLastPiquantOutputPath(userID string, datasetID string, piquantCommand string, fileName string) string { if len(fileName) > 0 { - return path.Join(RootUserContent, userID, datasetID, quantLastOutputSubPath, piquantCommand, fileName) + return path.Join(rootLastQuantOutput, datasetID, userID, piquantCommand, fileName) } - return path.Join(RootUserContent, userID, datasetID, quantLastOutputSubPath, piquantCommand) -} - -const viewStatePath = "ViewState" - -// TODO: Move files directly in ViewState/ into their own directory, eg Last/ because -// currently we have to check what we're deleting if resetting view state for example - -// Sub-dir containing all workspaces. These are saved copies of view states -const ViewStateSavedSubpath = "Workspaces" - -// Sub-dir containing all workspace collections. These are flat files containing all workspaces they were created from -const ViewStateCollectionsSubpath = "WorkspaceCollections" - -// Gets user content file path by user id and file name -func GetUserContentPath(userID string, fileName string) string { - return path.Join(RootUserContent, userID, fileName) -} - -// Gets user content file path for a given dataset id. Also requires user id, and the file name -func GetUserContentDatasetPath(userID string, datasetID string, fileName string) string { - return path.Join(RootUserContent, userID, datasetID, fileName) + return path.Join(rootLastQuantOutput, datasetID, userID, piquantCommand) } // Name of user manually entered diffraction peaks file. NOTE: this file only exists as a shared file! @@ -269,13 +170,6 @@ const DiffractionPeakManualFileName = "manual-diffraction-peaks.json" // OTE: this file only exists as a shared file! const DiffractionPeakStatusFileName = "diffraction-peak-statuses.json" -/* -Root directory containing all user activity stored, to track clicks and user flows for research purposes - - Activity/ - - -----------/.json - User activity files (things captured by middleware logger) -*/ -const RootUserActivity = "Activity" - //////////////////////////////////////////////////////////////////////////////////// // Job Bucket //////////////////////////////////////////////////////////////////////////////////// @@ -314,42 +208,6 @@ func GetJobDataPath(datasetID string, jobID string, fileName string) string { return path.Join(RootJobData, datasetID) } -/* -Root directory for all job statuses. These are stored separately to JobData so we can easily list all jobs and -query their statuses - - JobStatus/ - - ----/-status.json -*/ -const RootJobStatus = "JobStatus" - -// Job status file name suffix. Appended to the job ID -const JobStatusSuffix = "-status.json" - -// Retrieves the path of a job status file for given dataset id and job id. If job id is blank, this just -// returns the root of all job statuses for the given datset id -func GetJobStatusPath(datasetID string, jobID string) string { - if len(jobID) <= 0 { - return path.Join(RootJobStatus, datasetID) - } - return path.Join(RootJobStatus, datasetID, jobID+JobStatusSuffix) -} - -/* -Root directory for all job summaries. This is stored separately to JobData so we can easily list all jobs -and get their metadata (summary) files - - JobSummaries/ - - -----jobs.json - Summary files describing all jobs for a dataset -*/ -const RootJobSummaries = "JobSummaries" - -// Job summary file name suffix. Appended to dataset ID -const JobSummarySuffix = "-jobs.json" - -// Gets the job summary path for a given dataset ID -func GetJobSummaryPath(datasetID string) string { - return path.Join(RootJobSummaries, datasetID+JobSummarySuffix) -} - //////////////////////////////////////////////////////////////////////////////////// // Artifacts Manual Upload Data Source Bucket //////////////////////////////////////////////////////////////////////////////////// @@ -372,21 +230,6 @@ const DatasetCustomRoot = "dataset-addons" // File name for dataset custom meta file containing the title and other settings const DatasetCustomMetaFileName = "custom-meta.json" -// Get the custom meta file path for a given dataset ID -func GetCustomMetaPath(datasetID string) string { - return path.Join(DatasetCustomRoot, datasetID, DatasetCustomMetaFileName) -} - -// Get the custom image path for a given dataset ID. Note imageType must be one of UNALIGNED, MATCHED or RGBU -func GetCustomImagePath(datasetID string, imgType string, fileName string) string { - // NOTE: We assume imgType is valid! - s3Path := path.Join(DatasetCustomRoot, datasetID, strings.ToUpper(imgType)) - if len(fileName) > 0 { - s3Path = path.Join(s3Path, fileName) - } - return s3Path -} - /* Root directory to store uploaded dataset "raw" artifacts. These are then read by dataset importer to create a dataset in the dataset bucket @@ -399,102 +242,10 @@ to create a dataset in the dataset bucket */ const DatasetUploadRoot = "UploadedDatasets" -// Gets the path to a file in the dataset upload area, for a given dataset id and file name -func GetDatasetUploadPath(datasetID string, fileName string) string { - return path.Join(DatasetUploadRoot, datasetID, fileName) -} - -//////////////////////////////////////////////////////////////////////////////////// -// Artifacts Built Bucket - where we go to download built PIQUANT, etc -//////////////////////////////////////////////////////////////////////////////////// - -/* -PIQUANT binaries root file - this kind of went unused and is likely not working because -our build process doesn't write to the bucket any more - - piquant/ - - ----piquant-linux-*.zip - Built PIQUANT executables (zipped) -*/ -const PiquantDownloadPath = "piquant" - -//////////////////////////////////////////////////////////////////////////////////// -// Some specific helper functions for better searchability/grouping -//////////////////////////////////////////////////////////////////////////////////// - -// Getting element set file path for a user -func GetElementSetPath(userID string) string { - return GetUserContentPath(userID, elementSetFile) -} - -// Getting RGB mix file path for a user -func GetRGBMixPath(userID string) string { - return GetUserContentPath(userID, rgbMixFile) -} - -// Getting ROI file path for a user and dataset -func GetROIPath(userID string, datasetID string) string { - return GetUserContentDatasetPath(userID, datasetID, roiFile) -} - -// Getting tag file path for a user -func GetTagPath(userID string) string { - return GetUserContentPath(userID, tagFile) -} - -// Getting multi-quant z-stack file path for a user and dataset -func GetMultiQuantZStackPath(userID string, datasetID string) string { - return GetUserContentDatasetPath(userID, datasetID, MultiQuantZStackFile) -} - -// Getting spectrum annotations file path for a user and dataset -func GetAnnotationsPath(userID string, datasetID string) string { - return GetUserContentDatasetPath(userID, datasetID, annotationFile) -} - -// Getting view state file path for a user, dataset and file name. Note if file name is blank, this just returns the directory -func GetViewStatePath(userID string, datasetID string, fileName string) string { - if len(fileName) <= 0 { - // Just return the root of this - return path.Join(RootUserContent, userID, datasetID, viewStatePath) - } - return path.Join(RootUserContent, userID, datasetID, viewStatePath, fileName+".json") -} - -// Getting workspace file path for a user, dataset and workspace ID. Note if id is blank, this just returns the directory -// Validates ids to make sure they are valid (because the id is actually part of the file name) -func GetWorkspacePath(userID string, datasetID string, id string) string { - if len(id) <= 0 { - // Just return the root of this - return path.Join(RootUserContent, userID, datasetID, viewStatePath, ViewStateSavedSubpath) - } - // ensure it's a valid file name - id = fileaccess.MakeValidObjectName(id, true) - return path.Join(RootUserContent, userID, datasetID, viewStatePath, ViewStateSavedSubpath, id+".json") -} - -// Getting collection file path for a user, dataset and workspace ID. Note if id is blank, this just returns the directory -// Validates ids to make sure they are valid (because the id is actually part of the file name) -func GetCollectionPath(userID string, datasetID string, id string) string { - if len(id) <= 0 { - // Just return the root of this - return path.Join(RootUserContent, userID, datasetID, viewStatePath, ViewStateCollectionsSubpath) - } - // ensure it's a valid file name - id = fileaccess.MakeValidObjectName(id, true) - return path.Join(RootUserContent, userID, datasetID, viewStatePath, ViewStateCollectionsSubpath, id+".json") -} - //////////////////////////////////////////////////////////////////////////////////// // Helpers for forming certain file names //////////////////////////////////////////////////////////////////////////////////// -// QuantSummaryFilePrefix - summary files are summary-.json -const QuantSummaryFilePrefix = "summary-" - -// MakeQuantSummaryFileName - Given a quant ID, generates the file name: summary-.json (use this for searchability/consistency) -func MakeQuantSummaryFileName(quantID string) string { - return QuantSummaryFilePrefix + quantID + ".json" -} - // QuantFileSuffix - quant files are .bin const QuantFileSuffix = ".bin" diff --git a/api/filepaths/filepaths_test.go b/api/filepaths/filepaths_test.go deleted file mode 100644 index a555f5f1..00000000 --- a/api/filepaths/filepaths_test.go +++ /dev/null @@ -1,30 +0,0 @@ -// Licensed to NASA JPL under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. NASA JPL licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package filepaths - -import "fmt" - -// Check the summary filename gets created correctly -func Example_makeQuantSummaryFileName() { - fn := MakeQuantSummaryFileName("myquantid") - - fmt.Printf("%v", fn) - - // Output: - // summary-myquantid.json -} diff --git a/internal/cmd-line-tools/v3-importer/main.go b/internal/cmd-line-tools/v3-importer/main.go index 1d5ed3cb..90790d19 100644 --- a/internal/cmd-line-tools/v3-importer/main.go +++ b/internal/cmd-line-tools/v3-importer/main.go @@ -398,19 +398,6 @@ func main() { fmt.Println("Skipping migration of diffraction peaks...") } - // NOTE: we don't actually migrate them any more... new world is too different. Just left - // here for completeness/documentation of what was tried, and it does still clear the - // View States DB collection! - wg.Add(1) - go func() { - defer wg.Done() - fmt.Println("View States...") - err = migrateViewStates(userContentBucket, userContentPaths, fs, destDB) - if err != nil { - fatalError(err) - } - }() - // Wait for all wg.Wait() printFinishStats() diff --git a/internal/cmd-line-tools/v3-importer/piquantConfigs.go b/internal/cmd-line-tools/v3-importer/piquantConfigs.go index 3c262e54..9e30fd25 100644 --- a/internal/cmd-line-tools/v3-importer/piquantConfigs.go +++ b/internal/cmd-line-tools/v3-importer/piquantConfigs.go @@ -18,6 +18,9 @@ type SrcPiquantVersionConfig struct { Creator SrcUserInfo `json:"creator"` } +// Config contains the docker container to use for PIQUANT. Separate from config.json because users can configure this in UI +const piquantVersionFileName = "piquant-version.json" + func migratePiquantVersion(configBucket string, fs fileaccess.FileAccess, dest *mongo.Database) error { coll := dest.Collection(dbCollections.PiquantVersionName) err := coll.Drop(context.TODO()) @@ -26,7 +29,7 @@ func migratePiquantVersion(configBucket string, fs fileaccess.FileAccess, dest * } ver := SrcPiquantVersionConfig{} - err = fs.ReadJSON(configBucket, filepaths.GetConfigFilePath(filepaths.PiquantVersionFileName), &ver, false) + err = fs.ReadJSON(configBucket, filepaths.GetConfigFilePath(piquantVersionFileName), &ver, false) if err != nil { return err } diff --git a/internal/cmd-line-tools/v3-importer/viewStates.go b/internal/cmd-line-tools/v3-importer/viewStates.go deleted file mode 100644 index b2953914..00000000 --- a/internal/cmd-line-tools/v3-importer/viewStates.go +++ /dev/null @@ -1,437 +0,0 @@ -package main - -import ( - "context" - - "github.com/pixlise/core/v4/api/dbCollections" - "github.com/pixlise/core/v4/core/fileaccess" - "go.mongodb.org/mongo-driver/mongo" -) - -/* - func getViewStateFiles(state *wholeViewState, fs fileaccess.FileAccess, bucket string, datasetID string, userID string) error { - items, err := fs.ListObjects( - bucket, - filepaths.GetViewStatePath(userID, datasetID, "")+"/", - ) - if err != nil { - return err - } - - for _, path := range items { - if filepath.Ext(path) != ".json" { - continue // skip, it's not something we care about - } - - fileNameOnly := filepath.Base(path[0 : len(path)-5]) - dataType, whichInstance := splitWidgetFileName(fileNameOnly) - - // Try read in to ones which only return a single object (not looking for whichInstance) - if dataType == "spectrum" && len(whichInstance) <= 0 { - // OLD STYLE spectrum pre configurable spectrum position, only here for backwards compatibility with existing - // view states newer ones are saved with whichInstance set to something to identify their position - err := fs.ReadJSON(bucket, path, &state.Spectrum, false) - if err != nil && !fs.IsNotFoundError(err) { - return err - } - } else if dataType == "quantification" { - err = fs.ReadJSON(bucket, path, &state.Quantification, false) - if err != nil && !fs.IsNotFoundError(err) { - return err - } - - applyQuantByROIFallback(&state.Quantification) - } else if dataType == "selection" { - err = fs.ReadJSON(bucket, path, &state.Selection, false) - if err != nil && !fs.IsNotFoundError(err) { - return err - } - } else if dataType == "roi" { - err = fs.ReadJSON(bucket, path, &state.ROIs, false) - if err != nil && !fs.IsNotFoundError(err) { - return err - } - } else if dataType == "analysisLayout" { - err = fs.ReadJSON(bucket, path, &state.AnalysisLayout, false) - if err != nil && !fs.IsNotFoundError(err) { - return err - } - } else if dataType == "annotations" { - err = fs.ReadJSON(bucket, path, &state.Annotations, false) - if err != nil && !fs.IsNotFoundError(err) { - return err - } - } else if len(whichInstance) > 0 { - // Check the ones where we DO require an instance name - if dataType == "contextImage" { - contextImage := defaultContextImage() - err = fs.ReadJSON(bucket, path, &contextImage, false) - if err != nil && !fs.IsNotFoundError(err) { - return err - } - state.ContextImages[whichInstance] = contextImage - } else if dataType == "histogram" { - histogram := defaultHistogram() - err = fs.ReadJSON(bucket, path, &histogram, false) - if err != nil && !fs.IsNotFoundError(err) { - return err - } - state.Histograms[whichInstance] = histogram - } else if dataType == "chord" { - chord := defaultChordDiagram() - err = fs.ReadJSON(bucket, path, &chord, false) - if err != nil && !fs.IsNotFoundError(err) { - return err - } - state.ChordDiagrams[whichInstance] = chord - } else if dataType == "ternary" { - ternary := defaultTernaryPlot() - err = fs.ReadJSON(bucket, path, &ternary, false) - if err != nil && !fs.IsNotFoundError(err) { - return err - } - state.TernaryPlots[whichInstance] = ternary - } else if dataType == "binary" { - binary := defaultBinaryPlot() - err = fs.ReadJSON(bucket, path, &binary, false) - if err != nil && !fs.IsNotFoundError(err) { - return err - } - state.BinaryPlots[whichInstance] = binary - } else if dataType == "table" { - table := defaultTable() - err = fs.ReadJSON(bucket, path, &table, false) - if err != nil && !fs.IsNotFoundError(err) { - return err - } - state.Tables[whichInstance] = table - } else if dataType == "roiQuantTable" { - table := defaultROIQuantTable() - err = fs.ReadJSON(bucket, path, &table, false) - if err != nil && !fs.IsNotFoundError(err) { - return err - } - state.ROIQuantTables[whichInstance] = table - } else if dataType == "variogram" { - vario := defaultVariogram() - err = fs.ReadJSON(bucket, path, &vario, false) - if err != nil && !fs.IsNotFoundError(err) { - return err - } - state.Variograms[whichInstance] = vario - } else if dataType == "rgbuPlot" { - rgbu := defaultRGBUPlot() - err = fs.ReadJSON(bucket, path, &rgbu, false) - if err != nil && !fs.IsNotFoundError(err) { - return err - } - state.RGBUPlots[whichInstance] = rgbu - } else if dataType == "singleAxisRGBU" { - singleAxisRGBU := defaultSingleAxisRGBU() - err = fs.ReadJSON(bucket, path, &singleAxisRGBU, false) - if err != nil && !fs.IsNotFoundError(err) { - return err - } - state.SingleAxisRGBU[whichInstance] = singleAxisRGBU - } else if dataType == "rgbuImages" { - rgbu := defaultRGBUImages() - err = fs.ReadJSON(bucket, path, &rgbu, false) - if err != nil && !fs.IsNotFoundError(err) { - return err - } - state.RGBUImageViews[whichInstance] = rgbu - } else if dataType == "parallelogram" { - pgram := defaultParallelogram() - err = fs.ReadJSON(bucket, path, &pgram, false) - if err != nil && !fs.IsNotFoundError(err) { - return err - } - state.Parallelograms[whichInstance] = pgram - } else if dataType == "spectrum" { - spectrum := defaultSpectrum() - err := fs.ReadJSON(bucket, path, &spectrum, false) - if err != nil && !fs.IsNotFoundError(err) { - return err - } - state.Spectrums[whichInstance] = spectrum - } - } - } - - filterUnusedWidgetStates(state) - - return nil - } - - func filterUnusedWidgetStates(state *wholeViewState) { - // Make a list of allowed items from the analysis layout values - // Unfortunately the names of things don't match in analysis layout vs the file names for the widgets :( - // This is due to the evolution of our saving of view state, early on we didn't even have configurable layouts - // and things were added gradually. - - // NOTE: we don't do any filtering unless the layout sections have been filled out. For example, with a blank view - // state we want the UI to decide what to do with the widget data we pass in - if len(state.AnalysisLayout.TopWidgetSelectors) != 2 || len(state.AnalysisLayout.BottomWidgetSelectors) != 4 { - return - } - - layoutNameToWidgetFileName := map[string]string{ - "chord-view-widget": "chord", - "binary-plot-widget": "binary", - "ternary-plot-widget": "ternary", - "table-widget": "table", - "histogram-widget": "histogram", - "variogram-widget": "variogram", - "rgbu-viewer-widget": "rgbuImages", - "rgbu-plot-widget": "rgbuPlot", - "single-axis-rgbu-widget": "singleAxisRGBU", - "parallel-coords-widget": "parallelogram", - "spectrum-widget": "spectrum", - "context-image": "contextImage", - "roi-quant-table-widget": "roiQuantTable", - } - - // These are always allowed through... - allowedItems := []string{"contextImage-analysis", "contextImage-map"} - - for c := 0; c < len(state.AnalysisLayout.TopWidgetSelectors); c++ { - allowedItems = append(allowedItems, layoutNameToWidgetFileName[state.AnalysisLayout.TopWidgetSelectors[c]]+"-top"+strconv.Itoa(c)) - } - - for c := 0; c < len(state.AnalysisLayout.BottomWidgetSelectors); c++ { - posName := "undercontext" - if c > 0 { - posName = "underspectrum" + strconv.Itoa(c-1) - } - allowedItems = append(allowedItems, layoutNameToWidgetFileName[state.AnalysisLayout.BottomWidgetSelectors[c]]+"-"+posName) - } - - for widgetIdentifier := range state.ContextImages { - if !utils.ItemInSlice("contextImage-"+widgetIdentifier, allowedItems) { - delete(state.ContextImages, widgetIdentifier) - } - } - for widgetIdentifier := range state.Histograms { - if !utils.ItemInSlice("histogram-"+widgetIdentifier, allowedItems) { - delete(state.Histograms, widgetIdentifier) - } - } - for widgetIdentifier := range state.ChordDiagrams { - if !utils.ItemInSlice("chord-"+widgetIdentifier, allowedItems) { - delete(state.ChordDiagrams, widgetIdentifier) - } - } - for widgetIdentifier := range state.BinaryPlots { - if !utils.ItemInSlice("binary-"+widgetIdentifier, allowedItems) { - delete(state.BinaryPlots, widgetIdentifier) - } - } - for widgetIdentifier := range state.TernaryPlots { - if !utils.ItemInSlice("ternary-"+widgetIdentifier, allowedItems) { - delete(state.TernaryPlots, widgetIdentifier) - } - } - for widgetIdentifier := range state.Tables { - if !utils.ItemInSlice("table-"+widgetIdentifier, allowedItems) { - delete(state.Tables, widgetIdentifier) - } - } - for widgetIdentifier := range state.ROIQuantTables { - if !utils.ItemInSlice("roiQuantTable-"+widgetIdentifier, allowedItems) { - delete(state.ROIQuantTables, widgetIdentifier) - } - } - for widgetIdentifier := range state.Variograms { - if !utils.ItemInSlice("variogram-"+widgetIdentifier, allowedItems) { - delete(state.Variograms, widgetIdentifier) - } - } - for widgetIdentifier := range state.RGBUPlots { - if !utils.ItemInSlice("rgbuPlot-"+widgetIdentifier, allowedItems) { - delete(state.RGBUPlots, widgetIdentifier) - } - } - for widgetIdentifier := range state.SingleAxisRGBU { - if !utils.ItemInSlice("singleAxisRGBU-"+widgetIdentifier, allowedItems) { - delete(state.SingleAxisRGBU, widgetIdentifier) - } - } - for widgetIdentifier := range state.RGBUImageViews { - if !utils.ItemInSlice("rgbuImages-"+widgetIdentifier, allowedItems) { - delete(state.RGBUImageViews, widgetIdentifier) - } - } - for widgetIdentifier := range state.Parallelograms { - if !utils.ItemInSlice("parallelogram-"+widgetIdentifier, allowedItems) { - delete(state.Parallelograms, widgetIdentifier) - } - } - for widgetIdentifier := range state.Spectrums { - if !utils.ItemInSlice("spectrum-"+widgetIdentifier, allowedItems) { - delete(state.Spectrums, widgetIdentifier) - } - } - } - - func splitWidgetFileName(fileNameOnly string) (string, string) { - dataType := fileNameOnly - whichInstance := "" - - dashIdx := strings.Index(fileNameOnly, "-") - if dashIdx > 0 { - dataType = fileNameOnly[0:dashIdx] - whichInstance = fileNameOnly[dashIdx+1:] - } - - return dataType, whichInstance - } -*/ -func migrateViewStates(userContentBucket string, userContentFiles []string, fs fileaccess.FileAccess, dest *mongo.Database) error { - coll := dest.Collection(dbCollections.ViewStatesName) - err := coll.Drop(context.TODO()) - if err != nil { - return err - } - /* - destViewStates := []interface{}{} - - viewStatesToSave := map[string][]string{} - - for _, p := range userContentFiles { - if filepath.Base(filepath.Dir(p)) == "ViewState" { - if strings.HasPrefix(p, "UserContent/shared/") { - return fmt.Errorf("Unexpected view state path: %v\n", p) - } else { - datasetId := filepath.Base(filepath.Dir(filepath.Dir(p))) - userId := utils.FixUserId(filepath.Base(filepath.Dir(filepath.Dir(filepath.Dir(p))))) - - // Form an id that should make a scan unique - id := userId + "_" + datasetId - - viewStatesToSave[id] = []string{userId, datasetId} - } - } - } - - // We've found all the view states we want to save, now do those, because each view state - // spans multiple files - for id, bits := range viewStatesToSave { - if maxItemsToRead > 0 && len(destViewStates) >= maxItemsToRead { - break - } - - userId := bits[0] - datasetId := bits[1] - - state := defaultWholeViewState() - err = getViewStateFiles(&state, fs, userContentBucket, datasetId, userId) - if err != nil { - return err - } - - contextImages := map[string]*protos.ContextImageState{} - histograms := map[string]*protos.HistogramState{} - chordDiagrams := map[string]*protos.ChordState{} - ternaryPlots := map[string]*protos.TernaryState{} - binaryPlots := map[string]*protos.BinaryState{} - tables := map[string]*protos.TableState{} - roiQuantTables := map[string]*protos.ROIQuantTableState{} - variograms := map[string]*protos.VariogramState{} - spectrums := map[string]*protos.SpectrumWidgetState{} - rgbuPlots := map[string]*protos.RGBUPlotWidgetState{} - singleAxisRGBU := map[string]*protos.SingleAxisRGBUWidgetState{} - rgbuImages := map[string]*protos.RGBUImagesWidgetState{} - parallelograms := map[string]*protos.ParallelogramWidgetState{} - - destState := protos.ViewState{ - Id: id, - //ScanId: datasetId, - UserId: userId, - AnalysisLayout: &protos.AnalysisLayout{ - TopWidgetSelectors: state.AnalysisLayout.TopWidgetSelectors, - BottomWidgetSelectors: state.AnalysisLayout.BottomWidgetSelectors, - }, - ContextImages: contextImages, - Histograms: histograms, - ChordDiagrams: chordDiagrams, - TernaryPlots: ternaryPlots, - BinaryPlots: binaryPlots, - Tables: tables, - RoiQuantTables: roiQuantTables, - Variograms: variograms, - Spectrums: spectrums, - RgbuPlots: rgbuPlots, - SingleAxisRGBU: singleAxisRGBU, - RgbuImages: rgbuImages, - Parallelograms: parallelograms, - Annotations: &protos.AnnotationDisplayState{ - SavedAnnotations: makeAnnotationStates(state.Annotations.SavedAnnotations), - }, - Rois: &protos.ROIDisplayState{ - RoiColours: state.ROIs.ROIColours, - RoiShapes: state.ROIs.ROIShapes, - }, - Quantification: &protos.QuantificationState{ - AppliedQuantID: state.Quantification.AppliedQuantID, - }, - Selection: &protos.SelectionState{ - RoiID: state.Selection.SelectedROIID, - RoiName: state.Selection.SelectedROIName, - LocIdxs: toInt32s(state.Selection.SelectedLocIdxs), - PixelSelectionImageName: state.Selection.PixelSelectionImageName, - PixelIdxs: toInt32s(state.Selection.PixelIdxs), - CropPixelIdxs: toInt32s(state.Selection.CropPixelIdxs), - }, - } - - destViewStates = append(destViewStates, destState) - - if len(destViewStates)%10 == 0 { - fmt.Printf(" Read %v%% (%v of %v) view states...\n", len(destViewStates)*100/len(viewStatesToSave), len(destViewStates), len(viewStatesToSave)) - } - } - - result, err := coll.InsertMany(context.TODO(), destViewStates) - if err != nil { - return err - } - - fmt.Printf("View states inserted: %v\n", len(result.InsertedIDs)) - */ - return err -} - -/* -func toInt32s(a []int) []int32 { - result := []int32{} - for _, i := range a { - result = append(result, int32(i)) - } - return result -} - -func makeAnnotationStates(items []FullScreenAnnotationItem) []*protos.FullScreenAnnotationItem { - result := []*protos.FullScreenAnnotationItem{} - for _, item := range items { - pts := []*protos.AnnotationPoint{} - for _, pt := range item.Points { - pts = append(pts, &protos.AnnotationPoint{ - X: pt.X, - Y: pt.Y, - ScreenHeight: pt.ScreenHeight, - ScreenWidth: pt.ScreenWidth, - }) - } - result = append(result, &protos.FullScreenAnnotationItem{ - Type: item.Type, - Points: pts, - Colour: item.Colour, - Complete: item.Complete, - Text: item.Text, - FontSize: int32(item.FontSize), - Id: int32(item.ID), - }) - } - return result -} -*/