train with ups dataset3

francesco-vaselli · Aug 28, 2023 · b0dfaf9 · b0dfaf9
1 parent a95c55f
commit b0dfaf9
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 0 deletions.
diff --git a/src/data_processing/check_dataset.py b/src/data_processing/check_dataset.py
@@ -32,6 +32,11 @@ def filter_stationary_sequences_dataset(ds):
 ds = filter_stationary_sequences_dataset(ds)
 train_x = ds[:, :7]
 train_y = ds[:, 7:]
+# check for nans/infs
+print(np.any(np.isnan(train_x)))
+print(np.any(np.isnan(train_y)))
+print(np.any(np.isinf(train_x)))
+print(np.any(np.isinf(train_y)))
 # filter_stationary_sequences(train_x, train_y)
 print(train_x.shape, train_y.shape)
 # restore mean and std

diff --git a/src/data_processing/upsample_dataset.py b/src/data_processing/upsample_dataset.py
@@ -58,6 +58,12 @@ def upsample_dataset(
     dataset._scale(standardize=True)
     data = dataset.train_x
     targets = dataset.train_y
+
+    # shuffle the data
+    idx = np.random.permutation(len(data))
+    data = data[idx]
+    targets = targets[idx]
+
     return data, targets