From 061cf9fdd907959ce77ef2e32efda70748518096 Mon Sep 17 00:00:00 2001
From: Francesco Vaselli <francesco.vaselli@protonmail.com>
Date: Sun, 28 Jan 2024 23:09:44 +0100
Subject: [PATCH] 24h dataset

---
 configs/data_config.yaml           | 276 ++++++++++++++---------------
 src/data_processing/data_reader.py |  13 +-
 2 files changed, 147 insertions(+), 142 deletions(-)

diff --git a/configs/data_config.yaml b/configs/data_config.yaml
index 689fd5f..a569964 100644
--- a/configs/data_config.yaml
+++ b/configs/data_config.yaml
@@ -3,144 +3,144 @@ data_dir: "/home/fvaselli/Documents/TSA/data/data_oh"
 data_type: "OH_24h"
 # patients ids
 ids: ['72492570',
-      # '86323903',
-      # '05274556',
-      # '95851255',
-      # '69739062',
-      # '73398408',
-      # '80625186',
-      # '27526291',
-      # '86286077',
-      # '27819368',
-      # '94200862',
-      # '84109428',
-      # '15634563',
-      # '37948668',
-      # '76817975',
-      # '67208817',
-      # '49551394',
-      # '37998755',
-      # '57176789',
-      # '54280576',
-      # '19626656',
-      # '60844515',
-      # '62401782',
-      # '33470634',
-      # '70811987',
-      # '71397255',
-      # '20396154',
-      # '02199852',
-      # '56958831',
-      # '14092221',
-      # '70454270',
-      # '68267781',
-      # '85199788',
-      # '68026248',
-      # '13783771',
-      # '99712241',
-      # '97417885',
-      # '20649783',
-      # '86025410',
-      # '48509634',
-      # '96805916',
-      # '93839818',
-      # '67359234',
-      # '48540630',
-      # '79526193',
-      # '89727223',
-      # '40997757',
-      # '66836068',
-      # '41131654',
-      # '32997134',
-      # '71618088',
-      # '91161972',
-      # '37875431',
-      # '00221634',
-      # '28176124',
-      # '66773091',
-      # '17161370',
-      # '63725802',
-      # '74077367',
-      # '28608066',
-      # '65143138',
-      # '69587086',
-      # '13029224',
-      # '98340749',
-      # '51968000',
-      # '35719805',
-      # '33962890',
-      # '18991425',
-      # '98974339',
-      # '95614431',
-      # '90398368',
-      # '25692073',
-      # '38110191',
-      # '85653830',
-      # '69965708',
-      # '24448124',
-      # '33831564',
-      # '67167655',
-      # '43589707',
-      # '80501215',
-      # '93606058',
-      # '39986716',
-      # '77104076',
-      # '40237051',
-      # '21946407',
-      # '35533061',
-      # '41663654',
-      # '45120081',
-      # '47323535',
-      # '78420229',
-      # '73521474',
-      # '23428091',
-      # '84589080',
-      # '77411181',
-      # '28756888',
-      # '81099003',
-      # '46253612',
-      # '22961398',
-      # '99848889',
-      # '20216809',
-      # '89710417',
-      # '66019205',
-      # '99296581',
-      # '49796612',
-      # '56568290',
-      # '00897741',
-      # '13484299',
-      # '04762925',
-      # '64024750',
-      # '42052178',
-      # '26856617',
-      # '88252802',
-      # '81680176',
-      # '07886752',
-      # '89032650',
-      # '97872409',
-      # '28768536',
-      # '01352464',
-      # '62345070',
-      # '40634871',
-      # '02033176',
-      # '12689381',
-      # '61179686',
-      # '16975609',
-      # '00309157',
-      # '15558575',
-      # '84984656',
-      # '71236754',
-      # '50311906',
-      # '80373992',
-      # '32407882',
-      # '66937570',
-      # '87770486',
-      # '88004055',
-      # '96254963',
-      # '47750728',
-      # '99908129',
-      # '63047517',
-      # '80796147',
+      '86323903',
+      '05274556',
+      '95851255',
+      '69739062',
+      '73398408',
+      '80625186',
+      '27526291',
+      '86286077',
+      '27819368',
+      '94200862',
+      '84109428',
+      '15634563',
+      '37948668',
+      '76817975',
+      '67208817',
+      '49551394',
+      '37998755',
+      '57176789',
+      '54280576',
+      '19626656',
+      '60844515',
+      '62401782',
+      '33470634',
+      '70811987',
+      '71397255',
+      '20396154',
+      '02199852',
+      '56958831',
+      '14092221',
+      '70454270',
+      '68267781',
+      '85199788',
+      '68026248',
+      '13783771',
+      '99712241',
+      '97417885',
+      '20649783',
+      '86025410',
+      '48509634',
+      '96805916',
+      '93839818',
+      '67359234',
+      '48540630',
+      '79526193',
+      '89727223',
+      '40997757',
+      '66836068',
+      '41131654',
+      '32997134',
+      '71618088',
+      '91161972',
+      '37875431',
+      '00221634',
+      '28176124',
+      '66773091',
+      '17161370',
+      '63725802',
+      '74077367',
+      '28608066',
+      '65143138',
+      '69587086',
+      '13029224',
+      '98340749',
+      '51968000',
+      '35719805',
+      '33962890',
+      '18991425',
+      '98974339',
+      '95614431',
+      '90398368',
+      '25692073',
+      '38110191',
+      '85653830',
+      '69965708',
+      '24448124',
+      '33831564',
+      '67167655',
+      '43589707',
+      '80501215',
+      '93606058',
+      '39986716',
+      '77104076',
+      '40237051',
+      '21946407',
+      '35533061',
+      '41663654',
+      '45120081',
+      '47323535',
+      '78420229',
+      '73521474',
+      '23428091',
+      '84589080',
+      '77411181',
+      '28756888',
+      '81099003',
+      '46253612',
+      '22961398',
+      '99848889',
+      '20216809',
+      '89710417',
+      '66019205',
+      '99296581',
+      '49796612',
+      '56568290',
+      '00897741',
+      '13484299',
+      '04762925',
+      '64024750',
+      '42052178',
+      '26856617',
+      '88252802',
+      '81680176',
+      '07886752',
+      '89032650',
+      '97872409',
+      '28768536',
+      '01352464',
+      '62345070',
+      '40634871',
+      '02033176',
+      '12689381',
+      '61179686',
+      '16975609',
+      '00309157',
+      '15558575',
+      '84984656',
+      '71236754',
+      '50311906',
+      '80373992',
+      '32407882',
+      '66937570',
+      '87770486',
+      '88004055',
+      '96254963',
+      '47750728',
+      '99908129',
+      '63047517',
+      '80796147',
       '64406000',]
 test_ids: []
       # '99908129']
diff --git a/src/data_processing/data_reader.py b/src/data_processing/data_reader.py
index 012a9f3..dc0e7eb 100644
--- a/src/data_processing/data_reader.py
+++ b/src/data_processing/data_reader.py
@@ -117,12 +117,15 @@ def read_OH_24h(self):
         except IndexError:
             value_0 = None
 
-        t0 = parser.parse(data[0]["dateString"]).replace(tzinfo=None)
+        try:
+            t0 = parser.parse(data[0]["dateString"]).replace(tzinfo=None)
+        except (IndexError, KeyError, ParserError):
+            value_0 = None
 
         if value_0 is not None:
             # Initialize with the first reading
             # cast t0 to  a float of the seconds of the day
-            t0_to_save = t0.hour * 3600 + t0.minute * 60 + t0.second
+            t0_to_save = t0 # t0.hour * 3600 + t0.minute * 60 + t0.second
             res.append([(value_0, t0_to_save)])
             # set t0 to the beginning of the day
             t0 = t0.replace(hour=0, minute=0, second=0, microsecond=0)
@@ -132,7 +135,7 @@ def read_OH_24h(self):
                 # print(data[i]['dateString'])
                 try:
                     t1 = parser.parse(data[i]["dateString"]).replace(tzinfo=None)
-                except (KeyError, ParserError):
+                except (KeyError, ParserError, IndexError):
                     # skip one reading if 'dateString' is not present or in wrong format
                     continue
                 
@@ -144,6 +147,8 @@ def read_OH_24h(self):
                     value = data[i].get(
                         "mbg", None
                     )
+                except IndexError:
+                    value = None
                 
                 # Check if value is an empty string and set it to None
                 if value == '':
@@ -155,7 +160,7 @@ def read_OH_24h(self):
                         res[-1].append((value, t1_to_save))
                 else:
                     if value is not None:
-                        t1_to_save = t1.hour * 3600 + t1.minute * 60 + t1.second
+                        t1_to_save = t1 # t1.hour * 3600 + t1.minute * 60 + t1.second
                         res.append([(value, t1_to_save)])
                         # update t0
                         t0 = t1