From d55a07d618b1251892b74c09336468342cf3be59 Mon Sep 17 00:00:00 2001 From: Darryl Melander Date: Tue, 23 May 2023 09:55:33 -0600 Subject: [PATCH 1/3] Make sure look-ahead periods are included when parsing input files. The RTS-GMLC file format has a way to indicate how many look-ahead periods to include in a RUC and SCED (the Look_Ahead_Periods_Per_Step parameter in simulation_objects.csv). Prescient also has a way to indicate this same thing (--ruc-horizon and --sced-horizon). The two aren't guaranteed to match, which can lead to run-time errors due to missing data. This commit uses the Prescient horizon arguments to determine how much data to ask Egret to parse, and ignores the Look_Ahead_Periods in the CSV. It uses a recent Egret PR (https://github.com/grid-parity-exchange/Prescient/pull/180, which hasn't been accepted yet) to tell Egret to not parse additional data beyond the end date. Instead, Prescient advances the end date to include any necessary look-ahead periods. --- prescient/data/providers/gmlc_data_provider.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/prescient/data/providers/gmlc_data_provider.py b/prescient/data/providers/gmlc_data_provider.py index e8c7e65a..c2854f51 100644 --- a/prescient/data/providers/gmlc_data_provider.py +++ b/prescient/data/providers/gmlc_data_provider.py @@ -28,10 +28,21 @@ class GmlcDataProvider(DataProvider): ''' def __init__(self, options:Options): - # midnight start + # See how much extra time to parse for lookahead horizon + sced_extra_minutes = options.sced_horizon * options.sced_frequency_minutes + ruc_extra_minutes = (options.ruc_horizon - options.ruc_every_hours)*60 + extra_minutes = max(sced_extra_minutes, ruc_extra_minutes) + + # Move minutes to days as needed + days=options.num_days + while extra_minutes >= 24*60: + days += 1 + extra_minutes -= 24*60 + + # Start at midnight of start date self._start_time = datetime.combine(options.start_date, datetime.min.time()) - self._end_time = self._start_time + timedelta(days=options.num_days) - self._cache = parser.parse_to_cache(options.data_path, self._start_time, self._end_time) + self._end_time = self._start_time + timedelta(days=days, minutes=extra_minutes) + self._cache = parser.parse_to_cache(options.data_path, self._start_time, self._end_time, honor_lookahead=False) def negotiate_data_frequency(self, desired_frequency_minutes:int): ''' Get the number of minutes between each timestep of actuals data this provider will supply, From 12e05f128599c5a90007dae10837c0c69fa1602d Mon Sep 17 00:00:00 2001 From: Darryl Melander Date: Tue, 23 May 2023 17:51:37 -0600 Subject: [PATCH 2/3] Remove unused fields from documentation for simulation_objects.csv --- .../rts-gmlc/simulation_objects.rst | 57 ------------------- 1 file changed, 57 deletions(-) diff --git a/doc/OnlineDocs/source/reference/file_formats/rts-gmlc/simulation_objects.rst b/doc/OnlineDocs/source/reference/file_formats/rts-gmlc/simulation_objects.rst index 6eba43de..e62e4658 100644 --- a/doc/OnlineDocs/source/reference/file_formats/rts-gmlc/simulation_objects.rst +++ b/doc/OnlineDocs/source/reference/file_formats/rts-gmlc/simulation_objects.rst @@ -36,37 +36,6 @@ The following values of :col:`Simulation_Parameter` are supported: files - The number of seconds between values in *REAL_TIME* timeseries data files - * - *Date_From* - - Yes - - The date and time of the first value in each timeseries data file. - Most reasonable formats are accepted. - - The date and time of the first value in each DAY_AHEAD timeseries data - file - - The date and time of the first value in each REAL_TIME timeseries data - file - * - *Date_To* - - Yes - - The latest date and time for which we have enough data in timeseries - files to formulate a RUC or SCED. Most reasonable formats are - accepted. See `Date_To Details`_ below. - - The latest date and time for which we have enough data in DAY_AHEAD - timeseries data files to formulate a RUC - - The latest date and time for which we have enough data in REAL_TIME - timeseries data files to formulate a SCED - * - *Look_Ahead_Periods_Per_Step* - - Yes - - The default number of look-ahead periods to use in RUC or SCED formulations - - The default number of look-ahead periods to use in RUC formulations - - The default number of look-ahead periods to use in SCED formulations - * - *Look_Ahead_Resolution* - - Yes - - The default number of seconds between each look-ahead period used in - RUC or SCED formulations. See `Look_Ahead_Resolution Details`_ below. - - The default number of seconds between each look-ahead period in RUC - formulations - - The default number of seconds between each look-ahead period in SCED - formulations - formulations * - *Reserve_Products* - No - Which reserve products to enforce for RUC plans or SCED operations. @@ -75,32 +44,6 @@ The following values of :col:`Simulation_Parameter` are supported: - Which reserve products to enforce for SCED operations -Date_To Details -~~~~~~~~~~~~~~~ - -The value of *Date_To* identifies the latest time for which there is -enough data to formulate or RUC (for :col:`DAY_AHEAD`) or SCED (for -:col:`REAL_TIME`), including look-ahead periods. This is not the date -and time of the final value in timeseries data files. Instead, the -*Date_To* is *Look_Ahead_Periods_Per_Step \* Look_Ahead_Resolution* -before the date and time of the final value in the timeseries data files. - -For example, consider a data set with 24 look-ahead periods with a -look-ahead resolution of 1 hour. If the final value in a timeseries is -for April 10\ :sup:`th` at midnight, then *Date_To* is April 9\ :sup:`th` -at midnight, because that is the latest time for which we have enough -data to satisfy the 24 hour look-ahead requirement. - -Look_Ahead_Resolution Details -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The *Look_Ahead_Resolution* parameter is used to determine the date and time of the -final value in timeseries data files, as described in `Date_To Details`_. Despite its -name, it is not used to specify the look-ahead resolution used during simulation. The -actual look-ahead resolution used during simulation is determined by configuration parameters -passed to Prescient. Prescient will interpolate the available data as necessary to -honor the look-ahead resolution specified in its configuration parameters. - Reserve_Products Details ~~~~~~~~~~~~~~~~~~~~~~~~ From 46857b626b03f2c2ac6eca5859645a4803a94254 Mon Sep 17 00:00:00 2001 From: Darryl Melander Date: Thu, 25 May 2023 10:26:01 -0600 Subject: [PATCH 3/3] Update required Egret version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b0789c2c..22f9d7c1 100755 --- a/setup.py +++ b/setup.py @@ -46,6 +46,6 @@ 'prescient.simulator.tests':['regression_tests_data/**/*'], }, install_requires=['numpy','matplotlib','pandas','scipy','pyomo>=6.1.2', - 'python-dateutil','networkx','jupyter', 'gridx-egret==0.5.5', + 'python-dateutil','networkx','jupyter', 'gridx-egret==0.5.6.dev0', ], )