Skip to content

Commit

Permalink
ckpt
Browse files Browse the repository at this point in the history
  • Loading branch information
@jmoralez
jmoralez committed Feb 20, 2024
1 parent 1f259e6 commit 1568f3a
Show file tree
Hide file tree
Showing 10 changed files with 275 additions and 630 deletions.
12 changes: 12 additions & 0 deletions mlforecast/_modidx.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,8 @@
'mlforecast/lag_transforms.py'),
'mlforecast.lag_transforms._BaseLagTransform._set_core_tfm': ( 'lag_transforms.html#_baselagtransform._set_core_tfm',
'mlforecast/lag_transforms.py'),
'mlforecast.lag_transforms._BaseLagTransform.stack': ( 'lag_transforms.html#_baselagtransform.stack',
'mlforecast/lag_transforms.py'),
'mlforecast.lag_transforms._BaseLagTransform.take': ( 'lag_transforms.html#_baselagtransform.take',
'mlforecast/lag_transforms.py'),
'mlforecast.lag_transforms._BaseLagTransform.transform': ( 'lag_transforms.html#_baselagtransform.transform',
Expand Down Expand Up @@ -353,6 +355,8 @@
'mlforecast/target_transforms.py'),
'mlforecast.target_transforms.BaseTargetTransform.set_column_names': ( 'target_transforms.html#basetargettransform.set_column_names',
'mlforecast/target_transforms.py'),
'mlforecast.target_transforms.BaseTargetTransform.stack': ( 'target_transforms.html#basetargettransform.stack',
'mlforecast/target_transforms.py'),
'mlforecast.target_transforms.BaseTargetTransform.update': ( 'target_transforms.html#basetargettransform.update',
'mlforecast/target_transforms.py'),
'mlforecast.target_transforms.Differences': ( 'target_transforms.html#differences',
Expand All @@ -365,6 +369,8 @@
'mlforecast/target_transforms.py'),
'mlforecast.target_transforms.Differences.inverse_transform_fitted': ( 'target_transforms.html#differences.inverse_transform_fitted',
'mlforecast/target_transforms.py'),
'mlforecast.target_transforms.Differences.stack': ( 'target_transforms.html#differences.stack',
'mlforecast/target_transforms.py'),
'mlforecast.target_transforms.Differences.take': ( 'target_transforms.html#differences.take',
'mlforecast/target_transforms.py'),
'mlforecast.target_transforms.Differences.update': ( 'target_transforms.html#differences.update',
Expand All @@ -377,6 +383,8 @@
'mlforecast/target_transforms.py'),
'mlforecast.target_transforms.GlobalSklearnTransformer.inverse_transform': ( 'target_transforms.html#globalsklearntransformer.inverse_transform',
'mlforecast/target_transforms.py'),
'mlforecast.target_transforms.GlobalSklearnTransformer.stack': ( 'target_transforms.html#globalsklearntransformer.stack',
'mlforecast/target_transforms.py'),
'mlforecast.target_transforms.GlobalSklearnTransformer.update': ( 'target_transforms.html#globalsklearntransformer.update',
'mlforecast/target_transforms.py'),
'mlforecast.target_transforms.LocalBoxCox': ( 'target_transforms.html#localboxcox',
Expand All @@ -387,6 +395,8 @@
'mlforecast/target_transforms.py'),
'mlforecast.target_transforms.LocalBoxCox.inverse_transform': ( 'target_transforms.html#localboxcox.inverse_transform',
'mlforecast/target_transforms.py'),
'mlforecast.target_transforms.LocalBoxCox.stack': ( 'target_transforms.html#localboxcox.stack',
'mlforecast/target_transforms.py'),
'mlforecast.target_transforms.LocalBoxCox.take': ( 'target_transforms.html#localboxcox.take',
'mlforecast/target_transforms.py'),
'mlforecast.target_transforms.LocalMinMaxScaler': ( 'target_transforms.html#localminmaxscaler',
Expand All @@ -407,6 +417,8 @@
'mlforecast/target_transforms.py'),
'mlforecast.target_transforms._BaseGroupedArrayTargetTransform.set_num_threads': ( 'target_transforms.html#_basegroupedarraytargettransform.set_num_threads',
'mlforecast/target_transforms.py'),
'mlforecast.target_transforms._BaseGroupedArrayTargetTransform.stack': ( 'target_transforms.html#_basegroupedarraytargettransform.stack',
'mlforecast/target_transforms.py'),
'mlforecast.target_transforms._BaseGroupedArrayTargetTransform.take': ( 'target_transforms.html#_basegroupedarraytargettransform.take',
'mlforecast/target_transforms.py'),
'mlforecast.target_transforms._BaseGroupedArrayTargetTransform.update': ( 'target_transforms.html#_basegroupedarraytargettransform.update',
Expand Down
18 changes: 3 additions & 15 deletions mlforecast/compat.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,20 +5,8 @@

# %% ../nbs/compat.ipynb 1
try:
import coreforecast.lag_transforms as core_tfms
import coreforecast.scalers as core_scalers
from coreforecast.grouped_array import GroupedArray as CoreGroupedArray

from mlforecast.lag_transforms import BaseLagTransform, Lag

CORE_INSTALLED = True
from window_ops.shift import shift_array
except ImportError:
core_tfms = None
core_scalers = None
CoreGroupedArray = None

class BaseLagTransform: ...

Lag = None

CORE_INSTALLED = False
def shift_array(*args, **kwargs):
raise Exception
47 changes: 30 additions & 17 deletions mlforecast/distributed/forecast.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -750,22 +750,34 @@ def possibly_concat_indices(collection):
combined = pd.Index(combined)
return combined

def combine_target_tfms(by_partition):
by_transform = [
[part[i] for part in by_partition] for i in range(len(by_partition[0]))
]
out = []
for tfms in by_transform:
out.append(tfms[0].stack(tfms))
return out

def combine_core_lag_tfms(by_partition):
by_transform = [
(name, [part[name] for part in by_partition])
for name in by_partition[0].keys()
]
out = {}
for name, partition_tfms in by_transform:
out[name] = partition_tfms[0].stack(partition_tfms)
return out

uids = possibly_concat_indices([ts.uids for ts in all_ts])
last_dates = possibly_concat_indices([ts.last_dates for ts in all_ts])
statics = ufp.vertical_concat([ts.static_features_ for ts in all_ts])
core_tfms = [ts._get_core_lag_tfms() for ts in all_ts]
combined_core_tfms = {}
if core_tfms[0]: # at least one core_tfm
for name in core_tfms[0].keys():
first_tfm = core_tfms[0][name]
if not hasattr(first_tfm._core_tfm, "stats_"):
combined_tfm = first_tfm
else:
combined_tfm = copy.deepcopy(first_tfm)
combined_tfm._core_tfm.stats_ = np.vstack(
[tfm[name]._core_tfm.stats_ for tfm in core_tfms]
)
combined_core_tfms[name] = combined_tfm
combined_target_tfms = combine_target_tfms(
[ts.target_transforms for ts in all_ts]
)
combined_core_lag_tfms = combine_core_lag_tfms(
[ts._get_core_lag_tfms() for ts in all_ts]
)
sizes = np.hstack([np.diff(ts.ga.indptr) for ts in all_ts])
data = np.hstack([ts.ga.data for ts in all_ts])
indptr = np.append(0, sizes).cumsum()
Expand All @@ -782,9 +794,9 @@ def possibly_concat_indices(collection):
last_dates = last_dates[sort_idxs]
statics = ufp.take_rows(statics, sort_idxs)
statics = ufp.drop_index_if_pandas(statics)
for tfm in combined_core_tfms:
if hasattr(tfm, "stats_"):
tfm.stats_ = tfm.stats_[sort_idxs]
for tfm in combined_core_lag_tfms.values():
tfm._core_tfm = tfm._core_tfm.take(sort_idxs)
combined_target_tfms = [tfm.take(sort_idxs) for tfm in combined_target_tfms]
old_data = data.copy()
old_indptr = indptr.copy()
indptr = np.append(0, sizes[sort_idxs]).cumsum()
Expand All @@ -802,7 +814,8 @@ def possibly_concat_indices(collection):
ts.last_dates = last_dates
ts.ga = ga
ts.static_features_ = statics
ts.transforms.update(combined_core_tfms)
ts.transforms.update(combined_core_lag_tfms)
ts.target_transforms = combined_target_tfms
fcst = MLForecast(models=self.models_, freq=ts.freq)
fcst.ts = ts
fcst.models_ = self.models_
Expand Down
10 changes: 9 additions & 1 deletion mlforecast/lag_transforms.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import copy
import inspect
import re
from typing import Callable, Optional
from typing import Callable, Optional, Sequence

import numpy as np
import coreforecast.lag_transforms as core_tfms
Expand Down Expand Up @@ -60,6 +60,14 @@ def take(self, idxs: np.ndarray) -> "_BaseLagTransform":
out._core_tfm = self._core_tfm.take(idxs)
return out

@staticmethod
def stack(transforms: Sequence["_BaseLagTransform"]) -> "_BaseLagTransform":
out = copy.deepcopy(transforms[0])
out._core_tfm = transforms[0]._core_tfm.stack(
[tfm._core_tfm for tfm in transforms]
)
return out

# %% ../nbs/lag_transforms.ipynb 6
class Lag(_BaseLagTransform):
def __init__(self, lag: int):
Expand Down
56 changes: 38 additions & 18 deletions mlforecast/target_transforms.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
# %% ../nbs/target_transforms.ipynb 3
import abc
import copy
from typing import Iterable, List, Optional
from typing import Iterable, List, Optional, Sequence

import coreforecast.scalers as core_scalers
import numpy as np
Expand Down Expand Up @@ -40,6 +40,9 @@ def set_column_names(self, id_col: str, time_col: str, target_col: str):
def update(self, df: DataFrame) -> DataFrame:
raise NotImplementedError

def stack(transforms: Sequence["BaseTargetTransform"]) -> "BaseTargetTransform":
raise NotImplementedError

@abc.abstractmethod
def fit_transform(self, df: DataFrame) -> DataFrame: ...

Expand Down Expand Up @@ -68,8 +71,17 @@ def inverse_transform_fitted(self, ga: GroupedArray) -> GroupedArray:
return self.inverse_transform(ga)

@abc.abstractmethod
def take(self, idxs: np.ndarray) -> "_BaseGroupedArrayTargetTransform":
...
def take(self, idxs: np.ndarray) -> "_BaseGroupedArrayTargetTransform": ...

@staticmethod
def stack(
scalers: Sequence["_BaseGroupedArrayTargetTransform"],
) -> "_BaseGroupedArrayTargetTransform":
first_scaler = scalers[0]
core_scaler = first_scaler.scaler_
out = copy.deepcopy(first_scaler)
out.scaler_ = core_scaler.stack([sc.scaler_ for sc in scalers])
return out

# %% ../nbs/target_transforms.ipynb 7
class Differences(_BaseGroupedArrayTargetTransform):
Expand Down Expand Up @@ -129,6 +141,18 @@ def take(self, idxs: np.ndarray) -> "Differences":
out.scalers_ = [scaler.take(idxs) for scaler in self.scalers_]
return out

@staticmethod
def stack(scalers: Sequence["Differences"]) -> "Differences":
first_scaler = scalers[0]
core_scaler = first_scaler.scalers_[0]
diffs = first_scaler.differences
out = Differences(diffs)
out.scalers_ = [
core_scaler.stack([sc.scalers_[i] for sc in scalers])
for i in range(len(diffs))
]
return out

# %% ../nbs/target_transforms.ipynb 10
class AutoDifferences(_BaseGroupedArrayTargetTransform):
"""Find and apply the optimal number of differences to each serie.
Expand All @@ -139,11 +163,6 @@ class AutoDifferences(_BaseGroupedArrayTargetTransform):
Maximum number of differences to apply."""

def __init__(self, max_diffs: int):
if not CORE_INSTALLED:
raise ImportError(
"coreforecast is required for this transformation. "
"Please follow the installation instructions at https://github.com/Nixtla/coreforecast/#installation"
)
self.scaler_ = core_scalers.AutoDifferences(max_diffs)

def fit_transform(self, ga: GroupedArray) -> GroupedArray:
Expand Down Expand Up @@ -184,11 +203,6 @@ class AutoSeasonalDifferences(AutoDifferences):
def __init__(
self, season_length: int, max_diffs: int, n_seasons: Optional[int] = 10
):
if not CORE_INSTALLED:
raise ImportError(
"coreforecast is required for this transformation. "
"Please follow the installation instructions at https://github.com/Nixtla/coreforecast/#installation"
)
self.scaler_ = core_scalers.AutoSeasonalDifferences(
season_length=season_length,
max_diffs=max_diffs,
Expand All @@ -213,11 +227,6 @@ class AutoSeasonalityAndDifferences(AutoDifferences):
def __init__(
self, max_season_length: int, max_diffs: int, n_seasons: Optional[int] = 10
):
if not CORE_INSTALLED:
raise ImportError(
"coreforecast is required for this transformation. "
"Please follow the installation instructions at https://github.com/Nixtla/coreforecast/#installation"
)
self.scaler_ = core_scalers.AutoSeasonalityAndDifferences(
max_season_length=max_season_length,
max_diffs=max_diffs,
Expand Down Expand Up @@ -297,6 +306,12 @@ def take(self, idxs: np.ndarray) -> "LocalBoxCox":
out.scaler_.lmbdas_ = self.scaler_.lmbdas_[idxs]
return out

@staticmethod
def stack(scalers: Sequence["LocalBoxCox"]) -> "LocalBoxCox":
out = LocalBoxCox()
out.scaler_.lmbdas_ = np.hstack([sc.scaler_.lmbdas_ for sc in scalers])
return out

# %% ../nbs/target_transforms.ipynb 27
class GlobalSklearnTransformer(BaseTargetTransform):
"""Applies the same scikit-learn transformer to all series."""
Expand All @@ -323,3 +338,8 @@ def update(self, df: pd.DataFrame) -> pd.DataFrame:
df = df.copy(deep=False)
df[self.target_col] = self.transformer_.transform(df[[self.target_col]].values)
return df

def stack(
transforms: Sequence["GlobalSklearnTransformer"],
) -> "GlobalSklearnTransformer":
return transforms[0]
Loading

0 comments on commit 1568f3a

Please sign in to comment.