From c109091f223beb0a9469ed83d476f8a524a5ff4f Mon Sep 17 00:00:00 2001 From: Arno Strouwen Date: Sat, 24 Feb 2024 02:02:53 +0100 Subject: [PATCH] reapply formatter --- .JuliaFormatter.toml | 3 +- docs/pages.jl | 8 ++- docs/src/esn_tutorials/change_layers.md | 2 +- src/ReservoirComputing.jl | 2 +- src/esn/echostatenetwork.jl | 67 +++++++++++--------- src/esn/esn_input_layers.jl | 77 ++++++++++++++--------- src/esn/esn_reservoir_drivers.jl | 76 ++++++++++++----------- src/esn/esn_reservoirs.jl | 81 ++++++++++++++++--------- src/reca/reca_input_encodings.jl | 9 +-- src/states.jl | 21 ++++--- src/train/linear_regression.jl | 11 ++-- test/esn/test_drivers.jl | 6 +- test/esn/test_train.jl | 2 +- test/runtests.jl | 18 +++--- 14 files changed, 225 insertions(+), 158 deletions(-) diff --git a/.JuliaFormatter.toml b/.JuliaFormatter.toml index 9c793591..959ad88a 100644 --- a/.JuliaFormatter.toml +++ b/.JuliaFormatter.toml @@ -1,2 +1,3 @@ style = "sciml" -format_markdown = true \ No newline at end of file +format_markdown = true +format_docstrings = true \ No newline at end of file diff --git a/docs/pages.jl b/docs/pages.jl index 309f76fa..0c1465c7 100644 --- a/docs/pages.jl +++ b/docs/pages.jl @@ -1,9 +1,11 @@ pages = [ "ReservoirComputing.jl" => "index.md", - "General Settings" => Any["Changing Training Algorithms" => "general/different_training.md", + "General Settings" => Any[ + "Changing Training Algorithms" => "general/different_training.md", "Altering States" => "general/states_variation.md", "Generative vs Predictive" => "general/predictive_generative.md"], - "Echo State Network Tutorials" => Any["Lorenz System Forecasting" => "esn_tutorials/lorenz_basic.md", + "Echo State Network Tutorials" => Any[ + "Lorenz System Forecasting" => "esn_tutorials/lorenz_basic.md", #"Mackey-Glass Forecasting on GPU" => "esn_tutorials/mackeyglass_basic.md", "Using Different Layers" => "esn_tutorials/change_layers.md", "Using Different Reservoir Drivers" => "esn_tutorials/different_drivers.md", @@ -17,5 +19,5 @@ pages = [ "Echo State Networks" => "api/esn.md", "ESN Layers" => "api/esn_layers.md", "ESN Drivers" => "api/esn_drivers.md", - "ReCA" => "api/reca.md"], + "ReCA" => "api/reca.md"] ] diff --git a/docs/src/esn_tutorials/change_layers.md b/docs/src/esn_tutorials/change_layers.md index 5cfb65cf..86c70477 100644 --- a/docs/src/esn_tutorials/change_layers.md +++ b/docs/src/esn_tutorials/change_layers.md @@ -76,7 +76,7 @@ using ReservoirComputing, StatsBase res_size = 300 input_layer = [ MinimumLayer(0.85, IrrationalSample()), - MinimumLayer(0.95, IrrationalSample()), + MinimumLayer(0.95, IrrationalSample()) ] reservoirs = [SimpleCycleReservoir(res_size, 0.7), CycleJumpsReservoir(res_size, cycle_weight = 0.7, jump_weight = 0.2, jump_size = 5)] diff --git a/src/ReservoirComputing.jl b/src/ReservoirComputing.jl index 4c24427a..7a5d97ae 100644 --- a/src/ReservoirComputing.jl +++ b/src/ReservoirComputing.jl @@ -65,7 +65,7 @@ end """ Predictive(prediction_data) -Given a set of labels as ```prediction_data```, this method of prediction will return the corresponding labels in a standard Machine Learning fashion. +Given a set of labels as `prediction_data`, this method of prediction will return the corresponding labels in a standard Machine Learning fashion. """ function Predictive(prediction_data) prediction_len = size(prediction_data, 2) diff --git a/src/esn/echostatenetwork.jl b/src/esn/echostatenetwork.jl index 42fab481..bfa088bf 100644 --- a/src/esn/echostatenetwork.jl +++ b/src/esn/echostatenetwork.jl @@ -34,20 +34,23 @@ end Hybrid(prior_model, u0, tspan, datasize) Constructs a `Hybrid` variation of Echo State Networks (ESNs) integrating a knowledge-based model -(`prior_model`) with ESNs for advanced training and prediction in chaotic systems. +(`prior_model`) with ESNs for advanced training and prediction in chaotic systems. # Parameters -- `prior_model`: A knowledge-based model function for integration with ESNs. -- `u0`: Initial conditions for the model. -- `tspan`: Time span as a tuple, indicating the duration for model operation. -- `datasize`: The size of the data to be processed. + + - `prior_model`: A knowledge-based model function for integration with ESNs. + - `u0`: Initial conditions for the model. + - `tspan`: Time span as a tuple, indicating the duration for model operation. + - `datasize`: The size of the data to be processed. # Returns -- A `Hybrid` struct instance representing the combined ESN and knowledge-based model. + + - A `Hybrid` struct instance representing the combined ESN and knowledge-based model. This method is effective for chaotic processes as highlighted in [^Pathak]. Reference: + [^Pathak]: Jaideep Pathak et al. "Hybrid Forecasting of Chaotic Processes: Using Machine Learning in Conjunction with a Knowledge-Based Model" (2018). @@ -67,27 +70,30 @@ end Creates an Echo State Network (ESN) using specified parameters and training data, suitable for various machine learning tasks. # Parameters -- `train_data`: Matrix of training data (columns as time steps, rows as features). -- `variation`: Variation of ESN (default: `Default()`). -- `input_layer`: Input layer of ESN (default: `DenseLayer()`). -- `reservoir`: Reservoir of the ESN (default: `RandSparseReservoir(100)`). -- `bias`: Bias vector for each time step (default: `NullLayer()`). -- `reservoir_driver`: Mechanism for evolving reservoir states (default: `RNN()`). -- `nla_type`: Non-linear activation type (default: `NLADefault()`). -- `states_type`: Format for storing states (default: `StandardStates()`). -- `washout`: Initial time steps to discard (default: `0`). -- `matrix_type`: Type of matrices used internally (default: type of `train_data`). + + - `train_data`: Matrix of training data (columns as time steps, rows as features). + - `variation`: Variation of ESN (default: `Default()`). + - `input_layer`: Input layer of ESN (default: `DenseLayer()`). + - `reservoir`: Reservoir of the ESN (default: `RandSparseReservoir(100)`). + - `bias`: Bias vector for each time step (default: `NullLayer()`). + - `reservoir_driver`: Mechanism for evolving reservoir states (default: `RNN()`). + - `nla_type`: Non-linear activation type (default: `NLADefault()`). + - `states_type`: Format for storing states (default: `StandardStates()`). + - `washout`: Initial time steps to discard (default: `0`). + - `matrix_type`: Type of matrices used internally (default: type of `train_data`). # Returns -- An initialized ESN instance with specified parameters. + + - An initialized ESN instance with specified parameters. # Examples + ```julia using ReservoirComputing train_data = rand(10, 100) # 10 features, 100 time steps -esn = ESN(train_data, reservoir=RandSparseReservoir(200), washout=10) +esn = ESN(train_data, reservoir = RandSparseReservoir(200), washout = 10) ``` """ function ESN(train_data; @@ -159,16 +165,16 @@ function obtain_layers(in_size, if input_layer isa Array input_matrix = [create_layer(input_layer[j], input_res_sizes[j], in_sizes[j], - matrix_type = matrix_type) for j in 1:esn_depth] + matrix_type = matrix_type) for j in 1:esn_depth] else _input_layer = fill(input_layer, esn_depth) input_matrix = [create_layer(_input_layer[k], input_res_sizes[k], in_sizes[k], - matrix_type = matrix_type) for k in 1:esn_depth] + matrix_type = matrix_type) for k in 1:esn_depth] end res_sizes = [get_ressize(input_matrix[j]) for j in 1:esn_depth] reservoir_matrix = [create_reservoir(reservoir[k], res_sizes[k], - matrix_type = matrix_type) for k in 1:esn_depth] + matrix_type = matrix_type) for k in 1:esn_depth] if bias isa Array bias_vector = [create_layer(bias[j], res_sizes[j], 1, matrix_type = matrix_type) @@ -212,15 +218,17 @@ end Trains an Echo State Network (ESN) using the provided target data and a specified training method. # Parameters -- `esn::AbstractEchoStateNetwork`: The ESN instance to be trained. -- `target_data`: Supervised training data for the ESN. -- `training_method`: The method for training the ESN (default: `StandardRidge(0.0)`). + + - `esn::AbstractEchoStateNetwork`: The ESN instance to be trained. + - `target_data`: Supervised training data for the ESN. + - `training_method`: The method for training the ESN (default: `StandardRidge(0.0)`). # Returns -- The trained ESN model. Its type and structure depend on `training_method` and the ESN's implementation. + - The trained ESN model. Its type and structure depend on `training_method` and the ESN's implementation. # Returns + The trained ESN model. The exact type and structure of the return value depends on the `training_method` and the specific ESN implementation. @@ -228,20 +236,21 @@ The trained ESN model. The exact type and structure of the return value depends using ReservoirComputing # Initialize an ESN instance and target data -esn = ESN(train_data, reservoir=RandSparseReservoir(200), washout=10) +esn = ESN(train_data, reservoir = RandSparseReservoir(200), washout = 10) target_data = rand(size(train_data, 2)) # Train the ESN using the default training method trained_esn = train(esn, target_data) # Train the ESN using a custom training method -trained_esn = train(esn, target_data, training_method=StandardRidge(1.0)) +trained_esn = train(esn, target_data, training_method = StandardRidge(1.0)) ``` # Notes -- When using a `Hybrid` variation, the function extends the state matrix with data from the + + - When using a `Hybrid` variation, the function extends the state matrix with data from the physical model included in the `variation`. -- The training is handled by a lower-level `_train` function which takes the new state matrix + - The training is handled by a lower-level `_train` function which takes the new state matrix and performs the actual training using the specified `training_method`. """ function train(esn::AbstractEchoStateNetwork, diff --git a/src/esn/esn_input_layers.jl b/src/esn/esn_input_layers.jl index e7bb950c..36b553a1 100644 --- a/src/esn/esn_input_layers.jl +++ b/src/esn/esn_input_layers.jl @@ -14,12 +14,15 @@ elements distributed uniformly within the range [-`scaling`, `scaling`], following the approach in [^Lu]. # Parameters -- `scaling`: The scaling factor for the weight distribution (default: 0.1). + + - `scaling`: The scaling factor for the weight distribution (default: 0.1). # Returns -- A `WeightedInput` instance to be used for initializing the input layer of an ESN. + + - A `WeightedInput` instance to be used for initializing the input layer of an ESN. Reference: + [^Lu]: Lu, Zhixin, et al. "Reservoir observers: Model-free inference of unmeasured variables in chaotic systems." Chaos: An Interdisciplinary Journal of Nonlinear Science 27.4 (2017): 041102. @@ -59,10 +62,12 @@ This scaling factor can be provided either as an argument or a keyword argument. The `DenseLayer` is the default input layer in `ESN` construction. # Parameters -- `scaling`: The scaling factor for weight distribution (default: 0.1). + + - `scaling`: The scaling factor for weight distribution (default: 0.1). # Returns -- A `DenseLayer` instance for initializing the ESN's input layer. + + - A `DenseLayer` instance for initializing the ESN's input layer. """ struct DenseLayer{T} <: AbstractLayer scaling::T @@ -78,12 +83,14 @@ end Generates a matrix layer of size `res_size` x `in_size`, constructed according to the specifications of the `input_layer`. # Parameters -- `input_layer`: An instance of `AbstractLayer` determining the layer construction. -- `res_size`: The number of rows (reservoir size) for the layer. -- `in_size`: The number of columns (input size) for the layer. + + - `input_layer`: An instance of `AbstractLayer` determining the layer construction. + - `res_size`: The number of rows (reservoir size) for the layer. + - `in_size`: The number of columns (input size) for the layer. # Returns -- A matrix representing the constructed layer. + + - A matrix representing the constructed layer. """ function create_layer(input_layer::DenseLayer, res_size, @@ -104,11 +111,13 @@ The layer is initialized with weights distributed within [-`scaling`, `scaling`] and a specified `sparsity` level. Both `scaling` and `sparsity` can be set as arguments or keyword arguments. # Parameters -- `scaling`: Scaling factor for weight distribution (default: 0.1). -- `sparsity`: Sparsity level of the layer (default: 0.1). + + - `scaling`: Scaling factor for weight distribution (default: 0.1). + - `sparsity`: Sparsity level of the layer (default: 0.1). # Returns -- A `SparseLayer` instance for initializing ESN's input layer with sparse connections. + + - A `SparseLayer` instance for initializing ESN's input layer with sparse connections. """ struct SparseLayer{T} <: AbstractLayer scaling::T @@ -151,14 +160,17 @@ The parameter `p` sets the probability of a weight being positive, as per the `D This method of sign weight determination for input layers is based on the approach in [^Rodan]. # Parameters -- `p`: Probability of a positive weight (default: 0.5). + + - `p`: Probability of a positive weight (default: 0.5). # Returns -- A `BernoulliSample` instance for generating sign weights in `MinimumLayer`. + + - A `BernoulliSample` instance for generating sign weights in `MinimumLayer`. Reference: + [^Rodan]: Rodan, Ali, and Peter Tino. - "Minimum complexity echo state network." + "Minimum complexity echo state network." IEEE Transactions on Neural Networks 22.1 (2010): 131-144. """ function BernoulliSample(; p = 0.5) @@ -180,13 +192,16 @@ The `start` parameter sets the starting point in the decimal sequence. The signs are assigned based on the thresholding of each decimal digit against 4.5, as described in [^Rodan]. # Parameters -- `irrational`: An irrational number for weight sign determination (default: π). -- `start`: Starting index in the decimal expansion (default: 1). + + - `irrational`: An irrational number for weight sign determination (default: π). + - `start`: Starting index in the decimal expansion (default: 1). # Returns -- An `IrrationalSample` instance for generating sign weights in `MinimumLayer`. + + - An `IrrationalSample` instance for generating sign weights in `MinimumLayer`. Reference: + [^Rodan]: Rodan, Ali, and Peter Tiňo. "Simple deterministically constructed cycle reservoirs with regular jumps." Neural Computation 24.7 (2012): 1822-1852. @@ -211,13 +226,16 @@ weight determined by the `sampling` method. This approach, as detailed in [^Roda allows for controlled weight distribution in the layer. # Parameters -- `weight`: Absolute value of weights in the layer. -- `sampling`: Method for determining the sign of weights (default: `BernoulliSample(0.5)`). + + - `weight`: Absolute value of weights in the layer. + - `sampling`: Method for determining the sign of weights (default: `BernoulliSample(0.5)`). # Returns -- A `MinimumLayer` instance for initializing the ESN's input layer. + + - A `MinimumLayer` instance for initializing the ESN's input layer. References: + [^Rodan1]: Rodan, Ali, and Peter Tino. "Minimum complexity echo state network." IEEE Transactions on Neural Networks 22.1 (2010): 131-144. @@ -291,23 +309,27 @@ end Creates an `InformedLayer` initializer for Echo State Networks (ESNs) that generates a weighted input layer matrix. The matrix contains random non-zero elements drawn from -the range [-```scaling```, ```scaling```]. This initializer ensures that a fraction (`gamma`) +the range [-`scaling`, `scaling`]. This initializer ensures that a fraction (`gamma`) of reservoir nodes are exclusively connected to the raw inputs, while the rest are connected to the outputs of a prior knowledge model, as described in [^Pathak]. # Arguments -- `model_in_size`: The size of the prior knowledge model's output, + + - `model_in_size`: The size of the prior knowledge model's output, which determines the number of columns in the input layer matrix. # Keyword Arguments -- `scaling`: The absolute value of the weights (default: 0.1). -- `gamma`: The fraction of reservoir nodes connected exclusively to raw inputs (default: 0.5). + + - `scaling`: The absolute value of the weights (default: 0.1). + - `gamma`: The fraction of reservoir nodes connected exclusively to raw inputs (default: 0.5). # Returns -- An `InformedLayer` instance for initializing the ESN's input layer matrix. + + - An `InformedLayer` instance for initializing the ESN's input layer matrix. Reference: -[^Pathak]: Jaideep Pathak et al. + +[^Pathak]: Jaideep Pathak et al. "Hybrid Forecasting of Chaotic Processes: Using Machine Learning in Conjunction with a Knowledge-Based Model" (2018). """ function InformedLayer(model_in_size; scaling = 0.1, gamma = 0.5) @@ -359,7 +381,8 @@ end Creates a `NullLayer` initializer for Echo State Networks (ESNs) that generates a vector of zeros. # Returns -- A `NullLayer` instance for initializing the ESN's input layer matrix. + + - A `NullLayer` instance for initializing the ESN's input layer matrix. """ struct NullLayer <: AbstractLayer end diff --git a/src/esn/esn_reservoir_drivers.jl b/src/esn/esn_reservoir_drivers.jl index 6ab198d7..97e7b548 100644 --- a/src/esn/esn_reservoir_drivers.jl +++ b/src/esn/esn_reservoir_drivers.jl @@ -13,15 +13,17 @@ abstract type AbstractReservoirDriver end Create and return the trained Echo State Network (ESN) states according to the specified reservoir driver. # Arguments -- `reservoir_driver::AbstractReservoirDriver`: The reservoir driver that determines how the ESN states evolve over time. -- `train_data`: The training data used to train the ESN. -- `washout::Int`: The number of initial time steps to discard during training to allow the reservoir dynamics to wash out the initial conditions. -- `reservoir_matrix`: The reservoir matrix representing the dynamic, recurrent part of the ESN. -- `input_matrix`: The input matrix that defines the connections between input features and reservoir nodes. -- `bias_vector`: The bias vector to be added at each time step during the reservoir update. + + - `reservoir_driver::AbstractReservoirDriver`: The reservoir driver that determines how the ESN states evolve over time. + - `train_data`: The training data used to train the ESN. + - `washout::Int`: The number of initial time steps to discard during training to allow the reservoir dynamics to wash out the initial conditions. + - `reservoir_matrix`: The reservoir matrix representing the dynamic, recurrent part of the ESN. + - `input_matrix`: The input matrix that defines the connections between input features and reservoir nodes. + - `bias_vector`: The bias vector to be added at each time step during the reservoir update. # Returns -- A matrix of trained ESN states, where each column represents the state at a specific time step. + + - A matrix of trained ESN states, where each column represents the state at a specific time step. This function is responsible for creating and returning the states of the ESN during training based on the provided training data and parameters. """ @@ -100,16 +102,17 @@ end Returns a Recurrent Neural Network (RNN) initializer for the Echo State Network (ESN). # Arguments -- `activation_function`: The activation function used in the RNN. -- `leaky_coefficient`: The leaky coefficient used in the RNN. + + - `activation_function`: The activation function used in the RNN. + - `leaky_coefficient`: The leaky coefficient used in the RNN. # Keyword Arguments -- `activation_function`: The activation function used in the RNN. Defaults to `tanh`. -- `leaky_coefficient`: The leaky coefficient used in the RNN. Defaults to 1.0. + + - `activation_function`: The activation function used in the RNN. Defaults to `tanh`. + - `leaky_coefficient`: The leaky coefficient used in the RNN. Defaults to 1.0. This function creates an RNN object with the specified activation function and leaky coefficient, which can be used as a reservoir driver in the ESN. - """ function RNN(; activation_function = NNlib.fast_act(tanh), leaky_coefficient = 1.0) RNN(activation_function, leaky_coefficient) @@ -157,31 +160,32 @@ end """ MRNN(activation_function, leaky_coefficient, scaling_factor) - MRNN(;activation_function=[tanh, sigmoid], leaky_coefficient=1.0, + MRNN(;activation_function=[tanh, sigmoid], leaky_coefficient=1.0, scaling_factor=fill(leaky_coefficient, length(activation_function))) Returns a Multiple RNN (MRNN) initializer for the Echo State Network (ESN), introduced in [^lun]. # Arguments -- `activation_function`: A vector of activation functions used in the MRNN. -- `leaky_coefficient`: The leaky coefficient used in the MRNN. -- `scaling_factor`: A vector of scaling factors for combining activation functions. + + - `activation_function`: A vector of activation functions used in the MRNN. + - `leaky_coefficient`: The leaky coefficient used in the MRNN. + - `scaling_factor`: A vector of scaling factors for combining activation functions. # Keyword Arguments -- `activation_function`: A vector of activation functions used in the MRNN. Defaults to `[tanh, sigmoid]`. -- `leaky_coefficient`: The leaky coefficient used in the MRNN. Defaults to 1.0. -- `scaling_factor`: A vector of scaling factors for combining activation functions. Defaults to an array of the same size as `activation_function` with all elements set to `leaky_coefficient`. + + - `activation_function`: A vector of activation functions used in the MRNN. Defaults to `[tanh, sigmoid]`. + - `leaky_coefficient`: The leaky coefficient used in the MRNN. Defaults to 1.0. + - `scaling_factor`: A vector of scaling factors for combining activation functions. Defaults to an array of the same size as `activation_function` with all elements set to `leaky_coefficient`. This function creates an MRNN object with the specified activation functions, leaky coefficient, and scaling factors, which can be used as a reservoir driver in the ESN. # Reference: -[^lun]: Lun, Shu-Xian, et al. + +[^lun]: Lun, Shu-Xian, et al. "_A novel model of leaky integrator echo state network for time-series prediction._" Neurocomputing 159 (2015): 58-66. """ -function MRNN( - ; - activation_function = [tanh, sigmoid], +function MRNN(; activation_function = [tanh, sigmoid], leaky_coefficient = 1.0, scaling_factor = fill(leaky_coefficient, length(activation_function))) @assert length(activation_function) == length(scaling_factor) @@ -237,13 +241,14 @@ This function creates a FullyGated object, which can be used as a reservoir driv The FullyGated variant is described in the literature reference [^cho]. # Returns -- `FullyGated`: A FullyGated reservoir driver. + + - `FullyGated`: A FullyGated reservoir driver. # Reference + [^cho]: Cho, Kyunghyun, et al. "_Learning phrase representations using RNN encoder-decoder for statistical machine translation._" arXiv preprint arXiv:1406.1078 (2014). - """ struct FullyGated <: AbstractGRUVariant end @@ -252,8 +257,8 @@ struct FullyGated <: AbstractGRUVariant end Returns a minimal GRU ESN initializer as described in [^Zhou]. -[^Zhou]: Zhou, Guo-Bing, et al. "_Minimal gated unit for recurrent neural networks._" -International Journal of Automation and Computing 13.3 (2016): 226-234. +[^Zhou]: Zhou, Guo-Bing, et al. "_Minimal gated unit for recurrent neural networks._" + International Journal of Automation and Computing 13.3 (2016): 226-234. """ struct Minimal <: AbstractGRUVariant end @@ -268,23 +273,24 @@ struct Minimal <: AbstractGRUVariant end Returns a Gated Recurrent Unit (GRU) reservoir driver for Echo State Networks (ESNs). This driver is based on the GRU architecture [^Cho], which is designed to capture temporal dependencies in data and is commonly used in various machine learning applications. # Arguments -- `activation_function`: An array of activation functions for the GRU layers. By default, it uses sigmoid activation functions for the update gate, reset gate, and tanh for the hidden state. -- `inner_layer`: An array of inner layers used in the GRU architecture. By default, it uses two dense layers. -- `reservoir`: An array of reservoir layers. By default, it uses two random sparse reservoirs. -- `bias`: An array of bias layers for the GRU. By default, it uses two dense layers. -- `variant`: The GRU variant to use. By default, it uses the "FullyGated" variant. + + - `activation_function`: An array of activation functions for the GRU layers. By default, it uses sigmoid activation functions for the update gate, reset gate, and tanh for the hidden state. + - `inner_layer`: An array of inner layers used in the GRU architecture. By default, it uses two dense layers. + - `reservoir`: An array of reservoir layers. By default, it uses two random sparse reservoirs. + - `bias`: An array of bias layers for the GRU. By default, it uses two dense layers. + - `variant`: The GRU variant to use. By default, it uses the "FullyGated" variant. # Returns + A GRUParams object containing the parameters needed for the GRU-based reservoir driver. # References + [^Cho]: Cho, Kyunghyun, et al. "_Learning phrase representations using RNN encoder-decoder for statistical machine translation._" arXiv preprint arXiv:1406.1078 (2014). """ -function GRU( - ; - activation_function = [NNlib.sigmoid, NNlib.sigmoid, tanh], +function GRU(; activation_function = [NNlib.sigmoid, NNlib.sigmoid, tanh], inner_layer = fill(DenseLayer(), 2), reservoir = fill(RandSparseReservoir(0), 2), bias = fill(DenseLayer(), 2), diff --git a/src/esn/esn_reservoirs.jl b/src/esn/esn_reservoirs.jl index e014e4e7..2c727afb 100644 --- a/src/esn/esn_reservoirs.jl +++ b/src/esn/esn_reservoirs.jl @@ -18,18 +18,20 @@ end RandSparseReservoir(res_size, radius, sparsity) RandSparseReservoir(res_size; radius=1.0, sparsity=0.1) - Returns a random sparse reservoir initializer, which generates a matrix of size `res_size x res_size` with the specified `sparsity` and scaled spectral radius according to `radius`. This type of reservoir initializer is commonly used in Echo State Networks (ESNs) for capturing complex temporal dependencies. # Arguments -- `res_size`: The size of the reservoir matrix. -- `radius`: The desired spectral radius of the reservoir. By default, it is set to 1.0. -- `sparsity`: The sparsity level of the reservoir matrix, controlling the fraction of zero elements. By default, it is set to 0.1. + + - `res_size`: The size of the reservoir matrix. + - `radius`: The desired spectral radius of the reservoir. By default, it is set to 1.0. + - `sparsity`: The sparsity level of the reservoir matrix, controlling the fraction of zero elements. By default, it is set to 0.1. # Returns + A RandSparseReservoir object that can be used as a reservoir initializer in ESN construction. # References + This type of reservoir initialization is a common choice in ESN construction for its ability to capture temporal dependencies in data. However, there is no specific reference associated with this function. """ function RandSparseReservoir(res_size; radius = 1.0, sparsity = 0.1) @@ -43,14 +45,17 @@ end Given an `AbstractReservoir` constructor and the size of the reservoir (`res_size`), this function returns the corresponding reservoir matrix. Alternatively, it accepts a pre-generated matrix. # Arguments -- `reservoir`: An `AbstractReservoir` object or constructor. -- `res_size`: The size of the reservoir matrix. -- `matrix_type`: The type of the resulting matrix. By default, it is set to `Matrix{Float64}`. + + - `reservoir`: An `AbstractReservoir` object or constructor. + - `res_size`: The size of the reservoir matrix. + - `matrix_type`: The type of the resulting matrix. By default, it is set to `Matrix{Float64}`. # Returns + A matrix representing the reservoir, generated based on the properties of the specified `reservoir` object or constructor. # References + The choice of reservoir initialization is crucial in Echo State Networks (ESNs) for achieving effective temporal modeling. Specific references for reservoir initialization methods may vary based on the type of reservoir used, but the practice of initializing reservoirs for ESNs is widely documented in the ESN literature. """ function create_reservoir(reservoir::RandSparseReservoir, @@ -107,16 +112,19 @@ end Returns an initializer to build a sparse reservoir matrix with the given `sparsity` by using a pseudo-SVD approach as described in [^yang]. # Arguments -- `res_size`: The size of the reservoir matrix. -- `max_value`: The maximum absolute value of elements in the matrix. -- `sparsity`: The desired sparsity level of the reservoir matrix. -- `sorted`: A boolean indicating whether to sort the singular values before creating the diagonal matrix. By default, it is set to `true`. -- `reverse_sort`: A boolean indicating whether to reverse the sorted singular values. By default, it is set to `false`. + + - `res_size`: The size of the reservoir matrix. + - `max_value`: The maximum absolute value of elements in the matrix. + - `sparsity`: The desired sparsity level of the reservoir matrix. + - `sorted`: A boolean indicating whether to sort the singular values before creating the diagonal matrix. By default, it is set to `true`. + - `reverse_sort`: A boolean indicating whether to reverse the sorted singular values. By default, it is set to `false`. # Returns + A PseudoSVDReservoir object that can be used as a reservoir initializer in ESN construction. # References + This reservoir initialization method, based on a pseudo-SVD approach, is inspired by the work in [^yang], which focuses on designing polynomial echo state networks for time series prediction. [^yang]: Yang, Cuili, et al. "_Design of polynomial echo state networks for time series prediction._" Neurocomputing 290 (2018): 148-160. @@ -199,15 +207,18 @@ Returns a Delay Line Reservoir matrix constructor to obtain a deterministic rese described in [^Rodan2010]. # Arguments -- `res_size::Int`: The size of the reservoir. -- `weight::T`: The weight determines the absolute value of all the connections in the reservoir. + + - `res_size::Int`: The size of the reservoir. + - `weight::T`: The weight determines the absolute value of all the connections in the reservoir. # Returns + A `DelayLineReservoir` object. # References + [^Rodan2010]: Rodan, Ali, and Peter Tino. "Minimum complexity echo state network." -IEEE transactions on neural networks 22.1 (2010): 131-144. + IEEE transactions on neural networks 22.1 (2010): 131-144. """ function DelayLineReservoir(res_size; weight = 0.1) return DelayLineReservoir(res_size, weight) @@ -242,16 +253,19 @@ as described in [^Rodan2010]. The `weight` and `fb_weight` can be passed as eith keyword arguments, and they determine the absolute values of the connections in the reservoir. # Arguments -- `res_size::Int`: The size of the reservoir. -- `weight::T`: The weight determines the absolute value of forward connections in the reservoir. -- `fb_weight::T`: The `fb_weight` determines the absolute value of backward connections in the reservoir. + + - `res_size::Int`: The size of the reservoir. + - `weight::T`: The weight determines the absolute value of forward connections in the reservoir. + - `fb_weight::T`: The `fb_weight` determines the absolute value of backward connections in the reservoir. # Returns + A `DelayLineBackwardReservoir` object. # References + [^Rodan2010]: Rodan, Ali, and Peter Tino. "Minimum complexity echo state network." -IEEE transactions on neural networks 22.1 (2010): 131-144. + IEEE transactions on neural networks 22.1 (2010): 131-144. """ function DelayLineBackwardReservoir(res_size; weight = 0.1, fb_weight = 0.2) return DelayLineBackwardReservoir(res_size, weight, fb_weight) @@ -286,15 +300,18 @@ described in [^Rodan2010]. The `weight` can be passed as an argument or a keywor absolute value of all the connections in the reservoir. # Arguments -- `res_size::Int`: The size of the reservoir. -- `weight::T`: The weight determines the absolute value of connections in the reservoir. + + - `res_size::Int`: The size of the reservoir. + - `weight::T`: The weight determines the absolute value of connections in the reservoir. # Returns + A `SimpleCycleReservoir` object. # References + [^Rodan2010]: Rodan, Ali, and Peter Tino. "Minimum complexity echo state network." -IEEE transactions on neural networks 22.1 (2010): 131-144. + IEEE transactions on neural networks 22.1 (2010): 131-144. """ function SimpleCycleReservoir(res_size; weight = 0.1) return SimpleCycleReservoir(res_size, weight) @@ -331,17 +348,20 @@ in [^Rodan2012]. The `cycle_weight`, `jump_weight`, and `jump_size` can be passe determine the absolute values of connections in the reservoir. The `jump_size` determines the jumps between `jump_weight`s. # Arguments -- `res_size::Int`: The size of the reservoir. -- `cycle_weight::T`: The weight of cycle connections. -- `jump_weight::T`: The weight of jump connections. -- `jump_size::Int`: The number of steps between jump connections. + + - `res_size::Int`: The size of the reservoir. + - `cycle_weight::T`: The weight of cycle connections. + - `jump_weight::T`: The weight of jump connections. + - `jump_size::Int`: The number of steps between jump connections. # Returns + A `CycleJumpsReservoir` object. # References + [^Rodan2012]: Rodan, Ali, and Peter Tiňo. "Simple deterministically constructed cycle reservoirs -with regular jumps." Neural computation 24.7 (2012): 1822-1852. + with regular jumps." Neural computation 24.7 (2012): 1822-1852. """ function CycleJumpsReservoir(res_size; cycle_weight = 0.1, jump_weight = 0.1, jump_size = 3) return CycleJumpsReservoir(res_size, cycle_weight, jump_weight, jump_size) @@ -376,13 +396,16 @@ end Return a constructor for a matrix of zeros with dimensions `res_size x res_size`. # Arguments -- None + + - None # Returns + A `NullReservoir` object. # References -- None + + - None """ struct NullReservoir <: AbstractReservoir end diff --git a/src/reca/reca_input_encodings.jl b/src/reca/reca_input_encodings.jl index 8fdd5233..a26fb6f1 100644 --- a/src/reca/reca_input_encodings.jl +++ b/src/reca/reca_input_encodings.jl @@ -11,11 +11,11 @@ end RandomMapping(permutations; expansion_size=40) RandomMapping(;permutations=8, expansion_size=40) -Random mapping of the input data directly in the reservoir. The ```expansion_size``` -determines the dimension of the single reservoir, and ```permutations``` determines the +Random mapping of the input data directly in the reservoir. The `expansion_size` +determines the dimension of the single reservoir, and `permutations` determines the number of total reservoirs that will be connected, each with a different mapping. The detail of this implementation can be found in [1]. - + [1] Nichele, Stefano, and Andreas Molund. “Deep reservoir computing using cellular automata.” arXiv preprint arXiv:1703.02806 (2017). """ @@ -66,7 +66,8 @@ function encoding(rm::RandomMaps, input_vector, tot_encoded_vector) new_tot_enc_vec = copy(tot_encoded_vector) for i in 1:(rm.permutations) - new_tot_enc_vec[((i - 1) * rm.expansion_size + 1):(i * rm.expansion_size)] = single_encoding(input_vector, + new_tot_enc_vec[((i - 1) * rm.expansion_size + 1):(i * rm.expansion_size)] = single_encoding( + input_vector, new_tot_enc_vec[((i - 1) * rm.expansion_size + 1):(i * rm.expansion_size)], rm.maps[i, :]) diff --git a/src/states.jl b/src/states.jl index 2aac07b0..1de7952a 100644 --- a/src/states.jl +++ b/src/states.jl @@ -16,10 +16,10 @@ end """ StandardStates() -When this struct is employed, the states of the reservoir are not modified. It represents the default behavior -in scenarios where no specific state modification is required. This approach is ideal for applications -where the inherent dynamics of the reservoir are sufficient, and no external manipulation of the states -is necessary. It maintains the original state representation, ensuring that the reservoir's natural properties +When this struct is employed, the states of the reservoir are not modified. It represents the default behavior +in scenarios where no specific state modification is required. This approach is ideal for applications +where the inherent dynamics of the reservoir are sufficient, and no external manipulation of the states +is necessary. It maintains the original state representation, ensuring that the reservoir's natural properties are preserved and utilized in computations. """ struct StandardStates <: AbstractStates end @@ -27,9 +27,9 @@ struct StandardStates <: AbstractStates end """ ExtendedStates() -The `ExtendedStates` struct is used to extend the reservoir states by -vertically concatenating the input data (during training) and the prediction data (during the prediction phase). -This method enriches the state representation by integrating external data, enhancing the model's capability +The `ExtendedStates` struct is used to extend the reservoir states by +vertically concatenating the input data (during training) and the prediction data (during the prediction phase). +This method enriches the state representation by integrating external data, enhancing the model's capability to capture and utilize complex patterns in both training and prediction stages. """ struct ExtendedStates <: AbstractStates end @@ -103,7 +103,7 @@ end """ NLADefault() -`NLADefault` represents the default non-linear algorithm option. +`NLADefault` represents the default non-linear algorithm option. When used, it leaves the input array unchanged. This option is suitable in cases where no non-linear transformation of the data is required, maintaining the original state of the input array for further processing. @@ -126,7 +126,8 @@ making it a valuable tool in analyzing chaotic systems and improving the perform The T₁ transformation's uniqueness lies in its selective approach, allowing for a more nuanced manipulation of the input data. References: -[^Chattopadhyay]: Chattopadhyay, Ashesh, et al. + +[^Chattopadhyay]: Chattopadhyay, Ashesh, et al. "Data-driven prediction of a multi-scale Lorenz 96 chaotic system using a hierarchy of deep learning methods: Reservoir computing, ANN, and RNN-LSTM." (2019). [^Pathak]: Pathak, Jaideep, et al. @@ -158,6 +159,7 @@ enhancing complex patterns in the data, particularly beneficial in the analysis systems and in improving the dynamics within reservoir computing models. Reference: + [^Chattopadhyay]: Chattopadhyay, Ashesh, et al. "Data-driven prediction of a multi-scale Lorenz 96 chaotic system using a hierarchy of deep learning methods: Reservoir computing, ANN, and RNN-LSTM." (2019). @@ -186,6 +188,7 @@ useful for enhancing complex data patterns, thereby improving the modeling and a capabilities within reservoir computing, especially for chaotic and dynamic systems. Reference: + [^Chattopadhyay]: Chattopadhyay, Ashesh, et al. "Data-driven prediction of a multi-scale Lorenz 96 chaotic system using a hierarchy of deep learning methods: Reservoir computing, ANN, and RNN-LSTM." (2019). diff --git a/src/train/linear_regression.jl b/src/train/linear_regression.jl index f32b6601..7e9f6fca 100644 --- a/src/train/linear_regression.jl +++ b/src/train/linear_regression.jl @@ -7,7 +7,7 @@ end StandardRidge(;regularization_coeff=0.0) Ridge regression training for all the models in the library. The -```regularization_coeff``` is the regularization, it can be passed as an arg or kwarg. +`regularization_coeff` is the regularization, it can be passed as an arg or kwarg. """ function StandardRidge(; regularization_coeff = 0.0) return StandardRidge(regularization_coeff) @@ -29,18 +29,17 @@ struct LinearModel{T, S, K} <: AbstractLinearModel end """ - LinearModel(;regression=LinearRegression, - solver=Analytical(), + LinearModel(;regression=LinearRegression, + solver=Analytical(), regression_kwargs=(;)) Linear regression training based on [MLJLinearModels](https://juliaai.github.io/MLJLinearModels.jl/stable/) for all the -models in the library. All the parameters have to be passed into ```regression_kwargs```, +models in the library. All the parameters have to be passed into `regression_kwargs`, apart from the solver choice. MLJLinearModels.jl needs to be called in order to use these models. """ -function LinearModel( - ; regression = LinearRegression, +function LinearModel(; regression = LinearRegression, solver = Analytical(), regression_kwargs = (;)) return LinearModel(regression, solver, regression_kwargs) diff --git a/test/esn/test_drivers.jl b/test/esn/test_drivers.jl index db1c7b01..54045925 100644 --- a/test/esn/test_drivers.jl +++ b/test/esn/test_drivers.jl @@ -16,7 +16,7 @@ esn = ESN(input_data; reservoir_driver = GRU(variant = FullyGated(), reservoir = [ RandSparseReservoir(res_size, 1.0, 0.5), - RandSparseReservoir(res_size, 1.2, 0.1), + RandSparseReservoir(res_size, 1.2, 0.1) ])) output_layer = train(esn, target_data, training_method) @@ -47,7 +47,7 @@ output = esn(Predictive(target_data), output_layer, initial_conditions = target_ esn = ESN(input_data; reservoir = [ RandSparseReservoir(res_size, 1.2, 0.1), - RandSparseReservoir(res_size, 1.2, 0.1), + RandSparseReservoir(res_size, 1.2, 0.1) ]) output_layer = train(esn, target_data, training_method) output = esn(Predictive(target_data), output_layer, initial_conditions = target_data[1]) @@ -56,7 +56,7 @@ output = esn(Predictive(target_data), output_layer, initial_conditions = target_ esn = ESN(input_data; reservoir = [ RandSparseReservoir(res_size, 1.2, 0.1), - RandSparseReservoir(res_size, 1.2, 0.1), + RandSparseReservoir(res_size, 1.2, 0.1) ], input_layer = [DenseLayer(), DenseLayer()], bias = [NullLayer(), NullLayer()]) diff --git a/test/esn/test_train.jl b/test/esn/test_train.jl index e5140b97..178d4c75 100644 --- a/test/esn/test_train.jl +++ b/test/esn/test_train.jl @@ -18,7 +18,7 @@ training_methods = [ StandardRidge(regularization_coeff = reg), LinearModel(RidgeRegression, regression_kwargs = (; lambda = reg)), LinearModel(regression = RidgeRegression, regression_kwargs = (; lambda = reg)), - EpsilonSVR(), + EpsilonSVR() ] for t in training_methods diff --git a/test/runtests.jl b/test/runtests.jl index b1b28ad1..bac8443e 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,19 +2,19 @@ using SafeTestsets using Test @testset "Common Utilities" begin - @safetestset "Quality Assurance" begin include("qa.jl") end - @safetestset "States" begin include("test_states.jl") end + @safetestset "Quality Assurance" include("qa.jl") + @safetestset "States" include("test_states.jl") end @testset "Echo State Networks" begin - @safetestset "ESN Input Layers" begin include("esn/test_input_layers.jl") end - @safetestset "ESN Reservoirs" begin include("esn/test_reservoirs.jl") end - @safetestset "ESN States" begin include("esn/test_states.jl") end - @safetestset "ESN Train and Predict" begin include("esn/test_train.jl") end - @safetestset "ESN Drivers" begin include("esn/test_drivers.jl") end - @safetestset "Hybrid ESN" begin include("esn/test_hybrid.jl") end + @safetestset "ESN Input Layers" include("esn/test_input_layers.jl") + @safetestset "ESN Reservoirs" include("esn/test_reservoirs.jl") + @safetestset "ESN States" include("esn/test_states.jl") + @safetestset "ESN Train and Predict" include("esn/test_train.jl") + @safetestset "ESN Drivers" include("esn/test_drivers.jl") + @safetestset "Hybrid ESN" include("esn/test_hybrid.jl") end @testset "CA based Reservoirs" begin - @safetestset "RECA" begin include("reca/test_predictive.jl") end + @safetestset "RECA" include("reca/test_predictive.jl") end