Dataset and DataLoader

.github/workflows/linux.yml

@@ -32,7 +32,5 @@ jobs:
         run: |
           g++ examples/scalars.cpp -O3 --std=c++17
           ./a.out
-          g++ examples/xor_classification.cpp -O3 --std=c++17
-          ./a.out
           g++ examples/xor_regression.cpp -O3 --std=c++17
           ./a.out

.github/workflows/macos.yml

@@ -32,7 +32,5 @@ jobs:
         run: |
           g++ examples/scalars.cpp -O3 --std=c++17
           ./a.out
-          g++ examples/xor_classification.cpp -O3 --std=c++17
-          ./a.out
           g++ examples/xor_regression.cpp -O3 --std=c++17
           ./a.out

.github/workflows/windows.yml

@@ -30,5 +30,4 @@ jobs:
           CXX: ${{matrix.conf.compiler}}
         run: |
           g++ -o out examples/scalars.cpp -O3 --std=c++17
-          g++ -o out examples/xor_classification.cpp -O3 --std=c++17
           g++ -o out examples/xor_regression.cpp -O3 --std=c++17

.gitignore

@@ -1,5 +1,4 @@
 build/
-data/
 
 docs/html
 docs/latex

README.md

@@ -34,14 +34,17 @@ int main() {
     using namespace shkyera;
     using T = Type::float32;
 
-    std::vector<Vec32> xs;
-    std::vector<Vec32> ys;
-
-    // ---------- INPUT ----------- | -------- OUTPUT --------- //
-    xs.push_back(Vec32::of(0, 0)); ys.push_back(Vec32::of(0));
-    xs.push_back(Vec32::of(1, 0)); ys.push_back(Vec32::of(1));
-    xs.push_back(Vec32::of(0, 1)); ys.push_back(Vec32::of(1));
-    xs.push_back(Vec32::of(1, 1)); ys.push_back(Vec32::of(0));
+    // This is our XOR dataset. It maps from Vec32 to Vec32
+    Dataset<Vec32, Vec32> data;
+    data.addSample(Vec32::of(0, 0), Vec32::of(0));
+    data.addSample(Vec32::of(0, 1), Vec32::of(1));
+    data.addSample(Vec32::of(1, 0), Vec32::of(1));
+    data.addSample(Vec32::of(1, 1), Vec32::of(0));
+
+    // The is the data loader, it will take care of batching
+    size_t batchSize = 2;
+    bool shuffle = true;
+    DataLoader loader(data, batchSize, shuffle);
 
     auto network = SequentialBuilder<Type::float32>::begin()
                     .add(Linear32::create(2, 15))
@@ -52,29 +55,26 @@ int main() {
                     .add(Sigmoid32::create())
                     .build();
 
-
-    auto optimizer = Adam32(network->parameters(), 0.05);
+    auto optimizer = Adam32(network->parameters(), 0.1);
     auto lossFunction = Loss::MSE<T>;
 
     for (size_t epoch = 0; epoch < 100; epoch++) { // We train for 100 epochs
         auto epochLoss = Val32::create(0);
 
-        optimizer.reset();                                      // Reset the gradients
-        for (size_t sample = 0; sample < xs.size(); ++sample) { // We go through each sample
-            Vec32 pred = network->forward(xs[sample]);          // We get some prediction
-            auto loss = lossFunction(pred, ys[sample]);         // And calculate its error
-
-            epochLoss = epochLoss + loss; // Store the loss for feedback
+        optimizer.reset();                                                // Reset the gradients
+        for (const auto &[x, y] : loader) {                               // For each batch
+            auto pred = network->forward(x);                              // We get some prediction
+            epochLoss = epochLoss + Loss::compute(lossFunction, pred, y); // And calculate its error
         }
         optimizer.step(); // Update the parameters
 
-        auto averageLoss = epochLoss / Val32::create(xs.size());
+        auto averageLoss = epochLoss / Val32::create(loader.getTotalBatches());
         std::cout << "Epoch: " << epoch + 1 << " Loss: " << averageLoss->getValue() << std::endl;
     }
 
-    for (size_t sample = 0; sample < xs.size(); ++sample) { // Go through each example
-        Vec32 pred = network->forward(xs[sample]);          // Predict result
-        std::cout << xs[sample] << " -> " << pred[0] << "\t| True: " << ys[sample][0] << std::endl;
+    for (auto &[x, y] : data) {          // Go through each example
+        auto pred = network->forward(x); // We get some prediction
+        std::cout << x << " -> " << pred[0] << "\t| True: " << y[0] << std::endl;
     }
 }
 ```

examples/xor_regression.cpp

@@ -5,14 +5,17 @@ int main() {
     using T = Type::float32;
 
     // clang-format off
-    std::vector<Vec32> xs;
-    std::vector<Vec32> ys;
-
-    // ---------- INPUT ----------- | -------- OUTPUT --------- //
-    xs.push_back(Vec32::of(0, 0)); ys.push_back(Vec32::of(0));
-    xs.push_back(Vec32::of(1, 0)); ys.push_back(Vec32::of(1));
-    xs.push_back(Vec32::of(0, 1)); ys.push_back(Vec32::of(1));
-    xs.push_back(Vec32::of(1, 1)); ys.push_back(Vec32::of(0));
+    // This is our XOR dataset. It maps from Vec32 to Vec32
+    Dataset<Vec32, Vec32> data;
+    data.addSample(Vec32::of(0, 0), Vec32::of(0));
+    data.addSample(Vec32::of(0, 1), Vec32::of(1));
+    data.addSample(Vec32::of(1, 0), Vec32::of(1));
+    data.addSample(Vec32::of(1, 1), Vec32::of(0));
+
+    // The is the data loader, it will take care of batching
+    size_t batchSize = 2;
+    bool shuffle = true;
+    DataLoader loader(data, batchSize, shuffle);
 
     auto network = SequentialBuilder<Type::float32>::begin()
                     .add(Linear32::create(2, 15))
@@ -24,27 +27,25 @@ int main() {
                     .build();
     // clang-format on
 
-    auto optimizer = Adam32(network->parameters(), 0.05);
+    auto optimizer = Adam32(network->parameters(), 0.1);
     auto lossFunction = Loss::MSE<T>;
 
     for (size_t epoch = 0; epoch < 100; epoch++) { // We train for 100 epochs
         auto epochLoss = Val32::create(0);
 
-        optimizer.reset();                                      // Reset the gradients
-        for (size_t sample = 0; sample < xs.size(); ++sample) { // We go through each sample
-            Vec32 pred = network->forward(xs[sample]);          // We get some prediction
-            auto loss = lossFunction(pred, ys[sample]);         // And calculate its error
-
-            epochLoss = epochLoss + loss; // Store the loss for feedback
+        optimizer.reset();                                                // Reset the gradients
+        for (const auto &[x, y] : loader) {                               // For each batch
+            auto pred = network->forward(x);                              // We get some prediction
+            epochLoss = epochLoss + Loss::compute(lossFunction, pred, y); // And calculate its error
         }
         optimizer.step(); // Update the parameters
 
-        auto averageLoss = epochLoss / Val32::create(xs.size());
+        auto averageLoss = epochLoss / Val32::create(loader.getTotalBatches());
         std::cout << "Epoch: " << epoch + 1 << " Loss: " << averageLoss->getValue() << std::endl;
     }
 
-    for (size_t sample = 0; sample < xs.size(); ++sample) { // Go through each example
-        Vec32 pred = network->forward(xs[sample]);          // Predict result
-        std::cout << xs[sample] << " -> " << pred[0] << "\t| True: " << ys[sample][0] << std::endl;
+    for (auto &[x, y] : data) {          // Go through each example
+        auto pred = network->forward(x); // We get some prediction
+        std::cout << x << " -> " << pred[0] << "\t| True: " << y[0] << std::endl;
     }
 }

include/ShkyeraGrad.hpp

@@ -17,6 +17,9 @@
 #include "nn/Neuron.hpp"
 #include "nn/Sequential.hpp"
 
+#include "nn/data/DataLoader.hpp"
+#include "nn/data/Dataset.hpp"
+
 #include "nn/optimizers/AdaMax.hpp"
 #include "nn/optimizers/Adam.hpp"
 #include "nn/optimizers/NAG.hpp"

include/core/Utils.hpp

@@ -53,6 +53,8 @@ std::enable_if_t<std::is_integral_v<T>, std::vector<T>> sample(T from, T to, siz
     return sampled;
 }
 
+template <typename T> void shuffle(std::vector<T> &vec) { std::shuffle(vec.begin(), vec.end(), rand_dev); }
+
 template <typename Clock = std::chrono::high_resolution_clock> auto startTimer() { return Clock::now(); }
 
 template <typename Clock = std::chrono::high_resolution_clock>

include/core/Vector.hpp

@@ -18,6 +18,7 @@ template <typename T> class Vector;
 
 using Vec32 = Vector<Type::float32>;
 using Vec64 = Vector<Type::float64>;
+template <typename T> using Batch = std::vector<Vector<T>>;
 
 template <typename T> class Vector {
   private:

include/nn/Loss.hpp

@@ -8,6 +8,7 @@
 #pragma once
 
 #include "../core/Value.hpp"
+#include "../core/Vector.hpp"
 
 namespace shkyera::Loss {
 
@@ -31,8 +32,6 @@ Function<T> MSE = [](Vector<T> a, Vector<T> b) {
     if (a.size() > 0)
         loss = loss / Value<T>::create(a.size());
 
-    loss->backward();
-
     return loss;
 };
 
@@ -52,8 +51,6 @@ Function<T> MAE = [](Vector<T> a, Vector<T> b) {
     if (a.size() > 0)
         loss = loss / Value<T>::create(a.size());
 
-    loss->backward();
-
     return loss;
 };
 
@@ -80,9 +77,26 @@ Function<T> CrossEntropy = [](Vector<T> a, Vector<T> b) {
         loss = loss - (b[i] * (a[i]->log()));
     }
 
+    return loss;
+};
+
+template <typename T>
+ValuePtr<T> compute(Function<T> lossFunction, const Vector<T> prediction, const Vector<T> target) {
+    auto loss = lossFunction(prediction, target);
+    loss->backward();
+    return loss;
+}
+
+template <typename T> ValuePtr<T> compute(Function<T> lossFunction, const Batch<T> prediction, const Batch<T> target) {
+    ValuePtr<T> loss = Value<T>::create(0);
+    for (size_t i = 0; i < prediction.size(); ++i) {
+        loss = loss + lossFunction(prediction[i], target[i]);
+    }
+    loss = loss / Value<T>::create(prediction.size());
+
     loss->backward();
 
     return loss;
-};
+}
 
 } // namespace shkyera::Loss

include/nn/Module.hpp

@@ -19,8 +19,17 @@ template <typename T> class Module {
     Module() = default;
 
   public:
-    Vector<T> forward(const Vector<T> &x) const { return (*this)(x); }
+    template <typename U> U forward(const U &x) const { return (*this)(x); }
+
     virtual Vector<T> operator()(const Vector<T> &x) const { return x; }
+    std::vector<Vector<T>> operator()(const std::vector<Vector<T>> &x) const {
+        std::vector<Vector<T>> out(x.size());
+        for (size_t i = 0; i < x.size(); ++i) {
+            out[i] = this->operator()(x[i]);
+        }
+        return out;
+    }
+
     virtual std::vector<ValuePtr<T>> parameters() const { return {}; }
 };