Merge pull request #18 from fszewczyk/dataset

Dataset and DataLoader

.gitignore

@@ -1,5 +1,4 @@
 build/
-data/
 
 docs/html
 docs/latex

README.md

@@ -34,14 +34,17 @@ int main() {
     using namespace shkyera;
     using T = Type::float32;
 
-    std::vector<Vec32> xs;
-    std::vector<Vec32> ys;
-
-    // ---------- INPUT ----------- | -------- OUTPUT --------- //
-    xs.push_back(Vec32::of(0, 0)); ys.push_back(Vec32::of(0));
-    xs.push_back(Vec32::of(1, 0)); ys.push_back(Vec32::of(1));
-    xs.push_back(Vec32::of(0, 1)); ys.push_back(Vec32::of(1));
-    xs.push_back(Vec32::of(1, 1)); ys.push_back(Vec32::of(0));
+    // This is our XOR dataset. It maps from Vec32 to Vec32
+    Dataset<Vec32, Vec32> data;
+    data.addSample(Vec32::of(0, 0), Vec32::of(0));
+    data.addSample(Vec32::of(0, 1), Vec32::of(1));
+    data.addSample(Vec32::of(1, 0), Vec32::of(1));
+    data.addSample(Vec32::of(1, 1), Vec32::of(0));
+
+    // The is the data loader, it will take care of batching
+    size_t batchSize = 2;
+    bool shuffle = true;
+    DataLoader loader(data, batchSize, shuffle);
 
     auto network = SequentialBuilder<Type::float32>::begin()
                     .add(Linear32::create(2, 15))
@@ -52,29 +55,26 @@ int main() {
                     .add(Sigmoid32::create())
                     .build();
 
-
-    auto optimizer = Adam32(network->parameters(), 0.05);
+    auto optimizer = Adam32(network->parameters(), 0.1);
     auto lossFunction = Loss::MSE<T>;
 
     for (size_t epoch = 0; epoch < 100; epoch++) { // We train for 100 epochs
         auto epochLoss = Val32::create(0);
 
-        optimizer.reset();                                      // Reset the gradients
-        for (size_t sample = 0; sample < xs.size(); ++sample) { // We go through each sample
-            Vec32 pred = network->forward(xs[sample]);          // We get some prediction
-            auto loss = lossFunction(pred, ys[sample]);         // And calculate its error
-
-            epochLoss = epochLoss + loss; // Store the loss for feedback
+        optimizer.reset();                                                // Reset the gradients
+        for (const auto &[x, y] : loader) {                               // For each batch
+            auto pred = network->forward(x);                              // We get some prediction
+            epochLoss = epochLoss + Loss::compute(lossFunction, pred, y); // And calculate its error
         }
         optimizer.step(); // Update the parameters
 
-        auto averageLoss = epochLoss / Val32::create(xs.size());
+        auto averageLoss = epochLoss / Val32::create(loader.getTotalBatches());
         std::cout << "Epoch: " << epoch + 1 << " Loss: " << averageLoss->getValue() << std::endl;
     }
 
-    for (size_t sample = 0; sample < xs.size(); ++sample) { // Go through each example
-        Vec32 pred = network->forward(xs[sample]);          // Predict result
-        std::cout << xs[sample] << " -> " << pred[0] << "\t| True: " << ys[sample][0] << std::endl;
+    for (auto &[x, y] : data) {          // Go through each example
+        auto pred = network->forward(x); // We get some prediction
+        std::cout << x << " -> " << pred[0] << "\t| True: " << y[0] << std::endl;
     }
 }
 ```

examples/xor_regression.cpp

@@ -5,14 +5,17 @@ int main() {
     using T = Type::float32;
 
     // clang-format off
-    std::vector<Vec32> xs;
-    std::vector<Vec32> ys;
-
-    // ---------- INPUT ----------- | -------- OUTPUT --------- //
-    xs.push_back(Vec32::of(0, 0)); ys.push_back(Vec32::of(0));
-    xs.push_back(Vec32::of(1, 0)); ys.push_back(Vec32::of(1));
-    xs.push_back(Vec32::of(0, 1)); ys.push_back(Vec32::of(1));
-    xs.push_back(Vec32::of(1, 1)); ys.push_back(Vec32::of(0));
+    // This is our XOR dataset. It maps from Vec32 to Vec32
+    Dataset<Vec32, Vec32> data;
+    data.addSample(Vec32::of(0, 0), Vec32::of(0));
+    data.addSample(Vec32::of(0, 1), Vec32::of(1));
+    data.addSample(Vec32::of(1, 0), Vec32::of(1));
+    data.addSample(Vec32::of(1, 1), Vec32::of(0));
+
+    // The is the data loader, it will take care of batching
+    size_t batchSize = 2;
+    bool shuffle = true;
+    DataLoader loader(data, batchSize, shuffle);
 
     auto network = SequentialBuilder<Type::float32>::begin()
                     .add(Linear32::create(2, 15))
@@ -24,27 +27,25 @@ int main() {
                     .build();
     // clang-format on
 
-    auto optimizer = Adam32(network->parameters(), 0.05);
+    auto optimizer = Adam32(network->parameters(), 0.1);
     auto lossFunction = Loss::MSE<T>;
 
     for (size_t epoch = 0; epoch < 100; epoch++) { // We train for 100 epochs
         auto epochLoss = Val32::create(0);
 
-        optimizer.reset();                                      // Reset the gradients
-        for (size_t sample = 0; sample < xs.size(); ++sample) { // We go through each sample
-            Vec32 pred = network->forward(xs[sample]);          // We get some prediction
-            auto loss = lossFunction(pred, ys[sample]);         // And calculate its error
-
-            epochLoss = epochLoss + loss; // Store the loss for feedback
+        optimizer.reset();                                                // Reset the gradients
+        for (const auto &[x, y] : loader) {                               // For each batch
+            auto pred = network->forward(x);                              // We get some prediction
+            epochLoss = epochLoss + Loss::compute(lossFunction, pred, y); // And calculate its error
         }
         optimizer.step(); // Update the parameters
 
-        auto averageLoss = epochLoss / Val32::create(xs.size());
+        auto averageLoss = epochLoss / Val32::create(loader.getTotalBatches());
         std::cout << "Epoch: " << epoch + 1 << " Loss: " << averageLoss->getValue() << std::endl;
     }
 
-    for (size_t sample = 0; sample < xs.size(); ++sample) { // Go through each example
-        Vec32 pred = network->forward(xs[sample]);          // Predict result
-        std::cout << xs[sample] << " -> " << pred[0] << "\t| True: " << ys[sample][0] << std::endl;
+    for (auto &[x, y] : data) {          // Go through each example
+        auto pred = network->forward(x); // We get some prediction
+        std::cout << x << " -> " << pred[0] << "\t| True: " << y[0] << std::endl;
     }
 }

include/ShkyeraGrad.hpp

@@ -17,6 +17,9 @@
 #include "nn/Neuron.hpp"
 #include "nn/Sequential.hpp"
 
+#include "nn/data/DataLoader.hpp"
+#include "nn/data/Dataset.hpp"
+
 #include "nn/optimizers/AdaMax.hpp"
 #include "nn/optimizers/Adam.hpp"
 #include "nn/optimizers/NAG.hpp"

include/core/Utils.hpp

@@ -53,6 +53,8 @@ std::enable_if_t<std::is_integral_v<T>, std::vector<T>> sample(T from, T to, siz
     return sampled;
 }
 
+template <typename T> void shuffle(std::vector<T> &vec) { std::shuffle(vec.begin(), vec.end(), rand_dev); }
+
 template <typename Clock = std::chrono::high_resolution_clock> auto startTimer() { return Clock::now(); }
 
 template <typename Clock = std::chrono::high_resolution_clock>

include/core/Vector.hpp

@@ -18,6 +18,7 @@ template <typename T> class Vector;
 
 using Vec32 = Vector<Type::float32>;
 using Vec64 = Vector<Type::float64>;
+template <typename T> using Batch = std::vector<Vector<T>>;
 
 template <typename T> class Vector {
   private:

include/nn/Loss.hpp

@@ -8,6 +8,7 @@
 #pragma once
 
 #include "../core/Value.hpp"
+#include "../core/Vector.hpp"
 
 namespace shkyera::Loss {
 
@@ -31,8 +32,6 @@ Function<T> MSE = [](Vector<T> a, Vector<T> b) {
     if (a.size() > 0)
         loss = loss / Value<T>::create(a.size());
 
-    loss->backward();
-
     return loss;
 };
 
@@ -52,8 +51,6 @@ Function<T> MAE = [](Vector<T> a, Vector<T> b) {
     if (a.size() > 0)
         loss = loss / Value<T>::create(a.size());
 
-    loss->backward();
-
     return loss;
 };
 
@@ -80,9 +77,26 @@ Function<T> CrossEntropy = [](Vector<T> a, Vector<T> b) {
         loss = loss - (b[i] * (a[i]->log()));
     }
 
+    return loss;
+};
+
+template <typename T>
+ValuePtr<T> compute(Function<T> lossFunction, const Vector<T> prediction, const Vector<T> target) {
+    auto loss = lossFunction(prediction, target);
+    loss->backward();
+    return loss;
+}
+
+template <typename T> ValuePtr<T> compute(Function<T> lossFunction, const Batch<T> prediction, const Batch<T> target) {
+    ValuePtr<T> loss = Value<T>::create(0);
+    for (size_t i = 0; i < prediction.size(); ++i) {
+        loss = loss + lossFunction(prediction[i], target[i]);
+    }
+    loss = loss / Value<T>::create(prediction.size());
+
     loss->backward();
 
     return loss;
-};
+}
 
 } // namespace shkyera::Loss

include/nn/Module.hpp

@@ -19,8 +19,17 @@ template <typename T> class Module {
     Module() = default;
 
   public:
-    Vector<T> forward(const Vector<T> &x) const { return (*this)(x); }
+    template <typename U> U forward(const U &x) const { return (*this)(x); }
+
     virtual Vector<T> operator()(const Vector<T> &x) const { return x; }
+    std::vector<Vector<T>> operator()(const std::vector<Vector<T>> &x) const {
+        std::vector<Vector<T>> out(x.size());
+        for (size_t i = 0; i < x.size(); ++i) {
+            out[i] = this->operator()(x[i]);
+        }
+        return out;
+    }
+
     virtual std::vector<ValuePtr<T>> parameters() const { return {}; }
 };
 

include/nn/data/DataLoader.hpp

@@ -0,0 +1,101 @@
+/**
+ * Copyright © 2023 Franciszek Szewczyk. None of the rights reserved.
+ * This code is released under the Beerware License. If you find this code useful or you appreciate the work, you are
+ * encouraged to buy the author a beer in return.
+ * Contact the author at [email protected] for inquiries and support.
+ */
+
+#pragma once
+
+#include "Dataset.hpp"
+
+namespace shkyera {
+
+template <typename T, typename U> class DataLoader {
+  private:
+    const Dataset<T, U> &_dataset;
+    size_t _batchSize;
+    bool _shuffle;
+
+  public:
+    DataLoader(const Dataset<T, U> &dataset, size_t batchSize = 4, bool shuffle = false);
+
+    size_t getTotalBatches() const;
+
+    class ConstIterator {
+      private:
+        std::vector<size_t> _order;
+        size_t _index;
+
+        const DataLoader<T, U> &_dataLoader;
+
+      public:
+        ConstIterator(size_t index, const DataLoader<T, U> &dataLoader);
+
+        std::pair<std::vector<T>, std::vector<U>> operator*();
+        ConstIterator &operator++();
+        bool operator!=(const ConstIterator &other);
+    };
+
+    ConstIterator begin() const;
+    ConstIterator end() const;
+};
+
+template <typename T, typename U>
+DataLoader<T, U>::DataLoader(const Dataset<T, U> &dataset, size_t batchSize, bool shuffle)
+    : _dataset(dataset), _batchSize(batchSize), _shuffle(shuffle) {}
+
+template <typename T, typename U> size_t DataLoader<T, U>::getTotalBatches() const {
+    size_t batches = _dataset.size() / _batchSize;
+    if (_dataset.size() % _batchSize != 0)
+        batches++;
+    return batches;
+}
+
+template <typename T, typename U>
+DataLoader<T, U>::ConstIterator::ConstIterator(size_t index, const DataLoader<T, U> &dataLoader)
+    : _index(index), _dataLoader(dataLoader) {
+    _order.resize(_dataLoader._dataset.size(), 0);
+
+    std::iota(_order.begin(), _order.end(), 0);
+    if (_dataLoader._shuffle)
+        utils::shuffle(_order);
+}
+
+template <typename T, typename U>
+std::pair<std::vector<T>, std::vector<U>> DataLoader<T, U>::ConstIterator::operator*() {
+    size_t beginIndex = _index;
+    size_t endIndex = std::min(_index + _dataLoader._batchSize, _dataLoader._dataset.size());
+
+    std::vector<T> inputs(endIndex - beginIndex);
+    std::vector<U> outputs(endIndex - beginIndex);
+
+    for (size_t i = beginIndex; i < endIndex; ++i) {
+        auto [in, out] = _dataLoader._dataset[_order[i]];
+        inputs[i - beginIndex] = in;
+        outputs[i - beginIndex] = out;
+    }
+
+    return {inputs, outputs};
+}
+
+template <typename T, typename U>
+typename DataLoader<T, U>::ConstIterator &DataLoader<T, U>::ConstIterator::operator++() {
+    _index += _dataLoader._batchSize;
+    _index = std::min(_index, _dataLoader._dataset.size());
+    return *this;
+}
+
+template <typename T, typename U> bool DataLoader<T, U>::ConstIterator::operator!=(const ConstIterator &other) {
+    return _index != other._index;
+}
+
+template <typename T, typename U> typename DataLoader<T, U>::ConstIterator DataLoader<T, U>::begin() const {
+    return ConstIterator(0, *this);
+}
+
+template <typename T, typename U> typename DataLoader<T, U>::ConstIterator DataLoader<T, U>::end() const {
+    return ConstIterator(_dataset.size(), *this);
+}
+
+} // namespace shkyera