Image class and MNIST example

.github/workflows/linux.yml

@@ -32,5 +32,5 @@ jobs:
         run: |
           g++ examples/scalars.cpp -O3 --std=c++17
           ./a.out
-          g++ examples/xor_regression.cpp -O3 --std=c++17
+          g++ examples/xor.cpp -O3 --std=c++17
           ./a.out

.github/workflows/macos.yml

@@ -32,5 +32,5 @@ jobs:
         run: |
           g++ examples/scalars.cpp -O3 --std=c++17
           ./a.out
-          g++ examples/xor_regression.cpp -O3 --std=c++17
+          g++ examples/xor.cpp -O3 --std=c++17
           ./a.out

.github/workflows/windows.yml

@@ -30,4 +30,4 @@ jobs:
           CXX: ${{matrix.conf.compiler}}
         run: |
           g++ -o out examples/scalars.cpp -O3 --std=c++17
-          g++ -o out examples/xor_regression.cpp -O3 --std=c++17
+          g++ -o out examples/xor.cpp -O3 --std=c++17

docs/tutorials/GetStarted.md

@@ -233,7 +233,7 @@ for (size_t sample = 0; sample < xs.size(); ++sample) {         // Go through ea
 }
 ```
 
-In case you got lost along the way, check out the `examples/xor_regression.cpp` file. It contains the exact same code and is ready to run :)
+In case you got lost along the way, check out the `examples/xor.cpp` file. It contains the exact same code and is ready to run :)
 
 ### Results
 

examples/README.md

@@ -3,7 +3,7 @@
 To compile an example, simply run the following command:
 
 ```
-g++ --std=c++17 xor_nn.cpp
+g++ --std=c++17 xor.cpp
 ```
 
 Remember to replace the file name with the appropriate name :)

examples/mnist.cpp

@@ -0,0 +1,79 @@
+#include <filesystem>
+#include <iostream>
+
+#include "../include/ShkyeraGrad.hpp"
+
+namespace fs = std::filesystem;
+using namespace shkyera;
+
+Dataset<Vec32, Vec32> load(std::string directory) {
+    Dataset<Vec32, Vec32> dataset;
+
+    std::cerr << "Loading [" << std::flush;
+    for (size_t digit = 0; digit < 10; ++digit) {
+        std::cerr << "▮" << std::flush;
+        int added = 0;
+        for (const auto &entry : fs::directory_iterator(directory + std::to_string(digit))) {
+            Image image(entry.path().string());
+            auto target = Vec32::oneHotEncode(digit, 10);
+
+            dataset.addSample(image.flatten<Type::float32>() / 255.0f, target);
+        }
+    }
+    std::cerr << "]" << std::endl;
+
+    return dataset;
+}
+
+int main() {
+    Dataset<Vec32, Vec32> trainData = load("datasets/mnist/train/");
+    std::cerr << "Loaded training data." << std::endl;
+
+    DataLoader trainLoader(trainData, 16, true);
+
+    // clang-format off
+    auto mlp = SequentialBuilder32::begin()
+                .add(Linear32::create(784, 100))
+                .add(ReLU32::create())
+                .add(Linear32::create(100, 50))
+                .add(Sigmoid32::create())
+                .add(Linear32::create(50, 10))
+                .add(Softmax32::create())
+                .build();
+    // clang-format on
+
+    auto optimizer = Adam32(mlp->parameters(), 0.01, 0.99);
+    auto lossFunction = Loss::CrossEntropy<Type::float32>;
+
+    for (size_t epoch = 0; epoch < 50; epoch++) {
+        float epochLoss = 0;
+        double epochAccuracy = 0;
+
+        for (const auto [x, y] : trainLoader) {
+            optimizer.reset();
+
+            auto pred = mlp->forward(x);
+
+            double accuracy = 0;
+            for (size_t i = 0; i < pred.size(); ++i) {
+                size_t predictedDigit = pred[i].argMax();
+                size_t trueDigit = y[i].argMax();
+
+                if (predictedDigit == trueDigit)
+                    accuracy += 1;
+            }
+
+            accuracy /= pred.size();
+            epochAccuracy += accuracy;
+
+            auto loss = Loss::compute(lossFunction, pred, y);
+            epochLoss = epochLoss + loss->getValue();
+
+            optimizer.step();
+
+            std::cerr << "Loss: " << loss->getValue() << " Accuracy: " << accuracy << std::endl;
+        }
+        std::cerr << "Epoch: " << epoch + 1 << " Loss: " << epochLoss / trainLoader.getTotalBatches()
+                  << " Accuracy: " << epochAccuracy / trainLoader.getTotalBatches() << std::endl;
+    }
+}

include/ShkyeraGrad.hpp

@@ -7,6 +7,7 @@
 
 #pragma once
 
+#include "core/Image.hpp"
 #include "core/Type.hpp"
 #include "core/Utils.hpp"
 #include "core/Value.hpp"

include/core/Image.hpp

@@ -0,0 +1,61 @@
+/**
+ * Copyright © 2023 Franciszek Szewczyk. None of the rights reserved.
+ * This code is released under the Beerware License. If you find this code useful or you appreciate the work, you are
+ * encouraged to buy the author a beer in return.
+ * Contact the author at [email protected] for inquiries and support.
+ */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#define STB_IMAGE_IMPLEMENTATION
+#include "../external/stb_image.h"
+
+#include "Vector.hpp"
+
+namespace shkyera {
+
+class Image {
+  private:
+    std::vector<uint8_t> _data;
+
+  public:
+    Image() = default;
+    Image(std::string filename, bool grayscale = true);
+
+    template <typename T> Vector<T> flatten(size_t takeEvery = 1) const;
+};
+
+Image::Image(std::string filename, bool grayscale) {
+    int width, height, channels;
+    uint8_t *imageData = nullptr;
+
+    if (grayscale)
+        imageData = stbi_load(filename.c_str(), &width, &height, &channels, 1);
+    else
+        imageData = stbi_load(filename.c_str(), &width, &height, &channels, 3);
+
+    if (!imageData) {
+        std::cerr << "Error loading image: " << filename << std::endl;
+        return;
+    }
+
+    if (grayscale)
+        _data.assign(imageData, imageData + (width * height));
+    else
+        _data.assign(imageData, imageData + (width * height * 3));
+
+    stbi_image_free(imageData);
+}
+
+template <typename T> Vector<T> Image::flatten(size_t takeEvery) const {
+    std::vector<T> converted;
+    converted.reserve(_data.size());
+    for (size_t i = 0; i < _data.size(); i += takeEvery)
+        converted.push_back(static_cast<T>(_data[i]));
+    return Vector<T>::of(converted);
+}
+
+} // namespace shkyera

include/core/Value.hpp

@@ -238,6 +238,11 @@ template <typename T> void Value<T>::backward() {
     for (auto val = sorted.rbegin(); val != sorted.rend(); val++) {
         (*val)->_backward();
     }
+
+    for (auto s : sorted) {
+        s->_children = {};
+        s->_backward = []() {};
+    }
 }
 
 template <typename T> std::ostream &operator<<(std::ostream &os, const ValuePtr<T> &value) {

include/core/Vector.hpp

@@ -8,6 +8,7 @@
 #pragma once
 
 #include <exception>
+#include <numeric>
 
 #include "Type.hpp"
 #include "Value.hpp"
@@ -30,10 +31,12 @@ template <typename T> class Vector {
 
     static Vector<T> of(const std::vector<T> &values);
     template <typename... Args> static Vector<T> of(const Args &...args);
+    static Vector<T> oneHotEncode(size_t index, size_t size);
 
     ValuePtr<T> dot(const Vector<T> &other) const;
     ValuePtr<T> sum() const;
     size_t size() const;
+    size_t argMax() const;
 
     template <typename U> friend std::ostream &operator<<(std::ostream &os, const Vector<U> &vector);
 
@@ -85,8 +88,33 @@ template <typename T> template <typename... Args> Vector<T> Vector<T>::of(const
     return Vector<T>(valuePtrs);
 }
 
+template <typename T> Vector<T> Vector<T>::oneHotEncode(size_t index, size_t size) {
+    std::vector<ValuePtr<T>> valuePtrs(size);
+
+    for (size_t i = 0; i < size; ++i) {
+        if (i == index)
+            valuePtrs[i] = Value<T>::create(1);
+        else
+            valuePtrs[i] = Value<T>::create(0);
+    }
+
+    return valuePtrs;
+}
+
 template <typename T> size_t Vector<T>::size() const { return _values.size(); }
 
+template <typename T> size_t Vector<T>::argMax() const {
+    T largest = std::numeric_limits<T>::lowest();
+    size_t largestIndex = 0;
+    for (size_t i = 0; i < _values.size(); ++i) {
+        if (_values[i]->getValue() > largest) {
+            largest = _values[i]->getValue();
+            largestIndex = i;
+        }
+    }
+    return largestIndex;
+}
+
 template <typename T> ValuePtr<T> Vector<T>::dot(const Vector<T> &other) const {
     if (other.size() != size()) {
         throw std::invalid_argument("Vectors need to be of the same size to compute the dot product. Sizes are " +

include/nn/Loss.hpp

@@ -71,10 +71,13 @@ Function<T> CrossEntropy = [](Vector<T> a, Vector<T> b) {
                                     std::to_string(aSum->getValue()) + " and " + std::to_string(bSum->getValue()) +
                                     ".");
     }
-
+    auto eps = Value<T>::create(1e-8);
     auto loss = Value<T>::create(0);
     for (size_t i = 0; i < a.size(); ++i) {
-        loss = loss - (b[i] * (a[i]->log()));
+        if (a[i] < eps)
+            loss = loss - (b[i] * (eps->log()));
+        else
+            loss = loss - (b[i] * (a[i]->log()));
     }
 
     return loss;

include/nn/optimizers/Optimizer.hpp

@@ -36,7 +36,8 @@ Optimizer<T>::Optimizer(std::vector<ValuePtr<T>> params, T learningRate) : _lear
 }
 
 template <typename T> void Optimizer<T>::reset() {
-    std::for_each(_parameters.begin(), _parameters.end(), [](ValuePtr<T> val) { val->_gradient = 0; });
+    for (ValuePtr<T> &val : _parameters)
+        val->_gradient = 0;
 }
 
 template <typename T> void Optimizer<T>::step() {