New network trainers structure implemented

Vec structure checking implemented
Vec to_string improved

CMakeLists.txt

@@ -1,4 +1,5 @@
 cmake_minimum_required(VERSION 3.15)
+#TODO: rename all names to lowercase to make CLI port
 project(NeuralNetworkTry)
 set(CMAKE_CXX_STANDARD 14)
 

src/math/vec.h

@@ -9,6 +9,7 @@
 #include <initializer_list>
 #include <stdexcept>
 
+//TODO: get rid of the l and r values!
 template <typename T>
 class vec : public stringable {
 private:
@@ -18,6 +19,20 @@ class vec : public stringable {
     size_t vec_array_size;
     size_t vec_size;
 
+    template<typename T>
+    friend void _get_shape(vec<T>& v, vec<size_t>* res);
+
+    template<typename T>
+    friend void _fill_vec(const vec<vec<T>*>& to_fill, const vec<size_t>* shape, size_t ind);
+    template<typename T>
+    friend void _fill_vec(const vec<vec<vec<T>>*>& to_fill, const vec<size_t>* shape, size_t ind);
+
+    static const char* def_tab;
+    template<typename T>
+    friend std::string _vec_to_string(const vec<T>* v, const vec<size_t>* shape, const std::string* depth);
+    template<typename T>
+    friend std::string _vec_to_string(const vec<vec<T>>* v, const vec<size_t>* shape, const std::string* depth);
+
 public:
     vec();
     vec(size_t size, bool fill_zeros);
@@ -28,16 +43,28 @@ class vec : public stringable {
 
 	//Functions
     size_t size() const;
+    vec<size_t> get_shape() const;
+
     void resize(size_t new_size);
+    void reverse(int from = 0, int to = -1);
     void push_back(const T& element);
     void push_back(const vec& elements);
 
+	//TODO: create method `create_same_shape`
+    /// <summary>
+    /// Makes new vector with the shape of shape_vec
+    /// </summary>
+    static vec create_from_shape(const vec<size_t>& shape_vec);
+
     T max_element();
     size_t max_element_ind();
     T min_element();
     size_t min_element_ind();
     T sum() const;
 
+    vec square();
+    vec sqrt();
+
     /// <summary>
     /// Function to get pointer to the wrapped array.
     /// </summary>

src/math/vec.tpp

@@ -16,6 +16,8 @@
 
 template<typename T>
 const double vec<T>::resize_coof = 1.5;
+template<typename T>
+const char* vec<T>::def_tab = "   ";
 
 // Constructors :
 template<typename T>
@@ -60,6 +62,23 @@ vec<T>::~vec() {
 template <typename T>
 size_t vec<T>::size() const { return vec_size; }
 
+
+template<typename T>
+void _get_shape(T& v, vec<size_t>* res){}
+template<typename T>
+void _get_shape(vec<T>& v, vec<size_t>* res)
+{
+    res->push_back(v.vec_size);
+    _get_shape(v.vec_data[0], res);
+}
+template <typename T>
+vec<size_t> vec<T>::get_shape() const
+{
+    vec<size_t> ret({vec_size});
+    _get_shape(vec_data[0], &ret);
+    return ret;
+}
+
 template <typename T>
 void vec<T>::resize(size_t new_size)
 {
@@ -74,6 +93,22 @@ void vec<T>::resize(size_t new_size)
     vec_size = new_size;
 }
 
+template <typename T>
+void vec<T>::reverse(int from, int to)
+{
+    if (to == -1) to = vec_size;
+#ifdef _DEBUG_VEC
+	if(to < from){
+		std::stringstream ss;
+    	ss << "You\'ve tried to reverse array with invalid";
+    	ss << "from(" << from << ") and to(" << to << ") arguments!";
+    	std::cerr << ss.str();
+    	throw std::out_of_range(ss.str());
+	}
+#endif
+    std::reverse(vec_data + from, vec_data + to);
+}
+
 template <typename T>
 void vec<T>::push_back(const T& element)
 {
@@ -91,6 +126,49 @@ void vec<T>::push_back(const vec& elements)
         vec_data + buff);
 }
 
+template<typename T>
+void _fill_vec(const vec<vec<T>*>& to_fill, const vec<size_t>* shape, size_t ind)
+{
+    size_t to_fill_size = to_fill.size(),
+        prev_shape = shape->vec_data[ind - 1],
+        to_fill_shape = shape->vec_data[ind];
+    vec<T>* to_fill_p;
+    for (size_t i = 0; i < to_fill_size; i++) {
+        to_fill_p = to_fill.vec_data[i];
+        for (size_t j = 0; j < prev_shape; j++, to_fill_p++)
+            new (to_fill_p) vec<T>(to_fill_shape, false);
+    }
+}
+template<typename T>
+void _fill_vec(const vec<vec<vec<T>>*>& to_fill, const vec<size_t>* shape, size_t ind)
+{
+    size_t to_fill_size = to_fill.size(),
+        prev_shape = shape->vec_data[ind - 1],
+        to_fill_shape = shape->vec_data[ind],
+        next_to_fill_ind = 0;
+    vec<vec<T>*> next_to_fill(to_fill_size * prev_shape, false);
+    vec<vec<T>>* to_fill_p;
+    for (size_t i = 0; i < to_fill_size; i++) {
+        to_fill_p = to_fill.vec_data[i];
+        for (size_t j = 0; j < prev_shape; j++, to_fill_p++) {
+            new (to_fill_p) vec<vec<T>>(to_fill_shape, false);
+            next_to_fill.vec_data[next_to_fill_ind++] = to_fill_p->vec_data;
+        }
+    }
+    if (ind < shape->size())
+        _fill_vec(next_to_fill, shape, ind + 1);
+}
+template <typename T>
+vec<T> vec<T>::create_from_shape(const vec<size_t>& shape_vec)
+{
+    size_t size = shape_vec[0];
+    vec<T> ret(size, false);
+    vec<T*> to_fill({ret.vec_data});
+	if(shape_vec.size() > 1)
+		_fill_vec(to_fill, &shape_vec, 1);
+    return ret;
+}
+
 template <typename T>
 T vec<T>::max_element()
 {
@@ -380,16 +458,45 @@ vec<T> &operator/=(vec<T> &a, const vec<T> &b)
 }
 
 template<typename T>
-std::string vec<T>::to_string() const {
+std::string _vec_to_string(const vec<T>* v, const vec<size_t>* shape = NULL, const std::string* depth = NULL)
+{
+    std::stringstream ss;
+    ss << "v(" << v->vec_size << "){ ";
+    if (v->vec_size > 0) ss << v->vec_data[0];
+    for (size_t i = 1; i < v->vec_size; i++)
+        ss << ", " << v->vec_data[i];
+    ss << " }";
+    return ss.str();
+}
+template<typename T>
+std::string _vec_to_string(const vec<vec<T>>* v, const vec<size_t>* shape, const std::string* depth)
+{
     std::stringstream ss;
-    ss<<"v("<<vec_size<<"){ ";
-    if(vec_size >= 1)
-        ss<<vec_data[0];
-    for(size_t i = 1; i < vec_size; i++)
-        ss<<", "<<vec_data[i];
-    ss<<" }";
+    ss << "v(" << v->vec_size << "){ ";
+    if (v->vec_size > 0) {
+        depth++;
+        ss << *depth << _vec_to_string(&v->vec_data[0], shape, depth);
+        for (size_t i = 1; i < v->vec_size; i++)
+            ss << ',' << *depth <<
+				_vec_to_string(&v->vec_data[i], shape, depth);
+        depth--;
+        ss << *depth << "}";
+    }
+    else ss << " }";
     return ss.str();
 }
+template<typename T>
+std::string vec<T>::to_string() const {
+    vec<size_t> shape = get_shape();
+    size_t shape_size = shape.size();
+    if (shape_size == 1)
+        return _vec_to_string(this);
+    std::string* depth_arr = new std::string[shape_size + 1];
+    depth_arr[0] = "\n";
+    for (size_t i = 1; i <= shape_size; i++)
+        depth_arr[i] = depth_arr[i - 1] + def_tab;
+    return _vec_to_string(this, &shape, depth_arr);
+}
 
 template <typename T>
 vec<vec<T>> operator&(vec<T> &a, vec<T> &b)

src/neural_core/simple_network/simple_network.cpp → src/neural_core/simple_network.cpp

@@ -1,4 +1,4 @@
-#include <neural_core/simple_network/simple_network.h>
+#include <neural_core/simple_network.h>
 #include <neural_core/data_saver.h>
 #include <stdexcept>
 #include <sstream>

src/neural_core/trainer/adam_trainer.cpp

@@ -0,0 +1,17 @@
+#include <neural_core/trainer/adam_trainer.h>
+
+AdamTrainer::AdamTrainer(SimpleNetwork* network, I_SimpleData* train_data,
+	double beta1, double beta2, double alpha,
+	int t_stop1, int t_stop2) : NetworkTrainer(network, train_data)
+{
+	Beta1 = beta1;
+	Beta2 = beta2;
+	Alpha = alpha;
+	T_stop1 = t_stop1;
+	T_stop2 = t_stop2;
+}
+
+void AdamTrainer::TrainNetwork(int checks)
+{
+
+}

src/neural_core/trainer/adam_trainer.h

@@ -0,0 +1,46 @@
+#ifndef NEURALNETWORKTRY_ADAMTRAINER_H
+#define NEURALNETWORKTRY_ADAMTRAINER_H
+
+#include "network_trainer.h"
+
+/// <summary>
+/// https://habr.com/ru/post/318970/
+/// </summary>
+class AdamTrainer : NetworkTrainer
+{
+	/// <summary> Really small number </summary>
+	const double eta = 1e-8;
+
+public:
+	/// <summary> Training speed coefficient </summary>
+	double Alpha;
+	/// <summary> How much prev gradient is used </summary>
+	double Beta1;
+	/// <summary> How much more we edit unique weights </summary>
+	double Beta2;
+
+	/// <summary> How many times we artificially increase m </summary>
+	int T_stop1;
+	/// <summary> How many times we artificially increase v </summary>
+	int T_stop2;
+
+	/// <summary> Gradient average </summary>
+	vec<vec<vec<double>>> m;
+	/// <summary> Usage average </summary>
+	vec<vec<vec<double>>> v;
+
+	AdamTrainer(SimpleNetwork* network, I_SimpleData* train_data,
+		double beta1 = .9, double beta2 = .999, double alpha = .001,
+		int t_stop1 = 10, int t_stop2 = 1000);
+
+	double ComputeError(int test_index) override;
+	double ComputeAverageError() override;
+
+	double TrainNetwork() override;
+
+	void TrainNetwork(int checks) override;
+
+	void TrainNetwork(int checks, std::ostream* stream, int logging_rate = 10) override;
+};
+
+#endif //NEURALNETWORKTRY_ADAMTRAINER_H

src/neural_core/trainer/network_trainer.h

@@ -0,0 +1,45 @@
+#ifndef NEURALNETWORKTRY_NETWORKTRAINER_H
+#define NEURALNETWORKTRY_NETWORKTRAINER_H
+
+#include "math/random_element.h"
+#include "neural_core/simple_data.h"
+#include "neural_core/simple_network.h"
+
+class NetworkTrainer : protected RandomElement<>
+{
+protected:
+	SimpleNetwork* Network;
+	I_SimpleData* TrainData;
+
+public:
+	NetworkTrainer(SimpleNetwork* network, I_SimpleData* train_data)
+	{
+		Network = network;
+		TrainData = train_data;
+	}
+
+	virtual double ComputeError(int test_index) = 0;
+	virtual double ComputeAverageError() = 0;
+
+	/// <summary>
+	/// Do one network check
+	/// </summary>
+	virtual double TrainNetwork() = 0;
+
+	/// <summary>
+	/// Does several network checks.
+	/// </summary>
+	/// <param name="checks">Number of network checks</param>
+	virtual void TrainNetwork(int checks) = 0;
+
+	/// <summary>
+	/// Does several network checks.
+	///	Outputs log to the stream.
+	/// </summary>
+	/// <param name="checks">Number of network checks</param>
+	/// <param name="stream">The stream to write the logs</param>
+	/// <param name="logging_rate">Number of checks per log</param>
+	virtual void TrainNetwork(int checks, std::ostream* stream, int logging_rate = 10) = 0;
+};
+
+#endif //NEURALNETWORKTRY_NETWORKTRAINER_H