mlp-cpp/loss_activation.h at main · ysf-ad/mlp-cpp · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
#ifndef LOSS_ACTIVATION_H
#define LOSS_ACTIVATION_H

#include <vector>
#include <cmath>
using namespace std;

class ActivationFunction {
public:
    virtual vector<double> forward(const vector<double> x) = 0;
    virtual vector<double> backward(const vector<double> x) = 0;
};

class LossFunction {
public:
    virtual double forward(const vector<double> x, const vector<double> y) = 0;
    virtual vector<double> backward(const vector<double> x, const vector<double> y) = 0;
};

class CrossEntropy : public LossFunction {
public:
    double forward(const vector<double> x, const vector<double> y) override {
        double sum = 0.0;
        double eps = 1e-15;
        for (size_t i = 0; i < x.size(); i++) {
            sum += -y[i] * log(x[i] + eps);
        }
        return sum;
    }
    vector<double> backward(const vector<double> x, const vector<double> y) override {
        vector<double> grad(x.size());
        for (size_t i = 0; i < x.size(); i++) {
            grad[i] = x[i] - y[i];
        }
        return grad;
    }
};

class ReLU : public ActivationFunction {
public:
    vector<double> forward(const vector<double> x) override {
        vector<double> result = x;
        for (double& val : result) {
            if (val < 0.0) val = 0.0;
        }
        return result;
    }

    vector<double> backward(const vector<double> x) override {
        vector<double> result = x;
        for (double& val : result) {
            val = (val < 0.0) ? 0.0 : 1.0;
        }
        return result;
    }
};

class Softmax : public ActivationFunction {
public:
    vector<double> forward(const vector<double> x) override {
        vector<double> result = x;
        double max_val = result[0];
        for (size_t i = 1; i < result.size(); i++) {
            if (result[i] > max_val) max_val = result[i];
        }
        double sum = 0.0;
        for (double& val : result) {
            val = exp(val - max_val);
            sum += val;
        }
        for (double& val : result) {
            val = val / sum;
        }
        return result;
    }
    vector<double> backward(const vector<double> x) override {
        return vector<double>(x.size(), 1.0);
    }
};

class MSE : public LossFunction {
public:
    double forward(const vector<double> x, const vector<double> y) override {
        double sum = 0.0;
        for (size_t i = 0; i < x.size(); i++) {
            double diff = x[i] - y[i];
            sum += diff * diff;
        }
        return sum / x.size();
    }
    vector<double> backward(const vector<double> x, const vector<double> y) override {
        vector<double> grad(x.size());
        for (size_t i = 0; i < x.size(); i++) {
            grad[i] = 2.0 * (x[i] - y[i]) / x.size();
        }
        return grad;
    }
};

#endif