Program.cs

using System;
using LinearAlgebra;

namespace MonoLayerPerceptron
{
    /// <summary>
    /// A neural network with one hidden layer, written directly with
    /// SimpleLinearAlgebra matrices: manual forward pass, manual
    /// backpropagation and manual gradient descent.
    /// It learns the XNOR, AND, OR and XOR logic gates at the same time.
    /// </summary>
    class Program
    {
        static void Main(string[] args)
        {
            int epochs = args.Length > 0 ? int.Parse(args[0]) : 5000;

            //Parameters
            int inputCount = 2;
            int hiddenCount = 5;
            int outputCount = 4;
            int examplesCount = 4;
            double learningRate = 0.1;
            Random r = new Random(42);

            //Training data           //INPUT
            Matrix x = new double[,] { { 0, 0 },
                                       { 0, 1 },
                                       { 1, 0 },
                                       { 1, 1 }};
                                    //DESIRED OUTPUT
                                    //XNOR AND OR XOR
            Matrix y = new double[,] { { 1, 0, 0, 0 },
                                       { 0, 0, 1, 1 },
                                       { 0, 1, 1, 1 },
                                       { 1, 1, 1, 0 }};

            //Weight init
            Matrix w1 = (Matrix.Random(inputCount + 1, hiddenCount, r) - 0.5) * 2.0;
            Matrix w2 = (Matrix.Random(hiddenCount + 1, outputCount, r) - 0.5) * 2.0;

            double loss = double.MaxValue;
            for (int epoch = 0; epoch <= epochs; epoch++)
            {
                //Forward pass
                Matrix z1 = x.AddColumn(Matrix.Ones(examplesCount, 1));
                Matrix a1 = z1; //(examples, input + 1)
                Matrix z2 = (a1 * w1).AddColumn(Matrix.Ones(examplesCount, 1));
                Matrix a2 = Sigmoid(z2); //(examples, hidden neurons + 1)
                Matrix z3 = a2 * w2;
                Matrix a3 = Sigmoid(z3); //(examples, output)

                //Backpropagation
                Matrix a3Error = a3 - y; //(examples, output)
                Matrix delta3 = a3Error * Sigmoid(z3, true);

                Matrix a2Error = delta3 * w2.T;
                Matrix delta2 = a2Error * Sigmoid(z2, true);
                delta2 = delta2.Slice(0, 1, delta2.X, delta2.Y); //Drop the bias neuron delta

                //Gradient descent
                w2 -= (a2.T * delta3) * learningRate;
                w1 -= (a1.T * delta2) * learningRate;

                loss = a3Error.Abs.Avg * examplesCount;
                if (epoch % 1000 == 0)
                {
                    Console.WriteLine($"--------- epoch {epoch} ---------");
                    Console.WriteLine($"Loss: {loss}");
                    Console.WriteLine("Input:");
                    Console.WriteLine(x.ToString());
                    Console.WriteLine("Prediction (XNOR AND OR XOR):");
                    Console.WriteLine(a3.ToString());
                }
            }
            Console.WriteLine($"Final loss after {epochs} epochs: {loss}");
        }

        static Matrix Sigmoid(Matrix m, bool derivated = false)
        {
            double[,] a = m;
            Matrix.MatrixLoop((i, j) =>
            {
                if (derivated)
                {
                    double sig = 1.0 / (1.0 + Math.Exp(-a[i, j]));
                    a[i, j] = sig * (1.0 - sig);
                }
                else
                {
                    a[i, j] = 1.0 / (1.0 + Math.Exp(-a[i, j]));
                }
            }, m.X, m.Y);

            return a;
        }
    }
}