diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8cd469a..9424189 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,12 +35,12 @@ jobs: - name: Configure with CMake working-directory: ./build - run: cmake .. + run: cmake -DCMAKE_BUILD_TYPE=Debug .. - name: Build working-directory: ./build - run: cmake --build . + run: cmake --build . --config Debug - name: Run tests working-directory: ./build - run: ctest --output-on-failure + run: ctest -C Debug --output-on-failure diff --git a/.gitignore b/.gitignore index c9a65fa..15477da 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,6 @@ docs/doxygen/ .DS_Store Thumbs.db .cache/ + +# Remove solution +cpp-ml/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 47f3a24..236afc7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,14 +7,15 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) # Option to build examples (enabled by default) option(BUILD_EXAMPLES "Build examples" ON) +# Global include directories for headers +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/ml_library_include) + # Source files -file(GLOB_RECURSE SOURCES "src/*.cpp") +file(GLOB_RECURSE SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/*.cpp") # Define the library target add_library(cpp_ml_library STATIC ${SOURCES}) - -# Include directory for headers -target_include_directories(cpp_ml_library PUBLIC ${PROJECT_SOURCE_DIR}/ml_library_include) +target_include_directories(cpp_ml_library PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/ml_library_include) # Installation install(TARGETS cpp_ml_library DESTINATION lib) @@ -23,33 +24,41 @@ install(DIRECTORY ml_library_include/ DESTINATION include) # Enable testing enable_testing() -# Add Catch2 for testing -include(FetchContent) -FetchContent_Declare( - Catch2 - GIT_REPOSITORY https://github.com/catchorg/Catch2.git - GIT_TAG v3.0.1 # Use the latest stable release of Catch2 v3 for improved compatibility -) -FetchContent_MakeAvailable(Catch2) - -# Add tests with Catch2 v3 -file(GLOB_RECURSE TEST_SOURCES "tests/**/*.cpp") -foreach(TEST_SOURCE ${TEST_SOURCES}) - get_filename_component(TEST_NAME ${TEST_SOURCE} NAME_WE) - add_executable(${TEST_NAME} ${TEST_SOURCE}) - target_link_libraries(${TEST_NAME} cpp_ml_library Catch2::Catch2WithMain) - target_include_directories(${TEST_NAME} PUBLIC ${PROJECT_SOURCE_DIR}/ml_library_include) - add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) -endforeach() +# Add individual test files as separate executables and define a macro for each +add_executable(LogisticRegressionTest tests/regression/LogisticRegressionTest.cpp) +target_compile_definitions(LogisticRegressionTest PRIVATE TEST_LOGISTIC_REGRESSION) +target_link_libraries(LogisticRegressionTest cpp_ml_library) + +add_executable(PolynomialRegressionTest tests/regression/PolynomialRegressionTest.cpp) +target_compile_definitions(PolynomialRegressionTest PRIVATE TEST_POLYNOMIAL_REGRESSION) +target_link_libraries(PolynomialRegressionTest cpp_ml_library) + +add_executable(MultiLinearRegressionTest tests/regression/MultilinearRegressionTest.cpp) +target_compile_definitions(MultiLinearRegressionTest PRIVATE TEST_MULTILINEAR_REGRESSION) +target_link_libraries(MultiLinearRegressionTest cpp_ml_library) + +# Register individual tests +add_test(NAME LogisticRegressionTest COMMAND LogisticRegressionTest) +add_test(NAME PolynomialRegressionTest COMMAND PolynomialRegressionTest) +add_test(NAME MultiLinearRegressionTest COMMAND MultiLinearRegressionTest) # Add example executables if BUILD_EXAMPLES is ON if(BUILD_EXAMPLES) - file(GLOB_RECURSE EXAMPLE_SOURCES "examples/*.cpp") + file(GLOB_RECURSE EXAMPLE_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/examples/*.cpp") foreach(EXAMPLE_SOURCE ${EXAMPLE_SOURCES}) get_filename_component(EXAMPLE_NAME ${EXAMPLE_SOURCE} NAME_WE) set(EXAMPLE_TARGET "example_${EXAMPLE_NAME}") # Add a prefix to the executable name add_executable(${EXAMPLE_TARGET} ${EXAMPLE_SOURCE}) # Use prefixed name for executable target_link_libraries(${EXAMPLE_TARGET} cpp_ml_library) - target_include_directories(${EXAMPLE_TARGET} PUBLIC ${PROJECT_SOURCE_DIR}/ml_library_include) + target_include_directories(${EXAMPLE_TARGET} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/ml_library_include) + + # Define specific macros to control main() inclusion if necessary + if(EXAMPLE_NAME STREQUAL "LogisticRegressionExample") + target_compile_definitions(${EXAMPLE_TARGET} PRIVATE TEST_LOGISTIC_REGRESSION) + elseif(EXAMPLE_NAME STREQUAL "MultilinearRegressionExample") + target_compile_definitions(${EXAMPLE_TARGET} PRIVATE TEST_MULTILINEAR_REGRESSION) + elseif(EXAMPLE_NAME STREQUAL "PolynomialRegressionExample") + target_compile_definitions(${EXAMPLE_TARGET} PRIVATE TEST_POLYNOMIAL_REGRESSION) + endif() endforeach() endif() diff --git a/README.md b/README.md index 35fb267..bb343d3 100644 --- a/README.md +++ b/README.md @@ -58,8 +58,10 @@ To use this library in your C++ project, include the master header file: The following machine learning algorithms are planned, inspired by concepts and techniques taught in the Udemy course: 1. **Regression** - - [ ] Polynomial Regression - - [ ] Multi-Linear Regression + - [x] Polynomial Regression + - [x] Multi-Linear Regression + - [x] Logistic Regression + 2. **Classification** - [ ] Decision Tree Classifier @@ -85,8 +87,9 @@ The following machine learning algorithms are planned, inspired by concepts and | Algorithm Category | Algorithm | Implemented | Tests | Examples | |--------------------------|------------------------------|-------------|-------|----------| -| **Regression** | Polynomial Regression | [ ] | [ ] | [ ] | -| | Multi-Linear Regression | [ ] | [ ] | [ ] | +| **Regression** | Polynomial Regression | [x] | [ ] | [x] | +| | Logistic Regression | [x] | [ ] | [x] | +| | Multi-Linear Regression | [x] | [ ] | [x] | | **Classification** | Decision Tree Classifier | [ ] | [ ] | [ ] | | | Random Forest Classifier | [ ] | [ ] | [ ] | | | K-Nearest Neighbors | [ ] | [ ] | [ ] | diff --git a/examples/LogisticRegressionExample.cpp b/examples/LogisticRegressionExample.cpp new file mode 100644 index 0000000..b1660e5 --- /dev/null +++ b/examples/LogisticRegressionExample.cpp @@ -0,0 +1,41 @@ +#include "../ml_library_include/ml/regression/LogisticRegression.hpp" +#include +#include +#include + +// Test function for Logistic Regression +void testLogisticRegression() { + LogisticRegression model(0.1, 1000); + + std::vector> features = { + {0.0, 0.0}, + {1.0, 1.0}, + {1.0, 0.0}, + {0.0, 1.0} + }; + + std::vector labels = { 0, 1, 1, 0 }; + + model.train(features, labels); + + std::vector testInput = { 1.0, 1.0 }; + int predictedClass = model.predict(testInput); + + std::cout << "Predicted class for {1.0, 1.0}: " << predictedClass << std::endl; + + // Optionally add a check to verify the predicted class + if (predictedClass == 1) { + std::cout << "Test passed: Correct prediction." << std::endl; + } + else { + std::cerr << "Test failed: Prediction was " << predictedClass << ", expected 1." << std::endl; + } +} + +// Only include main if TEST_LOGISTIC_REGRESSION is defined +#ifdef TEST_LOGISTIC_REGRESSION +int main() { + testLogisticRegression(); + return 0; +} +#endif diff --git a/examples/MultilinearRegressionExample.cpp b/examples/MultilinearRegressionExample.cpp new file mode 100644 index 0000000..a6cd9c6 --- /dev/null +++ b/examples/MultilinearRegressionExample.cpp @@ -0,0 +1,49 @@ +#include "../ml_library_include/ml/regression/MultiLinearRegression.hpp" +#include +#include +#include + +// Helper function for approximate equality check +inline bool approxEqual(double a, double b, double tolerance = 0.1) { + return std::fabs(a - b) < tolerance; +} + +void testMultilinearRegression() { + MultilinearRegression model(0.01, 1000); + + std::vector> features = { + {1.0, 2.0}, + {2.0, 3.0}, + {3.0, 4.0}, + {4.0, 5.0} + }; + + std::vector target = { 3.0, 5.0, 7.0, 9.0 }; + + try { + model.train(features, target); + std::cout << "Training passed." << std::endl; + } + catch (...) { + std::cerr << "Training failed with an exception!" << std::endl; + return; + } + + std::vector testFeatures = { 5.0, 6.0 }; + double prediction = model.predict(testFeatures); + + if (approxEqual(prediction, 11.0)) { + std::cout << "Test passed: Prediction is within tolerance." << std::endl; + } + else { + std::cerr << "Test failed: Prediction is " << prediction << ", expected ~11.0." << std::endl; + } +} + +// Only include main if TEST_MULTILINEAR_REGRESSION is defined +#ifdef TEST_MULTILINEAR_REGRESSION +int main() { + testMultilinearRegression(); + return 0; +} +#endif diff --git a/examples/PolynomialRegressionExample.cpp b/examples/PolynomialRegressionExample.cpp new file mode 100644 index 0000000..dad180d --- /dev/null +++ b/examples/PolynomialRegressionExample.cpp @@ -0,0 +1,40 @@ +#include "../ml_library_include/ml/regression/PolynomialRegression.hpp" +#include +#include +#include + +// Helper function for approximate equality check +inline bool approxEqual(double a, double b, double tolerance = 0.1) { + return std::fabs(a - b) < tolerance; +} + +// Test function for Polynomial Regression +void testPolynomialRegression() { + PolynomialRegression model(2); // Quadratic regression + + std::vector x = { 1.0, 2.0, 3.0, 4.0 }; + std::vector y = { 3.0, 5.0, 7.0, 9.0 }; + + model.train(x, y); + + double testInput = 5.0; + double prediction = model.predict(testInput); + + std::cout << "Predicted value for " << testInput << ": " << prediction << std::endl; + + // Check if prediction is close to the expected value + if (approxEqual(prediction, 11.0)) { + std::cout << "Test passed: Prediction is within tolerance." << std::endl; + } + else { + std::cerr << "Test failed: Prediction is " << prediction << ", expected ~11.0." << std::endl; + } +} + +// Only include main if TEST_POLYNOMIAL_REGRESSION is defined +#ifdef TEST_POLYNOMIAL_REGRESSION +int main() { + testPolynomialRegression(); + return 0; +} +#endif diff --git a/ml_library_include/ml/association/Apriori.h b/ml_library_include/ml/association/Apriori.hpp similarity index 100% rename from ml_library_include/ml/association/Apriori.h rename to ml_library_include/ml/association/Apriori.hpp diff --git a/ml_library_include/ml/association/Eclat.h b/ml_library_include/ml/association/Eclat.hpp similarity index 100% rename from ml_library_include/ml/association/Eclat.h rename to ml_library_include/ml/association/Eclat.hpp diff --git a/ml_library_include/ml/clustering/KMeans.h b/ml_library_include/ml/clustering/KMeans.hpp similarity index 100% rename from ml_library_include/ml/clustering/KMeans.h rename to ml_library_include/ml/clustering/KMeans.hpp diff --git a/ml_library_include/ml/ml.h b/ml_library_include/ml/ml.h deleted file mode 100644 index ae9a337..0000000 --- a/ml_library_include/ml/ml.h +++ /dev/null @@ -1,17 +0,0 @@ -// ml.h - Master header file -#ifndef ML_H -#define ML_H - -#include "tree/DecisionTreeClassifier.h" -#include "tree/DecisionTreeRegressor.h" -#include "tree/RandomForestClassifier.h" -#include "tree/RandomForestRegressor.h" -#include "regression/PolynomialRegression.h" -#include "regression/MultiLinearRegression.h" -#include "neural_network/ANN.h" -#include "neural_network/CNN.h" -#include "clustering/KMeans.h" -#include "association/Apriori.h" -#include "association/Eclat.h" - -#endif // ML_H diff --git a/ml_library_include/ml/ml.hpp b/ml_library_include/ml/ml.hpp new file mode 100644 index 0000000..c12fb3b --- /dev/null +++ b/ml_library_include/ml/ml.hpp @@ -0,0 +1,17 @@ +// ml.h - Master header file +#ifndef ML_H +#define ML_H + +#include "tree/DecisionTreeClassifier.hpp" +#include "tree/DecisionTreeRegressor.hpp" +#include "tree/RandomForestClassifier.hpp" +#include "tree/RandomForestRegressor.hpp" +#include "regression/PolynomialRegression.hpp" +#include "regression/MultiLinearRegression.hpp" +#include "neural_network/ANN.hpp" +#include "neural_network/CNN.hpp" +#include "clustering/KMeans.hpp" +#include "association/Apriori.hpp" +#include "association/Eclat.hpp" + +#endif // ML_H diff --git a/ml_library_include/ml/neural_network/ANN.h b/ml_library_include/ml/neural_network/ANN.hpp similarity index 100% rename from ml_library_include/ml/neural_network/ANN.h rename to ml_library_include/ml/neural_network/ANN.hpp diff --git a/ml_library_include/ml/neural_network/CNN.h b/ml_library_include/ml/neural_network/CNN.hpp similarity index 100% rename from ml_library_include/ml/neural_network/CNN.h rename to ml_library_include/ml/neural_network/CNN.hpp diff --git a/ml_library_include/ml/neural_network/NN.h b/ml_library_include/ml/neural_network/NN.h deleted file mode 100644 index 03e5075..0000000 --- a/ml_library_include/ml/neural_network/NN.h +++ /dev/null @@ -1,195 +0,0 @@ -#ifndef NN_H -#define NN_H - -#include -#include -#include -#include -#include -#include -#include - -using namespace std; - -/** - * @brief A class to handle training data for the neural network. - */ -class TrainingData { -public: - /** - * @brief Constructor that opens the training data file. - * @param filename The name of the training data file. - */ - TrainingData(const string filename); - - /** - * @brief Check if end of file is reached. - * @return True if end of file, otherwise false. - */ - bool isEof(void) { return _trainingDataFile.eof(); } - - /** - * @brief Reads the topology of the neural network from the training data. - * @param topology A vector to store the topology data. - */ - void getTopology(vector& topology); - - /** - * @brief Gets the next set of input values from the training data. - * @param inputVals A vector to store the input values. - * @return The number of input values. - */ - unsigned getNextInputs(vector& inputVals); - - /** - * @brief Gets the target output values from the training data. - * @param targetOutputsVals A vector to store the target output values. - * @return The number of target output values. - */ - unsigned getTargetOutputs(vector& targetOutputsVals); - -private: - ifstream _trainingDataFile; ///< Input file stream for the training data. -}; - -/** - * @brief Represents a connection between neurons, storing weight and delta weight. - */ -struct Connection { - double weight; ///< Weight of the connection. - double deltaWeight; ///< Change in weight for the connection. -}; - -/** - * @brief Represents a single neuron in the neural network. - */ -class Neuron { -public: - /** - * @brief Constructs a neuron with a specified number of outputs. - * @param numOutputs The number of outputs for the neuron. - * @param myIdx The index of the neuron in its layer. - */ - Neuron(unsigned numOutputs, unsigned myIdx); - - /** - * @brief Sets the output value for this neuron. - * @param val The output value to set. - */ - void setOutputVal(double val) { _outputVal = val; } - - /** - * @brief Gets the output value of this neuron. - * @return The output value. - */ - double getOutputVal(void) const { return _outputVal; } - - /** - * @brief Feeds forward the input values from the previous layer. - * @param prevLayer The previous layer of neurons. - */ - void feedForward(const vector& prevLayer); - - /** - * @brief Calculates gradients for the output neurons. - * @param targetVal The target output value. - */ - void calcOutputGradients(double targetVal); - - /** - * @brief Calculates gradients for hidden layer neurons. - * @param nextLayer The next layer of neurons. - */ - void calcHiddenGradients(const vector& nextLayer); - - /** - * @brief Updates input weights for this neuron. - * @param prevLayer The previous layer of neurons. - */ - void updateInputWeights(vector& prevLayer); - - /** - * @brief Generates a random weight. - * @param The Output is a random double. - */ - static double randomWeight(); // Random weight initializer function - - -private: - static double eta; ///< Learning rate [0.0..1.0]. - static double alpha; ///< Momentum factor [0.0..1.0]. - double _outputVal; ///< Output value of the neuron. - vector _outputWeights; ///< Weights for connections to next layer. - unsigned _myIdx; ///< Index of this neuron within its layer. - double _gradient; ///< Gradient used in backpropagation. - - static double transferFunction(double x); ///< Activation function. - static double transferFunctionDerivative(double x); ///< Derivative of activation function. - double sumDOW(const vector& nextLayer) const; ///< Sum of derivatives of weights. -}; - -/** - * @brief Represents the neural network. - */ -class NN { -public: - /** - * @brief Constructs the neural network based on the given topology. - * @param topology A vector representing the number of neurons in each layer. - */ - NN(const vector& topology); - - /** - * @brief Feeds forward the input values through the network. - * @param inputVals The input values for the network. - */ - void feedForward(const vector& inputVals); - - /** - * @brief Performs backpropagation based on the target values. - * @param targetVals The target output values. - */ - void backProp(const vector& targetVals); - - /** - * @brief Gets the output values from the network. - * @param resultsVals A vector to store the output values. - */ - void getResults(vector& resultsVals) const; - - /** - * @brief Gets the recent average error of the network. - * @return The recent average error. - */ - double getRecentAverageError(void) const { return _recentAverageError; } - -private: - vector> _layers; ///< The layers of neurons in the network. - double _error; ///< The current error of the network. - double _recentAverageError; ///< The recent average error. - static double _recentAverageSmoothFactor; ///< Smoothing factor for the recent average error. -}; - -/** - * @brief Displays vector values in the console. - * @param label A label to display before the values. - * @param v The vector containing values. - */ -void showVectorVals(string label, vector& v); - -/** - * @brief Formats vector values as a string. - * @param label A label to display before the values. - * @param v The vector containing values. - * @return A formatted string of the vector values. - */ -string printVectorVals(string label, vector& v); - -/** - * @brief Saves a string to a specified file. - * @param filename The name of the file. - * @param content The content to write to the file. - */ -void saveStringToFile(const std::string& filename, const std::string& content); - -#endif // NN_H diff --git a/ml_library_include/ml/regression/LogisticRegression.hpp b/ml_library_include/ml/regression/LogisticRegression.hpp new file mode 100644 index 0000000..98fefd7 --- /dev/null +++ b/ml_library_include/ml/regression/LogisticRegression.hpp @@ -0,0 +1,76 @@ +#ifndef LOGISTIC_REGRESSION_HPP +#define LOGISTIC_REGRESSION_HPP + +#include +#include + +/** + * @brief Logistic Regression model for binary classification tasks. + */ +class LogisticRegression { +public: + /** + * @brief Constructor initializing the learning rate and iteration count. + * @param learningRate The rate at which the model learns. + * @param iterations Number of training iterations. + */ + LogisticRegression(double learningRate, int iterations) + : learningRate_(learningRate), iterations_(iterations) {} + + /** + * @brief Train the model using features and labels. + * @param features Input feature matrix. + * @param labels Binary labels (0 or 1). + */ + void train(const std::vector>& features, const std::vector& labels) { + int numFeatures = features[0].size(); + weights_ = std::vector(numFeatures, 0.0); + + for (int iter = 0; iter < iterations_; ++iter) { + for (size_t i = 0; i < features.size(); ++i) { + double prediction = predictProbability(features[i]); + for (int j = 0; j < numFeatures; ++j) { + weights_[j] += learningRate_ * (labels[i] - prediction) * features[i][j]; + } + } + } + } + + /** + * @brief Predicts the class label for a given input. + * @param features Input feature vector. + * @return Predicted class label (0 or 1). + */ + int predict(const std::vector& features) const { + return predictProbability(features) >= 0.5 ? 1 : 0; + } + +private: + double learningRate_; ///< Learning rate for gradient descent + int iterations_; ///< Number of training iterations + std::vector weights_; ///< Model weights + + /** + * @brief Sigmoid activation function. + * @param z Linear combination of inputs and weights. + * @return Result of applying the sigmoid function to z. + */ + double sigmoid(double z) const { + return 1.0 / (1.0 + std::exp(-z)); + } + + /** + * @brief Predicts the probability of class 1 for a given input. + * @param features Input feature vector. + * @return Probability of the input belonging to class 1. + */ + double predictProbability(const std::vector& features) const { + double z = 0.0; + for (size_t i = 0; i < features.size(); ++i) { + z += weights_[i] * features[i]; + } + return sigmoid(z); + } +}; + +#endif // LOGISTIC_REGRESSION_HPP diff --git a/ml_library_include/ml/regression/MultiLinearRegression.hpp b/ml_library_include/ml/regression/MultiLinearRegression.hpp new file mode 100644 index 0000000..814ec06 --- /dev/null +++ b/ml_library_include/ml/regression/MultiLinearRegression.hpp @@ -0,0 +1,83 @@ +#ifndef MULTILINEAR_REGRESSION_HPP +#define MULTILINEAR_REGRESSION_HPP + +#include +#include +#include +#include + +/** + * @brief A class that implements Multilinear Regression for predicting values + * based on multiple features. + */ +class MultilinearRegression { +public: + /** + * @brief Constructs the MultilinearRegression model with the given learning rate and number of iterations. + * + * @param learningRate The rate at which the model learns (default 0.01). + * @param iterations The number of iterations for the gradient descent (default 1000). + */ + MultilinearRegression(double learningRate = 0.01, int iterations = 1000) + : learningRate(learningRate), iterations(iterations) {} + + /** + * @brief Trains the Multilinear Regression model on the provided data. + * + * @param features A vector of vectors, where each sub-vector represents the features for one data point. + * @param target A vector containing the target values corresponding to each data point. + * @throw std::invalid_argument If the number of features does not match the target size. + */ + void train(const std::vector>& features, const std::vector& target) { + if (features.empty() || features.size() != target.size()) { + throw std::invalid_argument("Features and target data sizes do not match."); + } + + int numFeatures = features[0].size(); + weights.resize(numFeatures, 0.0); // Initialize weights + + for (int i = 0; i < iterations; ++i) { + gradientDescentStep(features, target); + } + } + + /** + * @brief Predicts the output for a given set of features. + * + * @param features A vector containing feature values for a single data point. + * @return The predicted value. + */ + double predict(const std::vector& features) const { + return std::inner_product(weights.begin(), weights.end(), features.begin(), 0.0); + } + +private: + double learningRate; ///< The learning rate for gradient descent. + int iterations; ///< The number of iterations for training. + std::vector weights; ///< The weights for the model. + + /** + * @brief Performs a single iteration of gradient descent to update the model weights. + * + * @param features A vector of vectors containing the feature data. + * @param target A vector containing the target values. + */ + void gradientDescentStep(const std::vector>& features, const std::vector& target) { + std::vector gradients(weights.size(), 0.0); + + for (size_t i = 0; i < features.size(); ++i) { + double prediction = predict(features[i]); + double error = prediction - target[i]; + + for (size_t j = 0; j < weights.size(); ++j) { + gradients[j] += error * features[i][j]; + } + } + + for (size_t j = 0; j < weights.size(); ++j) { + weights[j] -= (learningRate / features.size()) * gradients[j]; + } + } +}; + +#endif // MULTILINEAR_REGRESSION_HPP diff --git a/ml_library_include/ml/regression/PolynomialRegression.hpp b/ml_library_include/ml/regression/PolynomialRegression.hpp new file mode 100644 index 0000000..15eb395 --- /dev/null +++ b/ml_library_include/ml/regression/PolynomialRegression.hpp @@ -0,0 +1,117 @@ +#ifndef POLYNOMIAL_REGRESSION_HPP +#define POLYNOMIAL_REGRESSION_HPP + +#include +#include +#include + +/** + * @brief Polynomial Regression model for fitting polynomial curves. + */ +class PolynomialRegression { +public: + /** + * @brief Constructor initializing the polynomial degree. + * @param degree Degree of the polynomial. + */ + PolynomialRegression(int degree) : degree_(degree) {} + + /** + * @brief Train the model using features and target values. + * @param x Feature vector. + * @param y Target vector. + */ + void train(const std::vector& x, const std::vector& y) { + computeCoefficients(x, y); + } + + /** + * @brief Predicts the output for a given input value. + * @param x Input feature. + * @return Predicted value. + */ + double predict(double x) const { + double result = 0.0; + for (int i = 0; i <= degree_; ++i) { + result += coefficients_[i] * std::pow(x, i); + } + return result; + } + +private: + int degree_; ///< Degree of the polynomial + std::vector coefficients_; ///< Coefficients of the polynomial + + /** + * @brief Computes polynomial regression coefficients for the provided data. + * @param x Feature vector. + * @param y Target vector. + */ + void computeCoefficients(const std::vector& x, const std::vector& y) { + int n = x.size(); + int m = degree_ + 1; + + // Create the matrix X and vector Y for the normal equation + std::vector> X(m, std::vector(m, 0.0)); + std::vector Y(m, 0.0); + + // Populate X and Y using the x and y data points + for (int i = 0; i < n; ++i) { + for (int j = 0; j < m; ++j) { + Y[j] += std::pow(x[i], j) * y[i]; + for (int k = 0; k < m; ++k) { + X[j][k] += std::pow(x[i], j + k); + } + } + } + + // Solve the system using Gaussian elimination + coefficients_ = gaussianElimination(X, Y); + } + + /** + * @brief Solves the linear system using Gaussian elimination. + * @param A Matrix representing the system's coefficients. + * @param b Vector representing the constant terms. + * @return Solution vector containing the polynomial coefficients. + */ + std::vector gaussianElimination(std::vector>& A, std::vector& b) { + int n = A.size(); + + // Perform Gaussian elimination + for (int i = 0; i < n; ++i) { + // Partial pivoting + int maxRow = i; + for (int k = i + 1; k < n; ++k) { + if (std::fabs(A[k][i]) > std::fabs(A[maxRow][i])) { + maxRow = k; + } + } + std::swap(A[i], A[maxRow]); + std::swap(b[i], b[maxRow]); + + // Make all rows below this one 0 in the current column + for (int k = i + 1; k < n; ++k) { + double factor = A[k][i] / A[i][i]; + for (int j = i; j < n; ++j) { + A[k][j] -= factor * A[i][j]; + } + b[k] -= factor * b[i]; + } + } + + // Back substitution + std::vector x(n, 0.0); + for (int i = n - 1; i >= 0; --i) { + x[i] = b[i]; + for (int j = i + 1; j < n; ++j) { + x[i] -= A[i][j] * x[j]; + } + x[i] /= A[i][i]; + } + + return x; + } +}; + +#endif // POLYNOMIAL_REGRESSION_HPP diff --git a/ml_library_include/ml/regression/MultiLinearRegression.h b/ml_library_include/ml/tree/DecisionTreeClassifier.hpp similarity index 100% rename from ml_library_include/ml/regression/MultiLinearRegression.h rename to ml_library_include/ml/tree/DecisionTreeClassifier.hpp diff --git a/ml_library_include/ml/regression/PolynomialRegression.h b/ml_library_include/ml/tree/DecisionTreeRegressor.hpp similarity index 100% rename from ml_library_include/ml/regression/PolynomialRegression.h rename to ml_library_include/ml/tree/DecisionTreeRegressor.hpp diff --git a/ml_library_include/ml/tree/DecisionTreeClassifier.h b/ml_library_include/ml/tree/RandomForestClassifier.hpp similarity index 100% rename from ml_library_include/ml/tree/DecisionTreeClassifier.h rename to ml_library_include/ml/tree/RandomForestClassifier.hpp diff --git a/ml_library_include/ml/tree/RandomForestRegressor.h b/ml_library_include/ml/tree/RandomForestRegressor.h deleted file mode 100644 index e69de29..0000000 diff --git a/ml_library_include/ml/tree/DecisionTreeRegressor.h b/ml_library_include/ml/tree/RandomForestRegressor.hpp similarity index 100% rename from ml_library_include/ml/tree/DecisionTreeRegressor.h rename to ml_library_include/ml/tree/RandomForestRegressor.hpp diff --git a/src/neural_network/NN.cpp b/src/neural_network/NN.cpp deleted file mode 100644 index aa87fc9..0000000 --- a/src/neural_network/NN.cpp +++ /dev/null @@ -1,239 +0,0 @@ -#include "ml/neural_network/NN.h" - -#include -#include - -// ********** TrainingData Class Implementation ********** // - -/** - * @brief Constructor that opens the file containing training data. - * Ensures the file is ready for reading data sequentially. - */ -TrainingData::TrainingData(const std::string filename) { - _trainingDataFile.open(filename.c_str()); -} - -/** - * @brief Parses the topology of the neural network. - * - * This method reads a line from the training data file that starts with the keyword "topology:" - * followed by integers representing the number of neurons in each layer. For instance, a topology - * line of "topology: 3 2 1" describes a network with 3 neurons in the first layer, 2 in the second, - * and 1 in the final layer. - */ -void TrainingData::getTopology(std::vector& topology) { - std::string line, label; - if (!getline(_trainingDataFile, line)) { - std::cerr << "Error reading from the file." << std::endl; - abort(); - } - - std::stringstream ss(line); - ss >> label; - if (label != "topology:") { - std::cerr << "Invalid format. Expected 'topology:'." << std::endl; - abort(); - } - - unsigned numLayers; - while (ss >> numLayers) { - topology.push_back(numLayers); - } - - // Ensure that a valid topology was provided, otherwise halt execution. - if (topology.empty()) { - std::cerr << "No topology data found." << std::endl; - abort(); - } -} - -/** - * @brief Retrieves the next set of input values. - * - * Each line with the "in:" prefix contains input values for a training pass. - * This function parses those values and stores them in inputVals. - */ -unsigned TrainingData::getNextInputs(std::vector& inputVals) { - inputVals.clear(); - std::string line; - getline(_trainingDataFile, line); - std::stringstream ss(line); - std::string label; - ss >> label; - if (label.compare("in:") == 0) { - double oneVal; - while (ss >> oneVal) { - inputVals.push_back(oneVal); - } - } - return inputVals.size(); -} - -/** - * @brief Retrieves the target output values. - * - * Each line with the "out:" prefix contains the expected output values for a training pass. - * This function parses those values and stores them in targetOutputsVals. - */ -unsigned TrainingData::getTargetOutputs(std::vector& targetOutputsVals) { - targetOutputsVals.clear(); - std::string line; - getline(_trainingDataFile, line); - std::stringstream ss(line); - std::string label; - ss >> label; - if (label.compare("out:") == 0) { - double oneVal; - while (ss >> oneVal) { - targetOutputsVals.push_back(oneVal); - } - } - return targetOutputsVals.size(); -} - -// ********** Neuron Class Implementation ********** // - -/** - * @brief Static variables for learning rate (eta) and momentum (alpha). - * - * - eta controls the rate at which the network adjusts during backpropagation. - * - alpha applies momentum to reduce oscillations during training. - */ -double Neuron::eta = 0.15; -double Neuron::alpha = 0.5; - -/** - * @brief Neuron constructor that initializes output weights with random values. - * - * Each neuron is connected to neurons in the next layer, and these connections - * are initialized with random weights in the range [0, 1). Random initialization - * is essential for neural networks to avoid symmetry during training. - */ -Neuron::Neuron(unsigned numOutputs, unsigned myIdx) : _myIdx(myIdx), _outputVal(0.0) { - for (unsigned c = 0; c < numOutputs; ++c) { - _outputWeights.push_back(Connection()); - _outputWeights.back().weight = randomWeight(); - } -} - -/** - * @brief Generates a random weight for initializing connections. - * - * Random weights are critical for breaking symmetry in the network. - * Without randomness, neurons would learn identical features. - */ -double Neuron::randomWeight() { - return rand() / double(RAND_MAX); -} - -/** - * @brief Hyperbolic tangent (tanh) transfer function. - * - * The tanh function is commonly used as an activation function. It outputs - * values between -1 and 1, providing a non-linear transformation that enables - * the network to approximate complex functions. - */ -double Neuron::transferFunction(double x) { - return tanh(x); -} - -/** - * @brief Derivative of the tanh function for backpropagation. - * - * The derivative is used to calculate gradients during the backpropagation process. - * With tanh(x), the derivative is 1 - x^2, which facilitates efficient gradient calculation. - */ -double Neuron::transferFunctionDerivative(double x) { - return 1.0 - x * x; -} - -/** - * @brief Computes the neuron’s output by summing inputs from the previous layer. - * - * For each neuron in the previous layer, the output is weighted and summed. This - * summation is then passed through the transfer function to generate the final output. - */ -void Neuron::feedForward(const std::vector& prevLayer) { - double sum = 0.0; - for (unsigned n = 0; n < prevLayer.size(); ++n) { - sum += prevLayer[n].getOutputVal() * prevLayer[n]._outputWeights[_myIdx].weight; - } - _outputVal = Neuron::transferFunction(sum); -} - -// ********** NN Class Implementation ********** // - -/** - * @brief Smoothing factor for recent average error calculations. - * - * The `_recentAverageSmoothFactor` is used to smooth out the fluctuations in the - * recent average error, providing a more stable view of error trends during training. - */ -double NN::_recentAverageSmoothFactor = 100.0; - -/** - * @brief Neural network constructor that builds layers and initializes neurons. - * - * Each layer is constructed according to the topology provided. The constructor also - * creates bias neurons, which always output 1.0. Bias neurons help the network learn - * patterns that require a constant offset. - */ -NN::NN(const std::vector& topology) { - unsigned numLayers = topology.size(); - for (unsigned layerNum = 0; layerNum < numLayers; ++layerNum) { - _layers.push_back(std::vector()); - unsigned numOutputs = (layerNum == topology.size() - 1) ? 0 : topology[layerNum + 1]; - for (unsigned neuronNum = 0; neuronNum <= topology[layerNum]; ++neuronNum) { - _layers.back().push_back(Neuron(numOutputs, neuronNum)); - _layers.back().back().setOutputVal(1.0); // Bias neuron output - } - } -} - -// Utility functions - -/** - * @brief Utility function to print vector values with a label. - * - * Primarily used for debugging, this function outputs each value in a vector - * with an associated label. - */ -void showVectorVals(std::string label, std::vector& v) { - std::cout << label << " "; - for (unsigned i = 0; i < v.size(); ++i) { - std::cout << v[i] << " "; - } - std::cout << std::endl; -} - -/** - * @brief Converts vector values to a formatted string for easier inspection. - * - * Useful for debugging and logging, this function formats vector values - * into a labeled string for printing or saving to a file. - */ -std::string printVectorVals(std::string label, std::vector& v) { - std::string res = label + " "; - for (const auto& val : v) { - res += std::to_string(val) + " "; - } - return res + "\n"; -} - -/** - * @brief Saves a string to a file. - * - * This utility function opens a file in write mode and saves the content. - * It’s used to log results or intermediate outputs for analysis. - */ -void saveStringToFile(const std::string& filename, const std::string& content) { - std::ofstream outputFile(filename); - if (outputFile.is_open()) { - outputFile << content; - outputFile.close(); - std::cout << "String saved to " << filename << std::endl; - } - else { - std::cerr << "Error: Unable to open the file for writing." << std::endl; - } -} diff --git a/ml_library_include/ml/tree/RandomForestClassifier.h b/src/regression/LogisticRegression.cpp similarity index 100% rename from ml_library_include/ml/tree/RandomForestClassifier.h rename to src/regression/LogisticRegression.cpp diff --git a/tests/TestUtils.hpp b/tests/TestUtils.hpp new file mode 100644 index 0000000..3c15d84 --- /dev/null +++ b/tests/TestUtils.hpp @@ -0,0 +1,9 @@ +// TestUtils.hpp +#ifndef TEST_UTILS_HPP +#define TEST_UTILS_HPP +#include +inline bool approxEqual(double a, double b, double epsilon = 0.1) { + return std::fabs(a - b) < epsilon; +} + +#endif // TEST_UTILS_HPP diff --git a/tests/regression/LogisticRegressionTest.cpp b/tests/regression/LogisticRegressionTest.cpp new file mode 100644 index 0000000..1320144 --- /dev/null +++ b/tests/regression/LogisticRegressionTest.cpp @@ -0,0 +1,31 @@ +#include "../../ml_library_include/ml/regression/LogisticRegression.hpp" +#include +#include +#include + +int main() { + LogisticRegression model(0.1, 1000); + + std::vector> features = { + {0.0, 0.0}, + {1.0, 1.0}, + {1.0, 0.0}, + {0.0, 1.0} + }; + + std::vector labels = {0, 1, 1, 0}; + + // Ensure that training runs without errors + model.train(features, labels); + + // Test the prediction + std::vector testFeatures = {1.0, 1.0}; + int prediction = model.predict(testFeatures); + + // Check that prediction is as expected + assert(prediction == 1); + + // Inform user of successful test + std::cout << "Logistic Regression Basic Test passed." << std::endl; + return 0; +} diff --git a/tests/regression/MultilinearRegressionTest.cpp b/tests/regression/MultilinearRegressionTest.cpp new file mode 100644 index 0000000..15e5e4b --- /dev/null +++ b/tests/regression/MultilinearRegressionTest.cpp @@ -0,0 +1,33 @@ +#include "../../ml_library_include/ml/regression/MultiLinearRegression.hpp" +#include +#include +#include +#include "../TestUtils.hpp" + + +int main() { + MultilinearRegression model(0.01, 1000); + + std::vector> features = { + {1.0, 2.0}, + {2.0, 3.0}, + {3.0, 4.0}, + {4.0, 5.0} + }; + + std::vector target = {3.0, 5.0, 7.0, 9.0}; + + // Ensure that training runs without errors + model.train(features, target); + + // Test the prediction + std::vector testFeatures = {5.0, 6.0}; + double prediction = model.predict(testFeatures); + + // Assert with approximate equality + assert(approxEqual(prediction, 11.0)); + + // Inform user of successful test + std::cout << "MultilinearRegression Basic Test passed." << std::endl; + return 0; +} diff --git a/tests/regression/PolynomialRegressionTest.cpp b/tests/regression/PolynomialRegressionTest.cpp new file mode 100644 index 0000000..9c40b20 --- /dev/null +++ b/tests/regression/PolynomialRegressionTest.cpp @@ -0,0 +1,27 @@ +#include "../../ml_library_include/ml/regression/PolynomialRegression.hpp" +#include +#include +#include +#include +#include "../TestUtils.hpp" + +int main() { + PolynomialRegression model(2); // Quadratic regression + + std::vector x = {1.0, 2.0, 3.0, 4.0}; + std::vector y = {3.0, 5.0, 7.0, 9.0}; + + // Ensure that training runs without errors + model.train(x, y); + + // Test the prediction + double prediction = model.predict(5.0); + + // Check that the prediction is approximately as expected + assert(approxEqual(prediction, 11.0, 0.1)); + + // Inform user of successful test + std::cout << "Polynomial Regression Basic Test passed." << std::endl; + + return 0; +}