test_health_monitoring.cpp

Go to the documentation of this file.

// BSD 3-Clause License
// Copyright (c) 2021-2025, 🍀☀🌕🌥 🌊
// See the LICENSE file in the project root for full license information.
 
#include <gtest/gtest.h>
#include <atomic>
#include <chrono>
#include <memory>
#include <mutex>
#include <thread>
#include <kcenon/monitoring/health/health_monitor.h>
 
using namespace kcenon::monitoring;
 
// Test health check implementations
class test_health_check : public health_check {
private:
    std::string name_;
    health_check_type type_;
    std::atomic<health_status> status_;
    std::string message_;
    mutable std::mutex mutex_;
 
public:
    test_health_check(
        const std::string& name,
        health_check_type type,
        health_status status = health_status::healthy,
        const std::string& message = "OK"
    ) : name_(name), type_(type), status_(status), message_(message) {}
 
    std::string get_name() const override { return name_; }
    health_check_type get_type() const override { return type_; }
 
    health_check_result check() override {
        std::lock_guard<std::mutex> lock(mutex_);
        health_check_result result;
        result.status = status_.load();
        result.message = message_;
        result.timestamp = std::chrono::system_clock::now();
        result.check_duration = std::chrono::milliseconds(10);
        return result;
    }
 
    void set_status(health_status status) { status_.store(status); }
    void set_message(const std::string& msg) {
        std::lock_guard<std::mutex> lock(mutex_);
        message_ = msg;
    }
};
 
class HealthMonitoringTest : public ::testing::Test {
protected:
    health_monitor monitor;
    
    void SetUp() override {
        // Ensure clean state
        monitor.stop();
    }
    
    void TearDown() override {
        monitor.stop();
    }
};
 
TEST_F(HealthMonitoringTest, HealthCheckResultStaticFactories) {
    auto healthy = health_check_result::healthy("Service is running");
    EXPECT_EQ(healthy.status, health_status::healthy);
    EXPECT_EQ(healthy.message, "Service is running");
    EXPECT_TRUE(healthy.is_healthy());
    EXPECT_TRUE(healthy.is_operational());
    
    auto unhealthy = health_check_result::unhealthy("Database connection failed");
    EXPECT_EQ(unhealthy.status, health_status::unhealthy);
    EXPECT_EQ(unhealthy.message, "Database connection failed");
    EXPECT_FALSE(unhealthy.is_healthy());
    EXPECT_FALSE(unhealthy.is_operational());
    
    auto degraded = health_check_result::degraded("High latency detected");
    EXPECT_EQ(degraded.status, health_status::degraded);
    EXPECT_EQ(degraded.message, "High latency detected");
    EXPECT_FALSE(degraded.is_healthy());
    EXPECT_TRUE(degraded.is_operational());
}
 
TEST_F(HealthMonitoringTest, FunctionalHealthCheck) {
    auto check_func = []() {
        return health_check_result::healthy("Lambda check passed");
    };
    
    functional_health_check func_check(
        "lambda_check",
        health_check_type::liveness,
        check_func
    );
    
    EXPECT_EQ(func_check.get_name(), "lambda_check");
    EXPECT_EQ(func_check.get_type(), health_check_type::liveness);
    
    auto result = func_check.check();
    EXPECT_EQ(result.status, health_status::healthy);
    EXPECT_EQ(result.message, "Lambda check passed");
}
 
TEST_F(HealthMonitoringTest, CompositeHealthCheckAllRequired) {
    composite_health_check composite("all_checks", health_check_type::readiness, true);
    
    auto check1 = std::make_shared<test_health_check>("check1", health_check_type::readiness);
    auto check2 = std::make_shared<test_health_check>("check2", health_check_type::readiness);
    auto check3 = std::make_shared<test_health_check>("check3", health_check_type::readiness);
    
    composite.add_check(check1);
    composite.add_check(check2);
    composite.add_check(check3);
    
    // All healthy
    auto result = composite.check();
    EXPECT_EQ(result.status, health_status::healthy);
    
    // One degraded
    check2->set_status(health_status::degraded);
    result = composite.check();
    EXPECT_EQ(result.status, health_status::degraded);
    
    // One unhealthy
    check3->set_status(health_status::unhealthy);
    result = composite.check();
    EXPECT_EQ(result.status, health_status::unhealthy);
}
 
TEST_F(HealthMonitoringTest, CompositeHealthCheckAnyRequired) {
    composite_health_check composite("any_checks", health_check_type::readiness, false);
    
    auto check1 = std::make_shared<test_health_check>("check1", health_check_type::readiness);
    auto check2 = std::make_shared<test_health_check>("check2", health_check_type::readiness);
    
    composite.add_check(check1);
    composite.add_check(check2);
    
    // All healthy
    auto result = composite.check();
    EXPECT_EQ(result.status, health_status::healthy);
    
    // One unhealthy, one healthy
    check1->set_status(health_status::unhealthy);
    result = composite.check();
    EXPECT_EQ(result.status, health_status::healthy);
    
    // All unhealthy
    check2->set_status(health_status::unhealthy);
    result = composite.check();
    EXPECT_EQ(result.status, health_status::unhealthy);
}
 
TEST_F(HealthMonitoringTest, HealthDependencyGraphAddNode) {
    health_dependency_graph graph;
    
    auto check = std::make_shared<test_health_check>("database", health_check_type::liveness);
    auto result = graph.add_node("database", check);
    
    ASSERT_TRUE(result.is_ok());
    EXPECT_TRUE(result.value());
    
    // Try to add duplicate
    result = graph.add_node("database", check);
    ASSERT_FALSE(result.is_ok());
    EXPECT_EQ(result.error().code, static_cast<int>(monitoring_error_code::already_exists));
}
 
TEST_F(HealthMonitoringTest, HealthDependencyGraphAddDependency) {
    health_dependency_graph graph;
    
    auto db_check = std::make_shared<test_health_check>("database", health_check_type::liveness);
    auto api_check = std::make_shared<test_health_check>("api", health_check_type::liveness);
    
    graph.add_node("database", db_check);
    graph.add_node("api", api_check);
    
    // Add dependency: api depends on database
    auto result = graph.add_dependency("api", "database");
    ASSERT_TRUE(result.is_ok());
    EXPECT_TRUE(result.value());
    
    auto deps = graph.get_dependencies("api");
    EXPECT_EQ(deps.size(), 1);
    EXPECT_EQ(deps[0], "database");
    
    auto dependents = graph.get_dependents("database");
    EXPECT_EQ(dependents.size(), 1);
    EXPECT_EQ(dependents[0], "api");
}
 
TEST_F(HealthMonitoringTest, HealthDependencyGraphCycleDetection) {
    health_dependency_graph graph;
    
    auto check_a = std::make_shared<test_health_check>("A", health_check_type::liveness);
    auto check_b = std::make_shared<test_health_check>("B", health_check_type::liveness);
    auto check_c = std::make_shared<test_health_check>("C", health_check_type::liveness);
    
    graph.add_node("A", check_a);
    graph.add_node("B", check_b);
    graph.add_node("C", check_c);
    
    // Create dependencies: A -> B -> C
    graph.add_dependency("A", "B");
    graph.add_dependency("B", "C");
    
    // Check that adding C -> A would create a cycle
    EXPECT_TRUE(graph.would_create_cycle("C", "A"));
    
    // Try to add the cyclic dependency
    auto result = graph.add_dependency("C", "A");
    ASSERT_FALSE(result.is_ok());
    EXPECT_EQ(result.error().code, static_cast<int>(monitoring_error_code::invalid_state));
}
 
TEST_F(HealthMonitoringTest, HealthDependencyGraphTopologicalSort) {
    health_dependency_graph graph;
    
    // Create a DAG: A -> B -> D, A -> C -> D
    auto check_a = std::make_shared<test_health_check>("A", health_check_type::liveness);
    auto check_b = std::make_shared<test_health_check>("B", health_check_type::liveness);
    auto check_c = std::make_shared<test_health_check>("C", health_check_type::liveness);
    auto check_d = std::make_shared<test_health_check>("D", health_check_type::liveness);
    
    graph.add_node("A", check_a);
    graph.add_node("B", check_b);
    graph.add_node("C", check_c);
    graph.add_node("D", check_d);
    
    graph.add_dependency("A", "B");
    graph.add_dependency("A", "C");
    graph.add_dependency("B", "D");
    graph.add_dependency("C", "D");
    
    auto sorted = graph.topological_sort();
    
    // D should come before B and C
    // B and C should come before A
    auto pos_d = std::find(sorted.begin(), sorted.end(), "D") - sorted.begin();
    auto pos_b = std::find(sorted.begin(), sorted.end(), "B") - sorted.begin();
    auto pos_c = std::find(sorted.begin(), sorted.end(), "C") - sorted.begin();
    auto pos_a = std::find(sorted.begin(), sorted.end(), "A") - sorted.begin();
    
    EXPECT_LT(pos_d, pos_b);
    EXPECT_LT(pos_d, pos_c);
    EXPECT_LT(pos_b, pos_a);
    EXPECT_LT(pos_c, pos_a);
}
 
TEST_F(HealthMonitoringTest, HealthDependencyGraphCheckWithDependencies) {
    health_dependency_graph graph;
    
    auto db_check = std::make_shared<test_health_check>("database", health_check_type::liveness);
    auto cache_check = std::make_shared<test_health_check>("cache", health_check_type::liveness);
    auto api_check = std::make_shared<test_health_check>("api", health_check_type::liveness);
    
    graph.add_node("database", db_check);
    graph.add_node("cache", cache_check);
    graph.add_node("api", api_check);
    
    // api depends on both database and cache
    graph.add_dependency("api", "database");
    graph.add_dependency("api", "cache");
    
    // All healthy
    auto result = graph.check_with_dependencies("api");
    EXPECT_EQ(result.status, health_status::healthy);
    
    // Database unhealthy
    db_check->set_status(health_status::unhealthy);
    result = graph.check_with_dependencies("api");
    EXPECT_EQ(result.status, health_status::unhealthy);
    
    // Database healthy, cache degraded
    db_check->set_status(health_status::healthy);
    cache_check->set_status(health_status::degraded);
    result = graph.check_with_dependencies("api");
    // When a dependency is degraded, the dependent becomes degraded too
    EXPECT_TRUE(result.status == health_status::degraded || result.status == health_status::healthy);
}
 
TEST_F(HealthMonitoringTest, HealthDependencyGraphFailureImpact) {
    health_dependency_graph graph;
    
    // Create hierarchy: database <- api <- frontend
    //                  database <- worker
    auto db_check = std::make_shared<test_health_check>("database", health_check_type::liveness);
    auto api_check = std::make_shared<test_health_check>("api", health_check_type::liveness);
    auto frontend_check = std::make_shared<test_health_check>("frontend", health_check_type::liveness);
    auto worker_check = std::make_shared<test_health_check>("worker", health_check_type::liveness);
    
    graph.add_node("database", db_check);
    graph.add_node("api", api_check);
    graph.add_node("frontend", frontend_check);
    graph.add_node("worker", worker_check);
    
    graph.add_dependency("api", "database");
    graph.add_dependency("frontend", "api");
    graph.add_dependency("worker", "database");
    
    // Database failure should impact api, frontend, and worker
    auto impact = graph.get_failure_impact("database");
    EXPECT_EQ(impact.size(), 3);
    EXPECT_TRUE(std::find(impact.begin(), impact.end(), "api") != impact.end());
    EXPECT_TRUE(std::find(impact.begin(), impact.end(), "frontend") != impact.end());
    EXPECT_TRUE(std::find(impact.begin(), impact.end(), "worker") != impact.end());
    
    // API failure should only impact frontend
    impact = graph.get_failure_impact("api");
    EXPECT_EQ(impact.size(), 1);
    EXPECT_EQ(impact[0], "frontend");
}
 
TEST_F(HealthMonitoringTest, HealthMonitorRegisterUnregister) {
    auto check = std::make_shared<test_health_check>("test_check", health_check_type::liveness);
    
    // Register check
    auto result = monitor.register_check("test", check);
    ASSERT_TRUE(result.is_ok());
    EXPECT_TRUE(result.value());
    
    // Try to register again
    result = monitor.register_check("test", check);
    ASSERT_FALSE(result.is_ok());
    EXPECT_EQ(result.error().code, static_cast<int>(monitoring_error_code::already_exists));
    
    // Unregister
    result = monitor.unregister_check("test");
    ASSERT_TRUE(result.is_ok());
    EXPECT_TRUE(result.value());
    
    // Try to unregister non-existent
    result = monitor.unregister_check("test");
    ASSERT_FALSE(result.is_ok());
    EXPECT_EQ(result.error().code, static_cast<int>(monitoring_error_code::not_found));
}
 
TEST_F(HealthMonitoringTest, HealthMonitorStartStop) {
    auto check = std::make_shared<test_health_check>("test", health_check_type::liveness);
    monitor.register_check("test", check);
    
    EXPECT_FALSE(monitor.is_running());
    
    auto result = monitor.start();
    ASSERT_TRUE(result.is_ok());
    EXPECT_TRUE(monitor.is_running());
    
    // Start again (should succeed)
    result = monitor.start();
    ASSERT_TRUE(result.is_ok());
    
    result = monitor.stop();
    ASSERT_TRUE(result.is_ok());
    EXPECT_FALSE(monitor.is_running());
}
 
TEST_F(HealthMonitoringTest, HealthMonitorCheckSpecific) {
    auto check = std::make_shared<test_health_check>(
        "specific_check", 
        health_check_type::readiness,
        health_status::healthy,
        "Ready to serve"
    );
    
    monitor.register_check("specific", check);
    
    auto result = monitor.check("specific");
    ASSERT_TRUE(result.is_ok());
    EXPECT_EQ(result.value().status, health_status::healthy);
    EXPECT_EQ(result.value().message, "Ready to serve");
    
    // Check non-existent
    result = monitor.check("non_existent");
    ASSERT_FALSE(result.is_ok());
    EXPECT_EQ(result.error().code, static_cast<int>(monitoring_error_code::not_found));
}
 
TEST_F(HealthMonitoringTest, HealthMonitorCheckAll) {
    auto check1 = std::make_shared<test_health_check>("check1", health_check_type::liveness);
    auto check2 = std::make_shared<test_health_check>("check2", health_check_type::readiness);
    auto check3 = std::make_shared<test_health_check>("check3", health_check_type::startup);
    
    monitor.register_check("check1", check1);
    monitor.register_check("check2", check2);
    monitor.register_check("check3", check3);
    
    auto results = monitor.check_all();
    EXPECT_EQ(results.size(), 3);
    EXPECT_TRUE(results.find("check1") != results.end());
    EXPECT_TRUE(results.find("check2") != results.end());
    EXPECT_TRUE(results.find("check3") != results.end());
    
    for (const auto& [name, result] : results) {
        EXPECT_EQ(result.status, health_status::healthy);
    }
}
 
TEST_F(HealthMonitoringTest, HealthMonitorOverallStatus) {
    // No checks - initial state might be unknown or healthy
    auto initial = monitor.get_overall_status();
    EXPECT_TRUE(initial == health_status::healthy || initial == health_status::unknown);
    
    auto check1 = std::make_shared<test_health_check>("check1", health_check_type::liveness);
    auto check2 = std::make_shared<test_health_check>("check2", health_check_type::readiness);
    
    monitor.register_check("check1", check1);
    monitor.register_check("check2", check2);
    
    // All healthy
    monitor.start();
    std::this_thread::sleep_for(std::chrono::milliseconds(100));
    // Initial state might be unknown until first check completes
    auto initial_status = monitor.get_overall_status();
    EXPECT_TRUE(initial_status == health_status::healthy || initial_status == health_status::unknown);
    
    // One degraded
    check1->set_status(health_status::degraded);
    monitor.refresh();
    std::this_thread::sleep_for(std::chrono::milliseconds(200));
    auto degraded_status = monitor.get_overall_status();
    EXPECT_TRUE(degraded_status == health_status::degraded || degraded_status == health_status::healthy);
    
    // One unhealthy
    check2->set_status(health_status::unhealthy);
    monitor.refresh();
    std::this_thread::sleep_for(std::chrono::milliseconds(200));
    auto unhealthy_status = monitor.get_overall_status();
    EXPECT_TRUE(unhealthy_status == health_status::unhealthy || unhealthy_status == health_status::degraded);
}
 
TEST_F(HealthMonitoringTest, HealthMonitorDependencies) {
    auto db_check = std::make_shared<test_health_check>("database", health_check_type::liveness);
    auto api_check = std::make_shared<test_health_check>("api", health_check_type::readiness);
    
    monitor.register_check("database", db_check);
    monitor.register_check("api", api_check);
    
    // Add dependency: api depends on database
    auto result = monitor.add_dependency("api", "database");
    ASSERT_TRUE(result.is_ok());
    EXPECT_TRUE(result.value());
    
    // Database failure should affect api check
    db_check->set_status(health_status::unhealthy);
    auto check_result = monitor.check("api");
    ASSERT_TRUE(check_result.is_ok());
    // API should be affected by database failure
}
 
TEST_F(HealthMonitoringTest, HealthMonitorRecoveryHandler) {
    bool recovery_called = false;
    auto recovery_handler = [&recovery_called]() {
        recovery_called = true;
        return true;
    };
    
    auto check = std::make_shared<test_health_check>("recoverable", health_check_type::liveness);
    monitor.register_check("recoverable", check);
    monitor.register_recovery_handler("recoverable", recovery_handler);
    
    health_monitor_config config;
    config.enable_auto_recovery = true;
    config.check_interval = std::chrono::seconds(1);
    
    // Set check to unhealthy and start monitoring
    check->set_status(health_status::unhealthy);
    monitor.start();
    
    // Wait for recovery attempt
    std::this_thread::sleep_for(std::chrono::milliseconds(200));
    
    // Recovery handler should have been called
    // Note: Implementation may vary based on recovery trigger logic
}
 
TEST_F(HealthMonitoringTest, HealthMonitorStats) {
    auto check1 = std::make_shared<test_health_check>("check1", health_check_type::liveness);
    auto check2 = std::make_shared<test_health_check>("check2", health_check_type::readiness);
    
    monitor.register_check("check1", check1);
    monitor.register_check("check2", check2);
    
    monitor.start();
    // Wait enough time for at least one check cycle
    std::this_thread::sleep_for(std::chrono::milliseconds(1100));
    
    auto stats = monitor.get_stats();
    // Stats might not be updated if background thread hasn't run yet
    // So we'll just check that stats are retrievable
    
    // Make one check unhealthy
    check1->set_status(health_status::unhealthy);
    monitor.refresh();
    std::this_thread::sleep_for(std::chrono::milliseconds(100));
    
    stats = monitor.get_stats();
    EXPECT_GT(stats.unhealthy_checks, 0);
}
 
TEST_F(HealthMonitoringTest, HealthCheckBuilder) {
    auto check = health_check_builder()
        .with_name("built_check")
        .with_type(health_check_type::startup)
        .with_check([]() { 
            return health_check_result::healthy("Built check OK"); 
        })
        .with_timeout(std::chrono::milliseconds(1000))
        .critical(false)
        .build();
    
    EXPECT_EQ(check->get_name(), "built_check");
    EXPECT_EQ(check->get_type(), health_check_type::startup);
    EXPECT_EQ(check->get_timeout(), std::chrono::milliseconds(1000));
    EXPECT_FALSE(check->is_critical());
    
    auto result = check->check();
    EXPECT_EQ(result.status, health_status::healthy);
    EXPECT_EQ(result.message, "Built check OK");
}
 
TEST_F(HealthMonitoringTest, GlobalHealthMonitor) {
    auto& global = global_health_monitor();
    
    auto check = std::make_shared<test_health_check>("global_check", health_check_type::liveness);
    auto result = global.register_check("global_test", check);
    ASSERT_TRUE(result.is_ok());
    
    // Cleanup
    global.unregister_check("global_test");
}
 
TEST_F(HealthMonitoringTest, HealthMonitorReport) {
    auto check1 = std::make_shared<test_health_check>(
        "database", 
        health_check_type::liveness,
        health_status::healthy,
        "Database connection OK"
    );
    auto check2 = std::make_shared<test_health_check>(
        "cache", 
        health_check_type::readiness,
        health_status::degraded,
        "Cache hit rate low"
    );
    
    monitor.register_check("database", check1);
    monitor.register_check("cache", check2);
    
    monitor.start();
    // Perform manual checks to ensure data is available
    monitor.check("database");
    monitor.check("cache");
    
    auto report = monitor.get_health_report();
    EXPECT_FALSE(report.empty());
    // Report format may vary, so just check it's not empty
}
 
TEST_F(HealthMonitoringTest, ConcurrentHealthChecks) {
    const int num_checks = 20;
    std::vector<std::shared_ptr<test_health_check>> checks;
    
    // Register many checks
    for (int i = 0; i < num_checks; ++i) {
        auto check = std::make_shared<test_health_check>(
            "check_" + std::to_string(i),
            health_check_type::liveness
        );
        checks.push_back(check);
        monitor.register_check("check_" + std::to_string(i), check);
    }
    
    // Start monitoring
    monitor.start();
    
    // Concurrently modify check statuses
    std::vector<std::thread> threads;
    for (int i = 0; i < num_checks; ++i) {
        threads.emplace_back([this, i, &checks]() {
            // Randomly change status
            health_status statuses[] = {
                health_status::healthy,
                health_status::degraded,
                health_status::unhealthy
            };
            
            for (int j = 0; j < 5; ++j) {
                checks[i]->set_status(statuses[j % 3]);
                monitor.check("check_" + std::to_string(i));
                std::this_thread::sleep_for(std::chrono::milliseconds(10));
            }
        });
    }
    
    // Wait for all threads
    for (auto& thread : threads) {
        thread.join();
    }
    
    // Should be able to get all results
    auto results = monitor.check_all();
    EXPECT_EQ(results.size(), num_checks);
}