test_stability.cpp File Reference

Long-Running Stability Tests - 24-Hour Continuous Operation. More...

#include "test_fixtures.h"
#include "test_data_generator.h"
#include <catch2/catch_test_macros.hpp>
#include "kcenon/pacs/network/dimse/dimse_message.h"
#include "kcenon/pacs/services/query_scp.h"
#include "kcenon/pacs/services/storage_scp.h"
#include "kcenon/pacs/services/storage_scu.h"
#include "kcenon/pacs/services/verification_scp.h"
#include "kcenon/pacs/storage/file_storage.h"
#include "kcenon/pacs/storage/index_database.h"
#include <atomic>
#include <chrono>
#include <cstdlib>
#include <filesystem>
#include <fstream>
#include <future>
#include <iomanip>
#include <iostream>
#include <mutex>
#include <random>
#include <set>
#include <sstream>
#include <thread>
#include <vector>

Include dependency graph for test_stability.cpp:

Go to the source code of this file.

Functions
	TEST_CASE ("Continuous store/query operation", "[stability][.slow]")
	TEST_CASE ("Memory stability over iterations", "[stability][memory]")
	TEST_CASE ("Connection pool exhaustion recovery", "[stability][network]")
	TEST_CASE ("Database integrity under concurrent load", "[stability][database]")
	TEST_CASE ("Short stability smoke test", "[stability][smoke]")

Detailed Description

Long-Running Stability Tests - 24-Hour Continuous Operation.

Tests system reliability under extended operation:

1. Continuous Store/Query (configurable duration)
2. Memory Leak Detection
3. Connection Pool Stability
4. Database Integrity Under Load

These tests are tagged with [.slow] and excluded from normal CI runs. Use environment variables to configure test duration:

• PACS_STABILITY_TEST_DURATION: Duration in minutes (default: 60)
• PACS_STABILITY_STORE_RATE: Stores per second (default: 5)
• PACS_STABILITY_QUERY_RATE: Queries per second (default: 1)

See also

Issue #140 - Long-Running Stability Tests

Definition in file test_stability.cpp.

Function Documentation

TEST_CASE() [1/5]

TEST_CASE	(	"Connection pool exhaustion recovery"	,
		""	[stability][network] )

Definition at line 529 of file test_stability.cpp.

                                                                         {
    uint16_t port = find_available_port();
    stability_test_server server(port);
    REQUIRE(server.initialize());
    REQUIRE(server.start());
 
    constexpr size_t max_concurrent = 20;
    constexpr size_t cycles = 5;
 
    for (size_t cycle = 0; cycle < cycles; ++cycle) {
        INFO("Cycle " << (cycle + 1) << " of " << cycles);
 
        std::vector<association> associations;
        associations.reserve(max_concurrent);
 
        // Open many associations
        for (size_t i = 0; i < max_concurrent; ++i) {
            association_config assoc_config;
            assoc_config.calling_ae_title = "POOL_TEST_" + std::to_string(i);
            assoc_config.called_ae_title = server.ae_title();
            assoc_config.implementation_class_uid = "1.2.826.0.1.3680043.9.9999.11";
            assoc_config.proposed_contexts.push_back({1, "1.2.840.10008.1.1", {"1.2.840.10008.1.2"}});
 
            auto assoc_result = association::connect(
                "127.0.0.1", port, assoc_config, std::chrono::seconds{30});
 
            REQUIRE(assoc_result.is_ok());
            associations.push_back(std::move(assoc_result.value()));
        }
 
        // Release all associations
        for (auto& assoc : associations) {
            (void)assoc.release(std::chrono::seconds{5});
        }
        associations.clear();
 
        // Allow server to clean up
        std::this_thread::sleep_for(std::chrono::milliseconds{500});
 
        // Verify server can still accept new connections
        association_config verify_config;
        verify_config.calling_ae_title = "VERIFY_SCU";
        verify_config.called_ae_title = server.ae_title();
        verify_config.implementation_class_uid = "1.2.826.0.1.3680043.9.9999.12";
        verify_config.proposed_contexts.push_back({1, "1.2.840.10008.1.1", {"1.2.840.10008.1.2"}});
 
        auto test_result = association::connect(
            "127.0.0.1", port, verify_config, std::chrono::seconds{30});
 
        REQUIRE(test_result.is_ok());
        (void)test_result.value().release(std::chrono::seconds{5});
    }
 
    server.stop();
}

References kcenon::pacs::network::association_config::called_ae_title, kcenon::pacs::network::association_config::calling_ae_title, kcenon::pacs::integration_test::find_available_port(), kcenon::pacs::network::association_config::implementation_class_uid, and kcenon::pacs::network::association_config::proposed_contexts.

Here is the call graph for this function:

TEST_CASE() [2/5]

TEST_CASE	(	"Continuous store/query operation"	,
		""	[stability][.slow] )

Definition at line 366 of file test_stability.cpp.

                                                                    {
    auto config = stability_config::from_environment();
    stability_metrics metrics;
    metrics.reset();
 
    // Record initial memory
    metrics.initial_memory_kb = get_process_memory_kb();
    metrics.peak_memory_kb = metrics.initial_memory_kb;
 
    INFO("Starting stability test for " << config.duration.count() << " minutes");
 
    uint16_t port = find_available_port();
    stability_test_server server(port);
    REQUIRE(server.initialize());
    REQUIRE(server.start());
 
    std::atomic<bool> running{true};
    std::vector<std::thread> workers;
 
    // Store workers
    for (size_t i = 0; i < config.store_workers; ++i) {
        workers.emplace_back([&, i]() {
            auto interval = std::chrono::milliseconds{
                static_cast<long>(1000.0 / config.store_rate * config.store_workers)};
 
            while (running.load()) {
                try {
                    auto ds = generate_random_dataset();
 
                    association_config assoc_config;
                    assoc_config.calling_ae_title = "STORE_SCU_" + std::to_string(i);
                    assoc_config.called_ae_title = server.ae_title();
                    assoc_config.implementation_class_uid = "1.2.826.0.1.3680043.9.9999.9";
                    assoc_config.proposed_contexts.push_back({
                        1,
                        ds.get_string(tags::sop_class_uid),
                        {"1.2.840.10008.1.2.1", "1.2.840.10008.1.2"}
                    });
 
                    auto assoc_result = association::connect(
                        "127.0.0.1", port, assoc_config, std::chrono::seconds{30});
 
                    if (assoc_result.is_ok()) {
                        ++metrics.connections_opened;
 
                        auto& assoc = assoc_result.value();
                        storage_scu scu;
                        auto store_result = scu.store(assoc, ds);
 
                        if (store_result.is_ok()) {
                            ++metrics.stores_completed;
                        } else {
                            ++metrics.stores_failed;
                        }
 
                        (void)assoc.release(std::chrono::seconds{5});
                        ++metrics.connections_closed;
                    } else {
                        ++metrics.connection_errors;
                    }
                } catch (...) {
                    ++metrics.stores_failed;
                }
 
                std::this_thread::sleep_for(interval);
            }
        });
    }
 
    // Memory monitoring thread
    workers.emplace_back([&]() {
        while (running.load()) {
            size_t current = get_process_memory_kb();
            size_t peak = metrics.peak_memory_kb.load();
            while (current > peak &&
                   !metrics.peak_memory_kb.compare_exchange_weak(peak, current)) {
                // Retry until successful
            }
            std::this_thread::sleep_for(std::chrono::seconds{5});
        }
    });
 
    // Run for configured duration
    std::this_thread::sleep_for(config.duration);
 
    running = false;
    for (auto& w : workers) {
        if (w.joinable()) {
            w.join();
        }
    }
 
    server.stop();
 
    // Report results
    metrics.report(std::cout);
 
    // Save to file for CI analysis
    auto report_path = std::filesystem::temp_directory_path() / "stability_test_report.txt";
    metrics.save_to_file(report_path);
    INFO("Report saved to: " << report_path.string());
 
    // Verify no critical errors
    REQUIRE(metrics.stores_failed.load() == 0);
    REQUIRE(metrics.connection_errors.load() == 0);
    REQUIRE(server.error_count() == 0);
 
    // Verify memory growth is bounded
    size_t memory_growth = (metrics.peak_memory_kb - metrics.initial_memory_kb) / 1024;
    REQUIRE(memory_growth < config.max_memory_growth_mb);
}

References kcenon::pacs::network::association_config::called_ae_title, kcenon::pacs::network::association_config::calling_ae_title, kcenon::pacs::integration_test::find_available_port(), kcenon::pacs::network::association_config::implementation_class_uid, kcenon::pacs::network::association_config::proposed_contexts, and kcenon::pacs::services::storage_scu::store().

Here is the call graph for this function:

TEST_CASE() [3/5]

TEST_CASE	(	"Database integrity under concurrent load"	,
		""	[stability][database] )

Definition at line 585 of file test_stability.cpp.

                                                                               {
    uint16_t port = find_available_port();
    stability_test_server server(port);
    REQUIRE(server.initialize());
    REQUIRE(server.start());
 
    constexpr size_t num_workers = 4;
    constexpr size_t images_per_worker = 25;
    constexpr size_t total_images = num_workers * images_per_worker;
 
    std::atomic<size_t> completed{0};
    std::atomic<size_t> failed{0};
    std::vector<std::thread> workers;
    std::vector<std::string> all_instance_uids;
    std::mutex uid_mutex;
 
    for (size_t w = 0; w < num_workers; ++w) {
        workers.emplace_back([&, w]() {
            for (size_t i = 0; i < images_per_worker; ++i) {
                auto ds = generate_random_dataset();
                std::string instance_uid = ds.get_string(tags::sop_instance_uid);
 
                {
                    std::lock_guard<std::mutex> lock(uid_mutex);
                    all_instance_uids.push_back(instance_uid);
                }
 
                association_config assoc_config;
                assoc_config.calling_ae_title = "DB_TEST_" + std::to_string(w);
                assoc_config.called_ae_title = server.ae_title();
                assoc_config.implementation_class_uid = "1.2.826.0.1.3680043.9.9999.13";
                assoc_config.proposed_contexts.push_back({
                    1,
                    ds.get_string(tags::sop_class_uid),
                    {"1.2.840.10008.1.2.1", "1.2.840.10008.1.2"}
                });
 
                auto assoc_result = association::connect(
                    "127.0.0.1", port, assoc_config, std::chrono::seconds{30});
 
                if (assoc_result.is_ok()) {
                    auto& assoc = assoc_result.value();
                    storage_scu scu;
                    auto store_result = scu.store(assoc, ds);
 
                    if (store_result.is_ok()) {
                        ++completed;
                    } else {
                        ++failed;
                    }
 
                    (void)assoc.release(std::chrono::seconds{5});
                } else {
                    ++failed;
                }
            }
        });
    }
 
    for (auto& w : workers) {
        w.join();
    }
 
    server.stop();
 
    // Verify counts
    REQUIRE(completed.load() == total_images);
    REQUIRE(failed.load() == 0);
    REQUIRE(server.stored_count() == total_images);
 
    // Verify no duplicate UIDs were generated
    std::set<std::string> unique_uids(all_instance_uids.begin(), all_instance_uids.end());
    REQUIRE(unique_uids.size() == total_images);
 
    // Verify database consistency
    REQUIRE(server.verify_consistency());
}

References kcenon::pacs::network::association_config::called_ae_title, kcenon::pacs::network::association_config::calling_ae_title, kcenon::pacs::services::completed, kcenon::pacs::integration_test::find_available_port(), kcenon::pacs::network::association_config::implementation_class_uid, kcenon::pacs::network::association_config::proposed_contexts, and kcenon::pacs::services::storage_scu::store().

Here is the call graph for this function:

TEST_CASE() [4/5]

TEST_CASE	(	"Memory stability over iterations"	,
		""	[stability][memory] )

Definition at line 478 of file test_stability.cpp.

                                                                     {
    uint16_t port = find_available_port();
    stability_test_server server(port);
    REQUIRE(server.initialize());
    REQUIRE(server.start());
 
    size_t initial_memory = get_process_memory_kb();
    constexpr size_t num_iterations = 100;
    constexpr size_t max_growth_kb = 50 * 1024;  // 50 MB max growth
 
    for (size_t i = 0; i < num_iterations; ++i) {
        auto ds = generate_random_dataset();
 
        association_config assoc_config;
        assoc_config.calling_ae_title = "MEM_TEST_SCU";
        assoc_config.called_ae_title = server.ae_title();
        assoc_config.implementation_class_uid = "1.2.826.0.1.3680043.9.9999.10";
        assoc_config.proposed_contexts.push_back({
            1,
            ds.get_string(tags::sop_class_uid),
            {"1.2.840.10008.1.2.1", "1.2.840.10008.1.2"}
        });
 
        auto assoc_result = association::connect(
            "127.0.0.1", port, assoc_config, std::chrono::seconds{30});
 
        REQUIRE(assoc_result.is_ok());
 
        auto& assoc = assoc_result.value();
        storage_scu scu;
        auto store_result = scu.store(assoc, ds);
        REQUIRE(store_result.is_ok());
 
        (void)assoc.release(std::chrono::seconds{5});
 
        // Check memory growth periodically
        if ((i + 1) % 20 == 0) {
            size_t current_memory = get_process_memory_kb();
            size_t growth = current_memory - initial_memory;
 
            INFO("Iteration " << (i + 1) << ": Memory growth = "
                 << growth / 1024 << " MB");
 
            REQUIRE(growth < max_growth_kb);
        }
    }
 
    server.stop();
    REQUIRE(server.stored_count() == num_iterations);
}

References kcenon::pacs::network::association_config::called_ae_title, kcenon::pacs::network::association_config::calling_ae_title, kcenon::pacs::integration_test::find_available_port(), kcenon::pacs::network::association_config::implementation_class_uid, kcenon::pacs::network::association_config::proposed_contexts, and kcenon::pacs::services::storage_scu::store().

Here is the call graph for this function:

TEST_CASE() [5/5]

TEST_CASE	(	"Short stability smoke test"	,
		""	[stability][smoke] )

Definition at line 663 of file test_stability.cpp.

                                                              {
    // Quick 10-second test for CI validation
    auto config = stability_config::from_environment();
    config.duration = std::chrono::minutes{0};  // Override for smoke test
 
    stability_metrics metrics;
    metrics.reset();
    metrics.initial_memory_kb = get_process_memory_kb();
 
    uint16_t port = find_available_port();
    stability_test_server server(port);
    REQUIRE(server.initialize());
    REQUIRE(server.start());
 
    std::atomic<bool> running{true};
    std::thread store_worker([&]() {
        while (running.load()) {
            auto ds = test_data_generator::ct();
 
            association_config assoc_config;
            assoc_config.calling_ae_title = "SMOKE_SCU";
            assoc_config.called_ae_title = server.ae_title();
            assoc_config.implementation_class_uid = "1.2.826.0.1.3680043.9.9999.14";
            assoc_config.proposed_contexts.push_back({
                1,
                ds.get_string(tags::sop_class_uid),
                {"1.2.840.10008.1.2.1", "1.2.840.10008.1.2"}
            });
 
            auto assoc_result = association::connect(
                "127.0.0.1", port, assoc_config, std::chrono::seconds{10});
 
            if (assoc_result.is_ok()) {
                auto& assoc = assoc_result.value();
                storage_scu scu;
                if (scu.store(assoc, ds).is_ok()) {
                    ++metrics.stores_completed;
                } else {
                    ++metrics.stores_failed;
                }
                (void)assoc.release(std::chrono::seconds{5});
            } else {
                ++metrics.connection_errors;
            }
 
            std::this_thread::sleep_for(std::chrono::milliseconds{100});
        }
    });
 
    // Run for 10 seconds
    std::this_thread::sleep_for(std::chrono::seconds{10});
 
    running = false;
    store_worker.join();
    server.stop();
 
    // Verify basic functionality
    REQUIRE(metrics.stores_completed.load() > 0);
    REQUIRE(metrics.stores_failed.load() == 0);
    REQUIRE(metrics.connection_errors.load() == 0);
 
    INFO("Smoke test completed: " << metrics.stores_completed.load() << " stores in 10 seconds");
}

References kcenon::pacs::network::association_config::called_ae_title, kcenon::pacs::network::association_config::calling_ae_title, kcenon::pacs::integration_test::test_data_generator::ct(), kcenon::pacs::integration_test::find_available_port(), kcenon::pacs::network::association_config::implementation_class_uid, kcenon::pacs::network::association_config::proposed_contexts, and kcenon::pacs::services::storage_scu::store().

Here is the call graph for this function: