Monitoring System 0.1.0
System resource monitoring with pluggable collectors and alerting
Loading...
Searching...
No Matches
alert_notifiers.h
Go to the documentation of this file.
1// BSD 3-Clause License
2// Copyright (c) 2021-2025, 🍀☀🌕🌥 🌊
3// See the LICENSE file in the project root for full license information.
4
5#pragma once
6
15#include <chrono>
16#include <cstring>
17#include <fstream>
18#include <functional>
19#include <memory>
20#include <mutex>
21#include <queue>
22#include <sstream>
23#include <string>
24#include <thread>
25#include <vector>
26
27#include "alert_manager.h"
28#include "alert_types.h"
30
31namespace kcenon::monitoring {
32
38 std::string url;
39 std::string method = "POST";
40 std::chrono::milliseconds timeout{30000};
41 std::unordered_map<std::string, std::string> headers;
42 size_t max_retries = 3;
43 std::chrono::milliseconds retry_delay{1000};
44 bool send_resolved = true;
45 std::string content_type = "application/json";
46
50 webhook_config& add_header(const std::string& key, const std::string& value) {
51 headers[key] = value;
52 return *this;
53 }
54
58 bool validate() const {
59 return !url.empty() && timeout.count() > 0;
60 }
61};
62
71public:
72 virtual ~alert_formatter() = default;
73
79 virtual std::string format(const alert& a) const = 0;
80
86 virtual std::string format_group(const alert_group& group) const = 0;
87};
88
94public:
95 std::string format(const alert& a) const override {
96 std::ostringstream oss;
97 oss << "{";
98 oss << "\"name\":\"" << escape_json(a.name) << "\",";
99 oss << "\"state\":\"" << alert_state_to_string(a.state) << "\",";
100 oss << "\"severity\":\"" << alert_severity_to_string(a.severity) << "\",";
101 oss << "\"value\":" << a.value << ",";
102 oss << "\"summary\":\"" << escape_json(a.annotations.summary) << "\",";
103 oss << "\"description\":\"" << escape_json(a.annotations.description) << "\",";
104 oss << "\"fingerprint\":\"" << a.fingerprint() << "\",";
105 oss << "\"labels\":{";
106 bool first = true;
107 for (const auto& [key, value] : a.labels.labels) {
108 if (!first) oss << ",";
109 oss << "\"" << escape_json(key) << "\":\"" << escape_json(value) << "\"";
110 first = false;
111 }
112 oss << "}}";
113 return oss.str();
114 }
115
116 std::string format_group(const alert_group& group) const override {
117 std::ostringstream oss;
118 oss << "{";
119 oss << "\"group_key\":\"" << escape_json(group.group_key) << "\",";
120 oss << "\"severity\":\"" << alert_severity_to_string(group.max_severity()) << "\",";
121 oss << "\"alert_count\":" << group.size() << ",";
122 oss << "\"alerts\":[";
123 bool first = true;
124 for (const auto& a : group.alerts) {
125 if (!first) oss << ",";
126 oss << format(a);
127 first = false;
128 }
129 oss << "]}";
130 return oss.str();
131 }
132
133private:
134 static std::string escape_json(const std::string& s) {
135 std::ostringstream oss;
136 for (char c : s) {
137 switch (c) {
138 case '"': oss << "\\\""; break;
139 case '\\': oss << "\\\\"; break;
140 case '\n': oss << "\\n"; break;
141 case '\r': oss << "\\r"; break;
142 case '\t': oss << "\\t"; break;
143 default: oss << c; break;
144 }
145 }
146 return oss.str();
147 }
148};
149
155public:
156 std::string format(const alert& a) const override {
157 std::ostringstream oss;
158 oss << "[" << alert_state_to_string(a.state) << "] "
159 << a.name << " (" << alert_severity_to_string(a.severity) << ")\n"
160 << " Summary: " << a.annotations.summary << "\n"
161 << " Value: " << a.value << "\n"
162 << " Fingerprint: " << a.fingerprint();
163 return oss.str();
164 }
165
166 std::string format_group(const alert_group& group) const override {
167 std::ostringstream oss;
168 oss << "Alert Group: " << group.group_key << "\n"
169 << " Total alerts: " << group.size() << "\n"
170 << " Max severity: " << alert_severity_to_string(group.max_severity()) << "\n"
171 << " Alerts:\n";
172 for (const auto& a : group.alerts) {
173 oss << " - " << a.name << " (" << alert_state_to_string(a.state) << ")\n";
174 }
175 return oss.str();
176 }
177};
178
198public:
199 using http_sender_func = std::function<common::VoidResult(
200 const std::string& url,
201 const std::string& method,
202 const std::unordered_map<std::string, std::string>& headers,
203 const std::string& body
204 )>;
205
212 std::shared_ptr<alert_formatter> formatter = nullptr)
213 : config_(config)
214 , formatter_(formatter ? formatter : std::make_shared<json_alert_formatter>()) {}
215
216 std::string name() const override {
217 return "webhook:" + config_.url;
218 }
219
220 common::VoidResult notify(const alert& a) override {
222 return common::ok();
223 }
224
225 std::string payload = formatter_->format(a);
226 return send_with_retry(payload);
227 }
228
229 common::VoidResult notify_group(const alert_group& group) override {
230 std::string payload = formatter_->format_group(group);
231 return send_with_retry(payload);
232 }
233
234 bool is_ready() const override {
235 return config_.validate() && http_sender_ != nullptr;
236 }
237
254 http_sender_ = std::move(sender);
255 }
256
260 const webhook_config& config() const { return config_; }
261
262private:
263 common::VoidResult send_with_retry(const std::string& payload) {
264 if (!http_sender_) {
265 return common::VoidResult::err(error_info(monitoring_error_code::operation_failed, "No HTTP sender configured").to_common_error());
266 }
267
268 auto headers = config_.headers;
269 headers["Content-Type"] = config_.content_type;
270
271 for (size_t attempt = 0; attempt <= config_.max_retries; ++attempt) {
272 auto result = http_sender_(config_.url, config_.method, headers, payload);
273 if (result.is_ok()) {
274 return result;
275 }
276
277 if (attempt < config_.max_retries) {
278 std::this_thread::sleep_for(config_.retry_delay);
279 }
280 }
281
282 return common::VoidResult::err(static_cast<int>(monitoring_error_code::retry_attempts_exhausted),
283 "Failed to send webhook after " +
284 std::to_string(config_.max_retries) + " retries");
285 }
286
288 std::shared_ptr<alert_formatter> formatter_;
290};
291
300public:
306 explicit file_notifier(std::string file_path,
307 std::shared_ptr<alert_formatter> formatter = nullptr)
308 : file_path_(std::move(file_path))
309 , formatter_(formatter ? formatter : std::make_shared<text_alert_formatter>()) {}
310
311 std::string name() const override {
312 return "file:" + file_path_;
313 }
314
315 common::VoidResult notify(const alert& a) override {
316 return write_to_file(formatter_->format(a));
317 }
318
319 common::VoidResult notify_group(const alert_group& group) override {
320 return write_to_file(formatter_->format_group(group));
321 }
322
323 bool is_ready() const override {
324 return !file_path_.empty();
325 }
326
327private:
328 common::VoidResult write_to_file(const std::string& content) {
329 std::lock_guard<std::mutex> lock(mutex_);
330
331 std::ofstream file(file_path_, std::ios::app);
332 if (!file) {
333 return common::VoidResult::err(static_cast<int>(monitoring_error_code::storage_write_failed),
334 "Failed to open file: " + file_path_);
335 }
336
337 auto now = std::chrono::system_clock::now();
338 auto time_t_now = std::chrono::system_clock::to_time_t(now);
339
340 char time_buf[26];
341#ifdef _MSC_VER
342 ctime_s(time_buf, sizeof(time_buf), &time_t_now);
343#else
344 std::strncpy(time_buf, std::ctime(&time_t_now), sizeof(time_buf) - 1);
345 time_buf[sizeof(time_buf) - 1] = '\0';
346#endif
347 file << "=== " << time_buf;
348 file << content << "\n\n";
349
350 return common::ok();
351 }
352
353 std::string file_path_;
354 std::shared_ptr<alert_formatter> formatter_;
355 std::mutex mutex_;
356};
357
366public:
370 explicit multi_notifier(std::string notifier_name)
371 : name_(std::move(notifier_name)) {}
372
376 void add_notifier(std::shared_ptr<alert_notifier> notifier) {
377 std::lock_guard<std::mutex> lock(mutex_);
378 notifiers_.push_back(std::move(notifier));
379 }
380
381 std::string name() const override { return name_; }
382
383 common::VoidResult notify(const alert& a) override {
384 std::lock_guard<std::mutex> lock(mutex_);
385
386 std::vector<std::string> failures;
387 for (const auto& notifier : notifiers_) {
388 if (notifier && notifier->is_ready()) {
389 auto result = notifier->notify(a);
390 if (!result.is_ok()) {
391 failures.push_back(notifier->name());
392 }
393 }
394 }
395
396 if (!failures.empty()) {
397 return common::VoidResult::err(static_cast<int>(monitoring_error_code::operation_failed),
398 "Failed notifiers: " + join_strings(failures, ", "));
399 }
400 return common::ok();
401 }
402
403 common::VoidResult notify_group(const alert_group& group) override {
404 std::lock_guard<std::mutex> lock(mutex_);
405
406 std::vector<std::string> failures;
407 for (const auto& notifier : notifiers_) {
408 if (notifier && notifier->is_ready()) {
409 auto result = notifier->notify_group(group);
410 if (!result.is_ok()) {
411 failures.push_back(notifier->name());
412 }
413 }
414 }
415
416 if (!failures.empty()) {
417 return common::VoidResult::err(static_cast<int>(monitoring_error_code::operation_failed),
418 "Failed notifiers: " + join_strings(failures, ", "));
419 }
420 return common::ok();
421 }
422
423 bool is_ready() const override {
424 std::lock_guard<std::mutex> lock(mutex_);
425 return !notifiers_.empty();
426 }
427
428private:
429 static std::string join_strings(const std::vector<std::string>& strings,
430 const std::string& delimiter) {
431 std::ostringstream oss;
432 for (size_t i = 0; i < strings.size(); ++i) {
433 if (i > 0) oss << delimiter;
434 oss << strings[i];
435 }
436 return oss.str();
437 }
438
439 std::string name_;
440 mutable std::mutex mutex_;
441 std::vector<std::shared_ptr<alert_notifier>> notifiers_;
442};
443
452public:
459 buffered_notifier(std::shared_ptr<alert_notifier> inner,
460 size_t buffer_size = 100,
461 std::chrono::milliseconds flush_interval = std::chrono::seconds(30))
462 : inner_(std::move(inner))
463 , buffer_size_(buffer_size)
464 , flush_interval_(flush_interval)
465 , last_flush_(std::chrono::steady_clock::now()) {}
466
467 std::string name() const override {
468 return "buffered:" + (inner_ ? inner_->name() : "none");
469 }
470
471 common::VoidResult notify(const alert& a) override {
472 std::lock_guard<std::mutex> lock(mutex_);
473
474 buffer_.push_back(a);
475
476 if (should_flush()) {
477 return flush_internal();
478 }
479
480 return common::ok();
481 }
482
483 common::VoidResult notify_group(const alert_group& group) override {
484 std::lock_guard<std::mutex> lock(mutex_);
485
486 for (const auto& a : group.alerts) {
487 buffer_.push_back(a);
488 }
489
490 if (should_flush()) {
491 return flush_internal();
492 }
493
494 return common::ok();
495 }
496
497 bool is_ready() const override {
498 return inner_ && inner_->is_ready();
499 }
500
504 common::VoidResult flush() {
505 std::lock_guard<std::mutex> lock(mutex_);
506 return flush_internal();
507 }
508
512 size_t pending_count() const {
513 std::lock_guard<std::mutex> lock(mutex_);
514 return buffer_.size();
515 }
516
517private:
518 bool should_flush() const {
519 if (buffer_.size() >= buffer_size_) {
520 return true;
521 }
522 auto now = std::chrono::steady_clock::now();
523 return (now - last_flush_) >= flush_interval_;
524 }
525
526 common::VoidResult flush_internal() {
527 if (buffer_.empty() || !inner_) {
528 return common::ok();
529 }
530
531 // Create a group from buffered alerts
532 alert_group group("buffered");
533 for (auto& a : buffer_) {
534 group.add_alert(std::move(a));
535 }
536 buffer_.clear();
537
538 last_flush_ = std::chrono::steady_clock::now();
539
540 return inner_->notify_group(group);
541 }
542
543 std::shared_ptr<alert_notifier> inner_;
545 std::chrono::milliseconds flush_interval_;
546
547 mutable std::mutex mutex_;
548 std::vector<alert> buffer_;
549 std::chrono::steady_clock::time_point last_flush_;
550};
551
560public:
561 using route_condition = std::function<bool(const alert&)>;
562
566 explicit routing_notifier(std::string notifier_name)
567 : name_(std::move(notifier_name)) {}
568
575 std::shared_ptr<alert_notifier> notifier) {
576 std::lock_guard<std::mutex> lock(mutex_);
577 routes_.push_back({std::move(condition), std::move(notifier)});
578 }
579
583 void set_default_route(std::shared_ptr<alert_notifier> notifier) {
584 std::lock_guard<std::mutex> lock(mutex_);
585 default_route_ = std::move(notifier);
586 }
587
588 std::string name() const override { return name_; }
589
590 common::VoidResult notify(const alert& a) override {
591 std::lock_guard<std::mutex> lock(mutex_);
592
593 for (const auto& [condition, notifier] : routes_) {
594 if (condition(a) && notifier && notifier->is_ready()) {
595 return notifier->notify(a);
596 }
597 }
598
599 if (default_route_ && default_route_->is_ready()) {
600 return default_route_->notify(a);
601 }
602
603 return common::ok();
604 }
605
606 common::VoidResult notify_group(const alert_group& group) override {
607 // Route each alert individually
608 for (const auto& a : group.alerts) {
609 auto result = notify(a);
610 if (!result.is_ok()) {
611 return result;
612 }
613 }
614 return common::ok();
615 }
616
617 bool is_ready() const override {
618 std::lock_guard<std::mutex> lock(mutex_);
619 return !routes_.empty() || default_route_ != nullptr;
620 }
621
622 // Factory methods for common routes
627 std::shared_ptr<alert_notifier> notifier) {
628 add_route(
629 [severity](const alert& a) { return a.severity == severity; },
630 std::move(notifier)
631 );
632 }
633
637 void route_by_label(const std::string& key,
638 const std::string& value,
639 std::shared_ptr<alert_notifier> notifier) {
640 add_route(
641 [key, value](const alert& a) { return a.labels.get(key) == value; },
642 std::move(notifier)
643 );
644 }
645
646private:
647 struct route {
649 std::shared_ptr<alert_notifier> notifier;
650 };
651
652 std::string name_;
653 mutable std::mutex mutex_;
654 std::vector<route> routes_;
655 std::shared_ptr<alert_notifier> default_route_;
656};
657
658} // namespace kcenon::monitoring
Central coordinator for alert lifecycle management.
Core alert data structures for the monitoring system.
Formats alerts for notification payloads.
virtual std::string format(const alert &a) const =0
Format a single alert.
virtual std::string format_group(const alert_group &group) const =0
Format an alert group.
Base class for alert notification handlers.
Buffers alerts and sends in batches.
std::string name() const override
Get notifier name.
common::VoidResult notify_group(const alert_group &group) override
Send a notification for an alert group.
buffered_notifier(std::shared_ptr< alert_notifier > inner, size_t buffer_size=100, std::chrono::milliseconds flush_interval=std::chrono::seconds(30))
Construct buffered notifier.
size_t pending_count() const
Get current buffer size.
common::VoidResult flush()
Flush buffered alerts.
std::shared_ptr< alert_notifier > inner_
std::chrono::milliseconds flush_interval_
bool is_ready() const override
Check if notifier is ready.
common::VoidResult notify(const alert &a) override
Send a notification for an alert.
std::chrono::steady_clock::time_point last_flush_
common::VoidResult notify(const alert &a) override
Send a notification for an alert.
common::VoidResult write_to_file(const std::string &content)
bool is_ready() const override
Check if notifier is ready.
std::string name() const override
Get notifier name.
file_notifier(std::string file_path, std::shared_ptr< alert_formatter > formatter=nullptr)
Construct file notifier.
std::shared_ptr< alert_formatter > formatter_
common::VoidResult notify_group(const alert_group &group) override
Send a notification for an alert group.
std::string format_group(const alert_group &group) const override
Format an alert group.
static std::string escape_json(const std::string &s)
std::string format(const alert &a) const override
Format a single alert.
Sends alerts to multiple notifiers.
multi_notifier(std::string notifier_name)
Construct with name.
void add_notifier(std::shared_ptr< alert_notifier > notifier)
Add a child notifier.
common::VoidResult notify_group(const alert_group &group) override
Send a notification for an alert group.
common::VoidResult notify(const alert &a) override
Send a notification for an alert.
static std::string join_strings(const std::vector< std::string > &strings, const std::string &delimiter)
std::vector< std::shared_ptr< alert_notifier > > notifiers_
std::string name() const override
Get notifier name.
bool is_ready() const override
Check if notifier is ready.
Routes alerts to different notifiers based on criteria.
common::VoidResult notify(const alert &a) override
Send a notification for an alert.
routing_notifier(std::string notifier_name)
Construct routing notifier.
void add_route(route_condition condition, std::shared_ptr< alert_notifier > notifier)
Add a route.
void route_by_label(const std::string &key, const std::string &value, std::shared_ptr< alert_notifier > notifier)
Route by label.
bool is_ready() const override
Check if notifier is ready.
void route_by_severity(alert_severity severity, std::shared_ptr< alert_notifier > notifier)
Route by severity.
std::string name() const override
Get notifier name.
common::VoidResult notify_group(const alert_group &group) override
Send a notification for an alert group.
std::shared_ptr< alert_notifier > default_route_
std::function< bool(const alert &)> route_condition
void set_default_route(std::shared_ptr< alert_notifier > notifier)
Add a default route for non-matching alerts.
Formats alerts as human-readable text.
std::string format_group(const alert_group &group) const override
Format an alert group.
std::string format(const alert &a) const override
Format a single alert.
Sends alerts to a webhook endpoint.
std::shared_ptr< alert_formatter > formatter_
webhook_notifier(const webhook_config &config, std::shared_ptr< alert_formatter > formatter=nullptr)
Construct webhook notifier.
common::VoidResult send_with_retry(const std::string &payload)
std::string name() const override
Get notifier name.
void set_http_sender(http_sender_func sender)
Set HTTP sender function for actual HTTP calls.
const webhook_config & config() const
Get configuration.
common::VoidResult notify_group(const alert_group &group) override
Send a notification for an alert group.
bool is_ready() const override
Check if notifier is ready.
common::VoidResult notify(const alert &a) override
Send a notification for an alert.
std::function< common::VoidResult( const std::string &url, const std::string &method, const std::unordered_map< std::string, std::string > &headers, const std::string &body)> http_sender_func
alert_severity
Severity levels for alerts.
Definition alert_types.h:35
constexpr const char * alert_state_to_string(alert_state state) noexcept
Convert alert state to string.
Definition alert_types.h:82
constexpr const char * alert_severity_to_string(alert_severity severity) noexcept
Convert alert severity to string.
Definition alert_types.h:47
@ resolved
Alert condition cleared.
Result pattern type definitions for monitoring system.
std::string description
Detailed description.
std::string summary
Brief description.
Group of related alerts for batch notification.
std::string group_key
Common grouping key.
void add_alert(alert a)
Add an alert to the group.
std::vector< alert > alerts
Alerts in this group.
size_t size() const
Get count of alerts in the group.
alert_severity max_severity() const
Get highest severity in the group.
std::unordered_map< std::string, std::string > labels
std::string get(const std::string &key) const
Get a label value.
Core alert data structure.
alert_state state
Current state.
double value
Current metric value.
alert_severity severity
Alert severity level.
std::string name
Alert name/identifier.
alert_labels labels
Identifying labels.
alert_annotations annotations
Descriptive annotations.
std::string fingerprint() const
Get alert fingerprint for deduplication.
Extended error information with context.
std::shared_ptr< alert_notifier > notifier
Configuration for webhook notifier.
std::chrono::milliseconds retry_delay
Delay between retries.
std::chrono::milliseconds timeout
Request timeout.
bool send_resolved
Send resolved notifications.
size_t max_retries
Maximum retry attempts.
std::string content_type
Content type header.
bool validate() const
Validate configuration.
webhook_config & add_header(const std::string &key, const std::string &value)
Add a custom header.
std::unordered_map< std::string, std::string > headers
Custom headers.