Hello
I have a setup where some notifications are sent out and some are not.
To give more information, let me explain: we have a distributed setup with 2 Icingaweb ( director , Cube and business process ) , then 2 masters ( running Icinga2 2.10.5-1) and 12 Satellites in multiple zones.
We also have service and hose templates that control the notification ( enable /Disable) and allow selection of medium ( email /Text messages)
What we are seeing now is that some service checks send out the emails but not the text, and vice versa, and I can’t seem to figure out why.
The system data:
Icinga 2.10.5.1
Director 1.5.2
IcingaWeb 2.6.3
uname -a
Linux r-ic-m-01.mtr.labs 2.6.32-754.17.1.el6.x86_64 #1 SMP Tue Jul 2 12:42:48 UTC 2019 x86_64 x86_64 x86_64 GNU/Linux
[root@r-ic-m-01 ~]# cat /etc/redhat-release
CentOS release 6.10 (Final)
A service object definition
[root@l-ic-m-01 notification-apply-rules]# icinga2 object list --type Notification --name 'l-ansible-master-01!Swap!*'
Object 'l-ansible-master-01!Swap!service-sms' of type 'Notification':
% declared in '/etc/icinga2/static/notification-apply-rules/base-notification.conf', lines 14:1-14:43
* __name = "l-ansible-master-01!Swap!service-sms"
* command = "current-sms-service-notification"
% = modified in '/var/lib/icinga2/api/packages/director/4cbfc342-a1bc-4d00-b51c-cdc959217f83/zones.d/master/notification_templates.conf', lines 28:5-28:48
* command_endpoint = ""
* host_name = "l-ansible-master-01"
% = modified in '/etc/icinga2/static/notification-apply-rules/base-notification.conf', lines 14:1-14:43
* interval = 3600
% = modified in '/var/lib/icinga2/api/packages/director/4cbfc342-a1bc-4d00-b51c-cdc959217f83/zones.d/master/notification_templates.conf', lines 29:5-29:17
* name = "service-sms"
* package = "_etc"
% = modified in '/etc/icinga2/static/notification-apply-rules/base-notification.conf', lines 14:1-14:43
* period = "24x7"
% = modified in '/var/lib/icinga2/api/packages/director/4cbfc342-a1bc-4d00-b51c-cdc959217f83/zones.d/master/notification_templates.conf', lines 30:5-30:19
* service_name = "Swap"
% = modified in '/etc/icinga2/static/notification-apply-rules/base-notification.conf', lines 14:1-14:43
* source_location
* first_column = 1
* first_line = 14
* last_column = 43
* last_line = 14
* path = "/etc/icinga2/static/notification-apply-rules/base-notification.conf"
* states = [ "Critical", "OK", "Unknown", "Warning" ]
% = modified in '/var/lib/icinga2/api/packages/director/4cbfc342-a1bc-4d00-b51c-cdc959217f83/zones.d/master/notification_templates.conf', lines 31:5-31:47
* templates = [ "service-sms", "service-sms-tmpl" ]
% = modified in '/etc/icinga2/static/notification-apply-rules/base-notification.conf', lines 14:1-14:43
% = modified in '/var/lib/icinga2/api/packages/director/4cbfc342-a1bc-4d00-b51c-cdc959217f83/zones.d/master/notification_templates.conf', lines 24:1-24:40
* times
% = modified in '/var/lib/icinga2/api/packages/director/4cbfc342-a1bc-4d00-b51c-cdc959217f83/zones.d/master/notification_templates.conf', lines 25:5-27:5
* end = 86400
* type = "Notification"
* types = [ "Acknowledgement", "Custom", "Problem", "Recovery" ]
% = modified in '/var/lib/icinga2/api/packages/director/4cbfc342-a1bc-4d00-b51c-cdc959217f83/zones.d/master/notification_templates.conf', lines 32:5-32:58
* user_groups = null
* users = [ "sinana" ]
% = modified in '/etc/icinga2/static/notification-apply-rules/base-notification.conf', lines 17:9-17:41
* vars = null
* zone = "mtl"
% = modified in '/etc/icinga2/static/notification-apply-rules/base-notification.conf', lines 14:1-14:43
Object 'l-ansible-master-01!Swap!service-alert' of type 'Notification':
% declared in '/etc/icinga2/static/notification-apply-rules/base-notification.conf', lines 1:0-1:44
* __name = "l-ansible-master-01!Swap!service-alert"
* command = "current-mail-service-notification"
% = modified in '/var/lib/icinga2/api/packages/director/4cbfc342-a1bc-4d00-b51c-cdc959217f83/zones.d/master/notification_templates.conf', lines 5:5-5:49
* command_endpoint = ""
* host_name = "l-ansible-master-01"
% = modified in '/etc/icinga2/static/notification-apply-rules/base-notification.conf', lines 1:0-1:44
* interval = 3600
% = modified in '/var/lib/icinga2/api/packages/director/4cbfc342-a1bc-4d00-b51c-cdc959217f83/zones.d/master/notification_templates.conf', lines 6:5-6:17
* name = "service-alert"
* package = "_etc"
% = modified in '/etc/icinga2/static/notification-apply-rules/base-notification.conf', lines 1:0-1:44
* period = "24x7"
% = modified in '/var/lib/icinga2/api/packages/director/4cbfc342-a1bc-4d00-b51c-cdc959217f83/zones.d/master/notification_templates.conf', lines 7:5-7:19
* service_name = "Swap"
% = modified in '/etc/icinga2/static/notification-apply-rules/base-notification.conf', lines 1:0-1:44
* source_location
* first_column = 0
* first_line = 1
* last_column = 44
* last_line = 1
* path = "/etc/icinga2/static/notification-apply-rules/base-notification.conf"
* states = [ "Critical", "OK", "Warning" ]
% = modified in '/var/lib/icinga2/api/packages/director/4cbfc342-a1bc-4d00-b51c-cdc959217f83/zones.d/master/notification_templates.conf', lines 8:5-8:38
* templates = [ "service-alert", "srv-mail-tmpl" ]
% = modified in '/etc/icinga2/static/notification-apply-rules/base-notification.conf', lines 1:0-1:44
% = modified in '/var/lib/icinga2/api/packages/director/4cbfc342-a1bc-4d00-b51c-cdc959217f83/zones.d/master/notification_templates.conf', lines 1:0-1:36
* times
% = modified in '/var/lib/icinga2/api/packages/director/4cbfc342-a1bc-4d00-b51c-cdc959217f83/zones.d/master/notification_templates.conf', lines 2:5-4:5
* end = 360000
* type = "Notification"
* types = [ "Acknowledgement", "Custom", "Problem", "Recovery" ]
% = modified in '/var/lib/icinga2/api/packages/director/4cbfc342-a1bc-4d00-b51c-cdc959217f83/zones.d/master/notification_templates.conf', lines 9:5-9:58
* user_groups = null
* users = [ "sinana" ]
% = modified in '/etc/icinga2/static/notification-apply-rules/base-notification.conf', lines 4:9-4:41
* vars = null
* zone = "mtl"
% = modified in '/etc/icinga2/static/notification-apply-rules/base-notification.conf', lines 1:0-1:44
#############################################################
Notification apply rules:
apply Notification "service-alert" to Service {
import "srv-mail-tmpl"
if ( host.vars.notified_users ) {
users = host.vars.notified_users
}
if ( host.vars.user_groups ){
user_groups = host.vars.user_groups
}
assign where host.vars.notified_users && service.vars.email_alert == true
assign where host.vars.user_groups && service.vars.email_alert == true
}
apply Notification "service-sms" to Service {
import "service-sms-tmpl"
if ( host.vars.notified_users ) {
users = host.vars.notified_users
}
if ( host.vars.user_groups ){
user_groups = host.vars.user_groups
}
assign where host.vars.notified_users && service.vars.enable_sms == true
assign where host.vars.user_groups && service.vars.enable_sms == true
}
apply Notification "Host-alert" to Host {
import "host-notification-tml"
if ( host.vars.notified_users ) {
users = host.vars.notified_users
}
if ( host.vars.user_groups ){
user_groups = host.vars.user_groups
}
assign where host.vars.notified_users && host.vars.email_alert == true
assign where host.vars.user_groups && host.vars.email_alert == true
}
apply Notification "Host-sms-alert" to Host {
import "host-sms-tmpl"
if ( host.vars.notified_users ) {
users = host.vars.notified_users
}
if ( host.vars.user_groups ){
user_groups = host.vars.user_groups
}
assign where host.vars.notified_users && host.vars.enable_sms == true
assign where host.vars.user_groups && host.vars.enable_sms == true
}
output from the Debug log:
[2019-12-11 15:44:32 +0200] debug/Checkable: Update checkable 'l-ansible-master-01!Check Read Only Mounts' with check interval '240' from last check time at 2019-12-11 15:44:32 +0200 (1.57607e+09) to next check time at 2019-12-11 15:48:28 +0200(1.57607e+09).
[2019-12-11 15:44:32 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01!Check Read Only Mounts'
[2019-12-11 15:45:14 +0200] debug/Checkable: Update checkable 'l-ansible-master-01' with check interval '60' from last check time at 2019-12-11 15:45:14 +0200 (1.57607e+09) to next check time at 2019-12-11 15:46:13 +0200(1.57607e+09).
[2019-12-11 15:45:14 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01'
[2019-12-11 15:45:15 +0200] debug/Checkable: Update checkable 'l-ansible-master-01!Swap' with check interval '240' from last check time at 2019-12-11 15:45:15 +0200 (1.57607e+09) to next check time at 2019-12-11 15:46:13 +0200(1.57607e+09).
[2019-12-11 15:45:15 +0200] debug/DbEvents: add log entry history for 'l-ansible-master-01!Swap'
[2019-12-11 15:45:15 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01!Swap'
[2019-12-11 15:45:15 +0200] debug/DbEvents: add state change history for 'l-ansible-master-01!Swap'
[2019-12-11 15:45:15 +0200] notice/Checkable: State Change: Checkable 'l-ansible-master-01!Swap' soft state change from OK to CRITICAL detected.
[2019-12-11 15:45:15 +0200] debug/Checkable: Update checkable 'l-ansible-master-01!Disk_check root /' with check interval '240' from last check time at 2019-12-11 15:45:15 +0200 (1.57607e+09) to next check time at 2019-12-11 15:49:04 +0200(1.57607e+09).
[2019-12-11 15:45:15 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01!Disk_check root /'
[2019-12-11 15:45:26 +0200] debug/Checkable: Update checkable 'l-ansible-master-01!Check Memory' with check interval '240' from last check time at 2019-12-11 15:45:26 +0200 (1.57607e+09) to next check time at 2019-12-11 15:49:24 +0200(1.57607e+09).
[2019-12-11 15:45:26 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01!Check Memory'
[2019-12-11 15:45:57 +0200] debug/Checkable: Update checkable 'l-ansible-master-01!Check Load' with check interval '240' from last check time at 2019-12-11 15:45:57 +0200 (1.57607e+09) to next check time at 2019-12-11 15:49:48 +0200(1.57607e+09).
[2019-12-11 15:45:57 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01!Check Load'
[2019-12-11 15:46:14 +0200] debug/Checkable: Update checkable 'l-ansible-master-01!Swap' with check interval '240' from last check time at 2019-12-11 15:46:14 +0200 (1.57607e+09) to next check time at 2019-12-11 15:47:13 +0200(1.57607e+09).
[2019-12-11 15:46:14 +0200] debug/DbEvents: add log entry history for 'l-ansible-master-01!Swap'
[2019-12-11 15:46:14 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01!Swap'
[2019-12-11 15:46:14 +0200] debug/DbEvents: add state change history for 'l-ansible-master-01!Swap'
[2019-12-11 15:46:14 +0200] notice/Checkable: State Change: Checkable 'l-ansible-master-01!Swap' soft state change from CRITICAL to CRITICAL detected.
[2019-12-11 15:46:17 +0200] debug/Checkable: Update checkable 'l-ansible-master-01' with check interval '60' from last check time at 2019-12-11 15:46:17 +0200 (1.57607e+09) to next check time at 2019-12-11 15:47:16 +0200(1.57607e+09).
[2019-12-11 15:46:17 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01'
[2019-12-11 15:46:35 +0200] debug/Checkable: Update checkable 'l-ansible-master-01!check_SOA' with check interval '240' from last check time at 2019-12-11 15:46:35 +0200 (1.57607e+09) to next check time at 2019-12-11 15:50:32 +0200(1.57607e+09).
[2019-12-11 15:46:35 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01!check_SOA'
[2019-12-11 15:46:36 +0200] debug/IdoMysqlConnection: Query: UPDATE icinga_servicestatus SET acknowledgement_type = '0', active_checks_enabled = '1', check_command = 'nrpe', check_source = 'l-ic-s-01.mtl.labs.mlnx', check_timeperiod_object_id = 245, check_type = '0', current_check_attempt = '1', current_notification_number = '2', current_state = '1', endpoint_object_id = 239, event_handler_enabled = '0', execution_time = '0.275039', flap_detection_enabled = '1', has_been_checked = '1', instance_id = 1, is_flapping = '0', is_reachable = '1', last_check = FROM_UNIXTIME(1576071995), last_hard_state = '1', last_hard_state_change = FROM_UNIXTIME(1576069656), last_notification = FROM_UNIXTIME(1576069120), last_state_change = FROM_UNIXTIME(1576069656), last_time_ok = FROM_UNIXTIME(1576069573), last_time_unknown = FROM_UNIXTIME(1576069654), last_time_warning = FROM_UNIXTIME(1576071995), latency = '0.000398', long_output = 'Status TST: [11-Dec-19 15:46:35 IST]: Script is already running, or failed, but lock exists! Exiting... [11-Dec-19 15:46:35 IST]: Review /tmp/TST_GetSOA_ServicesStatus.log file, and remove /tmp/l-ansible-master-01_TST_GetSOA_ServicesStatus.lck in order to run this script.\\nStatus QA: [11-Dec-19 15:46:35 IST]: Script is already running, or failed, but lock exists! Exiting... [11-Dec-19 15:46:35 IST]: Review /tmp/QA_GetSOA_ServicesStatus.log file, and remove /tmp/l-ansible-master-01_QA_GetSOA_ServicesStatus.lck in order to run this script.\\nStatus PRD: [11-Dec-19 15:46:35 IST]: Script is already running, or failed, but lock exists! Exiting... [11-Dec-19 15:46:35 IST]: Review /tmp/PRD_GetSOA_ServicesStatus.log file, and remove /tmp/l-ansible-master-01_PRD_GetSOA_ServicesStatus.lck in order to run this script.', max_check_attempts = '3', next_check = FROM_UNIXTIME(1576072232), next_notification = FROM_UNIXTIME(1576072720), normal_check_interval = '4', notifications_enabled = '1', original_attributes = 'null', output = 'One or more Services seem to be malfunctioning:', passive_checks_enabled = '0', percent_state_change = '13.600000', perfdata = '', problem_has_been_acknowledged = '0', process_performance_data = '0', retry_check_interval = '1', scheduled_downtime_depth = '0', service_object_id = 26698, should_be_scheduled = '1', state_type = '1', status_update_time = FROM_UNIXTIME(1576071995) WHERE service_object_id = 26698
[2019-12-11 15:47:13 +0200] debug/Checkable: Update checkable 'l-ansible-master-01!Swap' with check interval '240' from last check time at 2019-12-11 15:47:13 +0200 (1.57607e+09) to next check time at 2019-12-11 15:51:12 +0200(1.57607e+09).
[2019-12-11 15:47:13 +0200] debug/DbEvents: add log entry history for 'l-ansible-master-01!Swap'
[2019-12-11 15:47:13 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01!Swap'
[2019-12-11 15:47:13 +0200] debug/DbEvents: add state change history for 'l-ansible-master-01!Swap'
[2019-12-11 15:47:13 +0200] notice/Checkable: State Change: Checkable 'l-ansible-master-01!Swap' hard state change from CRITICAL to CRITICAL detected.
[2019-12-11 15:47:13 +0200] notice/ClusterEvents: Discarding 'send custom notification' message for checkable 'l-ansible-master-01!Swap' from 'l-ic-s-01.mtl.labs.mlnx': Unauthorized access.
[2019-12-11 15:47:21 +0200] debug/Checkable: Update checkable 'l-ansible-master-01!SSH Check' with check interval '300' from last check time at 2019-12-11 15:47:21 +0200 (1.57607e+09) to next check time at 2019-12-11 15:52:17 +0200(1.57607e+09).
[2019-12-11 15:47:21 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01!SSH Check'
[2019-12-11 15:47:21 +0200] debug/Checkable: Update checkable 'l-ansible-master-01' with check interval '60' from last check time at 2019-12-11 15:47:21 +0200 (1.57607e+09) to next check time at 2019-12-11 15:48:20 +0200(1.57607e+09).
[2019-12-11 15:47:21 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01'
[2019-12-11 15:47:21 +0200] information/Checkable: Checking for configured notifications for object 'l-ansible-master-01!Swap'
[2019-12-11 15:47:21 +0200] debug/Checkable: Checkable 'l-ansible-master-01!Swap' has 2 notification(s).
[2019-12-11 15:47:21 +0200] notice/Notification: Attempting to send notifications for notification object 'l-ansible-master-01!Swap!service-alert'.
[2019-12-11 15:47:21 +0200] information/Notification: Sending 'Problem' notification 'l-ansible-master-01!Swap!service-alert' for user 'sinana'
[2019-12-11 15:47:21 +0200] debug/DbEvents: add notification history for 'l-ansible-master-01!Swap'
[2019-12-11 15:47:21 +0200] debug/DbEvents: add contact notification history for service 'l-ansible-master-01!Swap' and user 'sinana'.
[2019-12-11 15:47:21 +0200] debug/DbEvents: add notification history for 'l-ansible-master-01!Swap'
[2019-12-11 15:47:21 +0200] debug/DbEvents: add contact notification history for service 'l-ansible-master-01!Swap' and user 'sinana'.
[2019-12-11 15:47:21 +0200] debug/DbEvents: add log entry history for 'l-ansible-master-01!Swap'
[2019-12-11 15:47:21 +0200] notice/Process: Running command '/usr/local/bin/gen-notification-new.py' 'mail' '--service' '-a' '10.7.77.253' '-b' '' '-c' '' '-d' '2019-12-11 15:47:21' '-i' 'Swap' '-l' 'l-ansible-master-01' '-m' 'sinana@mellanox.com' '-o' 'SWAP CRITICAL - 0% free (0 MB out of 0 MB) - Swap is either disabled, not present, or of zero size. ' '-s' 'CRITICAL' '-t' 'PROBLEM': PID 23967
[2019-12-11 15:47:21 +0200] debug/DbEvents: add log entry history for 'l-ansible-master-01!Swap'
[2019-12-11 15:47:21 +0200] information/Notification: Completed sending 'Problem' notification 'l-ansible-master-01!Swap!service-alert' for checkable 'l-ansible-master-01!Swap' and user 'sinana'.
[2019-12-11 15:47:21 +0200] notice/Process: PID 23967 ('/usr/local/bin/gen-notification-new.py' 'mail' '--service' '-a' '10.7.77.253' '-b' '' '-c' '' '-d' '2019-12-11 15:47:21' '-i' 'Swap' '-l' 'l-ansible-master-01' '-m' 'sinana@mellanox.com' '-o' 'SWAP CRITICAL - 0% free (0 MB out of 0 MB) - Swap is either disabled, not present, or of zero size. ' '-s' 'CRITICAL' '-t' 'PROBLEM') terminated with exit code 0
[2019-12-11 15:47:38 +0200] debug/Checkable: Update checkable 'l-ansible-master-01!Check Zombie Procs' with check interval '240' from last check time at 2019-12-11 15:47:38 +0200 (1.57607e+09) to next check time at 2019-12-11 15:51:29 +0200(1.57607e+09).
[2019-12-11 15:47:38 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01!Check Zombie Procs'
[2019-12-11 15:48:34 +0200] debug/Checkable: Update checkable 'l-ansible-master-01' with check interval '60' from last check time at 2019-12-11 15:48:24 +0200 (1.57607e+09) to next check time at 2019-12-11 15:49:33 +0200(1.57607e+09).
[2019-12-11 15:48:34 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01'
[2019-12-11 15:48:47 +0200] debug/Checkable: Update checkable 'l-ansible-master-01!Check Read Only Mounts' with check interval '240' from last check time at 2019-12-11 15:48:32 +0200 (1.57607e+09) to next check time at 2019-12-11 15:52:43 +0200(1.57607e+09).
[2019-12-11 15:48:47 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01!Check Read Only Mounts'
[2019-12-11 15:49:16 +0200] debug/Checkable: Update checkable 'l-ansible-master-01!Disk_check root /' with check interval '240' from last check time at 2019-12-11 15:49:15 +0200 (1.57607e+09) to next check time at 2019-12-11 15:53:06 +0200(1.57607e+09).
[2019-12-11 15:49:16 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01!Disk_check root /'
[2019-12-11 15:49:26 +0200] debug/Checkable: Update checkable 'l-ansible-master-01!Check Memory' with check interval '240' from last check time at 2019-12-11 15:49:26 +0200 (1.57607e+09) to next check time at 2019-12-11 15:53:24 +0200(1.57607e+09).
[2019-12-11 15:49:26 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01!Check Memory'
[2019-12-11 15:49:27 +0200] debug/Checkable: Update checkable 'l-ansible-master-01' with check interval '60' from last check time at 2019-12-11 15:49:27 +0200 (1.57607e+09) to next check time at 2019-12-11 15:50:27 +0200(1.57607e+09).
[2019-12-11 15:49:27 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01'
[2019-12-11 15:49:55 +0200] debug/Checkable: Update checkable 'l-ansible-master-01!Check Load' with check interval '240' from last check time at 2019-12-11 15:49:55 +0200 (1.57607e+09) to next check time at 2019-12-11 15:53:46 +0200(1.57607e+09).
[2019-12-11 15:49:55 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01!Check Load'
[2019-12-11 15:50:30 +0200] debug/Checkable: Update checkable 'l-ansible-master-01' with check interval '60' from last check time at 2019-12-11 15:50:30 +0200 (1.57607e+09) to next check time at 2019-12-11 15:51:30 +0200(1.57607e+09).
[2019-12-11 15:50:30 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01'
[2019-12-11 15:50:35 +0200] debug/Checkable: Update checkable 'l-ansible-master-01!check_SOA' with check interval '240' from last check time at 2019-12-11 15:50:35 +0200 (1.57607e+09) to next check time at 2019-12-11 15:54:31 +0200(1.57607e+09).
[2019-12-11 15:50:35 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01!check_SOA'
[2019-12-11 15:50:36 +0200] debug/IdoMysqlConnection: Query: UPDATE icinga_servicestatus SET acknowledgement_type = '0', active_checks_enabled = '1', check_command = 'nrpe', check_source = 'l-ic-s-01.mtl.labs.mlnx', check_timeperiod_object_id = 245, check_type = '0', current_check_attempt = '1', current_notification_number = '2', current_state = '1', endpoint_object_id = 239, event_handler_enabled = '0', execution_time = '0.316119', flap_detection_enabled = '1', has_been_checked = '1', instance_id = 1, is_flapping = '0', is_reachable = '1', last_check = FROM_UNIXTIME(1576072235), last_hard_state = '1', last_hard_state_change = FROM_UNIXTIME(1576069656), last_notification = FROM_UNIXTIME(1576069120), last_state_change = FROM_UNIXTIME(1576069656), last_time_ok = FROM_UNIXTIME(1576069573), last_time_unknown = FROM_UNIXTIME(1576069654), last_time_warning = FROM_UNIXTIME(1576072235), latency = '0.000458', long_output = 'Status TST: [11-Dec-19 15:50:35 IST]: Script is already running, or failed, but lock exists! Exiting... [11-Dec-19 15:50:35 IST]: Review /tmp/TST_GetSOA_ServicesStatus.log file, and remove /tmp/l-ansible-master-01_TST_GetSOA_ServicesStatus.lck in order to run this script.\\nStatus QA: [11-Dec-19 15:50:35 IST]: Script is already running, or failed, but lock exists! Exiting... [11-Dec-19 15:50:35 IST]: Review /tmp/QA_GetSOA_ServicesStatus.log file, and remove /tmp/l-ansible-master-01_QA_GetSOA_ServicesStatus.lck in order to run this script.\\nStatus PRD: [11-Dec-19 15:50:35 IST]: Script is already running, or failed, but lock exists! Exiting... [11-Dec-19 15:50:35 IST]: Review /tmp/PRD_GetSOA_ServicesStatus.log file, and remove /tmp/l-ansible-master-01_PRD_GetSOA_ServicesStatus.lck in order to run this script.', max_check_attempts = '3', next_check = FROM_UNIXTIME(1576072471), next_notification = FROM_UNIXTIME(1576072720), normal_check_interval = '4', notifications_enabled = '1', original_attributes = 'null', output = 'One or more Services seem to be malfunctioning:', passive_checks_enabled = '0', percent_state_change = '9.400000', perfdata = '', problem_has_been_acknowledged = '0', process_performance_data = '0', retry_check_interval = '1', scheduled_downtime_depth = '0', service_object_id = 26698, should_be_scheduled = '1', state_type = '1', status_update_time = FROM_UNIXTIME(1576072235) WHERE service_object_id = 26698
[2019-12-11 15:51:07 +0200] debug/Checkable: Update checkable 'l-ansible-master-01!Swap' with check interval '240' from last check time at 2019-12-11 15:51:07 +0200 (1.57607e+09) to next check time at 2019-12-11 15:55:06 +0200(1.57607e+09).
[2019-12-11 15:51:07 +0200] debug/DbEvents: add log entry history for 'l-ansible-master-01!Swap'
[2019-12-11 15:51:07 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01!Swap'
[2019-12-11 15:51:07 +0200] debug/DbEvents: add state change history for 'l-ansible-master-01!Swap'
[2019-12-11 15:51:07 +0200] notice/Checkable: State Change: Checkable 'l-ansible-master-01!Swap' hard state change from CRITICAL to OK detected.
[2019-12-11 15:51:07 +0200] notice/ClusterEvents: Discarding 'send custom notification' message for checkable 'l-ansible-master-01!Swap' from 'l-ic-s-01.mtl.labs.mlnx': Unauthorized access.
[2019-12-11 15:51:07 +0200] information/Checkable: Checking for configured notifications for object 'l-ansible-master-01!Swap'
[2019-12-11 15:51:07 +0200] debug/Checkable: Checkable 'l-ansible-master-01!Swap' has 2 notification(s).
[2019-12-11 15:51:07 +0200] notice/Notification: Attempting to send notifications for notification object 'l-ansible-master-01!Swap!service-alert'.
[2019-12-11 15:51:07 +0200] information/Notification: Sending 'Recovery' notification 'l-ansible-master-01!Swap!service-alert' for user 'sinana'
[2019-12-11 15:51:07 +0200] debug/DbEvents: add notification history for 'l-ansible-master-01!Swap'
[2019-12-11 15:51:07 +0200] debug/DbEvents: add contact notification history for service 'l-ansible-master-01!Swap' and user 'sinana'.
[2019-12-11 15:51:07 +0200] debug/DbEvents: add notification history for 'l-ansible-master-01!Swap'
[2019-12-11 15:51:07 +0200] debug/DbEvents: add contact notification history for service 'l-ansible-master-01!Swap' and user 'sinana'.
[2019-12-11 15:51:07 +0200] debug/DbEvents: add log entry history for 'l-ansible-master-01!Swap'
[2019-12-11 15:51:07 +0200] notice/Process: Running command '/usr/local/bin/gen-notification-new.py' 'mail' '--service' '-a' '10.7.77.253' '-b' '' '-c' '' '-d' '2019-12-11 15:51:07' '-i' 'Swap' '-l' 'l-ansible-master-01' '-m' 'sinana@mellanox.com' '-o' 'SWAP OK - 100% free (1999 MB out of 1999 MB) ' '-s' 'OK' '-t' 'RECOVERY': PID 24869
[2019-12-11 15:51:07 +0200] debug/DbEvents: add log entry history for 'l-ansible-master-01!Swap'
[2019-12-11 15:51:07 +0200] information/Notification: Completed sending 'Recovery' notification 'l-ansible-master-01!Swap!service-alert' for checkable 'l-ansible-master-01!Swap' and user 'sinana'.
[2019-12-11 15:51:07 +0200] notice/Process: PID 24869 ('/usr/local/bin/gen-notification-new.py' 'mail' '--service' '-a' '10.7.77.253' '-b' '' '-c' '' '-d' '2019-12-11 15:51:07' '-i' 'Swap' '-l' 'l-ansible-master-01' '-m' 'sinana@mellanox.com' '-o' 'SWAP OK - 100% free (1999 MB out of 1999 MB) ' '-s' 'OK' '-t' 'RECOVERY') terminated with exit code 0
[2019-12-11 15:51:34 +0200] debug/Checkable: Update checkable 'l-ansible-master-01' with check interval '60' from last check time at 2019-12-11 15:51:34 +0200 (1.57607e+09) to next check time at 2019-12-11 15:52:33 +0200(1.57607e+09).
[2019-12-11 15:51:34 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01'
[2019-12-11 15:51:38 +0200] debug/Checkable: Update checkable 'l-ansible-master-01!Check Zombie Procs' with check interval '240' from last check time at 2019-12-11 15:51:38 +0200 (1.57607e+09) to next check time at 2019-12-11 15:55:29 +0200(1.57607e+09).
[2019-12-11 15:51:38 +0200] debug/DbEvents: add checkable check history for 'l-ansible-master-01!Check Zombie Procs'
Any Idea why it detects 2 notifications logic but triggers only 1 ?
Thanks