Duplicated notifications with notification interval 0

Hi everybody,

I have some duplicated notifications with notification interval 0, the log looks like:

[2019-12-03 11:50:07 +0100] information/Notification: Sending reminder 'Problem' notification 'server.local!disk!Zugangssysteme-Mail-Service-Notification' for user 'Zugangssysteme'
[2019-12-03 11:50:07 +0100] information/Notification: Completed sending 'Problem' notification 'server.local!disk!Zugangssysteme-Mail-Service-Notification' for checkable 'server.local!disk' and user 'Zugangssysteme' using command 'mail-service-prefdata-notification'.
[2019-12-03 11:50:21 +0100] information/Notification: Sending 'Problem' notification 'server.local!disk!Zugangssysteme-Mail-Service-Notification' for user 'Zugangssysteme'
[2019-12-03 11:50:21 +0100] information/Notification: Completed sending 'Problem' notification 'server.local!disk!Zugangssysteme-Mail-Service-Notification' for checkable 'server.local!disk' and user 'Zugangssysteme' using command 'mail-service-prefdata-notification'.

Has anybody a idea to stop this or is this maybe a bug?

icinga2 - The Icinga 2 network monitoring daemon (version: r2.11.2-1)

Best regards,
Alicia

Can you share the notification object from icinga2 object list --type Notification --name ...?

Sure:

Object 'server.local!disk!Zugangssysteme-Mail-Service-Notification' of type 'Notification':
  % declared in '/var/lib/icinga2/api/zones/master/director/notification_apply.conf', lines 366:1-366:72
  * __name = "server.local!disk!Zugangssysteme-Mail-Service-Notification"
  * command = "mail-service-prefdata-notification"
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_templates.conf', lines 38:5-38:50
  * command_endpoint = ""
  * host_name = "server.local"
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_apply.conf', lines 366:1-366:72
  * interval = 0
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_apply.conf', lines 372:5-372:17
  * name = "Zugangssysteme-Mail-Service-Notification"
  * package = "_cluster"
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_apply.conf', lines 366:1-366:72
  * period = "7x24"
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_templates.conf', lines 39:5-39:19
  * service_name = "disk"
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_apply.conf', lines 366:1-366:72
  * source_location
    * first_column = 1
    * first_line = 366
    * last_column = 72
    * last_line = 366
    * path = "/var/lib/icinga2/api/zones/master/director/notification_apply.conf"
  * states = [ "Critical", "OK", "Unknown", "Warning" ]
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_apply.conf', lines 374:5-374:47
  * templates = [ "Zugangssysteme-Mail-Service-Notification", "Mail-Service-Perfdata-Notification" ]
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_apply.conf', lines 366:1-366:72
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_templates.conf', lines 34:1-34:58
  * times
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_templates.conf', lines 35:5-37:5
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_apply.conf', lines 369:5-371:5
    * begin = 900
  * type = "Notification"
  * types = [ "Acknowledgement", "Custom", "Problem", "Recovery" ]
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_apply.conf', lines 375:5-375:58
  * user_groups = null
  * users = [ "Zugangssysteme" ]
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_apply.conf', lines 376:5-376:24
  * vars = null
  * zone = "master"
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_apply.conf', lines 366:1-366:72

So, interval = 0 and times.begin = 900. The logs look a bit like as if the first notification was sent delayed, therefore the reminder word in it. The second notification looks like as if only interval = 0 was configured. Was there a config change followed by a Director deployment in between?

I deleted manually a downtime, that was the reason of the delay. And I activated the notification short time before with a deployment. But 40 minutes later, without any deployments and without delay for sending the notification, I had the same issue:

[2019-12-03 12:24:58 +0100] information/Notification: Sending reminder 'Problem' notification 'master2!disk!Icinga-Admin-Mail-Service-Notification' for user 'icingaadmin'
[2019-12-03 12:24:58 +0100] information/Notification: Completed sending 'Problem' notification 'master2!disk!Icinga-Admin-Mail-Service-Notification' for checkable 'master2!disk' and user 'icingaadmin' using command 'mail-service-prefdata-notification'.
[2019-12-03 12:34:07 +0100] information/Notification: Sending 'Recovery' notification 'master2!disk!Icinga-Admin-Mail-Service-Notification' for user 'icingaadmin'
[2019-12-03 12:34:07 +0100] information/Notification: Completed sending 'Recovery' notification 'master2!disk!Icinga-Admin-Mail-Service-Notification' for checkable 'master2!disk' and user 'icingaadmin' using command 'mail-service-prefdata-notification'.

Object 'master2!disk!Icinga-Admin-Mail-Service-Notification' of type 'Notification':
  % declared in '/var/lib/icinga2/api/zones/master/director/notification_apply.conf', lines 183:1-183:70
  * __name = "master2!disk!Icinga-Admin-Mail-Service-Notification"
  * command = "mail-service-prefdata-notification"
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_templates.conf', lines 38:5-38:50
  * command_endpoint = ""
  * host_name = "master2"
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_apply.conf', lines 183:1-183:70
  * interval = 0
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_apply.conf', lines 186:5-186:17
  * name = "Icinga-Admin-Mail-Service-Notification"
  * package = "_cluster"
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_apply.conf', lines 183:1-183:70
  * period = "7x24"
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_templates.conf', lines 39:5-39:19
  * service_name = "disk"
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_apply.conf', lines 183:1-183:70
  * source_location
    * first_column = 1
    * first_line = 183
    * last_column = 70
    * last_line = 183
    * path = "/var/lib/icinga2/api/zones/master/director/notification_apply.conf"
  * states = [ "Critical", "OK", "Unknown", "Warning" ]
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_apply.conf', lines 188:5-188:47
  * templates = [ "Icinga-Admin-Mail-Service-Notification", "Mail-Service-Perfdata-Notification" ]
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_apply.conf', lines 183:1-183:70
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_templates.conf', lines 34:1-34:58
  * times
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_templates.conf', lines 35:5-37:5
    * begin = 900
  * type = "Notification"
  * types = [ "Acknowledgement", "Custom", "Problem", "Recovery" ]
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_apply.conf', lines 189:5-189:58
  * user_groups = null
  * users = [ "icingaadmin" ]
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_apply.conf', lines 190:5-190:26
  * vars = null
  * zone = "master"
    % = modified in '/var/lib/icinga2/api/zones/master/director/notification_apply.conf', lines 183:1-183:70

The logs say Problem followed by a Recovery. That looks normal to me.

Oh sorry, I haven’t read properly :flushed:

So this is a normal behavior if the notification is delayed by any reason?

The second thing you’re describing is totally valid. A Problem notification gets delayed by times.begin, with suppressing the initial send and later allowing it once, calling it “reminder”. Your scenario really is a special case not covered by the default log messages.

When a service returns to OK, the Recovery notification is sent.

Yes that is clear :slight_smile:

For the other part, if it works like planned, I have to handle it :slight_smile: thanks for your help!