Set up mail notification for icinga2 master node, only forced mail work particularly for custom mails

Hello everyone,
I have been setting up Icinga2 from scratch for the last weeks for the first time.
Currently i am facing an issue with mail notifications.
The mail function worked for me without any problem, when i tried them in the /conf.d directory, befor making icinga to master node
Since i changed my icinga2 to a master node and used the /zones.d directory with different file and folder structure i cannot sent mails without them being forced.

I particularly have that problem by trying to send custom mail. i can only send them when being forced, i do receive automatic mail warnings from my agent.

if anyone, would have an advice on what needs to be changed or added that would be great!

here are the icinga2 version infos:

icinga2 --version
icinga2 - The Icinga 2 network monitoring daemon (version: r2.13.7-1)

Copyright (c) 2012-2023 Icinga GmbH (https://icinga.com/)
License GPLv2+: GNU GPL version 2 or later <https://gnu.org/licenses/gpl2.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.

System information:
  Platform: Ubuntu
  Platform version: 22.04.2 LTS (Jammy Jellyfish)
  Kernel: Linux
  Kernel version: 5.15.0-71-generic
  Architecture: x86_64

Build information:
  Compiler: GNU 11.3.0
  Build host: runner-hh8q3bz2-project-575-concurrent-0
  OpenSSL version: OpenSSL 3.0.2 15 Mar 2022

Application information:

General paths:
  Config directory: /etc/icinga2
  Data directory: /var/lib/icinga2
  Log directory: /var/log/icinga2
  Cache directory: /var/cache/icinga2
  Spool directory: /var/spool/icinga2
  Run directory: /run/icinga2

Old paths (deprecated):
  Installation root: /usr
  Sysconf directory: /etc
  Run directory (base): /run
  Local state directory: /var

Internal paths:
  Package data directory: /usr/share/icinga2
  State path: /var/lib/icinga2/icinga2.state
  Modified attributes path: /var/lib/icinga2/modified-attributes.conf
  Objects path: /var/cache/icinga2/icinga2.debug
  Vars path: /var/cache/icinga2/icinga2.vars
  PID path: /run/icinga2/icinga2.pid

my overall config is in /etc/icinga2/zones.d/global-templates/test.conf
and looks like that:

template Host "generic-host" {
  max_check_attempts = 3
  check_interval = 1m
  retry_interval = 30s
  check_command = "hostalive"
}

apply Service "apt"{
  check_command = "apt"
  check_interval = 12h
  retry_interval = 1h
  command_endpoint = host.vars.client_endpoint
  assign where (host.address || host.address6) && host.vars.os == "Linux"
}

apply Service "ping4" {
  check_command = "ping4"
  assign where host.address
}

apply Service "ssh" {
  check_command = "ssh"
  assign where (host.address || host.address6) && host.vars.os == "Linux"
}

apply Service for (http_vhost => config in host.vars.http_vhosts) {
  check_command = "http"
  command_endpoint = host.vars.client_endpoint
  vars += config
  assign where host.vars.client_endpoint
}

apply Service "disk" {
  check_command = "disk"
  command_endpoint = host.vars.client_endpoint
  assign where host.vars.client_endpoint
}

apply Service "icinga" {
  check_command = "icinga"
  command_endpoint = host.vars.client_endpoint
  assign where host.vars.client_endpoint
}

apply Service "load" {
  check_command = "load"
  command_endpoint = host.vars.client_endpoint
  /* Used by the ScheduledDowntime apply rule in `downtimes.conf`. */
  vars.backup_downtime = "02:00-03:00"
  assign where (host.address || host.address6) && host.vars.os == "Linux"
}

apply Service for (procs in host.vars.procs) {
  check_command = "procs"
  vars.procs_critical = "1:"
  vars.procs_command = procs
  command_endpoint = host.vars.client_endpoint
  assign where host.vars.client_endpoint
}

apply Service for (argument in host.vars.runsv) {
  check_command = "procs"
  vars.procs_critical = "1:"
  vars.procs_command = "runsv"
  vars.procs_argument = argument
  command_endpoint = host.vars.client_endpoint
  assign where host.vars.client_endpoint
}

apply Service "procs" {
  check_command = "procs"
  command_endpoint = host.vars.client_endpoint
  assign where host.vars.client_endpoint
}

apply Service "swap" {
  check_command = "swap"
  command_endpoint = host.vars.client_endpoint
  assign where host.vars.client_endpoint
}

apply Service "users" {
  check_command = "users"
  command_endpoint = host.vars.client_endpoint
  assign where host.vars.client_endpoint
}

template User "generic-user" {
}

object User "icingaadmin" {
  import "generic-user"
  enable_notifications = true
  states = [ OK, Warning, Critical ]
  types = [ Problem, Recovery ]
  display_name = "Icinga 2 Admin"
  groups = [ "icingaadmins" ]
  email = "XXXXXXXX"
}

object UserGroup "icingaadmins" {
  display_name = "Icinga 2 Admin Group"
}

apply Notification "mail-icingaadmin" to Host {
  import "mail-host-notification"
  user_groups = host.vars.notification.mail.groups
  users = host.vars.notification.mail.users
  assign where host.vars.notification.mail

  interval = 60m

  vars.notification_logtosyslog = true
}

apply Notification "mail-icingaadmin" to Service {
  import "mail-service-notification"
  user_groups = host.vars.notification.mail.groups
  users = host.vars.notification.mail.users
  assign where host.vars.notification.mail

  interval = 60m
  vars.notification_logtosyslog = true
}

template Notification "mail-host-notification" {
  command = "mail-host-notification"
  states = [ Up, Down ]
  types = [ Problem, Acknowledgement, Recovery, Custom,
            FlappingStart, FlappingEnd,
            DowntimeStart, DowntimeEnd, DowntimeRemoved ]

  vars += {
    notification_from = "Icinga 2 Service Monitoring<XXX>"
    notification_logtosyslog = false
  }
  period = "24x7"
}
template Notification "mail-service-notification" {
  command = "mail-service-notification"

  states = [ OK, Warning, Critical, Unknown ]
  types = [ Problem, Acknowledgement, Recovery, Custom,
            FlappingStart, FlappingEnd,
            DowntimeStart, DowntimeEnd, DowntimeRemoved ]
  vars += {
    notification_logtosyslog = false
    notification_from = "Icinga 2 Service Monitoring<XXX>"
  }
  period = "24x7"
}

object NotificationCommand "mail-host-notification" {
  command = [ SysconfDir + "/icinga2/scripts/mail-host-notification.sh" ]
  arguments += {
    "-4" = {
      required = true
      value = "$notification_address$"
    }
    "-6" = "$notification_address6$"
    "-b" = "$notification_author$"
    "-c" = "$notification_comment$"
    "-d" = {
      required = true
      value = "$notification_date$"
    }
    "-f" = {
      value = "$notification_from$"
      description = "Set from address. Requires GNU mailutils (Debian/Ubuntu) or mailx (RHEL/SUSE)"
    }
    "-i" = "$notification_icingaweb2url$"
    "-l" = {
      required = true
      value = "$notification_hostname$"
    }
    "-n" = {
      required = true
      value = "$notification_hostdisplayname$"
    }
    "-o" = {
      required = true
      value = "$notification_hostoutput$"
    }
    "-r" = {
      required = true
      value = "$notification_useremail$"
    }
    "-s" = {
      required = true
      value = "$notification_hoststate$"
    }
    "-t" = {
      required = true
      value = "$notification_type$"
    }
    "-v" = "$notification_logtosyslog$"
  }
  vars += {
    notification_address = "$address$"
    notification_address6 = "$address6$"
    notification_author = "$notification.author$"
    notification_comment = "$notification.comment$"
    notification_type = "$notification.type$"
    notification_date = "$icinga.long_date_time$"
    notification_hostname = "$host.name$"
    notification_hostdisplayname = "$host.display_name$"
    notification_hostoutput = "$host.output$"
    notification_hoststate = "$host.state$"
    notification_useremail = "$user.email$"
  }
}

object NotificationCommand "mail-service-notification" {
  command = [ SysconfDir + "/icinga2/scripts/mail-service-notification.sh" ]
  arguments += {
    "-4" = {
      required = true
      value = "$notification_address$"
        }
    "-6" = "$notification_address6$"
    "-b" = "$notification_author$"
    "-c" = "$notification_comment$"
    "-d" = {
      required = true
      value = "$notification_date$"
    }
    "-e" = {
      required = true
      value = "$notification_servicename$"
    }
    "-f" = {
      value = "$notification_from$"
      description = "Set from address. Requires GNU mailutils (Debian/Ubuntu) or mailx (RHEL/SUSE)"
    }
    "-i" = "$notification_icingaweb2url$"
    "-l" = {
      required = true
      value = "$notification_hostname$"
    }
    "-n" = {
      required = true
      value = "$notification_hostdisplayname$"
    }
    "-o" = {
      required = true
      value = "$notification_serviceoutput$"
    }
    "-r" = {
      required = true
      value = "$notification_useremail$"
    }
    "-s" = {
      required = true
      value = "$notification_servicestate$"
    }
    "-t" = {
      required = true
      value = "$notification_type$"
    }
    "-u" = {
      required = true
      value = "$notification_servicedisplayname$"
    }
    "-v" = "$notification_logtosyslog$"
  }
  vars += {
    notification_address = "$address$"
    notification_address6 = "$address6$"
    notification_author = "$notification.author$"
    notification_comment = "$notification.comment$"
    notification_type = "$notification.type$"
    notification_date = "$icinga.long_date_time$"
    notification_hostname = "$host.name$"
    notification_hostdisplayname = "$host.display_name$"
    notification_servicename = "$service.name$"
    notification_serviceoutput = "$service.output$"
    notification_servicestate = "$service.state$"
    notification_useremail = "$user.email$"
    notification_servicedisplayname = "$service.display_name$"
  }
}

object TimePeriod "24x7" {
  import "legacy-timeperiod"
  display_name = "Icinga 2 24x7 TimePeriod"
  ranges = {
    "monday"    = "00:00-24:00"
    "tuesday"   = "00:00-24:00"
    "wednesday" = "00:00-24:00"
    "thursday"  = "00:00-24:00"
    "friday"    = "00:00-24:00"
    "saturday"  = "00:00-24:00"
    "sunday"    = "00:00-24:00"
  }
}

and the first configured hosts looks like that:

object Host "wks-test" {
  check_command = "hostalive"
  /* Import the default host template defined in `templates.conf`. */
  import "generic-host"

  vars.client_endpoint = name

  /* Specify the address attributes for checks e.g. `ssh` or `http`. */
  address = "XXX"
  address6 = "::1"

  /* Set custom attribute `os` for hostgroup assignment in `groups.conf`. */
  vars.os = "Linux"

  vars.disks["disk /"] = {
    disk_partitions = "/"
  }

  /* Define notification mail attributes for notification apply rules in `notifications.conf`. */
  vars.notification["mail"] = {
    /* The UserGroup `icingaadmins` is defined in `users.conf`. */
    groups = [ "icingaadmins" ]
  }
}

object Endpoint "wks-test" {
    host = "wks-test"
}

object Zone "wks-test" {
    endpoints = [ "wks-test" ]
    parent = "master"
}

Do you try to send notifications from every host with icinga installed or only from the master?

hey, i would like to get all the notifications from the master node.

zones.d/global-templates/ gets deploys to all hosts depending on your zones.conf not just the master zone.

ok thank you!
what would be the best approach?
like what to have in the zones.d/global-templates and what should be somewhere else?

Well I let the Director handle it for me but when I did it by hand, I put templates in global-templates and had the config for master in the master zone.

https://icinga.com/docs/icinga-2/latest/doc/06-distributed-monitoring/#zones

Think about the master zone as the conf.d directory under the zones.d you get one conf.d for every zone and as many global ones as you like.

i basically changed all those parts (mentioned below)now to, /zones.d/master/
i do get notifications from the agent, about a warning, i cant send forced custom notifications, but i cant send custom notifications, without forcing them. is there something still wrong with the conf?

template User "generic-user" {
}

object User "icingaadmin" {
  import "generic-user"
  enable_notifications = true
  states = [ OK, Warning, Critical ]
  types = [ Problem, Recovery ]
  display_name = "Icinga 2 Admin"
  groups = [ "icingaadmins" ]
  email = "XXX"
}

object UserGroup "icingaadmins" {
  display_name = "Icinga 2 Admin Group"
}

apply Notification "mail-icingaadmin" to Host {
  import "mail-host-notification"
  user_groups = host.vars.notification.mail.groups
  users = host.vars.notification.mail.users
  assign where host.vars.notification.mail

  interval = 60m

  vars.notification_logtosyslog = true
}

apply Notification "mail-icingaadmin" to Service {
  import "mail-service-notification"
  user_groups = host.vars.notification.mail.groups
  users = host.vars.notification.mail.users
  assign where host.vars.notification.mail

  interval = 60m
  vars.notification_logtosyslog = true
}

template Notification "mail-host-notification" {
  command = "mail-host-notification"
  states = [ Up, Down ]
  types = [ Problem, Acknowledgement, Recovery, Custom,
            FlappingStart, FlappingEnd,
            DowntimeStart, DowntimeEnd, DowntimeRemoved ]

  vars += {
    notification_from = "Icinga 2 Service Monitoring<XXX>"
    notification_logtosyslog = false
  }
  period = "24x7"
}

template Notification "mail-service-notification" {
  command = "mail-service-notification"

  states = [ OK, Warning, Critical, Unknown ]
  types = [ Problem, Acknowledgement, Recovery, Custom,
            FlappingStart, FlappingEnd,
            DowntimeStart, DowntimeEnd, DowntimeRemoved ]
  vars += {
    notification_logtosyslog = false
    notification_from = "Icinga 2 Service Monitoring<XXX>"
  }
  period = "24x7"
}

object NotificationCommand "mail-host-notification" {
  command = [ SysconfDir + "/icinga2/scripts/mail-host-notification.sh" ]
  arguments += {
    "-4" = {
     required = true
      value = "$notification_address$"
    }
    "-6" = "$notification_address6$"
    "-b" = "$notification_author$"
    "-c" = "$notification_comment$"
    "-d" = {
      required = true
      value = "$notification_date$"
    }
    "-f" = {
      value = "$notification_from$"
      description = "Set from address. Requires GNU mailutils (Debian/Ubuntu) or mailx (RHEL/SUSE)"
    }
    "-i" = "$notification_icingaweb2url$"
    "-l" = {
      required = true
      value = "$notification_hostname$"
    }
    "-n" = {
      required = true
      value = "$notification_hostdisplayname$"
    }
    "-o" = {
      required = true
      value = "$notification_hostoutput$"
    }
    "-r" = {
      required = true
      value = "$notification_useremail$"
    }
    "-s" = {
      required = true
      value = "$notification_hoststate$"
    }
    "-t" = {
      required = true
      value = "$notification_type$"
    }
    "-v" = "$notification_logtosyslog$"
  }
  vars += {
    notification_address = "$address$"
    notification_address6 = "$address6$"
    notification_author = "$notification.author$"
    notification_comment = "$notification.comment$"
    notification_type = "$notification.type$"
    notification_date = "$icinga.long_date_time$"
    notification_hostname = "$host.name$"
    notification_hostdisplayname = "$host.display_name$"
    notification_hostoutput = "$host.output$"
    notification_hoststate = "$host.state$"
    notification_useremail = "$user.email$"
  }
}

object NotificationCommand "mail-service-notification" {
  command = [ SysconfDir + "/icinga2/scripts/mail-service-notification.sh" ]
  arguments += {
    "-4" = {
      required = true
      value = "$notification_address$"
        }
    "-6" = "$notification_address6$"
    "-b" = "$notification_author$"
    "-c" = "$notification_comment$"
    "-d" = {
      required = true
     value = "$notification_date$"
    }
    "-e" = {
      required = true
      value = "$notification_servicename$"
    }
    "-f" = {
      value = "$notification_from$"
      description = "Set from address. Requires GNU mailutils (Debian/Ubuntu) or mailx (RHEL/SUSE)"
    }
    "-i" = "$notification_icingaweb2url$"
    "-l" = {
      required = true
      value = "$notification_hostname$"
    }
    "-n" = {
      required = true
      value = "$notification_hostdisplayname$"
    }
    "-o" = {
      required = true
      value = "$notification_serviceoutput$"
    }
    "-r" = {
      required = true
      value = "$notification_useremail$"
    }
    "-s" = {
      required = true
      value = "$notification_servicestate$"
    }
    "-t" = {
      required = true
      value = "$notification_type$"
    }
    "-u" = {
      required = true
      value = "$notification_servicedisplayname$"
    }
    "-v" = "$notification_logtosyslog$"
  }
  vars += {
    notification_address = "$address$"
    notification_address6 = "$address6$"
    notification_author = "$notification.author$"
    notification_comment = "$notification.comment$"
    notification_type = "$notification.type$"
    notification_date = "$icinga.long_date_time$"
    notification_hostname = "$host.name$"
    notification_hostdisplayname = "$host.display_name$"
    notification_servicename = "$service.name$"
    notification_serviceoutput = "$service.output$"
    notification_servicestate = "$service.state$"
    notification_useremail = "$user.email$"
    notification_servicedisplayname = "$service.display_name$"
  }
}

object TimePeriod "24x7" {
  import "legacy-timeperiod"
  display_name = "Icinga 2 24x7 TimePeriod"
  ranges = {
    "monday"    = "00:00-24:00"
    "tuesday"   = "00:00-24:00"
    "wednesday" = "00:00-24:00"
    "thursday"  = "00:00-24:00"
    "friday"    = "00:00-24:00"
    "saturday"  = "00:00-24:00"
   "sunday"    = "00:00-24:00"
  }
}

I dont get what is wrong, why do i receive mails from agents about warning, but then i cant send custom mails, without forcing them.

What is icinga2 feature list returning on the master and the agents?

Also is notification enabled, in Icingaweb2, on the host or service, you have to force?

master feature list:

Disabled features: command compatlog elasticsearch gelf graphite influxdb influxdb2 livestatus opentsdb perfdata statusdata syslog
Enabled features: api checker debuglog icingadb ido-pgsql mainlog notification

agent feature list:

Disabled features: command compatlog debuglog elasticsearch gelf graphite influxdb livestatus opentsdb perfdata statusdata syslog
Enabled features: api checker mainlog notification

and yes, notification in Icingaweb2 is enabled.

thank you!

Any filters at the notification or the user object level?

that here is all my configuration: (besides them i made no changes to notifications or users)

i hope i did not get your question wrong.

the following two .conf files are in my zones.d/master/ folder

object Host "wks-test" {
  check_command = "hostalive"
  /* Import the default host template defined in `templates.conf`. */
  import "generic-host"

  vars.client_endpoint = name

  /* Specify the address attributes for checks e.g. `ssh` or `http`. */
  address = "XXX"
  address6 = "::1"

  /* Set custom attribute `os` for hostgroup assignment in `groups.conf`. */
  vars.os = "Linux"

  vars.disks["disk /"] = {
    disk_partitions = "/"
  }
 /* Define notification mail attributes for notification apply rules in `notifications.conf`. */
  vars.notification["mail"] = {
    /* The UserGroup `icingaadmins` is defined in `users.conf`. */
    groups = [ "icingaadmins" ]
  }
}

object Endpoint "wks-test" {
    host = "wks-test"
}
template User "generic-user" {
}

object User "icingaadmin" {
  import "generic-user"
  enable_notifications = true
  states = [ OK, Warning, Critical ]
  types = [ Problem, Recovery ]
  display_name = "Icinga 2 Admin"
  groups = [ "icingaadmins" ]
  email = "XXX"
}

object UserGroup "icingaadmins" {
  display_name = "Icinga 2 Admin Group"
}

apply Notification "mail-icingaadmin" to Host {
  import "mail-host-notification"
  user_groups = host.vars.notification.mail.groups
  users = host.vars.notification.mail.users
  assign where host.vars.notification.mail

  interval = 60m

  vars.notification_logtosyslog = true
}

apply Notification "mail-icingaadmin" to Service {
  import "mail-service-notification"
  user_groups = host.vars.notification.mail.groups
  users = host.vars.notification.mail.users
  assign where host.vars.notification.mail

  interval = 60m
  vars.notification_logtosyslog = true
}

template Notification "mail-host-notification" {
  command = "mail-host-notification"
  states = [ Up, Down ]
  types = [ Problem, Acknowledgement, Recovery, Custom,
            FlappingStart, FlappingEnd,
            DowntimeStart, DowntimeEnd, DowntimeRemoved ]

  vars += {
    notification_from = "Icinga 2 Service Monitoring<XXX>"
    notification_logtosyslog = false
  }
  period = "24x7"
}

template Notification "mail-service-notification" {
  command = "mail-service-notification"

  states = [ OK, Warning, Critical, Unknown ]
  types = [ Problem, Acknowledgement, Recovery, Custom,
            FlappingStart, FlappingEnd,
            DowntimeStart, DowntimeEnd, DowntimeRemoved ]
  vars += {
    notification_logtosyslog = false
    notification_from = "Icinga 2 Service Monitoring<XXX>"
  }
  period = "24x7"
}

object NotificationCommand "mail-host-notification" {
  command = [ SysconfDir + "/icinga2/scripts/mail-host-notification.sh" ]
  arguments += {
    "-4" = {
     required = true
      value = "$notification_address$"
    }
    "-6" = "$notification_address6$"
    "-b" = "$notification_author$"
    "-c" = "$notification_comment$"
    "-d" = {
      required = true
      value = "$notification_date$"
    }
    "-f" = {
      value = "$notification_from$"
      description = "Set from address. Requires GNU mailutils (Debian/Ubuntu) or mailx (RHEL/SUSE)"
    }
    "-i" = "$notification_icingaweb2url$"
    "-l" = {
      required = true
      value = "$notification_hostname$"
    }
    "-n" = {
      required = true
      value = "$notification_hostdisplayname$"
    }
    "-o" = {
      required = true
      value = "$notification_hostoutput$"
    }
    "-r" = {
      required = true
      value = "$notification_useremail$"
    }
    "-s" = {
      required = true
      value = "$notification_hoststate$"
    }
    "-t" = {
      required = true
      value = "$notification_type$"
    }
    "-v" = "$notification_logtosyslog$"
  }
  vars += {
    notification_address = "$address$"
    notification_address6 = "$address6$"
    notification_author = "$notification.author$"
    notification_comment = "$notification.comment$"
    notification_type = "$notification.type$"
    notification_date = "$icinga.long_date_time$"
    notification_hostname = "$host.name$"
    notification_hostdisplayname = "$host.display_name$"
    notification_hostoutput = "$host.output$"
    notification_hoststate = "$host.state$"
    notification_useremail = "$user.email$"
  }
}

object NotificationCommand "mail-service-notification" {
  command = [ SysconfDir + "/icinga2/scripts/mail-service-notification.sh" ]
  arguments += {
    "-4" = {
      required = true
      value = "$notification_address$"
        }
    "-6" = "$notification_address6$"
    "-b" = "$notification_author$"
    "-c" = "$notification_comment$"
    "-d" = {
      required = true
     value = "$notification_date$"
    }
    "-e" = {
      required = true
      value = "$notification_servicename$"
    }
    "-f" = {
      value = "$notification_from$"
      description = "Set from address. Requires GNU mailutils (Debian/Ubuntu) or mailx (RHEL/SUSE)"
    }
    "-i" = "$notification_icingaweb2url$"
    "-l" = {
      required = true
      value = "$notification_hostname$"
    }
    "-n" = {
      required = true
      value = "$notification_hostdisplayname$"
    }
    "-o" = {
      required = true
      value = "$notification_serviceoutput$"
    }
    "-r" = {
      required = true
      value = "$notification_useremail$"
    }
    "-s" = {
      required = true
      value = "$notification_servicestate$"
    }
    "-t" = {
      required = true
      value = "$notification_type$"
    }
    "-u" = {
      required = true
      value = "$notification_servicedisplayname$"
    }
    "-v" = "$notification_logtosyslog$"
  }
  vars += {
    notification_address = "$address$"
    notification_address6 = "$address6$"
    notification_author = "$notification.author$"
    notification_comment = "$notification.comment$"
    notification_type = "$notification.type$"
    notification_date = "$icinga.long_date_time$"
    notification_hostname = "$host.name$"
    notification_hostdisplayname = "$host.display_name$"
    notification_servicename = "$service.name$"
    notification_serviceoutput = "$service.output$"
    notification_servicestate = "$service.state$"
    notification_useremail = "$user.email$"
    notification_servicedisplayname = "$service.display_name$"
  }
}

object TimePeriod "24x7" {
  import "legacy-timeperiod"
  display_name = "Icinga 2 24x7 TimePeriod"
  ranges = {
    "monday"    = "00:00-24:00"
    "tuesday"   = "00:00-24:00"
    "wednesday" = "00:00-24:00"
    "thursday"  = "00:00-24:00"
    "friday"    = "00:00-24:00"
    "saturday"  = "00:00-24:00"
   "sunday"    = "00:00-24:00"
  }
}

that following .conf file is in /zones.d/global-templates/

template Host "generic-host" {
  max_check_attempts = 3
  check_interval = 1m
  retry_interval = 30s
  check_command = "hostalive"
}

apply Service "apt"{
  check_command = "apt"
  check_interval = 12h
  retry_interval = 1h
  command_endpoint = host.vars.client_endpoint
  assign where (host.address || host.address6) && host.vars.os == "Linux"
}

apply Service "ping4" {
  check_command = "ping4"
  assign where host.address
}

apply Service "ssh" {
  check_command = "ssh"
  assign where (host.address || host.address6) && host.vars.os == "Linux"
}

//apply Service for (http_vhost => config in host.vars.http_vhosts) {
//  check_command = "http"
//  command_endpoint = host.vars.client_endpoint
//  vars += config
//  assign where host.vars.client_endpoint
//}

apply Service "disk" {
  check_command = "disk"
  command_endpoint = host.vars.client_endpoint
  assign where host.vars.client_endpoint
}

apply Service "icinga" {
  check_command = "icinga"
  command_endpoint = host.vars.client_endpoint
  assign where host.vars.client_endpoint
}

apply Service "load" {
  check_command = "load"
  command_endpoint = host.vars.client_endpoint
  /* Used by the ScheduledDowntime apply rule in `downtimes.conf`. */
  vars.backup_downtime = "02:00-03:00"
  assign where (host.address || host.address6) && host.vars.os == "Linux"
}

//apply Service for (procs in host.vars.procs) {
//  check_command = "procs"
//  vars.procs_critical = "1:"
//  vars.procs_command = procs
//  command_endpoint = host.vars.client_endpoint
//  assign where host.vars.client_endpoint
//}

//apply Service for (argument in host.vars.runsv) {
//  check_command = "procs"
//  vars.procs_critical = "1:"
//  vars.procs_command = "runsv"
//  vars.procs_argument = argument
//  command_endpoint = host.vars.client_endpoint
//  assign where host.vars.client_endpoint
//}

apply Service "procs" {
  check_command = "procs"
  command_endpoint = host.vars.client_endpoint
  assign where host.vars.client_endpoint
}

apply Service "swap" {
  check_command = "swap"
  command_endpoint = host.vars.client_endpoint
  assign where host.vars.client_endpoint
}

apply Service "users" {
  check_command = "users"
  command_endpoint = host.vars.client_endpoint
  assign where host.vars.client_endpoint
}

my zones.conf file:

object Endpoint "icidev" {
        host = "192.168.50.200"
}

object Zone "master" {
        name = "master"
        endpoints = [ "icidev" ]
}

object Zone "global-templates" {
        global = true
}

object Zone "director-global" {
        global = true
}

This is why normal notifications works and custom notifications needs to be forced.
You filtered out Custom.

here the full set:

template Notification "generic-notification" {

  states = [ OK, Warning, Critical, Unknown ]
  types = [ Problem, Acknowledgement, Recovery, Custom, FlappingStart,
            FlappingEnd, DowntimeStart, DowntimeEnd, DowntimeRemoved ]
}

https://icinga.com/docs/icinga-2/latest/doc/03-monitoring-basics/#notification-filters-by-state-and-type

thank you! that worked for me.
after i saw it it was quite obvious :slight_smile: