I still don’t understand what is happening. However I found some things:
- The default values are defined in: /usr/local/share/icinga2/include/command-plugins.conf
- The problem only happens in some servers, not all. The host definition are the same, the only thing that changes related to load monitoring are the variables like wload1, etc…
- I only have issues with FreeBSD servers (the majority), and not with Linux.
- I have the hosts define in zones.d/master/staging-hosts.conf and zones.d/master/production-hosts.conf. The configuration is the same, however I have no issues with the staging servers. Could it be related with the order that the files are included?
- The variables in the hosts keep the default values. In both, in hosts that work fine and those that don’t work.
Here, what I have done:
I inspected load in the master server. It has the default values.
Object 'load' of type 'CheckCommand':
% declared in '/usr/local/share/icinga2/include/command-plugins.conf', lines 1665:1-1665:26
* __name = "load"
* arguments
% = modified in '/usr/local/share/icinga2/include/command-plugins.conf', lines 1668:2-1681:2
* -c
* description = "Exit with CRITICAL status if load average exceed CLOADn; the load average format is the same used by 'uptime' and 'w'"
* value = "$load_cload1$,$load_cload5$,$load_cload15$"
* -r
* description = "Divide the load averages by the number of CPUs (when possible)"
* set_if = "$load_percpu$"
* -w
* description = "Exit with WARNING status if load average exceeds WLOADn"
* value = "$load_wload1$,$load_wload5$,$load_wload15$"
* command = [ "/usr/local/libexec/nagios/check_load" ]
% = modified in '/usr/local/share/icinga2/include/command-plugins.conf', lines 1666:2-1666:40
* env = null
* execute
% = modified in 'methods-itl.conf', lines 19:3-19:23
* arguments = [ "checkable", "cr", "resolvedMacros", "useResolvedMacros" ]
* deprecated = false
* name = "Internal#PluginCheck"
* side_effect_free = false
* type = "Function"
* name = "load"
* package = "_etc"
* source_location
* first_column = 1
* first_line = 1665
* last_column = 26
* last_line = 1665
* path = "/usr/local/share/icinga2/include/command-plugins.conf"
* templates = [ "load", "plugin-check-command" ]
% = modified in '/usr/local/share/icinga2/include/command-plugins.conf', lines 1665:1-1665:26
% = modified in 'methods-itl.conf', lines 18:2-18:94
* timeout = 60
* type = "CheckCommand"
* vars
* load_cload1 = 10
% = modified in '/usr/local/share/icinga2/include/command-plugins.conf', lines 1687:2-1687:24
* load_cload15 = 4
% = modified in '/usr/local/share/icinga2/include/command-plugins.conf', lines 1689:2-1689:24
* load_cload5 = 6
% = modified in '/usr/local/share/icinga2/include/command-plugins.conf', lines 1688:2-1688:23
* load_percpu = false
% = modified in '/usr/local/share/icinga2/include/command-plugins.conf', lines 1691:2-1691:25
* load_wload1 = 5
% = modified in '/usr/local/share/icinga2/include/command-plugins.conf', lines 1683:2-1683:23
* load_wload15 = 3
% = modified in '/usr/local/share/icinga2/include/command-plugins.conf', lines 1685:2-1685:24
* load_wload5 = 4
% = modified in '/usr/local/share/icinga2/include/command-plugins.conf', lines 1684:2-1684:23
* zone = ""
From this output I learn about /usr/share/icinga2/include/command-plugins.conf. I found the default values defined there:
<...>
object CheckCommand "load" {
command = [ PluginDir + "/check_load" ]
arguments = {
"-w" = {
value = "$load_wload1$,$load_wload5$,$load_wload15$"
description = "Exit with WARNING status if load average exceeds WLOADn"
}
"-c" = {
value = "$load_cload1$,$load_cload5$,$load_cload15$"
description = "Exit with CRITICAL status if load average exceed CLOADn; the load average format is the same used by 'uptime' and 'w'"
}
"-r" = {
set_if = "$load_percpu$"
description = "Divide the load averages by the number of CPUs (when possible)"
}
}
vars.load_wload1 = 5.0
vars.load_wload5 = 4.0
vars.load_wload15 = 3.0
vars.load_cload1 = 10.0
vars.load_cload5 = 6.0
vars.load_cload15 = 4.0
vars.load_percpu = false
}
<...>
However, when I see the host object I can see that the values have been set as expected.
Object 'hostx.foo.bar' of type 'Host':
% declared in '/usr/local/etc/icinga2/zones.d/master/production-host.conf', lines 198:1-198:36
* __name = "hostx.foo.bar"
* action_url = ""
* address = "xyz.xyz.xyz.xyz"
% = modified in '/usr/local/etc/icinga2/zones.d/master/production-host.conf', lines 201:5-201:29
* address6 = ""
* check_command = "hostalive"
% = modified in '/usr/local/etc/icinga2/conf.d/templates/generic-host.conf', lines 6:3-6:29
* check_interval = 60
% = modified in '/usr/local/etc/icinga2/conf.d/templates/generic-host.conf', lines 3:3-3:21
* check_period = ""
* check_timeout = null
* command_endpoint = ""
* display_name = "hostx.foo.bar"
* enable_active_checks = true
* enable_event_handler = true
* enable_flapping = false
* enable_notifications = true
* enable_passive_checks = true
* enable_perfdata = true
* event_command = ""
* flapping_threshold = 0
* flapping_threshold_high = 30
* flapping_threshold_low = 25
* groups = [ ]
* icon_image = ""
* icon_image_alt = ""
* max_check_attempts = 2
% = modified in '/usr/local/etc/icinga2/conf.d/templates/generic-host.conf', lines 2:3-2:24
* name = "hostx.foo.bar"
* notes = ""
* notes_url = ""
* package = "_etc"
* retry_interval = 30
% = modified in '/usr/local/etc/icinga2/conf.d/templates/generic-host.conf', lines 4:3-4:22
* source_location
* first_column = 1
* first_line = 198
* last_column = 36
* last_line = 198
* path = "/usr/local/etc/icinga2/zones.d/master/production-host.conf"
* templates = [ "hostx.foo.bar", "generic-host" ]
% = modified in '/usr/local/etc/icinga2/zones.d/master/production-host.conf', lines 198:1-198:36
% = modified in '/usr/local/etc/icinga2/conf.d/templates/generic-host.conf', lines 1:0-1:27
* type = "Host"
* vars
* disks
* disk /
% = modified in '/usr/local/etc/icinga2/conf.d/templates/generic-host.conf', lines 8:3-10:3
* disk_partitions = "/"
* load_cload1 = 20
% = modified in '/usr/local/etc/icinga2/zones.d/master/production-host.conf', lines 211:5-211:25
* load_cload15 = 8
% = modified in '/usr/local/etc/icinga2/zones.d/master/production-host.conf', lines 213:5-213:25
* load_cload5 = 12
% = modified in '/usr/local/etc/icinga2/zones.d/master/production-host.conf', lines 212:5-212:25
* load_wload1 = 10
% = modified in '/usr/local/etc/icinga2/zones.d/master/production-host.conf', lines 208:1-208:21
* load_wload15 = 6
% = modified in '/usr/local/etc/icinga2/zones.d/master/production-host.conf', lines 210:5-210:25
* load_wload5 = 8
% = modified in '/usr/local/etc/icinga2/zones.d/master/production-host.conf', lines 209:5-209:24
* os = "FreeBSD"
% = modified in '/usr/local/etc/icinga2/zones.d/master/production-host.conf', lines 204:5-204:23
* ssh_port = 2389
% = modified in '/usr/local/etc/icinga2/zones.d/master/production-host.conf', lines 202:5-202:24
* volatile = false
* zone = "master"
However when I retrieve the load object from the client I have the default values.
Object 'load' of type 'CheckCommand':
% declared in '/usr/local/share/icinga2/include/command-plugins.conf', lines 1665:1-1665:26
* __name = "load"
* arguments
% = modified in '/usr/local/share/icinga2/include/command-plugins.conf', lines 1668:2-1681:2
* -c
* description = "Exit with CRITICAL status if load average exceed CLOADn; the load average format is the same used by 'uptime' and 'w'"
* value = "$load_cload1$,$load_cload5$,$load_cload15$"
* -r
* description = "Divide the load averages by the number of CPUs (when possible)"
* set_if = "$load_percpu$"
* -w
* description = "Exit with WARNING status if load average exceeds WLOADn"
* value = "$load_wload1$,$load_wload5$,$load_wload15$"
* command = [ "/usr/local/libexec/nagios/check_load" ]
% = modified in '/usr/local/share/icinga2/include/command-plugins.conf', lines 1666:2-1666:40
* env = null
* execute
% = modified in 'methods-itl.conf', lines 19:3-19:23
* arguments = [ "checkable", "cr", "resolvedMacros", "useResolvedMacros" ]
* deprecated = false
* name = "Internal#PluginCheck"
* side_effect_free = false
* type = "Function"
* name = "load"
* package = "_etc"
* source_location
* first_column = 1
* first_line = 1665
* last_column = 26
* last_line = 1665
* path = "/usr/local/share/icinga2/include/command-plugins.conf"
* templates = [ "load", "plugin-check-command" ]
% = modified in '/usr/local/share/icinga2/include/command-plugins.conf', lines 1665:1-1665:26
% = modified in 'methods-itl.conf', lines 18:2-18:94
* timeout = 60
* type = "CheckCommand"
* vars
* load_cload1 = 10
% = modified in '/usr/local/share/icinga2/include/command-plugins.conf', lines 1687:2-1687:24
* load_cload15 = 4
% = modified in '/usr/local/share/icinga2/include/command-plugins.conf', lines 1689:2-1689:24
* load_cload5 = 6
% = modified in '/usr/local/share/icinga2/include/command-plugins.conf', lines 1688:2-1688:23
* load_percpu = false
% = modified in '/usr/local/share/icinga2/include/command-plugins.conf', lines 1691:2-1691:25
* load_wload1 = 5
% = modified in '/usr/local/share/icinga2/include/command-plugins.conf', lines 1683:2-1683:23
* load_wload15 = 3
% = modified in '/usr/local/share/icinga2/include/command-plugins.conf', lines 1685:2-1685:24
* load_wload5 = 4
% = modified in '/usr/local/share/icinga2/include/command-plugins.conf', lines 1684:2-1684:23
* zone = ""