I have strange flapping behavior with a custom check_command = “http” Service apply. It is flapping a crazy way:
***** Service Monitoring on mon *****
INERSERVICE LB server 8080 on testsite.example.com is CRITICAL!
Info: <A HREF="http://172.20.1.16:8080/" target="_blank">connect to address testsite.example.com and port 80: Connection refused
HTTP CRITICAL - Unable to open TCP socket
When: 2021-08-03 16:00:35 +0200
Service: website INERSERVICE check
Host: testsite.example.com
IPv4: 172.20.1.16
***** Service Monitoring on mon *****
INERSERVICE LB server 8080 on iier2-tst-wf.alig.hu is OK!
Info: <A HREF="http://172.20.1.16:8080/" target="_blank">HTTP OK: HTTP/1.1 200 OK - 22382 bytes in 0.033 second response time </A>
When: 2021-08-03 16:02:33 +0200
Service: website INERSERVICE check
Host: testsite.example.com
IPv4: 172.20.1.16
The check work on 8080 port by default but the one able to redefine to other port. I tried to use implicitly and explicitly the port number 8080. If you look the error message above you can see check on port 8080 but the try was on port 80. Then the error is recovering after few minutes.
Icinga version information:
icinga2 --version
icinga2 - The Icinga 2 network monitoring daemon (version: 2.12.4-1)
Copyright (c) 2012-2021 Icinga GmbH (https://icinga.com/)
License GPLv2+: GNU GPL version 2 or later <http://gnu.org/licenses/gpl2.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
System information:
Platform: CentOS Linux
Platform version: 7 (Core)
Kernel: Linux
Kernel version: 3.10.0-1127.8.2.el7.x86_64
Architecture: x86_64
Build information:
Compiler: GNU 4.8.5
Build host: runner-hh8q3bz2-project-322-concurrent-0
OpenSSL version: OpenSSL 1.0.2k-fips 26 Jan 2017
Application information:
General paths:
Config directory: /etc/icinga2
Data directory: /var/lib/icinga2
Log directory: /var/log/icinga2
Cache directory: /var/cache/icinga2
Spool directory: /var/spool/icinga2
Run directory: /run/icinga2
Old paths (deprecated):
Installation root: /usr
Sysconf directory: /etc
Run directory (base): /run
Local state directory: /var
Internal paths:
Package data directory: /usr/share/icinga2
State path: /var/lib/icinga2/icinga2.state
Modified attributes path: /var/lib/icinga2/modified-attributes.conf
Objects path: /var/cache/icinga2/icinga2.debug
Vars path: /var/cache/icinga2/icinga2.vars
PID path: /run/icinga2/icinga2.pid
Service definition looks like this:
apply Service "website INERSERVICE check" {
import "generic-service"
check_command = "http"
vars.http_uri = "/"
vars.http_link = true
vars.http_onredirect = "follow"
vars.http_string = "INERSERVICE - Bejelentkez"
vars.http_pagesize = 11000
vars.http_vhost = "$check_address$"
vars.http_ssl = false
vars.http_sni = false
vars.http_linespan = false
vars.check_ipv4 = "$http_ipv4$"
vars.check_ipv6 = "$http_ipv6$"
vars.http_verbose = false
vars.http_timeout = 20
vars.http_extendedperfdata = true
if (host.vars.iner_frontend_port != "" ) { vars.http_port = host.vars.iner_frontend_port } else { vars.http_port = 8080 }
display_name = "INERSERVICE LB server " + vars.http_port
log("INERSERVICE LB server: " + host.name + ":" + vars.http_port)
assign where (host.address || host.address6) && "prj:iner-frontend" in host.groups
}
The object dump look like this:
Object 'testsite.example.com!website INERSERVICE check' of type 'Service':
% declared in '/etc/icinga2/conf.d/services/INERSERVICE_services.conf', lines 2:1-2:35
* __name = "testsite.example.com!website INERSERVICE check"
* action_url = ""
* check_command = "http"
% = modified in '/etc/icinga2/conf.d/services/INERSERVICE_services.conf', lines 5:3-5:24
* check_interval = 60
% = modified in '/etc/icinga2/conf.d/templates.conf', lines 41:3-41:21
* check_period = ""
* check_timeout = null
* command_endpoint = ""
* display_name = "INERSERVICE LB server 8080"
% = modified in '/etc/icinga2/conf.d/services/INERSERVICE_services.conf', lines 21:3-21:52
* enable_active_checks = true
* enable_event_handler = true
* enable_flapping = true
% = modified in '/etc/icinga2/conf.d/templates.conf', lines 44:3-44:21
* enable_notifications = true
* enable_passive_checks = true
* enable_perfdata = true
* event_command = ""
* flapping_threshold = 0
* flapping_threshold_high = 40
% = modified in '/etc/icinga2/conf.d/templates.conf', lines 46:3-46:30
* flapping_threshold_low = 20
% = modified in '/etc/icinga2/conf.d/templates.conf', lines 45:3-45:29
* groups = [ ]
* host_name = "testsite.example.com"
% = modified in '/etc/icinga2/conf.d/services/INERSERVICE_services.conf', lines 2:1-2:35
* icon_image = ""
* icon_image_alt = ""
* max_check_attempts = 10
% = modified in '/etc/icinga2/conf.d/templates.conf', lines 40:3-40:25
* name = "website INERSERVICE check"
% = modified in '/etc/icinga2/conf.d/services/INERSERVICE_services.conf', lines 2:1-2:35
* notes = ""
* notes_url = ""
* package = "_etc"
% = modified in '/etc/icinga2/conf.d/services/INERSERVICE_services.conf', lines 2:1-2:35
* retry_interval = 30
% = modified in '/etc/icinga2/conf.d/templates.conf', lines 42:3-42:22
* source_location
* first_column = 1
* first_line = 2
* last_column = 35
* last_line = 2
* path = "/etc/icinga2/conf.d/services/INERSERVICE_services.conf"
* templates = [ "website INERSERVICE check", "generic-service" ]
% = modified in '/etc/icinga2/conf.d/services/INERSERVICE_services.conf', lines 2:1-2:35
% = modified in '/etc/icinga2/conf.d/templates.conf', lines 39:1-39:34
* type = "Service"
* vars
* http_extendedperfdata = true
% = modified in '/etc/icinga2/conf.d/services/INERSERVICE_services.conf', lines 19:3-19:35
* http_linespan = false
% = modified in '/etc/icinga2/conf.d/services/INERSERVICE_services.conf', lines 14:3-14:28
* http_link = true
% = modified in '/etc/icinga2/conf.d/services/INERSERVICE_services.conf', lines 7:3-7:23
* http_onredirect = "follow"
% = modified in '/etc/icinga2/conf.d/services/INERSERVICE_services.conf', lines 8:3-8:33
* http_pagesize = 11000
% = modified in '/etc/icinga2/conf.d/services/INERSERVICE_services.conf', lines 10:3-10:28
* http_port = 8080
% = modified in '/etc/icinga2/conf.d/services/INERSERVICE_services.conf', lines 20:47-20:92
* http_sni = false
% = modified in '/etc/icinga2/conf.d/services/INERSERVICE_services.conf', lines 13:3-13:23
* http_ssl = false
% = modified in '/etc/icinga2/conf.d/services/INERSERVICE_services.conf', lines 12:3-12:23
* http_string = "INERSERVICE - Bejelentkez"
% = modified in '/etc/icinga2/conf.d/services/INERSERVICE_services.conf', lines 9:3-9:42
* http_timeout = 20
% = modified in '/etc/icinga2/conf.d/services/INERSERVICE_services.conf', lines 18:3-18:24
* http_uri = "/"
% = modified in '/etc/icinga2/conf.d/services/INERSERVICE_services.conf', lines 6:3-6:21
* http_verbose = false
% = modified in '/etc/icinga2/conf.d/services/INERSERVICE_services.conf', lines 17:3-17:27
* volatile = false
* zone = "mon.example.com"
% = modified in '/etc/icinga2/conf.d/services/INERSERVICE_services.conf', lines 2:1-2:35