Satellite check pending

Hello everyone,

I’ve been trying for a few days to have my satellite run real checks other than ping. But the checks get stuck in the status pending. A synchronization between master and satellite works. Pings are performed from the satellite. It’s strange that a check shows a check source that doesn’t exist in the configurations. This is my test environment. I only set up the director for testing, but it is not really used. Thanks for the help in advance.

Best regards
Rico

here my configuration:
Both (master and satellite) has version: r2.13.3-1
Both Ubuntu 20.04
on master:
icinga2 feature list
Disabled features: compatlog debuglog elasticsearch gelf icingadb influxdb influxdb2 livestatus opentsdb perfdata statusdata syslog
Enabled features: api checker command graphite ido-mysql ido-pgsql mainlog notification
on satellite:
icinga2 feature list
Disabled features: command compatlog debuglog elasticsearch gelf graphite icingadb influxdb influxdb2 livestatus notification opentsdb perfdata statusdata syslog
Enabled features: api checker mainlog

my zones.conf on master

/*
 * Generated by Icinga 2 node setup commands
 * on 2022-03-08 14:36:22 +0100
 */
object Endpoint "esn-svm037" {
}
object Endpoint "nbb-vm-013" {
}
object Zone "master" {
	endpoints = [ "esn-svm037" ]
}
object Zone "global-templates" {
	global = true
}
object Zone "director-global" {
	global = true
}
object Zone "nbb-vm-013" {
        endpoints = [ "nbb-vm-013" ]
        parent = "master"
}

zones.conf on satellite

/*
 * Generated by Icinga 2 node setup commands
 * on 2022-05-09 07:54:59 +0200
 */
object Endpoint "esn-svm037" {
        host = "10.88.183.37"
        port = "5665"
}
object Endpoint "nbb-vm-013" {
}
object Zone "master" {
        endpoints = [ "esn-svm037" ]
}
object Zone "nbb-vm-013" {
        endpoints = [ "nbb-vm-013" ]
        parent = "master"
}
object Zone "global-templates" {
        global = true
}
object Zone "director-global" {
        global = true
}

the Endpointlist on satellite

icinga2 object list --type Endpoint
Object 'esn-svm037' of type 'Endpoint':
  % declared in '/etc/icinga2/zones.conf', lines 6:1-6:28
  * __name = "esn-svm037"
  * host = ""
  * log_duration = 86400
  * name = "esn-svm037"
  * package = "_etc"
  * port = "5665"
  * source_location
    * first_column = 1
    * first_line = 6
    * last_column = 28
    * last_line = 6
    * path = "/etc/icinga2/zones.conf"
  * templates = [ "esn-svm037" ]
    % = modified in '/etc/icinga2/zones.conf', lines 6:1-6:28
  * type = "Endpoint"
  * zone = ""

Object 'nbb-vm-013' of type 'Endpoint':
  % declared in '/etc/icinga2/zones.conf', lines 9:1-9:28
  * __name = "nbb-vm-013"
  * host = ""
  * log_duration = 86400
  * name = "nbb-vm-013"
  * package = "_etc"
  * port = "5665"
  * source_location
    * first_column = 1
    * first_line = 9
    * last_column = 28
    * last_line = 9
    * path = "/etc/icinga2/zones.conf"
  * templates = [ "nbb-vm-013" ]
    % = modified in '/etc/icinga2/zones.conf', lines 9:1-9:28
  * type = "Endpoint"
  * zone = ""

on master
icinga2 object list --type Zone

Object 'nbb-vm-013' of type 'Zone':
  % declared in '/etc/icinga2/zones.conf', lines 18:1-18:24
  * __name = "nbb-vm-013"
  * endpoints = [ "nbb-vm-013" ]
    % = modified in '/etc/icinga2/zones.conf', lines 19:2-19:29
  * global = false
  * name = "nbb-vm-013"
  * package = "_etc"
  * parent = "master"
    % = modified in '/etc/icinga2/zones.conf', lines 20:2-20:18
  * source_location
    * first_column = 1
    * first_line = 18
    * last_column = 24
    * last_line = 18
    * path = "/etc/icinga2/zones.conf"
  * templates = [ "nbb-vm-013" ]
    % = modified in '/etc/icinga2/zones.conf', lines 18:1-18:24
  * type = "Zone"
  * zone = ""

Object 'master' of type 'Zone':
  % declared in '/etc/icinga2/zones.conf', lines 14:1-14:20
  * __name = "master"
  * endpoints = [ "esn-svm037" ]
    % = modified in '/etc/icinga2/zones.conf', lines 15:9-15:36
  * global = false
  * name = "master"
  * package = "_etc"
  * parent = ""
  * source_location
    * first_column = 1
    * first_line = 14
    * last_column = 20
    * last_line = 14
    * path = "/etc/icinga2/zones.conf"
  * templates = [ "master" ]
    % = modified in '/etc/icinga2/zones.conf', lines 14:1-14:20
  * type = "Zone"
  * zone = ""

Object 'global-templates' of type 'Zone':
  % declared in '/etc/icinga2/zones.conf', lines 23:1-23:30
  * __name = "global-templates"
  * endpoints = null
  * global = true
    % = modified in '/etc/icinga2/zones.conf', lines 24:2-24:14
  * name = "global-templates"
  * package = "_etc"
  * parent = ""
  * source_location
    * first_column = 1
    * first_line = 23
    * last_column = 30
    * last_line = 23
    * path = "/etc/icinga2/zones.conf"
  * templates = [ "global-templates" ]
    % = modified in '/etc/icinga2/zones.conf', lines 23:1-23:30
  * type = "Zone"
  * zone = ""

Object 'director-global' of type 'Zone':
  % declared in '/etc/icinga2/zones.conf', lines 27:1-27:29
  * __name = "director-global"
  * endpoints = null
  * global = true
    % = modified in '/etc/icinga2/zones.conf', lines 28:2-28:14
  * name = "director-global"
  * package = "_etc"
  * parent = ""
  * source_location
    * first_column = 1
    * first_line = 27
    * last_column = 29
    * last_line = 27
    * path = "/etc/icinga2/zones.conf"
  * templates = [ "director-global" ]
    % = modified in '/etc/icinga2/zones.conf', lines 27:1-27:29
  * type = "Zone"
  * zone = ""

this works on satellite

object Host "imos_mc1_nbb" {
        import "generic-host"
        address = "imos1.multiconnect.de"
        vars.agent_endpoint = "nbb-vm-013"
        vars.location = "nb"
}
object Host "imos_mc2_nbb" {
        import "generic-host"
        address = "imos2.multiconnect.de"
        vars.agent_endpoint = "nbb-vm-013"
        vars.location = "nb"
}

This not

object Host "nbb-vm-73-019" {
   import "generic-linux"
   address = "10.120.73.19"
        vars.os = "Linux"
        vars.agent_endpoint = "nbb-vm-013"
        vars.department = "tools"
        vars.location = "nb"
        vars.procs_linux = true
        vars.hostgroup = "yatu"
}

apply Service "mem-sat" {
   import "generic-service"
   check_command = "nrpe"
   vars.nrpe_command = "check_memory"
   vars.nrpe_arguments = [ "-w $mem_warning$ -c $mem_critical$" ]
   assign where host.vars.os == "Linux" && host.vars.agent_endpoint
   ignore where host.address == "127.0.0.1"
}


there is no checksource notified
grafik

here is completely wrong check source inside but the right agent_endpoint
grafik

Hey,

what does systemctl status icinga2 say? Can you ping the satelite with ping YOUR_IP over the cli of the master?

Hello,
sure can I ping the satellite and the icinga2 status is running.
Both are sync here the latest part icinga2.log of my first satellite.
Regards Rico

[2022-05-10 13:11:53 +0200] warning/JsonRpcConnection: API client disconnected for identity 'esn-svm037'                                                                               │
│[2022-05-10 13:11:53 +0200] warning/ApiListener: Removing API client for endpoint 'esn-svm037'. 0 API clients left.                                                                    │
│[2022-05-10 13:12:02 +0200] information/ApiListener: Reconnecting to endpoint 'esn-svm037' via host '10.88.183.37' and port '5665'                                                     │
│[2022-05-10 13:12:02 +0200] information/ApiListener: New client connection for identity 'esn-svm037' to [10.88.183.37]:5665                                                            │
│[2022-05-10 13:12:02 +0200] information/ApiListener: Finished reconnecting to endpoint 'esn-svm037' via host '10.88.183.37' and port '5665'                                            │
│[2022-05-10 13:12:02 +0200] information/JsonRpcConnection: Requesting new certificate for this Icinga instance from endpoint 'esn-svm037'.                                             │
│[2022-05-10 13:12:02 +0200] information/ApiListener: Sending config updates for endpoint 'esn-svm037' in zone 'master'.                                                                │
│[2022-05-10 13:12:02 +0200] information/ApiListener: Finished sending config file updates for endpoint 'esn-svm037' in zone 'master'.                                                  │
│[2022-05-10 13:12:02 +0200] information/ApiListener: Syncing runtime objects to endpoint 'esn-svm037'.                                                                                 │
│[2022-05-10 13:12:02 +0200] information/ApiListener: Finished syncing runtime objects to endpoint 'esn-svm037'.                                                                        │
│[2022-05-10 13:12:02 +0200] information/ApiListener: Finished sending runtime config updates for endpoint 'esn-svm037' in zone 'master'.                                               │
│[2022-05-10 13:12:02 +0200] information/ApiListener: Sending replay log for endpoint 'esn-svm037' in zone 'master'.                                                                    │
│[2022-05-10 13:12:02 +0200] information/ApiListener: Replayed 14 messages.                                                                                                             │
│[2022-05-10 13:12:02 +0200] information/ApiListener: Finished sending replay log for endpoint 'esn-svm037' in zone 'master'.                                                           │
│[2022-05-10 13:12:02 +0200] information/ApiListener: Finished syncing endpoint 'esn-svm037' in zone 'master'.                                                                          │
│[2022-05-10 13:12:02 +0200] information/ApiListener: Applying config update from endpoint 'esn-svm037' of zone 'master'.                                                               │
│[2022-05-10 13:12:02 +0200] information/ApiListener: Received configuration for zone 'director-global' from endpoint 'esn-svm037'. Comparing the timestamp and checksums.              │
│[2022-05-10 13:12:02 +0200] information/ApiListener: Stage: Updating received configuration file '/var/lib/icinga2/api/zones-stage/director-global//director/001-director-basics.conf' │
│[2022-05-10 13:12:02 +0200] information/ApiListener: Stage: Updating received configuration file '/var/lib/icinga2/api/zones-stage/director-global//director/host_templates.conf' for z│
│[2022-05-10 13:12:02 +0200] information/ApiListener: Stage: Updating received configuration file '/var/lib/icinga2/api/zones-stage/director-global//director/service_apply.conf' for zo│
│[2022-05-10 13:12:02 +0200] information/ApiListener: Stage: Updating received configuration file '/var/lib/icinga2/api/zones-stage/director-global//director/service_templates.conf' fo│
│[2022-05-10 13:12:02 +0200] information/ApiListener: Applying configuration file update for path '/var/lib/icinga2/api/zones-stage/director-global' (5183 Bytes).                      │
│[2022-05-10 13:12:02 +0200] information/ApiListener: Received configuration for zone 'global-templates' from endpoint 'esn-svm037'. Comparing the timestamp and checksums.             │
│[2022-05-10 13:12:02 +0200] information/ApiListener: Stage: Updating received configuration file '/var/lib/icinga2/api/zones-stage/global-templates//_etc/allnet-temp.conf' for zone 'g│
│[2022-05-10 13:12:02 +0200] information/ApiListener: Stage: Updating received configuration file '/var/lib/icinga2/api/zones-stage/global-templates//_etc/cisco.conf' for zone 'global-│
│[2022-05-10 13:12:02 +0200] information/ApiListener: Stage: Updating received configuration file '/var/lib/icinga2/api/zones-stage/global-templates//_etc/development.conf' for zone 'g│
│[2022-05-10 13:12:02 +0200] information/ApiListener: Stage: Updating received configuration file '/var/lib/icinga2/api/zones-stage/global-templates//_etc/generic-host-no-ping.conf' fo│
│[2022-05-10 13:12:02 +0200] information/ApiListener: Stage: Updating received configuration file '/var/lib/icinga2/api/zones-stage/global-templates//_etc/generic-host.conf' for zone '│
│[2022-05-10 13:12:02 +0200] information/ApiListener: Stage: Updating received configuration file '/var/lib/icinga2/api/zones-stage/global-templates//_etc/generic-linux.conf' for zone │
│[2022-05-10 13:12:02 +0200] information/ApiListener: Stage: Updating received configuration file '/var/lib/icinga2/api/zones-stage/global-templates//_etc/generic-ubuntu.conf' for zone│
│[2022-05-10 13:12:02 +0200] information/ApiListener: Stage: Updating received configuration file '/var/lib/icinga2/api/zones-stage/global-templates//_etc/generic-windows.conf' for zon│
│[2022-05-10 13:12:02 +0200] information/ApiListener: Stage: Updating received configuration file '/var/lib/icinga2/api/zones-stage/global-templates//_etc/notification.conf' for zone '│
│[2022-05-10 13:12:02 +0200] information/ApiListener: Stage: Updating received configuration file '/var/lib/icinga2/api/zones-stage/global-templates//_etc/ntp.conf' for zone 'global-te│
│[2022-05-10 13:12:02 +0200] information/ApiListener: Stage: Updating received configuration file '/var/lib/icinga2/api/zones-stage/global-templates//_etc/raritan-pdu.conf' for zone 'g│
│[2022-05-10 13:12:02 +0200] information/ApiListener: Applying configuration file update for path '/var/lib/icinga2/api/zones-stage/global-templates' (5003 Bytes).                     │
│[2022-05-10 13:12:02 +0200] information/ApiListener: Received configuration for zone 'nbb-vm-013' from endpoint 'esn-svm037'. Comparing the timestamp and checksums.                   │
│[2022-05-10 13:12:02 +0200] information/ApiListener: Stage: Updating received configuration file '/var/lib/icinga2/api/zones-stage/nbb-vm-013//_etc/links.conf' for zone 'nbb-vm-013'. │
│[2022-05-10 13:12:02 +0200] information/ApiListener: Stage: Updating received configuration file '/var/lib/icinga2/api/zones-stage/nbb-vm-013//_etc/satellite.conf' for zone 'nbb-vm-01│
│[2022-05-10 13:12:02 +0200] information/ApiListener: Stage: Updating received configuration file '/var/lib/icinga2/api/zones-stage/nbb-vm-013//_etc/test.conf' for zone 'nbb-vm-013'.  │
│[2022-05-10 13:12:02 +0200] information/ApiListener: Stage: Updating received configuration file '/var/lib/icinga2/api/zones-stage/nbb-vm-013//director/hosts.conf' for zone 'nbb-vm-01│
│[2022-05-10 13:12:02 +0200] information/ApiListener: Applying configuration file update for path '/var/lib/icinga2/api/zones-stage/nbb-vm-013' (7875 Bytes).                           │
│[2022-05-10 13:12:02 +0200] information/ApiListener: Received configuration updates (3) from endpoint 'esn-svm037' are different to production, triggering validation and reload.      │
│[2022-05-10 13:12:02 +0200] critical/ApiListener: Config validation failed for staged cluster config sync in '/var/lib/icinga2/api/zones-stage/'. Aborting. Logs: '/var/lib/icinga2/api│
│[2022-05-10 13:16:01 +0200] information/ConfigObject: Dumping program state to file '/var/lib/icinga2/icinga2.state'                                                                   │
│[2022-05-10 13:16:11 +0200] information/WorkQueue: #6 (ApiListener, SyncQueue) items: 0, rate:  0/s (0/min 0/5min 0/15min);                                                            │
│[2022-05-10 13:16:11 +0200] information/WorkQueue: #5 (ApiListener, RelayQueue) items: 0, rate: 1.06667/s (64/min 320/5min 968/15min); 

There is sth. wrong in the config so it cannot be synced to the zone. maybe you have defined sth. that is not known in the specific zone, what say the logs?

Hello Marcus,
this was the right hint. This is my first satellite. I don’t know where additional log’s on satellite exist.
One template on master was in the wrong folder (gerneric-service). That was the reason, why no checks runing.

Thank you and best regards
Rico