=== RUN TestMultiControlPlane/serial/HAppyAfterSecondaryNodeRestart
ha_test.go:281: (dbg) Run: out/minikube-linux-arm64 profile list --output json
ha_test.go:281: (dbg) Done: out/minikube-linux-arm64 profile list --output json: (1.19106438s)
ha_test.go:309: expected profile "ha-472819" in json of 'profile list' to have "HAppy" status but have "Degraded" status. got *"{\"invalid\":[],\"valid\":[{\"Name\":\"ha-472819\",\"Status\":\"Degraded\",\"Config\":{\"Name\":\"ha-472819\",\"KeepContext\":false,\"EmbedCerts\":false,\"MinikubeISO\":\"\",\"KicBaseImage\":\"gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8\",\"Memory\":3072,\"CPUs\":2,\"DiskSize\":20000,\"Driver\":\"docker\",\"HyperkitVpnKitSock\":\"\",\"HyperkitVSockPorts\":[],\"DockerEnv\":null,\"ContainerVolumeMounts\":null,\"InsecureRegistry\":null,\"RegistryMirror\":[],\"HostOnlyCIDR\":\"192.168.59.1/24\",\"HypervVirtualSwitch\":\"\",\"HypervUseExternalSwitch\":false,\"HypervExternalAdapter\":\"\",\"KVMNetwork\":\"default\",\"KVMQemuURI\":\"qemu:///system\",\"KVMGPU\":false,\"KVMHidden\":false,\"KVMNUMACount\":1,\"APIServerPort\":8443,\"DockerOpt\":null,\"DisableDriverMounts\":false,\"NFSShare\":[],\"NFSShar
esRoot\":\"/nfsshares\",\"UUID\":\"\",\"NoVTXCheck\":false,\"DNSProxy\":false,\"HostDNSResolver\":true,\"HostOnlyNicType\":\"virtio\",\"NatNicType\":\"virtio\",\"SSHIPAddress\":\"\",\"SSHUser\":\"root\",\"SSHKey\":\"\",\"SSHPort\":22,\"KubernetesConfig\":{\"KubernetesVersion\":\"v1.34.1\",\"ClusterName\":\"ha-472819\",\"Namespace\":\"default\",\"APIServerHAVIP\":\"192.168.49.254\",\"APIServerName\":\"minikubeCA\",\"APIServerNames\":null,\"APIServerIPs\":null,\"DNSDomain\":\"cluster.local\",\"ContainerRuntime\":\"crio\",\"CRISocket\":\"\",\"NetworkPlugin\":\"cni\",\"FeatureGates\":\"\",\"ServiceCIDR\":\"10.96.0.0/12\",\"ImageRepository\":\"\",\"LoadBalancerStartIP\":\"\",\"LoadBalancerEndIP\":\"\",\"CustomIngressCert\":\"\",\"RegistryAliases\":\"\",\"ExtraOptions\":null,\"ShouldLoadCachedImages\":true,\"EnableDefaultCNI\":false,\"CNI\":\"\"},\"Nodes\":[{\"Name\":\"\",\"IP\":\"192.168.49.2\",\"Port\":8443,\"KubernetesVersion\":\"v1.34.1\",\"ContainerRuntime\":\"crio\",\"ControlPlane\":true,\"Worker\":true},{\"N
ame\":\"m02\",\"IP\":\"192.168.49.3\",\"Port\":8443,\"KubernetesVersion\":\"v1.34.1\",\"ContainerRuntime\":\"crio\",\"ControlPlane\":true,\"Worker\":true},{\"Name\":\"m03\",\"IP\":\"192.168.49.4\",\"Port\":8443,\"KubernetesVersion\":\"v1.34.1\",\"ContainerRuntime\":\"crio\",\"ControlPlane\":true,\"Worker\":true},{\"Name\":\"m04\",\"IP\":\"192.168.49.5\",\"Port\":0,\"KubernetesVersion\":\"v1.34.1\",\"ContainerRuntime\":\"\",\"ControlPlane\":false,\"Worker\":true}],\"Addons\":{\"ambassador\":false,\"amd-gpu-device-plugin\":false,\"auto-pause\":false,\"cloud-spanner\":false,\"csi-hostpath-driver\":false,\"dashboard\":false,\"default-storageclass\":false,\"efk\":false,\"freshpod\":false,\"gcp-auth\":false,\"gvisor\":false,\"headlamp\":false,\"inaccel\":false,\"ingress\":false,\"ingress-dns\":false,\"inspektor-gadget\":false,\"istio\":false,\"istio-provisioner\":false,\"kong\":false,\"kubeflow\":false,\"kubetail\":false,\"kubevirt\":false,\"logviewer\":false,\"metallb\":false,\"metrics-server\":false,\"nvidia-devi
ce-plugin\":false,\"nvidia-driver-installer\":false,\"nvidia-gpu-device-plugin\":false,\"olm\":false,\"pod-security-policy\":false,\"portainer\":false,\"registry\":false,\"registry-aliases\":false,\"registry-creds\":false,\"storage-provisioner\":false,\"storage-provisioner-rancher\":false,\"volcano\":false,\"volumesnapshots\":false,\"yakd\":false},\"CustomAddonImages\":null,\"CustomAddonRegistries\":null,\"VerifyComponents\":{\"apiserver\":true,\"apps_running\":true,\"default_sa\":true,\"extra\":true,\"kubelet\":true,\"node_ready\":true,\"system_pods\":true},\"StartHostTimeout\":360000000000,\"ScheduledStop\":null,\"ExposedPorts\":[],\"ListenAddress\":\"\",\"Network\":\"\",\"Subnet\":\"\",\"MultiNodeRequested\":true,\"ExtraDisks\":0,\"CertExpiration\":94608000000000000,\"MountString\":\"\",\"Mount9PVersion\":\"9p2000.L\",\"MountGID\":\"docker\",\"MountIP\":\"\",\"MountMSize\":262144,\"MountOptions\":[],\"MountPort\":0,\"MountType\":\"9p\",\"MountUID\":\"docker\",\"BinaryMirror\":\"\",\"DisableOptimizations\":
false,\"DisableMetrics\":false,\"DisableCoreDNSLog\":false,\"CustomQemuFirmwarePath\":\"\",\"SocketVMnetClientPath\":\"\",\"SocketVMnetPath\":\"\",\"StaticIP\":\"\",\"SSHAuthSock\":\"\",\"SSHAgentPID\":0,\"GPUs\":\"\",\"AutoPauseInterval\":60000000000},\"Active\":false,\"ActiveKubeContext\":true}]}"*. args: "out/minikube-linux-arm64 profile list --output json"
helpers_test.go:222: -----------------------post-mortem--------------------------------
helpers_test.go:223: ======> post-mortem[TestMultiControlPlane/serial/HAppyAfterSecondaryNodeRestart]: network settings <======
helpers_test.go:230: HOST ENV snapshots: PROXY env: HTTP_PROXY="<empty>" HTTPS_PROXY="<empty>" NO_PROXY="<empty>"
helpers_test.go:238: ======> post-mortem[TestMultiControlPlane/serial/HAppyAfterSecondaryNodeRestart]: docker inspect <======
helpers_test.go:239: (dbg) Run: docker inspect ha-472819
helpers_test.go:243: (dbg) docker inspect ha-472819:
-- stdout --
[
{
"Id": "66de5fe90fef65b9b7fdcec88f1cb31289b5fd1d95cc881b8beb6ec5f94ceb5c",
"Created": "2025-11-01T11:09:20.899997169Z",
"Path": "/usr/local/bin/entrypoint",
"Args": [
"/sbin/init"
],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 564549,
"ExitCode": 0,
"Error": "",
"StartedAt": "2025-11-01T11:09:20.960423395Z",
"FinishedAt": "0001-01-01T00:00:00Z"
},
"Image": "sha256:1b8004df0b408966a254b2ecd4551aa85aaac4627e7e9cb1cefc14dfe51ec273",
"ResolvConfPath": "/var/lib/docker/containers/66de5fe90fef65b9b7fdcec88f1cb31289b5fd1d95cc881b8beb6ec5f94ceb5c/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/66de5fe90fef65b9b7fdcec88f1cb31289b5fd1d95cc881b8beb6ec5f94ceb5c/hostname",
"HostsPath": "/var/lib/docker/containers/66de5fe90fef65b9b7fdcec88f1cb31289b5fd1d95cc881b8beb6ec5f94ceb5c/hosts",
"LogPath": "/var/lib/docker/containers/66de5fe90fef65b9b7fdcec88f1cb31289b5fd1d95cc881b8beb6ec5f94ceb5c/66de5fe90fef65b9b7fdcec88f1cb31289b5fd1d95cc881b8beb6ec5f94ceb5c-json.log",
"Name": "/ha-472819",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "unconfined",
"ExecIDs": null,
"HostConfig": {
"Binds": [
"/lib/modules:/lib/modules:ro",
"ha-472819:/var"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {}
},
"NetworkMode": "ha-472819",
"PortBindings": {
"22/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": ""
}
],
"2376/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": ""
}
],
"32443/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": ""
}
],
"5000/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": ""
}
],
"8443/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": ""
}
]
},
"RestartPolicy": {
"Name": "no",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"ConsoleSize": [
0,
0
],
"CapAdd": null,
"CapDrop": null,
"CgroupnsMode": "host",
"Dns": [],
"DnsOptions": [],
"DnsSearch": [],
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "private",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": true,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": [
"seccomp=unconfined",
"apparmor=unconfined",
"label=disable"
],
"Tmpfs": {
"/run": "",
"/tmp": ""
},
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"Isolation": "",
"CpuShares": 0,
"Memory": 3221225472,
"NanoCpus": 2000000000,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": [],
"BlkioDeviceReadBps": [],
"BlkioDeviceWriteBps": [],
"BlkioDeviceReadIOps": [],
"BlkioDeviceWriteIOps": [],
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": [],
"DeviceCgroupRules": null,
"DeviceRequests": null,
"MemoryReservation": 0,
"MemorySwap": 6442450944,
"MemorySwappiness": null,
"OomKillDisable": false,
"PidsLimit": null,
"Ulimits": [],
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"MaskedPaths": null,
"ReadonlyPaths": null
},
"GraphDriver": {
"Data": {
"ID": "66de5fe90fef65b9b7fdcec88f1cb31289b5fd1d95cc881b8beb6ec5f94ceb5c",
"LowerDir": "/var/lib/docker/overlay2/b2b4ec64838dd5e359c9159df7be29d4c92c2974901ee3965fdfb4d3899d9b98-init/diff:/var/lib/docker/overlay2/21d61574f17b4b99b161ba06788eed27ff2ed4cd88f8f323107c5ef7407644f1/diff",
"MergedDir": "/var/lib/docker/overlay2/b2b4ec64838dd5e359c9159df7be29d4c92c2974901ee3965fdfb4d3899d9b98/merged",
"UpperDir": "/var/lib/docker/overlay2/b2b4ec64838dd5e359c9159df7be29d4c92c2974901ee3965fdfb4d3899d9b98/diff",
"WorkDir": "/var/lib/docker/overlay2/b2b4ec64838dd5e359c9159df7be29d4c92c2974901ee3965fdfb4d3899d9b98/work"
},
"Name": "overlay2"
},
"Mounts": [
{
"Type": "bind",
"Source": "/lib/modules",
"Destination": "/lib/modules",
"Mode": "ro",
"RW": false,
"Propagation": "rprivate"
},
{
"Type": "volume",
"Name": "ha-472819",
"Source": "/var/lib/docker/volumes/ha-472819/_data",
"Destination": "/var",
"Driver": "local",
"Mode": "z",
"RW": true,
"Propagation": ""
}
],
"Config": {
"Hostname": "ha-472819",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"22/tcp": {},
"2376/tcp": {},
"32443/tcp": {},
"5000/tcp": {},
"8443/tcp": {}
},
"Tty": true,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"container=docker",
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
],
"Cmd": null,
"Image": "gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8",
"Volumes": null,
"WorkingDir": "/",
"Entrypoint": [
"/usr/local/bin/entrypoint",
"/sbin/init"
],
"OnBuild": null,
"Labels": {
"created_by.minikube.sigs.k8s.io": "true",
"mode.minikube.sigs.k8s.io": "ha-472819",
"name.minikube.sigs.k8s.io": "ha-472819",
"role.minikube.sigs.k8s.io": ""
},
"StopSignal": "SIGRTMIN+3"
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "2d06c75db1f88cbad3b99e1d3febd830132bbd4294bd314a091e234e9ed41115",
"SandboxKey": "/var/run/docker/netns/2d06c75db1f8",
"Ports": {
"22/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "33510"
}
],
"2376/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "33511"
}
],
"32443/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "33514"
}
],
"5000/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "33512"
}
],
"8443/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "33513"
}
]
},
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "",
"Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"MacAddress": "",
"Networks": {
"ha-472819": {
"IPAMConfig": {
"IPv4Address": "192.168.49.2"
},
"Links": null,
"Aliases": null,
"MacAddress": "0a:12:7f:3f:18:1d",
"DriverOpts": null,
"GwPriority": 0,
"NetworkID": "fad877b9a6cbf2fecd3371f8a88631aadb56e394476f97473ad152037f12fe08",
"EndpointID": "3d75015284989adc37a7194f7d4e42693d55ecd110cde90b4ea89049faa60f3e",
"Gateway": "192.168.49.1",
"IPAddress": "192.168.49.2",
"IPPrefixLen": 24,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"DNSNames": [
"ha-472819",
"66de5fe90fef"
]
}
}
}
}
]
-- /stdout --
helpers_test.go:247: (dbg) Run: out/minikube-linux-arm64 status --format={{.Host}} -p ha-472819 -n ha-472819
helpers_test.go:252: <<< TestMultiControlPlane/serial/HAppyAfterSecondaryNodeRestart FAILED: start of post-mortem logs <<<
helpers_test.go:253: ======> post-mortem[TestMultiControlPlane/serial/HAppyAfterSecondaryNodeRestart]: minikube logs <======
helpers_test.go:255: (dbg) Run: out/minikube-linux-arm64 -p ha-472819 logs -n 25
helpers_test.go:255: (dbg) Done: out/minikube-linux-arm64 -p ha-472819 logs -n 25: (1.375189806s)
helpers_test.go:260: TestMultiControlPlane/serial/HAppyAfterSecondaryNodeRestart logs:
-- stdout --
==> Audit <==
┌─────────┬──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┬───────────┬─────────┬─────────┬─────────────────────┬─────────────────────┐
│ COMMAND │ ARGS │ PROFILE │ USER │ VERSION │ START TIME │ END TIME │
├─────────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┼───────────┼─────────┼─────────┼─────────────────────┼─────────────────────┤
│ ssh │ ha-472819 ssh -n ha-472819-m03 sudo cat /home/docker/cp-test.txt │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:13 UTC │ 01 Nov 25 11:13 UTC │
│ cp │ ha-472819 cp ha-472819-m03:/home/docker/cp-test.txt ha-472819:/home/docker/cp-test_ha-472819-m03_ha-472819.txt │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:13 UTC │ 01 Nov 25 11:13 UTC │
│ ssh │ ha-472819 ssh -n ha-472819-m03 sudo cat /home/docker/cp-test.txt │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:13 UTC │ 01 Nov 25 11:13 UTC │
│ ssh │ ha-472819 ssh -n ha-472819 sudo cat /home/docker/cp-test_ha-472819-m03_ha-472819.txt │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:13 UTC │ 01 Nov 25 11:13 UTC │
│ cp │ ha-472819 cp ha-472819-m03:/home/docker/cp-test.txt ha-472819-m02:/home/docker/cp-test_ha-472819-m03_ha-472819-m02.txt │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:13 UTC │ 01 Nov 25 11:13 UTC │
│ ssh │ ha-472819 ssh -n ha-472819-m03 sudo cat /home/docker/cp-test.txt │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:13 UTC │ 01 Nov 25 11:13 UTC │
│ ssh │ ha-472819 ssh -n ha-472819-m02 sudo cat /home/docker/cp-test_ha-472819-m03_ha-472819-m02.txt │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:13 UTC │ 01 Nov 25 11:13 UTC │
│ cp │ ha-472819 cp ha-472819-m03:/home/docker/cp-test.txt ha-472819-m04:/home/docker/cp-test_ha-472819-m03_ha-472819-m04.txt │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:13 UTC │ 01 Nov 25 11:13 UTC │
│ ssh │ ha-472819 ssh -n ha-472819-m03 sudo cat /home/docker/cp-test.txt │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:13 UTC │ 01 Nov 25 11:13 UTC │
│ ssh │ ha-472819 ssh -n ha-472819-m04 sudo cat /home/docker/cp-test_ha-472819-m03_ha-472819-m04.txt │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:13 UTC │ 01 Nov 25 11:14 UTC │
│ cp │ ha-472819 cp testdata/cp-test.txt ha-472819-m04:/home/docker/cp-test.txt │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:14 UTC │ 01 Nov 25 11:14 UTC │
│ ssh │ ha-472819 ssh -n ha-472819-m04 sudo cat /home/docker/cp-test.txt │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:14 UTC │ 01 Nov 25 11:14 UTC │
│ cp │ ha-472819 cp ha-472819-m04:/home/docker/cp-test.txt /tmp/TestMultiControlPlaneserialCopyFile3224874569/001/cp-test_ha-472819-m04.txt │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:14 UTC │ 01 Nov 25 11:14 UTC │
│ ssh │ ha-472819 ssh -n ha-472819-m04 sudo cat /home/docker/cp-test.txt │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:14 UTC │ 01 Nov 25 11:14 UTC │
│ cp │ ha-472819 cp ha-472819-m04:/home/docker/cp-test.txt ha-472819:/home/docker/cp-test_ha-472819-m04_ha-472819.txt │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:14 UTC │ 01 Nov 25 11:14 UTC │
│ ssh │ ha-472819 ssh -n ha-472819-m04 sudo cat /home/docker/cp-test.txt │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:14 UTC │ 01 Nov 25 11:14 UTC │
│ ssh │ ha-472819 ssh -n ha-472819 sudo cat /home/docker/cp-test_ha-472819-m04_ha-472819.txt │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:14 UTC │ 01 Nov 25 11:14 UTC │
│ cp │ ha-472819 cp ha-472819-m04:/home/docker/cp-test.txt ha-472819-m02:/home/docker/cp-test_ha-472819-m04_ha-472819-m02.txt │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:14 UTC │ 01 Nov 25 11:14 UTC │
│ ssh │ ha-472819 ssh -n ha-472819-m04 sudo cat /home/docker/cp-test.txt │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:14 UTC │ 01 Nov 25 11:14 UTC │
│ ssh │ ha-472819 ssh -n ha-472819-m02 sudo cat /home/docker/cp-test_ha-472819-m04_ha-472819-m02.txt │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:14 UTC │ 01 Nov 25 11:14 UTC │
│ cp │ ha-472819 cp ha-472819-m04:/home/docker/cp-test.txt ha-472819-m03:/home/docker/cp-test_ha-472819-m04_ha-472819-m03.txt │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:14 UTC │ 01 Nov 25 11:14 UTC │
│ ssh │ ha-472819 ssh -n ha-472819-m04 sudo cat /home/docker/cp-test.txt │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:14 UTC │ 01 Nov 25 11:14 UTC │
│ ssh │ ha-472819 ssh -n ha-472819-m03 sudo cat /home/docker/cp-test_ha-472819-m04_ha-472819-m03.txt │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:14 UTC │ 01 Nov 25 11:14 UTC │
│ node │ ha-472819 node stop m02 --alsologtostderr -v 5 │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:14 UTC │ 01 Nov 25 11:14 UTC │
│ node │ ha-472819 node start m02 --alsologtostderr -v 5 │ ha-472819 │ jenkins │ v1.37.0 │ 01 Nov 25 11:14 UTC │ │
└─────────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┴───────────┴─────────┴─────────┴─────────────────────┴─────────────────────┘
==> Last Start <==
Log file created at: 2025/11/01 11:09:15
Running on machine: ip-172-31-31-251
Binary: Built with gc go1.24.6 for linux/arm64
Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg
I1101 11:09:15.424948 564163 out.go:360] Setting OutFile to fd 1 ...
I1101 11:09:15.425098 564163 out.go:408] TERM=,COLORTERM=, which probably does not support color
I1101 11:09:15.425112 564163 out.go:374] Setting ErrFile to fd 2...
I1101 11:09:15.425118 564163 out.go:408] TERM=,COLORTERM=, which probably does not support color
I1101 11:09:15.425408 564163 root.go:338] Updating PATH: /home/jenkins/minikube-integration/21830-532863/.minikube/bin
I1101 11:09:15.425949 564163 out.go:368] Setting JSON to false
I1101 11:09:15.426851 564163 start.go:133] hostinfo: {"hostname":"ip-172-31-31-251","uptime":10305,"bootTime":1761985051,"procs":150,"os":"linux","platform":"ubuntu","platformFamily":"debian","platformVersion":"20.04","kernelVersion":"5.15.0-1084-aws","kernelArch":"aarch64","virtualizationSystem":"","virtualizationRole":"","hostId":"982e3628-3742-4b3e-bb63-ac1b07660ec7"}
I1101 11:09:15.426921 564163 start.go:143] virtualization:
I1101 11:09:15.433995 564163 out.go:179] * [ha-472819] minikube v1.37.0 on Ubuntu 20.04 (arm64)
I1101 11:09:15.437863 564163 out.go:179] - MINIKUBE_LOCATION=21830
I1101 11:09:15.437980 564163 notify.go:221] Checking for updates...
I1101 11:09:15.444935 564163 out.go:179] - MINIKUBE_SUPPRESS_DOCKER_PERFORMANCE=true
I1101 11:09:15.448333 564163 out.go:179] - KUBECONFIG=/home/jenkins/minikube-integration/21830-532863/kubeconfig
I1101 11:09:15.451645 564163 out.go:179] - MINIKUBE_HOME=/home/jenkins/minikube-integration/21830-532863/.minikube
I1101 11:09:15.454845 564163 out.go:179] - MINIKUBE_BIN=out/minikube-linux-arm64
I1101 11:09:15.458142 564163 out.go:179] - MINIKUBE_FORCE_SYSTEMD=
I1101 11:09:15.461449 564163 driver.go:422] Setting default libvirt URI to qemu:///system
I1101 11:09:15.483754 564163 docker.go:124] docker version: linux-28.1.1:Docker Engine - Community
I1101 11:09:15.483880 564163 cli_runner.go:164] Run: docker system info --format "{{json .}}"
I1101 11:09:15.548573 564163 info.go:266] docker info: {ID:EOU5:DNGX:XN6V:L2FZ:UXRM:5TWK:EVUR:KC2F:GT7Z:Y4O4:GB77:5PD3 Containers:0 ContainersRunning:0 ContainersPaused:0 ContainersStopped:0 Images:3 Driver:overlay2 DriverStatus:[[Backing Filesystem extfs] [Supports d_type true] [Using metacopy false] [Native Overlay Diff true] [userxattr false]] SystemStatus:<nil> Plugins:{Volume:[local] Network:[bridge host ipvlan macvlan null overlay] Authorization:<nil> Log:[awslogs fluentd gcplogs gelf journald json-file local splunk syslog]} MemoryLimit:true SwapLimit:true KernelMemory:false KernelMemoryTCP:true CPUCfsPeriod:true CPUCfsQuota:true CPUShares:true CPUSet:true PidsLimit:true IPv4Forwarding:true BridgeNfIptables:false BridgeNfIP6Tables:false Debug:false NFd:23 OomKillDisable:true NGoroutines:42 SystemTime:2025-11-01 11:09:15.539247568 +0000 UTC LoggingDriver:json-file CgroupDriver:cgroupfs NEventsListener:0 KernelVersion:5.15.0-1084-aws OperatingSystem:Ubuntu 20.04.6 LTS OSType:linux Architecture:a
arch64 IndexServerAddress:https://index.docker.io/v1/ RegistryConfig:{AllowNondistributableArtifactsCIDRs:[] AllowNondistributableArtifactsHostnames:[] InsecureRegistryCIDRs:[::1/128 127.0.0.0/8] IndexConfigs:{DockerIo:{Name:docker.io Mirrors:[] Secure:true Official:true}} Mirrors:[]} NCPU:2 MemTotal:8214831104 GenericResources:<nil> DockerRootDir:/var/lib/docker HTTPProxy: HTTPSProxy: NoProxy: Name:ip-172-31-31-251 Labels:[] ExperimentalBuild:false ServerVersion:28.1.1 ClusterStore: ClusterAdvertise: Runtimes:{Runc:{Path:runc}} DefaultRuntime:runc Swarm:{NodeID: NodeAddr: LocalNodeState:inactive ControlAvailable:false Error: RemoteManagers:<nil>} LiveRestoreEnabled:false Isolation: InitBinary:docker-init ContainerdCommit:{ID:05044ec0a9a75232cad458027ca83437aae3f4da Expected:} RuncCommit:{ID:v1.2.5-0-g59923ef Expected:} InitCommit:{ID:de40ad0 Expected:} SecurityOptions:[name=apparmor name=seccomp,profile=builtin] ProductLicense: Warnings:<nil> ServerErrors:[] ClientInfo:{Debug:false Plugins:[map[Name:buildx P
ath:/usr/libexec/docker/cli-plugins/docker-buildx SchemaVersion:0.1.0 ShortDescription:Docker Buildx Vendor:Docker Inc. Version:v0.23.0] map[Name:compose Path:/usr/libexec/docker/cli-plugins/docker-compose SchemaVersion:0.1.0 ShortDescription:Docker Compose Vendor:Docker Inc. Version:v2.35.1]] Warnings:<nil>}}
I1101 11:09:15.548693 564163 docker.go:319] overlay module found
I1101 11:09:15.552124 564163 out.go:179] * Using the docker driver based on user configuration
I1101 11:09:15.555278 564163 start.go:309] selected driver: docker
I1101 11:09:15.555310 564163 start.go:930] validating driver "docker" against <nil>
I1101 11:09:15.555327 564163 start.go:941] status for docker: {Installed:true Healthy:true Running:false NeedsImprovement:false Error:<nil> Reason: Fix: Doc: Version:}
I1101 11:09:15.556133 564163 cli_runner.go:164] Run: docker system info --format "{{json .}}"
I1101 11:09:15.623357 564163 info.go:266] docker info: {ID:EOU5:DNGX:XN6V:L2FZ:UXRM:5TWK:EVUR:KC2F:GT7Z:Y4O4:GB77:5PD3 Containers:0 ContainersRunning:0 ContainersPaused:0 ContainersStopped:0 Images:3 Driver:overlay2 DriverStatus:[[Backing Filesystem extfs] [Supports d_type true] [Using metacopy false] [Native Overlay Diff true] [userxattr false]] SystemStatus:<nil> Plugins:{Volume:[local] Network:[bridge host ipvlan macvlan null overlay] Authorization:<nil> Log:[awslogs fluentd gcplogs gelf journald json-file local splunk syslog]} MemoryLimit:true SwapLimit:true KernelMemory:false KernelMemoryTCP:true CPUCfsPeriod:true CPUCfsQuota:true CPUShares:true CPUSet:true PidsLimit:true IPv4Forwarding:true BridgeNfIptables:false BridgeNfIP6Tables:false Debug:false NFd:23 OomKillDisable:true NGoroutines:42 SystemTime:2025-11-01 11:09:15.613378806 +0000 UTC LoggingDriver:json-file CgroupDriver:cgroupfs NEventsListener:0 KernelVersion:5.15.0-1084-aws OperatingSystem:Ubuntu 20.04.6 LTS OSType:linux Architecture:a
arch64 IndexServerAddress:https://index.docker.io/v1/ RegistryConfig:{AllowNondistributableArtifactsCIDRs:[] AllowNondistributableArtifactsHostnames:[] InsecureRegistryCIDRs:[::1/128 127.0.0.0/8] IndexConfigs:{DockerIo:{Name:docker.io Mirrors:[] Secure:true Official:true}} Mirrors:[]} NCPU:2 MemTotal:8214831104 GenericResources:<nil> DockerRootDir:/var/lib/docker HTTPProxy: HTTPSProxy: NoProxy: Name:ip-172-31-31-251 Labels:[] ExperimentalBuild:false ServerVersion:28.1.1 ClusterStore: ClusterAdvertise: Runtimes:{Runc:{Path:runc}} DefaultRuntime:runc Swarm:{NodeID: NodeAddr: LocalNodeState:inactive ControlAvailable:false Error: RemoteManagers:<nil>} LiveRestoreEnabled:false Isolation: InitBinary:docker-init ContainerdCommit:{ID:05044ec0a9a75232cad458027ca83437aae3f4da Expected:} RuncCommit:{ID:v1.2.5-0-g59923ef Expected:} InitCommit:{ID:de40ad0 Expected:} SecurityOptions:[name=apparmor name=seccomp,profile=builtin] ProductLicense: Warnings:<nil> ServerErrors:[] ClientInfo:{Debug:false Plugins:[map[Name:buildx P
ath:/usr/libexec/docker/cli-plugins/docker-buildx SchemaVersion:0.1.0 ShortDescription:Docker Buildx Vendor:Docker Inc. Version:v0.23.0] map[Name:compose Path:/usr/libexec/docker/cli-plugins/docker-compose SchemaVersion:0.1.0 ShortDescription:Docker Compose Vendor:Docker Inc. Version:v2.35.1]] Warnings:<nil>}}
I1101 11:09:15.623518 564163 start_flags.go:327] no existing cluster config was found, will generate one from the flags
I1101 11:09:15.623751 564163 start_flags.go:992] Waiting for all components: map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true]
I1101 11:09:15.626927 564163 out.go:179] * Using Docker driver with root privileges
I1101 11:09:15.629780 564163 cni.go:84] Creating CNI manager for ""
I1101 11:09:15.629849 564163 cni.go:136] multinode detected (0 nodes found), recommending kindnet
I1101 11:09:15.629863 564163 start_flags.go:336] Found "CNI" CNI - setting NetworkPlugin=cni
I1101 11:09:15.629952 564163 start.go:353] cluster config:
{Name:ha-472819 KeepContext:false EmbedCerts:false MinikubeISO: KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 Memory:3072 CPUs:2 DiskSize:20000 Driver:docker HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.34.1 ClusterName:ha-472819 Namespace:default APIServerHAVIP: APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:crio CR
ISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP: Port:8443 KubernetesVersion:v1.34.1 ContainerRuntime:crio ControlPlane:true Worker:true}] Addons:map[] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s MountString: Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:false DisableMetrics:false DisableCoreDNSLog:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPaus
eInterval:1m0s}
I1101 11:09:15.634889 564163 out.go:179] * Starting "ha-472819" primary control-plane node in "ha-472819" cluster
I1101 11:09:15.637899 564163 cache.go:124] Beginning downloading kic base image for docker with crio
I1101 11:09:15.640856 564163 out.go:179] * Pulling base image v0.0.48-1760939008-21773 ...
I1101 11:09:15.643596 564163 preload.go:183] Checking if preload exists for k8s version v1.34.1 and runtime crio
I1101 11:09:15.643660 564163 preload.go:198] Found local preload: /home/jenkins/minikube-integration/21830-532863/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.34.1-cri-o-overlay-arm64.tar.lz4
I1101 11:09:15.643675 564163 cache.go:59] Caching tarball of preloaded images
I1101 11:09:15.643690 564163 image.go:81] Checking for gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 in local docker daemon
I1101 11:09:15.643766 564163 preload.go:233] Found /home/jenkins/minikube-integration/21830-532863/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.34.1-cri-o-overlay-arm64.tar.lz4 in cache, skipping download
I1101 11:09:15.643778 564163 cache.go:62] Finished verifying existence of preloaded tar for v1.34.1 on crio
I1101 11:09:15.644121 564163 profile.go:143] Saving config to /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/config.json ...
I1101 11:09:15.644152 564163 lock.go:35] WriteFile acquiring /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/config.json: {Name:mk1ba5f23dfb700a1a8e1eba67301a5ea1e7302e Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1101 11:09:15.663055 564163 image.go:100] Found gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 in local docker daemon, skipping pull
I1101 11:09:15.663081 564163 cache.go:148] gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 exists in daemon, skipping load
I1101 11:09:15.663095 564163 cache.go:233] Successfully downloaded all kic artifacts
I1101 11:09:15.663121 564163 start.go:360] acquireMachinesLock for ha-472819: {Name:mke8efbc22a0e700799c27ca313f26b1261a26ea Clock:{} Delay:500ms Timeout:10m0s Cancel:<nil>}
I1101 11:09:15.663233 564163 start.go:364] duration metric: took 92.735µs to acquireMachinesLock for "ha-472819"
I1101 11:09:15.663263 564163 start.go:93] Provisioning new machine with config: &{Name:ha-472819 KeepContext:false EmbedCerts:false MinikubeISO: KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 Memory:3072 CPUs:2 DiskSize:20000 Driver:docker HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.34.1 ClusterName:ha-472819 Namespace:default APIServerHAVIP: APIServerName:minikubeCA
APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:crio CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP: Port:8443 KubernetesVersion:v1.34.1 ContainerRuntime:crio ControlPlane:true Worker:true}] Addons:map[] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s MountString: Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:false DisableMetrics:false DisableCoreDNSLog:false CustomQemuFirmwarePath: SocketVMn
etClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s} &{Name: IP: Port:8443 KubernetesVersion:v1.34.1 ContainerRuntime:crio ControlPlane:true Worker:true}
I1101 11:09:15.663334 564163 start.go:125] createHost starting for "" (driver="docker")
I1101 11:09:15.666819 564163 out.go:252] * Creating docker container (CPUs=2, Memory=3072MB) ...
I1101 11:09:15.667057 564163 start.go:159] libmachine.API.Create for "ha-472819" (driver="docker")
I1101 11:09:15.667098 564163 client.go:173] LocalClient.Create starting
I1101 11:09:15.667168 564163 main.go:143] libmachine: Reading certificate data from /home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem
I1101 11:09:15.667207 564163 main.go:143] libmachine: Decoding PEM data...
I1101 11:09:15.667225 564163 main.go:143] libmachine: Parsing certificate...
I1101 11:09:15.667289 564163 main.go:143] libmachine: Reading certificate data from /home/jenkins/minikube-integration/21830-532863/.minikube/certs/cert.pem
I1101 11:09:15.667320 564163 main.go:143] libmachine: Decoding PEM data...
I1101 11:09:15.667334 564163 main.go:143] libmachine: Parsing certificate...
I1101 11:09:15.667716 564163 cli_runner.go:164] Run: docker network inspect ha-472819 --format "{"Name": "{{.Name}}","Driver": "{{.Driver}}","Subnet": "{{range .IPAM.Config}}{{.Subnet}}{{end}}","Gateway": "{{range .IPAM.Config}}{{.Gateway}}{{end}}","MTU": {{if (index .Options "com.docker.network.driver.mtu")}}{{(index .Options "com.docker.network.driver.mtu")}}{{else}}0{{end}}, "ContainerIPs": [{{range $k,$v := .Containers }}"{{$v.IPv4Address}}",{{end}}]}"
W1101 11:09:15.683755 564163 cli_runner.go:211] docker network inspect ha-472819 --format "{"Name": "{{.Name}}","Driver": "{{.Driver}}","Subnet": "{{range .IPAM.Config}}{{.Subnet}}{{end}}","Gateway": "{{range .IPAM.Config}}{{.Gateway}}{{end}}","MTU": {{if (index .Options "com.docker.network.driver.mtu")}}{{(index .Options "com.docker.network.driver.mtu")}}{{else}}0{{end}}, "ContainerIPs": [{{range $k,$v := .Containers }}"{{$v.IPv4Address}}",{{end}}]}" returned with exit code 1
I1101 11:09:15.683839 564163 network_create.go:284] running [docker network inspect ha-472819] to gather additional debugging logs...
I1101 11:09:15.683861 564163 cli_runner.go:164] Run: docker network inspect ha-472819
W1101 11:09:15.699243 564163 cli_runner.go:211] docker network inspect ha-472819 returned with exit code 1
I1101 11:09:15.699272 564163 network_create.go:287] error running [docker network inspect ha-472819]: docker network inspect ha-472819: exit status 1
stdout:
[]
stderr:
Error response from daemon: network ha-472819 not found
I1101 11:09:15.699286 564163 network_create.go:289] output of [docker network inspect ha-472819]: -- stdout --
[]
-- /stdout --
** stderr **
Error response from daemon: network ha-472819 not found
** /stderr **
I1101 11:09:15.699396 564163 cli_runner.go:164] Run: docker network inspect bridge --format "{"Name": "{{.Name}}","Driver": "{{.Driver}}","Subnet": "{{range .IPAM.Config}}{{.Subnet}}{{end}}","Gateway": "{{range .IPAM.Config}}{{.Gateway}}{{end}}","MTU": {{if (index .Options "com.docker.network.driver.mtu")}}{{(index .Options "com.docker.network.driver.mtu")}}{{else}}0{{end}}, "ContainerIPs": [{{range $k,$v := .Containers }}"{{$v.IPv4Address}}",{{end}}]}"
I1101 11:09:15.715873 564163 network.go:206] using free private subnet 192.168.49.0/24: &{IP:192.168.49.0 Netmask:255.255.255.0 Prefix:24 CIDR:192.168.49.0/24 Gateway:192.168.49.1 ClientMin:192.168.49.2 ClientMax:192.168.49.254 Broadcast:192.168.49.255 IsPrivate:true Interface:{IfaceName: IfaceIPv4: IfaceMTU:0 IfaceMAC:} reservation:0x400191bd20}
I1101 11:09:15.715912 564163 network_create.go:124] attempt to create docker network ha-472819 192.168.49.0/24 with gateway 192.168.49.1 and MTU of 1500 ...
I1101 11:09:15.715972 564163 cli_runner.go:164] Run: docker network create --driver=bridge --subnet=192.168.49.0/24 --gateway=192.168.49.1 -o --ip-masq -o --icc -o com.docker.network.driver.mtu=1500 --label=created_by.minikube.sigs.k8s.io=true --label=name.minikube.sigs.k8s.io=ha-472819 ha-472819
I1101 11:09:15.771899 564163 network_create.go:108] docker network ha-472819 192.168.49.0/24 created
I1101 11:09:15.771935 564163 kic.go:121] calculated static IP "192.168.49.2" for the "ha-472819" container
I1101 11:09:15.772049 564163 cli_runner.go:164] Run: docker ps -a --format {{.Names}}
I1101 11:09:15.788100 564163 cli_runner.go:164] Run: docker volume create ha-472819 --label name.minikube.sigs.k8s.io=ha-472819 --label created_by.minikube.sigs.k8s.io=true
I1101 11:09:15.806575 564163 oci.go:103] Successfully created a docker volume ha-472819
I1101 11:09:15.806667 564163 cli_runner.go:164] Run: docker run --rm --name ha-472819-preload-sidecar --label created_by.minikube.sigs.k8s.io=true --label name.minikube.sigs.k8s.io=ha-472819 --entrypoint /usr/bin/test -v ha-472819:/var gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 -d /var/lib
I1101 11:09:16.355704 564163 oci.go:107] Successfully prepared a docker volume ha-472819
I1101 11:09:16.355755 564163 preload.go:183] Checking if preload exists for k8s version v1.34.1 and runtime crio
I1101 11:09:16.355776 564163 kic.go:194] Starting extracting preloaded images to volume ...
I1101 11:09:16.355856 564163 cli_runner.go:164] Run: docker run --rm --entrypoint /usr/bin/tar -v /home/jenkins/minikube-integration/21830-532863/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.34.1-cri-o-overlay-arm64.tar.lz4:/preloaded.tar:ro -v ha-472819:/extractDir gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 -I lz4 -xf /preloaded.tar -C /extractDir
I1101 11:09:20.827013 564163 cli_runner.go:217] Completed: docker run --rm --entrypoint /usr/bin/tar -v /home/jenkins/minikube-integration/21830-532863/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.34.1-cri-o-overlay-arm64.tar.lz4:/preloaded.tar:ro -v ha-472819:/extractDir gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 -I lz4 -xf /preloaded.tar -C /extractDir: (4.471121984s)
I1101 11:09:20.827045 564163 kic.go:203] duration metric: took 4.471266092s to extract preloaded images to volume ...
W1101 11:09:20.827184 564163 cgroups_linux.go:77] Your kernel does not support swap limit capabilities or the cgroup is not mounted.
I1101 11:09:20.827293 564163 cli_runner.go:164] Run: docker info --format "'{{json .SecurityOptions}}'"
I1101 11:09:20.885378 564163 cli_runner.go:164] Run: docker run -d -t --privileged --security-opt seccomp=unconfined --tmpfs /tmp --tmpfs /run -v /lib/modules:/lib/modules:ro --hostname ha-472819 --name ha-472819 --label created_by.minikube.sigs.k8s.io=true --label name.minikube.sigs.k8s.io=ha-472819 --label role.minikube.sigs.k8s.io= --label mode.minikube.sigs.k8s.io=ha-472819 --network ha-472819 --ip 192.168.49.2 --volume ha-472819:/var --security-opt apparmor=unconfined --memory=3072mb --cpus=2 -e container=docker --expose 8443 --publish=127.0.0.1::8443 --publish=127.0.0.1::22 --publish=127.0.0.1::2376 --publish=127.0.0.1::5000 --publish=127.0.0.1::32443 gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8
I1101 11:09:21.160814 564163 cli_runner.go:164] Run: docker container inspect ha-472819 --format={{.State.Running}}
I1101 11:09:21.184777 564163 cli_runner.go:164] Run: docker container inspect ha-472819 --format={{.State.Status}}
I1101 11:09:21.209851 564163 cli_runner.go:164] Run: docker exec ha-472819 stat /var/lib/dpkg/alternatives/iptables
I1101 11:09:21.261057 564163 oci.go:144] the created container "ha-472819" has a running status.
I1101 11:09:21.261091 564163 kic.go:225] Creating ssh key for kic: /home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819/id_rsa...
I1101 11:09:21.772918 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819/id_rsa.pub -> /home/docker/.ssh/authorized_keys
I1101 11:09:21.772974 564163 kic_runner.go:191] docker (temp): /home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819/id_rsa.pub --> /home/docker/.ssh/authorized_keys (381 bytes)
I1101 11:09:21.803408 564163 cli_runner.go:164] Run: docker container inspect ha-472819 --format={{.State.Status}}
I1101 11:09:21.833811 564163 kic_runner.go:93] Run: chown docker:docker /home/docker/.ssh/authorized_keys
I1101 11:09:21.833836 564163 kic_runner.go:114] Args: [docker exec --privileged ha-472819 chown docker:docker /home/docker/.ssh/authorized_keys]
I1101 11:09:21.893208 564163 cli_runner.go:164] Run: docker container inspect ha-472819 --format={{.State.Status}}
I1101 11:09:21.920802 564163 machine.go:94] provisionDockerMachine start ...
I1101 11:09:21.920914 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819
I1101 11:09:21.951309 564163 main.go:143] libmachine: Using SSH client type: native
I1101 11:09:21.951650 564163 main.go:143] libmachine: &{{{<nil> 0 [] [] []} docker [0x3ef110] 0x3f18d0 <nil> [] 0s} 127.0.0.1 33510 <nil> <nil>}
I1101 11:09:21.951667 564163 main.go:143] libmachine: About to run SSH command:
hostname
I1101 11:09:22.133550 564163 main.go:143] libmachine: SSH cmd err, output: <nil>: ha-472819
I1101 11:09:22.133576 564163 ubuntu.go:182] provisioning hostname "ha-472819"
I1101 11:09:22.133648 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819
I1101 11:09:22.153259 564163 main.go:143] libmachine: Using SSH client type: native
I1101 11:09:22.153580 564163 main.go:143] libmachine: &{{{<nil> 0 [] [] []} docker [0x3ef110] 0x3f18d0 <nil> [] 0s} 127.0.0.1 33510 <nil> <nil>}
I1101 11:09:22.153595 564163 main.go:143] libmachine: About to run SSH command:
sudo hostname ha-472819 && echo "ha-472819" | sudo tee /etc/hostname
I1101 11:09:22.321278 564163 main.go:143] libmachine: SSH cmd err, output: <nil>: ha-472819
I1101 11:09:22.321359 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819
I1101 11:09:22.340531 564163 main.go:143] libmachine: Using SSH client type: native
I1101 11:09:22.340844 564163 main.go:143] libmachine: &{{{<nil> 0 [] [] []} docker [0x3ef110] 0x3f18d0 <nil> [] 0s} 127.0.0.1 33510 <nil> <nil>}
I1101 11:09:22.340864 564163 main.go:143] libmachine: About to run SSH command:
if ! grep -xq '.*\sha-472819' /etc/hosts; then
if grep -xq '127.0.1.1\s.*' /etc/hosts; then
sudo sed -i 's/^127.0.1.1\s.*/127.0.1.1 ha-472819/g' /etc/hosts;
else
echo '127.0.1.1 ha-472819' | sudo tee -a /etc/hosts;
fi
fi
I1101 11:09:22.493831 564163 main.go:143] libmachine: SSH cmd err, output: <nil>:
I1101 11:09:22.493861 564163 ubuntu.go:188] set auth options {CertDir:/home/jenkins/minikube-integration/21830-532863/.minikube CaCertPath:/home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem CaPrivateKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca-key.pem CaCertRemotePath:/etc/docker/ca.pem ServerCertPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/server.pem ServerKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/server-key.pem ClientKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/certs/key.pem ServerCertRemotePath:/etc/docker/server.pem ServerKeyRemotePath:/etc/docker/server-key.pem ClientCertPath:/home/jenkins/minikube-integration/21830-532863/.minikube/certs/cert.pem ServerCertSANs:[] StorePath:/home/jenkins/minikube-integration/21830-532863/.minikube}
I1101 11:09:22.493890 564163 ubuntu.go:190] setting up certificates
I1101 11:09:22.493900 564163 provision.go:84] configureAuth start
I1101 11:09:22.493964 564163 cli_runner.go:164] Run: docker container inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}" ha-472819
I1101 11:09:22.511064 564163 provision.go:143] copyHostCerts
I1101 11:09:22.511110 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/key.pem -> /home/jenkins/minikube-integration/21830-532863/.minikube/key.pem
I1101 11:09:22.511144 564163 exec_runner.go:144] found /home/jenkins/minikube-integration/21830-532863/.minikube/key.pem, removing ...
I1101 11:09:22.511156 564163 exec_runner.go:203] rm: /home/jenkins/minikube-integration/21830-532863/.minikube/key.pem
I1101 11:09:22.511234 564163 exec_runner.go:151] cp: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/key.pem --> /home/jenkins/minikube-integration/21830-532863/.minikube/key.pem (1675 bytes)
I1101 11:09:22.511330 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem -> /home/jenkins/minikube-integration/21830-532863/.minikube/ca.pem
I1101 11:09:22.511353 564163 exec_runner.go:144] found /home/jenkins/minikube-integration/21830-532863/.minikube/ca.pem, removing ...
I1101 11:09:22.511363 564163 exec_runner.go:203] rm: /home/jenkins/minikube-integration/21830-532863/.minikube/ca.pem
I1101 11:09:22.511399 564163 exec_runner.go:151] cp: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem --> /home/jenkins/minikube-integration/21830-532863/.minikube/ca.pem (1078 bytes)
I1101 11:09:22.511453 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/cert.pem -> /home/jenkins/minikube-integration/21830-532863/.minikube/cert.pem
I1101 11:09:22.511474 564163 exec_runner.go:144] found /home/jenkins/minikube-integration/21830-532863/.minikube/cert.pem, removing ...
I1101 11:09:22.511481 564163 exec_runner.go:203] rm: /home/jenkins/minikube-integration/21830-532863/.minikube/cert.pem
I1101 11:09:22.511507 564163 exec_runner.go:151] cp: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/cert.pem --> /home/jenkins/minikube-integration/21830-532863/.minikube/cert.pem (1123 bytes)
I1101 11:09:22.511573 564163 provision.go:117] generating server cert: /home/jenkins/minikube-integration/21830-532863/.minikube/machines/server.pem ca-key=/home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem private-key=/home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca-key.pem org=jenkins.ha-472819 san=[127.0.0.1 192.168.49.2 ha-472819 localhost minikube]
I1101 11:09:23.107819 564163 provision.go:177] copyRemoteCerts
I1101 11:09:23.107890 564163 ssh_runner.go:195] Run: sudo mkdir -p /etc/docker /etc/docker /etc/docker
I1101 11:09:23.107933 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819
I1101 11:09:23.125061 564163 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33510 SSHKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819/id_rsa Username:docker}
I1101 11:09:23.229470 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem -> /etc/docker/ca.pem
I1101 11:09:23.229557 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem --> /etc/docker/ca.pem (1078 bytes)
I1101 11:09:23.247096 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/machines/server.pem -> /etc/docker/server.pem
I1101 11:09:23.247159 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/machines/server.pem --> /etc/docker/server.pem (1200 bytes)
I1101 11:09:23.264591 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/machines/server-key.pem -> /etc/docker/server-key.pem
I1101 11:09:23.264659 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/machines/server-key.pem --> /etc/docker/server-key.pem (1679 bytes)
I1101 11:09:23.282625 564163 provision.go:87] duration metric: took 788.694673ms to configureAuth
I1101 11:09:23.282653 564163 ubuntu.go:206] setting minikube options for container-runtime
I1101 11:09:23.282872 564163 config.go:182] Loaded profile config "ha-472819": Driver=docker, ContainerRuntime=crio, KubernetesVersion=v1.34.1
I1101 11:09:23.282984 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819
I1101 11:09:23.300232 564163 main.go:143] libmachine: Using SSH client type: native
I1101 11:09:23.300543 564163 main.go:143] libmachine: &{{{<nil> 0 [] [] []} docker [0x3ef110] 0x3f18d0 <nil> [] 0s} 127.0.0.1 33510 <nil> <nil>}
I1101 11:09:23.300570 564163 main.go:143] libmachine: About to run SSH command:
sudo mkdir -p /etc/sysconfig && printf %s "
CRIO_MINIKUBE_OPTIONS='--insecure-registry 10.96.0.0/12 '
" | sudo tee /etc/sysconfig/crio.minikube && sudo systemctl restart crio
I1101 11:09:23.560046 564163 main.go:143] libmachine: SSH cmd err, output: <nil>:
CRIO_MINIKUBE_OPTIONS='--insecure-registry 10.96.0.0/12 '
I1101 11:09:23.560073 564163 machine.go:97] duration metric: took 1.639247203s to provisionDockerMachine
I1101 11:09:23.560084 564163 client.go:176] duration metric: took 7.892975355s to LocalClient.Create
I1101 11:09:23.560098 564163 start.go:167] duration metric: took 7.893042884s to libmachine.API.Create "ha-472819"
I1101 11:09:23.560105 564163 start.go:293] postStartSetup for "ha-472819" (driver="docker")
I1101 11:09:23.560115 564163 start.go:322] creating required directories: [/etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs]
I1101 11:09:23.560191 564163 ssh_runner.go:195] Run: sudo mkdir -p /etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs
I1101 11:09:23.560242 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819
I1101 11:09:23.577371 564163 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33510 SSHKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819/id_rsa Username:docker}
I1101 11:09:23.681857 564163 ssh_runner.go:195] Run: cat /etc/os-release
I1101 11:09:23.685148 564163 main.go:143] libmachine: Couldn't set key VERSION_CODENAME, no corresponding struct field found
I1101 11:09:23.685219 564163 info.go:137] Remote host: Debian GNU/Linux 12 (bookworm)
I1101 11:09:23.685238 564163 filesync.go:126] Scanning /home/jenkins/minikube-integration/21830-532863/.minikube/addons for local assets ...
I1101 11:09:23.685303 564163 filesync.go:126] Scanning /home/jenkins/minikube-integration/21830-532863/.minikube/files for local assets ...
I1101 11:09:23.685388 564163 filesync.go:149] local asset: /home/jenkins/minikube-integration/21830-532863/.minikube/files/etc/ssl/certs/5347202.pem -> 5347202.pem in /etc/ssl/certs
I1101 11:09:23.685400 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/files/etc/ssl/certs/5347202.pem -> /etc/ssl/certs/5347202.pem
I1101 11:09:23.685527 564163 ssh_runner.go:195] Run: sudo mkdir -p /etc/ssl/certs
I1101 11:09:23.692811 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/files/etc/ssl/certs/5347202.pem --> /etc/ssl/certs/5347202.pem (1708 bytes)
I1101 11:09:23.710406 564163 start.go:296] duration metric: took 150.285888ms for postStartSetup
I1101 11:09:23.710773 564163 cli_runner.go:164] Run: docker container inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}" ha-472819
I1101 11:09:23.727204 564163 profile.go:143] Saving config to /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/config.json ...
I1101 11:09:23.727491 564163 ssh_runner.go:195] Run: sh -c "df -h /var | awk 'NR==2{print $5}'"
I1101 11:09:23.727555 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819
I1101 11:09:23.744722 564163 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33510 SSHKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819/id_rsa Username:docker}
I1101 11:09:23.847065 564163 ssh_runner.go:195] Run: sh -c "df -BG /var | awk 'NR==2{print $4}'"
I1101 11:09:23.851940 564163 start.go:128] duration metric: took 8.188589867s to createHost
I1101 11:09:23.851964 564163 start.go:83] releasing machines lock for "ha-472819", held for 8.188717483s
I1101 11:09:23.852042 564163 cli_runner.go:164] Run: docker container inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}" ha-472819
I1101 11:09:23.868795 564163 ssh_runner.go:195] Run: cat /version.json
I1101 11:09:23.868846 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819
I1101 11:09:23.869109 564163 ssh_runner.go:195] Run: curl -sS -m 2 https://registry.k8s.io/
I1101 11:09:23.869169 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819
I1101 11:09:23.887234 564163 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33510 SSHKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819/id_rsa Username:docker}
I1101 11:09:23.888188 564163 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33510 SSHKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819/id_rsa Username:docker}
I1101 11:09:24.071485 564163 ssh_runner.go:195] Run: systemctl --version
I1101 11:09:24.078091 564163 ssh_runner.go:195] Run: sudo sh -c "podman version >/dev/null"
I1101 11:09:24.116837 564163 ssh_runner.go:195] Run: sh -c "stat /etc/cni/net.d/*loopback.conf*"
W1101 11:09:24.121129 564163 cni.go:209] loopback cni configuration skipped: "/etc/cni/net.d/*loopback.conf*" not found
I1101 11:09:24.121205 564163 ssh_runner.go:195] Run: sudo find /etc/cni/net.d -maxdepth 1 -type f ( ( -name *bridge* -or -name *podman* ) -and -not -name *.mk_disabled ) -printf "%p, " -exec sh -c "sudo mv {} {}.mk_disabled" ;
I1101 11:09:24.150584 564163 cni.go:262] disabled [/etc/cni/net.d/87-podman-bridge.conflist, /etc/cni/net.d/10-crio-bridge.conflist.disabled] bridge cni config(s)
I1101 11:09:24.150648 564163 start.go:496] detecting cgroup driver to use...
I1101 11:09:24.150698 564163 detect.go:187] detected "cgroupfs" cgroup driver on host os
I1101 11:09:24.150758 564163 ssh_runner.go:195] Run: sudo systemctl stop -f containerd
I1101 11:09:24.168976 564163 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I1101 11:09:24.181753 564163 docker.go:218] disabling cri-docker service (if available) ...
I1101 11:09:24.181845 564163 ssh_runner.go:195] Run: sudo systemctl stop -f cri-docker.socket
I1101 11:09:24.198141 564163 ssh_runner.go:195] Run: sudo systemctl stop -f cri-docker.service
I1101 11:09:24.216855 564163 ssh_runner.go:195] Run: sudo systemctl disable cri-docker.socket
I1101 11:09:24.341682 564163 ssh_runner.go:195] Run: sudo systemctl mask cri-docker.service
I1101 11:09:24.476025 564163 docker.go:234] disabling docker service ...
I1101 11:09:24.476133 564163 ssh_runner.go:195] Run: sudo systemctl stop -f docker.socket
I1101 11:09:24.497860 564163 ssh_runner.go:195] Run: sudo systemctl stop -f docker.service
I1101 11:09:24.511210 564163 ssh_runner.go:195] Run: sudo systemctl disable docker.socket
I1101 11:09:24.628871 564163 ssh_runner.go:195] Run: sudo systemctl mask docker.service
I1101 11:09:24.751818 564163 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service docker
I1101 11:09:24.765436 564163 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///var/run/crio/crio.sock
" | sudo tee /etc/crictl.yaml"
I1101 11:09:24.779636 564163 crio.go:59] configure cri-o to use "registry.k8s.io/pause:3.10.1" pause image...
I1101 11:09:24.779754 564163 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|^.*pause_image = .*$|pause_image = "registry.k8s.io/pause:3.10.1"|' /etc/crio/crio.conf.d/02-crio.conf"
I1101 11:09:24.788816 564163 crio.go:70] configuring cri-o to use "cgroupfs" as cgroup driver...
I1101 11:09:24.788939 564163 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|^.*cgroup_manager = .*$|cgroup_manager = "cgroupfs"|' /etc/crio/crio.conf.d/02-crio.conf"
I1101 11:09:24.797807 564163 ssh_runner.go:195] Run: sh -c "sudo sed -i '/conmon_cgroup = .*/d' /etc/crio/crio.conf.d/02-crio.conf"
I1101 11:09:24.806546 564163 ssh_runner.go:195] Run: sh -c "sudo sed -i '/cgroup_manager = .*/a conmon_cgroup = "pod"' /etc/crio/crio.conf.d/02-crio.conf"
I1101 11:09:24.815271 564163 ssh_runner.go:195] Run: sh -c "sudo rm -rf /etc/cni/net.mk"
I1101 11:09:24.823860 564163 ssh_runner.go:195] Run: sh -c "sudo sed -i '/^ *"net.ipv4.ip_unprivileged_port_start=.*"/d' /etc/crio/crio.conf.d/02-crio.conf"
I1101 11:09:24.832662 564163 ssh_runner.go:195] Run: sh -c "sudo grep -q "^ *default_sysctls" /etc/crio/crio.conf.d/02-crio.conf || sudo sed -i '/conmon_cgroup = .*/a default_sysctls = \[\n\]' /etc/crio/crio.conf.d/02-crio.conf"
I1101 11:09:24.846440 564163 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^default_sysctls *= *\[|&\n "net.ipv4.ip_unprivileged_port_start=0",|' /etc/crio/crio.conf.d/02-crio.conf"
I1101 11:09:24.855087 564163 ssh_runner.go:195] Run: sudo sysctl net.bridge.bridge-nf-call-iptables
I1101 11:09:24.862971 564163 ssh_runner.go:195] Run: sudo sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"
I1101 11:09:24.870471 564163 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1101 11:09:24.977089 564163 ssh_runner.go:195] Run: sudo systemctl restart crio
I1101 11:09:25.118783 564163 start.go:543] Will wait 60s for socket path /var/run/crio/crio.sock
I1101 11:09:25.118897 564163 ssh_runner.go:195] Run: stat /var/run/crio/crio.sock
I1101 11:09:25.122853 564163 start.go:564] Will wait 60s for crictl version
I1101 11:09:25.122960 564163 ssh_runner.go:195] Run: which crictl
I1101 11:09:25.126642 564163 ssh_runner.go:195] Run: sudo /usr/local/bin/crictl version
I1101 11:09:25.155501 564163 start.go:580] Version: 0.1.0
RuntimeName: cri-o
RuntimeVersion: 1.34.1
RuntimeApiVersion: v1
I1101 11:09:25.155650 564163 ssh_runner.go:195] Run: crio --version
I1101 11:09:25.191623 564163 ssh_runner.go:195] Run: crio --version
I1101 11:09:25.225393 564163 out.go:179] * Preparing Kubernetes v1.34.1 on CRI-O 1.34.1 ...
I1101 11:09:25.228286 564163 cli_runner.go:164] Run: docker network inspect ha-472819 --format "{"Name": "{{.Name}}","Driver": "{{.Driver}}","Subnet": "{{range .IPAM.Config}}{{.Subnet}}{{end}}","Gateway": "{{range .IPAM.Config}}{{.Gateway}}{{end}}","MTU": {{if (index .Options "com.docker.network.driver.mtu")}}{{(index .Options "com.docker.network.driver.mtu")}}{{else}}0{{end}}, "ContainerIPs": [{{range $k,$v := .Containers }}"{{$v.IPv4Address}}",{{end}}]}"
I1101 11:09:25.249910 564163 ssh_runner.go:195] Run: grep 192.168.49.1 host.minikube.internal$ /etc/hosts
I1101 11:09:25.253806 564163 ssh_runner.go:195] Run: /bin/bash -c "{ grep -v $'\thost.minikube.internal$' "/etc/hosts"; echo "192.168.49.1 host.minikube.internal"; } > /tmp/h.$$; sudo cp /tmp/h.$$ "/etc/hosts""
I1101 11:09:25.264172 564163 kubeadm.go:884] updating cluster {Name:ha-472819 KeepContext:false EmbedCerts:false MinikubeISO: KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 Memory:3072 CPUs:2 DiskSize:20000 Driver:docker HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.34.1 ClusterName:ha-472819 Namespace:default APIServerHAVIP:192.168.49.254 APIServerName:minikubeCA APISe
rverNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:crio CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.168.49.2 Port:8443 KubernetesVersion:v1.34.1 ContainerRuntime:crio ControlPlane:true Worker:true}] Addons:map[] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s MountString: Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:false DisableMetrics:false DisableCoreDNSLog:false CustomQemuFirmwarePath: So
cketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s} ...
I1101 11:09:25.264297 564163 preload.go:183] Checking if preload exists for k8s version v1.34.1 and runtime crio
I1101 11:09:25.264354 564163 ssh_runner.go:195] Run: sudo crictl images --output json
I1101 11:09:25.296986 564163 crio.go:514] all images are preloaded for cri-o runtime.
I1101 11:09:25.297011 564163 crio.go:433] Images already preloaded, skipping extraction
I1101 11:09:25.297070 564163 ssh_runner.go:195] Run: sudo crictl images --output json
I1101 11:09:25.321789 564163 crio.go:514] all images are preloaded for cri-o runtime.
I1101 11:09:25.321813 564163 cache_images.go:86] Images are preloaded, skipping loading
I1101 11:09:25.321821 564163 kubeadm.go:935] updating node { 192.168.49.2 8443 v1.34.1 crio true true} ...
I1101 11:09:25.321912 564163 kubeadm.go:947] kubelet [Unit]
Wants=crio.service
[Service]
ExecStart=
ExecStart=/var/lib/minikube/binaries/v1.34.1/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroups-per-qos=false --config=/var/lib/kubelet/config.yaml --enforce-node-allocatable= --hostname-override=ha-472819 --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.168.49.2
[Install]
config:
{KubernetesVersion:v1.34.1 ClusterName:ha-472819 Namespace:default APIServerHAVIP:192.168.49.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:crio CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:}
I1101 11:09:25.321999 564163 ssh_runner.go:195] Run: crio config
I1101 11:09:25.380528 564163 cni.go:84] Creating CNI manager for ""
I1101 11:09:25.380597 564163 cni.go:136] multinode detected (1 nodes found), recommending kindnet
I1101 11:09:25.380638 564163 kubeadm.go:85] Using pod CIDR: 10.244.0.0/16
I1101 11:09:25.380700 564163 kubeadm.go:190] kubeadm options: {CertDir:/var/lib/minikube/certs ServiceCIDR:10.96.0.0/12 PodSubnet:10.244.0.0/16 AdvertiseAddress:192.168.49.2 APIServerPort:8443 KubernetesVersion:v1.34.1 EtcdDataDir:/var/lib/minikube/etcd EtcdExtraArgs:map[] ClusterName:ha-472819 NodeName:ha-472819 DNSDomain:cluster.local CRISocket:/var/run/crio/crio.sock ImageRepository: ComponentOptions:[{Component:apiServer ExtraArgs:map[enable-admission-plugins:NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota] Pairs:map[certSANs:["127.0.0.1", "localhost", "192.168.49.2"]]} {Component:controllerManager ExtraArgs:map[allocate-node-cidrs:true leader-elect:false] Pairs:map[]} {Component:scheduler ExtraArgs:map[leader-elect:false] Pairs:map[]}] FeatureArgs:map[] NodeIP:192.168.49.2 CgroupDriver:cgroupfs ClientCAFile:/var/lib/minikube/certs/ca.crt StaticPodPath:/etc/kubernetes/mani
fests ControlPlaneAddress:control-plane.minikube.internal KubeProxyOptions:map[] ResolvConfSearchRegression:false KubeletConfigOpts:map[containerRuntimeEndpoint:unix:///var/run/crio/crio.sock hairpinMode:hairpin-veth runtimeRequestTimeout:15m] PrependCriSocketUnix:true}
I1101 11:09:25.380877 564163 kubeadm.go:196] kubeadm config:
apiVersion: kubeadm.k8s.io/v1beta4
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 192.168.49.2
bindPort: 8443
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
ttl: 24h0m0s
usages:
- signing
- authentication
nodeRegistration:
criSocket: unix:///var/run/crio/crio.sock
name: "ha-472819"
kubeletExtraArgs:
- name: "node-ip"
value: "192.168.49.2"
taints: []
---
apiVersion: kubeadm.k8s.io/v1beta4
kind: ClusterConfiguration
apiServer:
certSANs: ["127.0.0.1", "localhost", "192.168.49.2"]
extraArgs:
- name: "enable-admission-plugins"
value: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
controllerManager:
extraArgs:
- name: "allocate-node-cidrs"
value: "true"
- name: "leader-elect"
value: "false"
scheduler:
extraArgs:
- name: "leader-elect"
value: "false"
certificatesDir: /var/lib/minikube/certs
clusterName: mk
controlPlaneEndpoint: control-plane.minikube.internal:8443
etcd:
local:
dataDir: /var/lib/minikube/etcd
kubernetesVersion: v1.34.1
networking:
dnsDomain: cluster.local
podSubnet: "10.244.0.0/16"
serviceSubnet: 10.96.0.0/12
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
authentication:
x509:
clientCAFile: /var/lib/minikube/certs/ca.crt
cgroupDriver: cgroupfs
containerRuntimeEndpoint: unix:///var/run/crio/crio.sock
hairpinMode: hairpin-veth
runtimeRequestTimeout: 15m
clusterDomain: "cluster.local"
# disable disk resource management by default
imageGCHighThresholdPercent: 100
evictionHard:
nodefs.available: "0%"
nodefs.inodesFree: "0%"
imagefs.available: "0%"
failSwapOn: false
staticPodPath: /etc/kubernetes/manifests
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
clusterCIDR: "10.244.0.0/16"
metricsBindAddress: 0.0.0.0:10249
conntrack:
maxPerCore: 0
# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_established"
tcpEstablishedTimeout: 0s
# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_close"
tcpCloseWaitTimeout: 0s
I1101 11:09:25.380929 564163 kube-vip.go:115] generating kube-vip config ...
I1101 11:09:25.381014 564163 ssh_runner.go:195] Run: sudo sh -c "lsmod | grep ip_vs"
I1101 11:09:25.393108 564163 kube-vip.go:163] giving up enabling control-plane load-balancing as ipvs kernel modules appears not to be available: sudo sh -c "lsmod | grep ip_vs": Process exited with status 1
stdout:
stderr:
I1101 11:09:25.393218 564163 kube-vip.go:137] kube-vip config:
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
name: kube-vip
namespace: kube-system
spec:
containers:
- args:
- manager
env:
- name: vip_arp
value: "true"
- name: port
value: "8443"
- name: vip_nodename
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: vip_interface
value: eth0
- name: vip_cidr
value: "32"
- name: dns_mode
value: first
- name: cp_enable
value: "true"
- name: cp_namespace
value: kube-system
- name: vip_leaderelection
value: "true"
- name: vip_leasename
value: plndr-cp-lock
- name: vip_leaseduration
value: "5"
- name: vip_renewdeadline
value: "3"
- name: vip_retryperiod
value: "1"
- name: address
value: 192.168.49.254
- name: prometheus_server
value: :2112
image: ghcr.io/kube-vip/kube-vip:v1.0.1
imagePullPolicy: IfNotPresent
name: kube-vip
resources: {}
securityContext:
capabilities:
add:
- NET_ADMIN
- NET_RAW
volumeMounts:
- mountPath: /etc/kubernetes/admin.conf
name: kubeconfig
hostAliases:
- hostnames:
- kubernetes
ip: 127.0.0.1
hostNetwork: true
volumes:
- hostPath:
path: "/etc/kubernetes/super-admin.conf"
name: kubeconfig
status: {}
I1101 11:09:25.393286 564163 ssh_runner.go:195] Run: sudo ls /var/lib/minikube/binaries/v1.34.1
I1101 11:09:25.401262 564163 binaries.go:44] Found k8s binaries, skipping transfer
I1101 11:09:25.401379 564163 ssh_runner.go:195] Run: sudo mkdir -p /etc/systemd/system/kubelet.service.d /lib/systemd/system /var/tmp/minikube /etc/kubernetes/manifests
I1101 11:09:25.409393 564163 ssh_runner.go:362] scp memory --> /etc/systemd/system/kubelet.service.d/10-kubeadm.conf (359 bytes)
I1101 11:09:25.422954 564163 ssh_runner.go:362] scp memory --> /lib/systemd/system/kubelet.service (352 bytes)
I1101 11:09:25.436690 564163 ssh_runner.go:362] scp memory --> /var/tmp/minikube/kubeadm.yaml.new (2206 bytes)
I1101 11:09:25.449600 564163 ssh_runner.go:362] scp memory --> /etc/kubernetes/manifests/kube-vip.yaml (1364 bytes)
I1101 11:09:25.462548 564163 ssh_runner.go:195] Run: grep 192.168.49.254 control-plane.minikube.internal$ /etc/hosts
I1101 11:09:25.466270 564163 ssh_runner.go:195] Run: /bin/bash -c "{ grep -v $'\tcontrol-plane.minikube.internal$' "/etc/hosts"; echo "192.168.49.254 control-plane.minikube.internal"; } > /tmp/h.$$; sudo cp /tmp/h.$$ "/etc/hosts""
I1101 11:09:25.475862 564163 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1101 11:09:25.601826 564163 ssh_runner.go:195] Run: sudo systemctl start kubelet
I1101 11:09:25.617309 564163 certs.go:69] Setting up /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819 for IP: 192.168.49.2
I1101 11:09:25.617340 564163 certs.go:195] generating shared ca certs ...
I1101 11:09:25.617373 564163 certs.go:227] acquiring lock for ca certs: {Name:mkf1eb1b0a157a52860366e1243b59ec23d70467 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1101 11:09:25.617566 564163 certs.go:236] skipping valid "minikubeCA" ca cert: /home/jenkins/minikube-integration/21830-532863/.minikube/ca.key
I1101 11:09:25.617633 564163 certs.go:236] skipping valid "proxyClientCA" ca cert: /home/jenkins/minikube-integration/21830-532863/.minikube/proxy-client-ca.key
I1101 11:09:25.617663 564163 certs.go:257] generating profile certs ...
I1101 11:09:25.617789 564163 certs.go:364] generating signed profile cert for "minikube-user": /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/client.key
I1101 11:09:25.617814 564163 crypto.go:68] Generating cert /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/client.crt with IP's: []
I1101 11:09:25.970419 564163 crypto.go:156] Writing cert to /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/client.crt ...
I1101 11:09:25.970452 564163 lock.go:35] WriteFile acquiring /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/client.crt: {Name:mk2f41d01137bc613681198561d475471e9b313e Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1101 11:09:25.970692 564163 crypto.go:164] Writing key to /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/client.key ...
I1101 11:09:25.970711 564163 lock.go:35] WriteFile acquiring /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/client.key: {Name:mkb2f404cf11e9ff6d4974de312113eaa2c2831e Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1101 11:09:25.970817 564163 certs.go:364] generating signed profile cert for "minikube": /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.key.985e35c4
I1101 11:09:25.970839 564163 crypto.go:68] Generating cert /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.crt.985e35c4 with IP's: [10.96.0.1 127.0.0.1 10.0.0.1 192.168.49.2 192.168.49.254]
I1101 11:09:26.521666 564163 crypto.go:156] Writing cert to /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.crt.985e35c4 ...
I1101 11:09:26.521708 564163 lock.go:35] WriteFile acquiring /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.crt.985e35c4: {Name:mkb42da9edce8a3a5d96bc6e579423c0b2c406c2 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1101 11:09:26.521948 564163 crypto.go:164] Writing key to /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.key.985e35c4 ...
I1101 11:09:26.521970 564163 lock.go:35] WriteFile acquiring /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.key.985e35c4: {Name:mkdb988833aef0d64a2a617d4983ef55d86bf204 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1101 11:09:26.522099 564163 certs.go:382] copying /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.crt.985e35c4 -> /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.crt
I1101 11:09:26.522188 564163 certs.go:386] copying /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.key.985e35c4 -> /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.key
I1101 11:09:26.522253 564163 certs.go:364] generating signed profile cert for "aggregator": /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/proxy-client.key
I1101 11:09:26.522271 564163 crypto.go:68] Generating cert /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/proxy-client.crt with IP's: []
I1101 11:09:26.790462 564163 crypto.go:156] Writing cert to /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/proxy-client.crt ...
I1101 11:09:26.790530 564163 lock.go:35] WriteFile acquiring /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/proxy-client.crt: {Name:mk06b0a4635c2902eec5ac65c88e17411a71c735 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1101 11:09:26.790721 564163 crypto.go:164] Writing key to /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/proxy-client.key ...
I1101 11:09:26.790734 564163 lock.go:35] WriteFile acquiring /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/proxy-client.key: {Name:mkba7a7f05214b82bdfe102379d16ce7f31a4fa3 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1101 11:09:26.790827 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/ca.crt -> /var/lib/minikube/certs/ca.crt
I1101 11:09:26.790855 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/ca.key -> /var/lib/minikube/certs/ca.key
I1101 11:09:26.790874 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/proxy-client-ca.crt -> /var/lib/minikube/certs/proxy-client-ca.crt
I1101 11:09:26.790886 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/proxy-client-ca.key -> /var/lib/minikube/certs/proxy-client-ca.key
I1101 11:09:26.790901 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.crt -> /var/lib/minikube/certs/apiserver.crt
I1101 11:09:26.790913 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.key -> /var/lib/minikube/certs/apiserver.key
I1101 11:09:26.790928 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/proxy-client.crt -> /var/lib/minikube/certs/proxy-client.crt
I1101 11:09:26.790938 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/proxy-client.key -> /var/lib/minikube/certs/proxy-client.key
I1101 11:09:26.790995 564163 certs.go:484] found cert: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/534720.pem (1338 bytes)
W1101 11:09:26.791032 564163 certs.go:480] ignoring /home/jenkins/minikube-integration/21830-532863/.minikube/certs/534720_empty.pem, impossibly tiny 0 bytes
I1101 11:09:26.791044 564163 certs.go:484] found cert: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca-key.pem (1679 bytes)
I1101 11:09:26.791066 564163 certs.go:484] found cert: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem (1078 bytes)
I1101 11:09:26.791099 564163 certs.go:484] found cert: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/cert.pem (1123 bytes)
I1101 11:09:26.791128 564163 certs.go:484] found cert: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/key.pem (1675 bytes)
I1101 11:09:26.791177 564163 certs.go:484] found cert: /home/jenkins/minikube-integration/21830-532863/.minikube/files/etc/ssl/certs/5347202.pem (1708 bytes)
I1101 11:09:26.791207 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/ca.crt -> /usr/share/ca-certificates/minikubeCA.pem
I1101 11:09:26.791225 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/534720.pem -> /usr/share/ca-certificates/534720.pem
I1101 11:09:26.791237 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/files/etc/ssl/certs/5347202.pem -> /usr/share/ca-certificates/5347202.pem
I1101 11:09:26.791808 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/ca.crt --> /var/lib/minikube/certs/ca.crt (1111 bytes)
I1101 11:09:26.811389 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/ca.key --> /var/lib/minikube/certs/ca.key (1679 bytes)
I1101 11:09:26.830648 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/proxy-client-ca.crt --> /var/lib/minikube/certs/proxy-client-ca.crt (1119 bytes)
I1101 11:09:26.848734 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/proxy-client-ca.key --> /var/lib/minikube/certs/proxy-client-ca.key (1679 bytes)
I1101 11:09:26.866857 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.crt --> /var/lib/minikube/certs/apiserver.crt (1424 bytes)
I1101 11:09:26.884898 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.key --> /var/lib/minikube/certs/apiserver.key (1679 bytes)
I1101 11:09:26.902647 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/proxy-client.crt --> /var/lib/minikube/certs/proxy-client.crt (1147 bytes)
I1101 11:09:26.923006 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/proxy-client.key --> /var/lib/minikube/certs/proxy-client.key (1675 bytes)
I1101 11:09:26.940928 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/ca.crt --> /usr/share/ca-certificates/minikubeCA.pem (1111 bytes)
I1101 11:09:26.958978 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/certs/534720.pem --> /usr/share/ca-certificates/534720.pem (1338 bytes)
I1101 11:09:26.977277 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/files/etc/ssl/certs/5347202.pem --> /usr/share/ca-certificates/5347202.pem (1708 bytes)
I1101 11:09:26.995250 564163 ssh_runner.go:362] scp memory --> /var/lib/minikube/kubeconfig (738 bytes)
I1101 11:09:27.009661 564163 ssh_runner.go:195] Run: openssl version
I1101 11:09:27.016856 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/minikubeCA.pem && ln -fs /usr/share/ca-certificates/minikubeCA.pem /etc/ssl/certs/minikubeCA.pem"
I1101 11:09:27.026824 564163 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/minikubeCA.pem
I1101 11:09:27.030812 564163 certs.go:528] hashing: -rw-r--r-- 1 root root 1111 Nov 1 10:49 /usr/share/ca-certificates/minikubeCA.pem
I1101 11:09:27.030886 564163 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/minikubeCA.pem
I1101 11:09:27.071933 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/b5213941.0 || ln -fs /etc/ssl/certs/minikubeCA.pem /etc/ssl/certs/b5213941.0"
I1101 11:09:27.080875 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/534720.pem && ln -fs /usr/share/ca-certificates/534720.pem /etc/ssl/certs/534720.pem"
I1101 11:09:27.089342 564163 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/534720.pem
I1101 11:09:27.093244 564163 certs.go:528] hashing: -rw-r--r-- 1 root root 1338 Nov 1 10:55 /usr/share/ca-certificates/534720.pem
I1101 11:09:27.093330 564163 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/534720.pem
I1101 11:09:27.135042 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/51391683.0 || ln -fs /etc/ssl/certs/534720.pem /etc/ssl/certs/51391683.0"
I1101 11:09:27.143864 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/5347202.pem && ln -fs /usr/share/ca-certificates/5347202.pem /etc/ssl/certs/5347202.pem"
I1101 11:09:27.152568 564163 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/5347202.pem
I1101 11:09:27.157253 564163 certs.go:528] hashing: -rw-r--r-- 1 root root 1708 Nov 1 10:55 /usr/share/ca-certificates/5347202.pem
I1101 11:09:27.157336 564163 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/5347202.pem
I1101 11:09:27.203904 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/3ec20f2e.0 || ln -fs /etc/ssl/certs/5347202.pem /etc/ssl/certs/3ec20f2e.0"
I1101 11:09:27.213041 564163 ssh_runner.go:195] Run: stat /var/lib/minikube/certs/apiserver-kubelet-client.crt
I1101 11:09:27.216525 564163 certs.go:400] 'apiserver-kubelet-client' cert doesn't exist, likely first start: stat /var/lib/minikube/certs/apiserver-kubelet-client.crt: Process exited with status 1
stdout:
stderr:
stat: cannot statx '/var/lib/minikube/certs/apiserver-kubelet-client.crt': No such file or directory
I1101 11:09:27.216581 564163 kubeadm.go:401] StartCluster: {Name:ha-472819 KeepContext:false EmbedCerts:false MinikubeISO: KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 Memory:3072 CPUs:2 DiskSize:20000 Driver:docker HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.34.1 ClusterName:ha-472819 Namespace:default APIServerHAVIP:192.168.49.254 APIServerName:minikubeCA APIServe
rNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:crio CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.168.49.2 Port:8443 KubernetesVersion:v1.34.1 ContainerRuntime:crio ControlPlane:true Worker:true}] Addons:map[] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s MountString: Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:false DisableMetrics:false DisableCoreDNSLog:false CustomQemuFirmwarePath: Socke
tVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s}
I1101 11:09:27.216654 564163 cri.go:54] listing CRI containers in root : {State:paused Name: Namespaces:[kube-system]}
I1101 11:09:27.216709 564163 ssh_runner.go:195] Run: sudo -s eval "crictl ps -a --quiet --label io.kubernetes.pod.namespace=kube-system"
I1101 11:09:27.244145 564163 cri.go:89] found id: ""
I1101 11:09:27.244250 564163 ssh_runner.go:195] Run: sudo ls /var/lib/kubelet/kubeadm-flags.env /var/lib/kubelet/config.yaml /var/lib/minikube/etcd
I1101 11:09:27.252322 564163 ssh_runner.go:195] Run: sudo cp /var/tmp/minikube/kubeadm.yaml.new /var/tmp/minikube/kubeadm.yaml
I1101 11:09:27.260600 564163 kubeadm.go:215] ignoring SystemVerification for kubeadm because of docker driver
I1101 11:09:27.260684 564163 ssh_runner.go:195] Run: sudo ls -la /etc/kubernetes/admin.conf /etc/kubernetes/kubelet.conf /etc/kubernetes/controller-manager.conf /etc/kubernetes/scheduler.conf
I1101 11:09:27.268298 564163 kubeadm.go:156] config check failed, skipping stale config cleanup: sudo ls -la /etc/kubernetes/admin.conf /etc/kubernetes/kubelet.conf /etc/kubernetes/controller-manager.conf /etc/kubernetes/scheduler.conf: Process exited with status 2
stdout:
stderr:
ls: cannot access '/etc/kubernetes/admin.conf': No such file or directory
ls: cannot access '/etc/kubernetes/kubelet.conf': No such file or directory
ls: cannot access '/etc/kubernetes/controller-manager.conf': No such file or directory
ls: cannot access '/etc/kubernetes/scheduler.conf': No such file or directory
I1101 11:09:27.268321 564163 kubeadm.go:158] found existing configuration files:
I1101 11:09:27.268382 564163 ssh_runner.go:195] Run: sudo grep https://control-plane.minikube.internal:8443 /etc/kubernetes/admin.conf
I1101 11:09:27.276280 564163 kubeadm.go:164] "https://control-plane.minikube.internal:8443" may not be in /etc/kubernetes/admin.conf - will remove: sudo grep https://control-plane.minikube.internal:8443 /etc/kubernetes/admin.conf: Process exited with status 2
stdout:
stderr:
grep: /etc/kubernetes/admin.conf: No such file or directory
I1101 11:09:27.276376 564163 ssh_runner.go:195] Run: sudo rm -f /etc/kubernetes/admin.conf
I1101 11:09:27.283648 564163 ssh_runner.go:195] Run: sudo grep https://control-plane.minikube.internal:8443 /etc/kubernetes/kubelet.conf
I1101 11:09:27.291436 564163 kubeadm.go:164] "https://control-plane.minikube.internal:8443" may not be in /etc/kubernetes/kubelet.conf - will remove: sudo grep https://control-plane.minikube.internal:8443 /etc/kubernetes/kubelet.conf: Process exited with status 2
stdout:
stderr:
grep: /etc/kubernetes/kubelet.conf: No such file or directory
I1101 11:09:27.291503 564163 ssh_runner.go:195] Run: sudo rm -f /etc/kubernetes/kubelet.conf
I1101 11:09:27.298997 564163 ssh_runner.go:195] Run: sudo grep https://control-plane.minikube.internal:8443 /etc/kubernetes/controller-manager.conf
I1101 11:09:27.306741 564163 kubeadm.go:164] "https://control-plane.minikube.internal:8443" may not be in /etc/kubernetes/controller-manager.conf - will remove: sudo grep https://control-plane.minikube.internal:8443 /etc/kubernetes/controller-manager.conf: Process exited with status 2
stdout:
stderr:
grep: /etc/kubernetes/controller-manager.conf: No such file or directory
I1101 11:09:27.306823 564163 ssh_runner.go:195] Run: sudo rm -f /etc/kubernetes/controller-manager.conf
I1101 11:09:27.314212 564163 ssh_runner.go:195] Run: sudo grep https://control-plane.minikube.internal:8443 /etc/kubernetes/scheduler.conf
I1101 11:09:27.322170 564163 kubeadm.go:164] "https://control-plane.minikube.internal:8443" may not be in /etc/kubernetes/scheduler.conf - will remove: sudo grep https://control-plane.minikube.internal:8443 /etc/kubernetes/scheduler.conf: Process exited with status 2
stdout:
stderr:
grep: /etc/kubernetes/scheduler.conf: No such file or directory
I1101 11:09:27.322240 564163 ssh_runner.go:195] Run: sudo rm -f /etc/kubernetes/scheduler.conf
I1101 11:09:27.329643 564163 ssh_runner.go:286] Start: sudo /bin/bash -c "env PATH="/var/lib/minikube/binaries/v1.34.1:$PATH" kubeadm init --config /var/tmp/minikube/kubeadm.yaml --ignore-preflight-errors=DirAvailable--etc-kubernetes-manifests,DirAvailable--var-lib-minikube,DirAvailable--var-lib-minikube-etcd,FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml,FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml,FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml,FileAvailable--etc-kubernetes-manifests-etcd.yaml,Port-10250,Swap,NumCPU,Mem,SystemVerification,FileContent--proc-sys-net-bridge-bridge-nf-call-iptables"
I1101 11:09:27.373103 564163 kubeadm.go:319] [init] Using Kubernetes version: v1.34.1
I1101 11:09:27.373161 564163 kubeadm.go:319] [preflight] Running pre-flight checks
I1101 11:09:27.398676 564163 kubeadm.go:319] [preflight] The system verification failed. Printing the output from the verification:
I1101 11:09:27.398752 564163 kubeadm.go:319] [0;37mKERNEL_VERSION[0m: [0;32m5.15.0-1084-aws[0m
I1101 11:09:27.398789 564163 kubeadm.go:319] [0;37mOS[0m: [0;32mLinux[0m
I1101 11:09:27.398836 564163 kubeadm.go:319] [0;37mCGROUPS_CPU[0m: [0;32menabled[0m
I1101 11:09:27.398892 564163 kubeadm.go:319] [0;37mCGROUPS_CPUACCT[0m: [0;32menabled[0m
I1101 11:09:27.398942 564163 kubeadm.go:319] [0;37mCGROUPS_CPUSET[0m: [0;32menabled[0m
I1101 11:09:27.398993 564163 kubeadm.go:319] [0;37mCGROUPS_DEVICES[0m: [0;32menabled[0m
I1101 11:09:27.399043 564163 kubeadm.go:319] [0;37mCGROUPS_FREEZER[0m: [0;32menabled[0m
I1101 11:09:27.399093 564163 kubeadm.go:319] [0;37mCGROUPS_MEMORY[0m: [0;32menabled[0m
I1101 11:09:27.399142 564163 kubeadm.go:319] [0;37mCGROUPS_PIDS[0m: [0;32menabled[0m
I1101 11:09:27.399193 564163 kubeadm.go:319] [0;37mCGROUPS_HUGETLB[0m: [0;32menabled[0m
I1101 11:09:27.399241 564163 kubeadm.go:319] [0;37mCGROUPS_BLKIO[0m: [0;32menabled[0m
I1101 11:09:27.468424 564163 kubeadm.go:319] [preflight] Pulling images required for setting up a Kubernetes cluster
I1101 11:09:27.468540 564163 kubeadm.go:319] [preflight] This might take a minute or two, depending on the speed of your internet connection
I1101 11:09:27.468642 564163 kubeadm.go:319] [preflight] You can also perform this action beforehand using 'kubeadm config images pull'
I1101 11:09:27.479009 564163 kubeadm.go:319] [certs] Using certificateDir folder "/var/lib/minikube/certs"
I1101 11:09:27.485386 564163 out.go:252] - Generating certificates and keys ...
I1101 11:09:27.485566 564163 kubeadm.go:319] [certs] Using existing ca certificate authority
I1101 11:09:27.485677 564163 kubeadm.go:319] [certs] Using existing apiserver certificate and key on disk
I1101 11:09:28.146819 564163 kubeadm.go:319] [certs] Generating "apiserver-kubelet-client" certificate and key
I1101 11:09:29.237838 564163 kubeadm.go:319] [certs] Generating "front-proxy-ca" certificate and key
I1101 11:09:29.375619 564163 kubeadm.go:319] [certs] Generating "front-proxy-client" certificate and key
I1101 11:09:29.669873 564163 kubeadm.go:319] [certs] Generating "etcd/ca" certificate and key
I1101 11:09:30.116711 564163 kubeadm.go:319] [certs] Generating "etcd/server" certificate and key
I1101 11:09:30.116850 564163 kubeadm.go:319] [certs] etcd/server serving cert is signed for DNS names [ha-472819 localhost] and IPs [192.168.49.2 127.0.0.1 ::1]
I1101 11:09:30.327488 564163 kubeadm.go:319] [certs] Generating "etcd/peer" certificate and key
I1101 11:09:30.327663 564163 kubeadm.go:319] [certs] etcd/peer serving cert is signed for DNS names [ha-472819 localhost] and IPs [192.168.49.2 127.0.0.1 ::1]
I1101 11:09:30.738429 564163 kubeadm.go:319] [certs] Generating "etcd/healthcheck-client" certificate and key
I1101 11:09:30.840698 564163 kubeadm.go:319] [certs] Generating "apiserver-etcd-client" certificate and key
I1101 11:09:31.445175 564163 kubeadm.go:319] [certs] Generating "sa" key and public key
I1101 11:09:31.445453 564163 kubeadm.go:319] [kubeconfig] Using kubeconfig folder "/etc/kubernetes"
I1101 11:09:31.867483 564163 kubeadm.go:319] [kubeconfig] Writing "admin.conf" kubeconfig file
I1101 11:09:32.130887 564163 kubeadm.go:319] [kubeconfig] Writing "super-admin.conf" kubeconfig file
I1101 11:09:32.613555 564163 kubeadm.go:319] [kubeconfig] Writing "kubelet.conf" kubeconfig file
I1101 11:09:33.124550 564163 kubeadm.go:319] [kubeconfig] Writing "controller-manager.conf" kubeconfig file
I1101 11:09:33.371630 564163 kubeadm.go:319] [kubeconfig] Writing "scheduler.conf" kubeconfig file
I1101 11:09:33.372352 564163 kubeadm.go:319] [etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
I1101 11:09:33.374952 564163 kubeadm.go:319] [control-plane] Using manifest folder "/etc/kubernetes/manifests"
I1101 11:09:33.378411 564163 out.go:252] - Booting up control plane ...
I1101 11:09:33.378530 564163 kubeadm.go:319] [control-plane] Creating static Pod manifest for "kube-apiserver"
I1101 11:09:33.378616 564163 kubeadm.go:319] [control-plane] Creating static Pod manifest for "kube-controller-manager"
I1101 11:09:33.379704 564163 kubeadm.go:319] [control-plane] Creating static Pod manifest for "kube-scheduler"
I1101 11:09:33.398248 564163 kubeadm.go:319] [kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
I1101 11:09:33.398586 564163 kubeadm.go:319] [kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/instance-config.yaml"
I1101 11:09:33.406765 564163 kubeadm.go:319] [patches] Applied patch of type "application/strategic-merge-patch+json" to target "kubeletconfiguration"
I1101 11:09:33.407161 564163 kubeadm.go:319] [kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
I1101 11:09:33.407368 564163 kubeadm.go:319] [kubelet-start] Starting the kubelet
I1101 11:09:33.546255 564163 kubeadm.go:319] [wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests"
I1101 11:09:33.546390 564163 kubeadm.go:319] [kubelet-check] Waiting for a healthy kubelet at http://127.0.0.1:10248/healthz. This can take up to 4m0s
I1101 11:09:34.539783 564163 kubeadm.go:319] [kubelet-check] The kubelet is healthy after 1.000858349s
I1101 11:09:34.543235 564163 kubeadm.go:319] [control-plane-check] Waiting for healthy control plane components. This can take up to 4m0s
I1101 11:09:34.543333 564163 kubeadm.go:319] [control-plane-check] Checking kube-apiserver at https://192.168.49.2:8443/livez
I1101 11:09:34.543583 564163 kubeadm.go:319] [control-plane-check] Checking kube-controller-manager at https://127.0.0.1:10257/healthz
I1101 11:09:34.543675 564163 kubeadm.go:319] [control-plane-check] Checking kube-scheduler at https://127.0.0.1:10259/livez
I1101 11:09:37.986079 564163 kubeadm.go:319] [control-plane-check] kube-controller-manager is healthy after 3.442309072s
I1101 11:09:39.112325 564163 kubeadm.go:319] [control-plane-check] kube-scheduler is healthy after 4.569041285s
I1101 11:09:41.044837 564163 kubeadm.go:319] [control-plane-check] kube-apiserver is healthy after 6.501498887s
I1101 11:09:41.064487 564163 kubeadm.go:319] [upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
I1101 11:09:41.079898 564163 kubeadm.go:319] [kubelet] Creating a ConfigMap "kubelet-config" in namespace kube-system with the configuration for the kubelets in the cluster
I1101 11:09:41.101331 564163 kubeadm.go:319] [upload-certs] Skipping phase. Please see --upload-certs
I1101 11:09:41.101559 564163 kubeadm.go:319] [mark-control-plane] Marking the node ha-472819 as control-plane by adding the labels: [node-role.kubernetes.io/control-plane node.kubernetes.io/exclude-from-external-load-balancers]
I1101 11:09:41.119154 564163 kubeadm.go:319] [bootstrap-token] Using token: btb653.26s7hd24i40lgq1y
I1101 11:09:41.122135 564163 out.go:252] - Configuring RBAC rules ...
I1101 11:09:41.122261 564163 kubeadm.go:319] [bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
I1101 11:09:41.131456 564163 kubeadm.go:319] [bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to get nodes
I1101 11:09:41.147503 564163 kubeadm.go:319] [bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
I1101 11:09:41.154473 564163 kubeadm.go:319] [bootstrap-token] Configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
I1101 11:09:41.161216 564163 kubeadm.go:319] [bootstrap-token] Configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
I1101 11:09:41.167042 564163 kubeadm.go:319] [bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
I1101 11:09:41.452979 564163 kubeadm.go:319] [kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key
I1101 11:09:41.879928 564163 kubeadm.go:319] [addons] Applied essential addon: CoreDNS
I1101 11:09:42.452390 564163 kubeadm.go:319] [addons] Applied essential addon: kube-proxy
I1101 11:09:42.453615 564163 kubeadm.go:319]
I1101 11:09:42.453715 564163 kubeadm.go:319] Your Kubernetes control-plane has initialized successfully!
I1101 11:09:42.453728 564163 kubeadm.go:319]
I1101 11:09:42.453805 564163 kubeadm.go:319] To start using your cluster, you need to run the following as a regular user:
I1101 11:09:42.453812 564163 kubeadm.go:319]
I1101 11:09:42.453838 564163 kubeadm.go:319] mkdir -p $HOME/.kube
I1101 11:09:42.453902 564163 kubeadm.go:319] sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
I1101 11:09:42.453956 564163 kubeadm.go:319] sudo chown $(id -u):$(id -g) $HOME/.kube/config
I1101 11:09:42.453964 564163 kubeadm.go:319]
I1101 11:09:42.454018 564163 kubeadm.go:319] Alternatively, if you are the root user, you can run:
I1101 11:09:42.454026 564163 kubeadm.go:319]
I1101 11:09:42.454074 564163 kubeadm.go:319] export KUBECONFIG=/etc/kubernetes/admin.conf
I1101 11:09:42.454082 564163 kubeadm.go:319]
I1101 11:09:42.454134 564163 kubeadm.go:319] You should now deploy a pod network to the cluster.
I1101 11:09:42.454210 564163 kubeadm.go:319] Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
I1101 11:09:42.454282 564163 kubeadm.go:319] https://kubernetes.io/docs/concepts/cluster-administration/addons/
I1101 11:09:42.454290 564163 kubeadm.go:319]
I1101 11:09:42.454374 564163 kubeadm.go:319] You can now join any number of control-plane nodes by copying certificate authorities
I1101 11:09:42.454453 564163 kubeadm.go:319] and service account keys on each node and then running the following as root:
I1101 11:09:42.454464 564163 kubeadm.go:319]
I1101 11:09:42.454806 564163 kubeadm.go:319] kubeadm join control-plane.minikube.internal:8443 --token btb653.26s7hd24i40lgq1y \
I1101 11:09:42.454943 564163 kubeadm.go:319] --discovery-token-ca-cert-hash sha256:6154fe00c4b3b6d1ce4f3500ef815797b79de90371950bebbded24106e2601a8 \
I1101 11:09:42.454976 564163 kubeadm.go:319] --control-plane
I1101 11:09:42.454986 564163 kubeadm.go:319]
I1101 11:09:42.455099 564163 kubeadm.go:319] Then you can join any number of worker nodes by running the following on each as root:
I1101 11:09:42.455108 564163 kubeadm.go:319]
I1101 11:09:42.455198 564163 kubeadm.go:319] kubeadm join control-plane.minikube.internal:8443 --token btb653.26s7hd24i40lgq1y \
I1101 11:09:42.455312 564163 kubeadm.go:319] --discovery-token-ca-cert-hash sha256:6154fe00c4b3b6d1ce4f3500ef815797b79de90371950bebbded24106e2601a8
I1101 11:09:42.459886 564163 kubeadm.go:319] [WARNING SystemVerification]: cgroups v1 support is in maintenance mode, please migrate to cgroups v2
I1101 11:09:42.460131 564163 kubeadm.go:319] [WARNING SystemVerification]: failed to parse kernel config: unable to load kernel module: "configs", output: "modprobe: FATAL: Module configs not found in directory /lib/modules/5.15.0-1084-aws\n", err: exit status 1
I1101 11:09:42.460249 564163 kubeadm.go:319] [WARNING Service-Kubelet]: kubelet service is not enabled, please run 'systemctl enable kubelet.service'
I1101 11:09:42.460270 564163 cni.go:84] Creating CNI manager for ""
I1101 11:09:42.460281 564163 cni.go:136] multinode detected (1 nodes found), recommending kindnet
I1101 11:09:42.463474 564163 out.go:179] * Configuring CNI (Container Networking Interface) ...
I1101 11:09:42.466360 564163 ssh_runner.go:195] Run: stat /opt/cni/bin/portmap
I1101 11:09:42.471008 564163 cni.go:182] applying CNI manifest using /var/lib/minikube/binaries/v1.34.1/kubectl ...
I1101 11:09:42.471032 564163 ssh_runner.go:362] scp memory --> /var/tmp/minikube/cni.yaml (2601 bytes)
I1101 11:09:42.485436 564163 ssh_runner.go:195] Run: sudo /var/lib/minikube/binaries/v1.34.1/kubectl apply --kubeconfig=/var/lib/minikube/kubeconfig -f /var/tmp/minikube/cni.yaml
I1101 11:09:42.779960 564163 ssh_runner.go:195] Run: /bin/bash -c "cat /proc/$(pgrep kube-apiserver)/oom_adj"
I1101 11:09:42.780051 564163 ssh_runner.go:195] Run: sudo /var/lib/minikube/binaries/v1.34.1/kubectl create clusterrolebinding minikube-rbac --clusterrole=cluster-admin --serviceaccount=kube-system:default --kubeconfig=/var/lib/minikube/kubeconfig
I1101 11:09:42.780097 564163 ssh_runner.go:195] Run: sudo /var/lib/minikube/binaries/v1.34.1/kubectl --kubeconfig=/var/lib/minikube/kubeconfig label --overwrite nodes ha-472819 minikube.k8s.io/updated_at=2025_11_01T11_09_42_0700 minikube.k8s.io/version=v1.37.0 minikube.k8s.io/commit=8d0f47abe6720ae55a5722df67bba0ddd12c8845 minikube.k8s.io/name=ha-472819 minikube.k8s.io/primary=true
I1101 11:09:42.946730 564163 ssh_runner.go:195] Run: sudo /var/lib/minikube/binaries/v1.34.1/kubectl get sa default --kubeconfig=/var/lib/minikube/kubeconfig
I1101 11:09:42.946790 564163 ops.go:34] apiserver oom_adj: -16
I1101 11:09:43.447305 564163 ssh_runner.go:195] Run: sudo /var/lib/minikube/binaries/v1.34.1/kubectl get sa default --kubeconfig=/var/lib/minikube/kubeconfig
I1101 11:09:43.947632 564163 ssh_runner.go:195] Run: sudo /var/lib/minikube/binaries/v1.34.1/kubectl get sa default --kubeconfig=/var/lib/minikube/kubeconfig
I1101 11:09:44.447625 564163 ssh_runner.go:195] Run: sudo /var/lib/minikube/binaries/v1.34.1/kubectl get sa default --kubeconfig=/var/lib/minikube/kubeconfig
I1101 11:09:44.946864 564163 ssh_runner.go:195] Run: sudo /var/lib/minikube/binaries/v1.34.1/kubectl get sa default --kubeconfig=/var/lib/minikube/kubeconfig
I1101 11:09:45.447661 564163 ssh_runner.go:195] Run: sudo /var/lib/minikube/binaries/v1.34.1/kubectl get sa default --kubeconfig=/var/lib/minikube/kubeconfig
I1101 11:09:45.947118 564163 ssh_runner.go:195] Run: sudo /var/lib/minikube/binaries/v1.34.1/kubectl get sa default --kubeconfig=/var/lib/minikube/kubeconfig
I1101 11:09:46.446849 564163 ssh_runner.go:195] Run: sudo /var/lib/minikube/binaries/v1.34.1/kubectl get sa default --kubeconfig=/var/lib/minikube/kubeconfig
I1101 11:09:46.947087 564163 ssh_runner.go:195] Run: sudo /var/lib/minikube/binaries/v1.34.1/kubectl get sa default --kubeconfig=/var/lib/minikube/kubeconfig
I1101 11:09:47.057163 564163 kubeadm.go:1114] duration metric: took 4.277184864s to wait for elevateKubeSystemPrivileges
I1101 11:09:47.057195 564163 kubeadm.go:403] duration metric: took 19.840620587s to StartCluster
I1101 11:09:47.057212 564163 settings.go:142] acquiring lock: {Name:mkcec05b3b9abd727f12cc8fc6d8b8719f9d2893 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1101 11:09:47.057272 564163 settings.go:150] Updating kubeconfig: /home/jenkins/minikube-integration/21830-532863/kubeconfig
I1101 11:09:47.057994 564163 lock.go:35] WriteFile acquiring /home/jenkins/minikube-integration/21830-532863/kubeconfig: {Name:mk48b340ab8169449b11ec70cb4900037359d91f Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1101 11:09:47.058207 564163 start.go:234] HA (multi-control plane) cluster: will skip waiting for primary control-plane node &{Name: IP:192.168.49.2 Port:8443 KubernetesVersion:v1.34.1 ContainerRuntime:crio ControlPlane:true Worker:true}
I1101 11:09:47.058246 564163 start.go:242] waiting for startup goroutines ...
I1101 11:09:47.058255 564163 addons.go:512] enable addons start: toEnable=map[ambassador:false amd-gpu-device-plugin:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:true efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false inaccel:false ingress:false ingress-dns:false inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow:false kubetail:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:true storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false]
I1101 11:09:47.058318 564163 addons.go:70] Setting storage-provisioner=true in profile "ha-472819"
I1101 11:09:47.058338 564163 addons.go:239] Setting addon storage-provisioner=true in "ha-472819"
I1101 11:09:47.058365 564163 host.go:66] Checking if "ha-472819" exists ...
I1101 11:09:47.058832 564163 cli_runner.go:164] Run: docker container inspect ha-472819 --format={{.State.Status}}
I1101 11:09:47.058997 564163 ssh_runner.go:195] Run: /bin/bash -c "sudo /var/lib/minikube/binaries/v1.34.1/kubectl --kubeconfig=/var/lib/minikube/kubeconfig -n kube-system get configmap coredns -o yaml"
I1101 11:09:47.059257 564163 config.go:182] Loaded profile config "ha-472819": Driver=docker, ContainerRuntime=crio, KubernetesVersion=v1.34.1
I1101 11:09:47.059298 564163 addons.go:70] Setting default-storageclass=true in profile "ha-472819"
I1101 11:09:47.059315 564163 addons_storage_classes.go:34] enableOrDisableStorageClasses default-storageclass=true on "ha-472819"
I1101 11:09:47.059542 564163 cli_runner.go:164] Run: docker container inspect ha-472819 --format={{.State.Status}}
I1101 11:09:47.094113 564163 kapi.go:59] client config for ha-472819: &rest.Config{Host:"https://192.168.49.254:8443", APIPath:"", ContentConfig:rest.ContentConfig{AcceptContentTypes:"", ContentType:"", GroupVersion:(*schema.GroupVersion)(nil), NegotiatedSerializer:runtime.NegotiatedSerializer(nil)}, Username:"", Password:"", BearerToken:"", BearerTokenFile:"", Impersonate:rest.ImpersonationConfig{UserName:"", UID:"", Groups:[]string(nil), Extra:map[string][]string(nil)}, AuthProvider:<nil>, AuthConfigPersister:rest.AuthProviderConfigPersister(nil), ExecProvider:<nil>, TLSClientConfig:rest.sanitizedTLSClientConfig{Insecure:false, ServerName:"", CertFile:"/home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/client.crt", KeyFile:"/home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/client.key", CAFile:"/home/jenkins/minikube-integration/21830-532863/.minikube/ca.crt", CertData:[]uint8(nil), KeyData:[]uint8(nil), CAData:[]uint8(nil), NextProtos:[]string(n
il)}, UserAgent:"", DisableCompression:false, Transport:http.RoundTripper(nil), WrapTransport:(transport.WrapperFunc)(0x21203d0), QPS:0, Burst:0, RateLimiter:flowcontrol.RateLimiter(nil), WarningHandler:rest.WarningHandler(nil), WarningHandlerWithContext:rest.WarningHandlerWithContext(nil), Timeout:0, Dial:(func(context.Context, string, string) (net.Conn, error))(nil), Proxy:(func(*http.Request) (*url.URL, error))(nil)}
I1101 11:09:47.094672 564163 envvar.go:172] "Feature gate default state" feature="ClientsAllowCBOR" enabled=false
I1101 11:09:47.094686 564163 envvar.go:172] "Feature gate default state" feature="ClientsPreferCBOR" enabled=false
I1101 11:09:47.094691 564163 envvar.go:172] "Feature gate default state" feature="InformerResourceVersion" enabled=false
I1101 11:09:47.094696 564163 envvar.go:172] "Feature gate default state" feature="InOrderInformers" enabled=true
I1101 11:09:47.094700 564163 envvar.go:172] "Feature gate default state" feature="WatchListClient" enabled=false
I1101 11:09:47.095052 564163 addons.go:239] Setting addon default-storageclass=true in "ha-472819"
I1101 11:09:47.099545 564163 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache"
I1101 11:09:47.099640 564163 host.go:66] Checking if "ha-472819" exists ...
I1101 11:09:47.100113 564163 cli_runner.go:164] Run: docker container inspect ha-472819 --format={{.State.Status}}
I1101 11:09:47.107324 564163 out.go:179] - Using image gcr.io/k8s-minikube/storage-provisioner:v5
I1101 11:09:47.110259 564163 addons.go:436] installing /etc/kubernetes/addons/storage-provisioner.yaml
I1101 11:09:47.110280 564163 ssh_runner.go:362] scp memory --> /etc/kubernetes/addons/storage-provisioner.yaml (2676 bytes)
I1101 11:09:47.110343 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819
I1101 11:09:47.139417 564163 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33510 SSHKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819/id_rsa Username:docker}
I1101 11:09:47.148358 564163 addons.go:436] installing /etc/kubernetes/addons/storageclass.yaml
I1101 11:09:47.148379 564163 ssh_runner.go:362] scp storageclass/storageclass.yaml --> /etc/kubernetes/addons/storageclass.yaml (271 bytes)
I1101 11:09:47.148446 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819
I1101 11:09:47.178909 564163 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33510 SSHKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819/id_rsa Username:docker}
I1101 11:09:47.268521 564163 ssh_runner.go:195] Run: /bin/bash -c "sudo /var/lib/minikube/binaries/v1.34.1/kubectl --kubeconfig=/var/lib/minikube/kubeconfig -n kube-system get configmap coredns -o yaml | sed -e '/^ forward . \/etc\/resolv.conf.*/i \ hosts {\n 192.168.49.1 host.minikube.internal\n fallthrough\n }' -e '/^ errors *$/i \ log' | sudo /var/lib/minikube/binaries/v1.34.1/kubectl --kubeconfig=/var/lib/minikube/kubeconfig replace -f -"
I1101 11:09:47.290532 564163 ssh_runner.go:195] Run: sudo KUBECONFIG=/var/lib/minikube/kubeconfig /var/lib/minikube/binaries/v1.34.1/kubectl apply -f /etc/kubernetes/addons/storage-provisioner.yaml
I1101 11:09:47.506464 564163 ssh_runner.go:195] Run: sudo KUBECONFIG=/var/lib/minikube/kubeconfig /var/lib/minikube/binaries/v1.34.1/kubectl apply -f /etc/kubernetes/addons/storageclass.yaml
I1101 11:09:47.743529 564163 start.go:977] {"host.minikube.internal": 192.168.49.1} host record injected into CoreDNS's ConfigMap
I1101 11:09:47.978148 564163 out.go:179] * Enabled addons: storage-provisioner, default-storageclass
I1101 11:09:47.981058 564163 addons.go:515] duration metric: took 922.780782ms for enable addons: enabled=[storage-provisioner default-storageclass]
I1101 11:09:47.981103 564163 start.go:247] waiting for cluster config update ...
I1101 11:09:47.981118 564163 start.go:256] writing updated cluster config ...
I1101 11:09:47.984275 564163 out.go:203]
I1101 11:09:47.987383 564163 config.go:182] Loaded profile config "ha-472819": Driver=docker, ContainerRuntime=crio, KubernetesVersion=v1.34.1
I1101 11:09:47.987471 564163 profile.go:143] Saving config to /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/config.json ...
I1101 11:09:47.990706 564163 out.go:179] * Starting "ha-472819-m02" control-plane node in "ha-472819" cluster
I1101 11:09:47.993384 564163 cache.go:124] Beginning downloading kic base image for docker with crio
I1101 11:09:47.996301 564163 out.go:179] * Pulling base image v0.0.48-1760939008-21773 ...
I1101 11:09:47.999988 564163 preload.go:183] Checking if preload exists for k8s version v1.34.1 and runtime crio
I1101 11:09:48.000022 564163 cache.go:59] Caching tarball of preloaded images
I1101 11:09:48.000067 564163 image.go:81] Checking for gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 in local docker daemon
I1101 11:09:48.000117 564163 preload.go:233] Found /home/jenkins/minikube-integration/21830-532863/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.34.1-cri-o-overlay-arm64.tar.lz4 in cache, skipping download
I1101 11:09:48.000134 564163 cache.go:62] Finished verifying existence of preloaded tar for v1.34.1 on crio
I1101 11:09:48.000258 564163 profile.go:143] Saving config to /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/config.json ...
I1101 11:09:48.023925 564163 image.go:100] Found gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 in local docker daemon, skipping pull
I1101 11:09:48.023946 564163 cache.go:148] gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 exists in daemon, skipping load
I1101 11:09:48.023960 564163 cache.go:233] Successfully downloaded all kic artifacts
I1101 11:09:48.023985 564163 start.go:360] acquireMachinesLock for ha-472819-m02: {Name:mkd9b09c2f5958eb6cf9785ab2b809fc6e14102e Clock:{} Delay:500ms Timeout:10m0s Cancel:<nil>}
I1101 11:09:48.024101 564163 start.go:364] duration metric: took 98.758µs to acquireMachinesLock for "ha-472819-m02"
I1101 11:09:48.024126 564163 start.go:93] Provisioning new machine with config: &{Name:ha-472819 KeepContext:false EmbedCerts:false MinikubeISO: KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 Memory:3072 CPUs:2 DiskSize:20000 Driver:docker HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.34.1 ClusterName:ha-472819 Namespace:default APIServerHAVIP:192.168.49.254 APIServerNa
me:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:crio CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.168.49.2 Port:8443 KubernetesVersion:v1.34.1 ContainerRuntime:crio ControlPlane:true Worker:true} {Name:m02 IP: Port:8443 KubernetesVersion:v1.34.1 ContainerRuntime:crio ControlPlane:true Worker:true}] Addons:map[default-storageclass:true storage-provisioner:true] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s MountString: Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:2621
44 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:false DisableMetrics:false DisableCoreDNSLog:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s} &{Name:m02 IP: Port:8443 KubernetesVersion:v1.34.1 ContainerRuntime:crio ControlPlane:true Worker:true}
I1101 11:09:48.024211 564163 start.go:125] createHost starting for "m02" (driver="docker")
I1101 11:09:48.027535 564163 out.go:252] * Creating docker container (CPUs=2, Memory=3072MB) ...
I1101 11:09:48.027682 564163 start.go:159] libmachine.API.Create for "ha-472819" (driver="docker")
I1101 11:09:48.027709 564163 client.go:173] LocalClient.Create starting
I1101 11:09:48.027787 564163 main.go:143] libmachine: Reading certificate data from /home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem
I1101 11:09:48.027824 564163 main.go:143] libmachine: Decoding PEM data...
I1101 11:09:48.027850 564163 main.go:143] libmachine: Parsing certificate...
I1101 11:09:48.027911 564163 main.go:143] libmachine: Reading certificate data from /home/jenkins/minikube-integration/21830-532863/.minikube/certs/cert.pem
I1101 11:09:48.027933 564163 main.go:143] libmachine: Decoding PEM data...
I1101 11:09:48.027947 564163 main.go:143] libmachine: Parsing certificate...
I1101 11:09:48.028229 564163 cli_runner.go:164] Run: docker network inspect ha-472819 --format "{"Name": "{{.Name}}","Driver": "{{.Driver}}","Subnet": "{{range .IPAM.Config}}{{.Subnet}}{{end}}","Gateway": "{{range .IPAM.Config}}{{.Gateway}}{{end}}","MTU": {{if (index .Options "com.docker.network.driver.mtu")}}{{(index .Options "com.docker.network.driver.mtu")}}{{else}}0{{end}}, "ContainerIPs": [{{range $k,$v := .Containers }}"{{$v.IPv4Address}}",{{end}}]}"
I1101 11:09:48.045847 564163 network_create.go:77] Found existing network {name:ha-472819 subnet:0x4001e9bf20 gateway:[0 0 0 0 0 0 0 0 0 0 255 255 192 168 49 1] mtu:1500}
I1101 11:09:48.045902 564163 kic.go:121] calculated static IP "192.168.49.3" for the "ha-472819-m02" container
I1101 11:09:48.045995 564163 cli_runner.go:164] Run: docker ps -a --format {{.Names}}
I1101 11:09:48.063585 564163 cli_runner.go:164] Run: docker volume create ha-472819-m02 --label name.minikube.sigs.k8s.io=ha-472819-m02 --label created_by.minikube.sigs.k8s.io=true
I1101 11:09:48.081625 564163 oci.go:103] Successfully created a docker volume ha-472819-m02
I1101 11:09:48.081759 564163 cli_runner.go:164] Run: docker run --rm --name ha-472819-m02-preload-sidecar --label created_by.minikube.sigs.k8s.io=true --label name.minikube.sigs.k8s.io=ha-472819-m02 --entrypoint /usr/bin/test -v ha-472819-m02:/var gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 -d /var/lib
I1101 11:09:48.719367 564163 oci.go:107] Successfully prepared a docker volume ha-472819-m02
I1101 11:09:48.719403 564163 preload.go:183] Checking if preload exists for k8s version v1.34.1 and runtime crio
I1101 11:09:48.719424 564163 kic.go:194] Starting extracting preloaded images to volume ...
I1101 11:09:48.719499 564163 cli_runner.go:164] Run: docker run --rm --entrypoint /usr/bin/tar -v /home/jenkins/minikube-integration/21830-532863/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.34.1-cri-o-overlay-arm64.tar.lz4:/preloaded.tar:ro -v ha-472819-m02:/extractDir gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 -I lz4 -xf /preloaded.tar -C /extractDir
I1101 11:09:53.148805 564163 cli_runner.go:217] Completed: docker run --rm --entrypoint /usr/bin/tar -v /home/jenkins/minikube-integration/21830-532863/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.34.1-cri-o-overlay-arm64.tar.lz4:/preloaded.tar:ro -v ha-472819-m02:/extractDir gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 -I lz4 -xf /preloaded.tar -C /extractDir: (4.429267175s)
I1101 11:09:53.148842 564163 kic.go:203] duration metric: took 4.429414598s to extract preloaded images to volume ...
W1101 11:09:53.148976 564163 cgroups_linux.go:77] Your kernel does not support swap limit capabilities or the cgroup is not mounted.
I1101 11:09:53.149102 564163 cli_runner.go:164] Run: docker info --format "'{{json .SecurityOptions}}'"
I1101 11:09:53.205412 564163 cli_runner.go:164] Run: docker run -d -t --privileged --security-opt seccomp=unconfined --tmpfs /tmp --tmpfs /run -v /lib/modules:/lib/modules:ro --hostname ha-472819-m02 --name ha-472819-m02 --label created_by.minikube.sigs.k8s.io=true --label name.minikube.sigs.k8s.io=ha-472819-m02 --label role.minikube.sigs.k8s.io= --label mode.minikube.sigs.k8s.io=ha-472819-m02 --network ha-472819 --ip 192.168.49.3 --volume ha-472819-m02:/var --security-opt apparmor=unconfined --memory=3072mb --cpus=2 -e container=docker --expose 8443 --publish=127.0.0.1::8443 --publish=127.0.0.1::22 --publish=127.0.0.1::2376 --publish=127.0.0.1::5000 --publish=127.0.0.1::32443 gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8
I1101 11:09:53.540381 564163 cli_runner.go:164] Run: docker container inspect ha-472819-m02 --format={{.State.Running}}
I1101 11:09:53.560268 564163 cli_runner.go:164] Run: docker container inspect ha-472819-m02 --format={{.State.Status}}
I1101 11:09:53.587014 564163 cli_runner.go:164] Run: docker exec ha-472819-m02 stat /var/lib/dpkg/alternatives/iptables
I1101 11:09:53.644476 564163 oci.go:144] the created container "ha-472819-m02" has a running status.
I1101 11:09:53.644505 564163 kic.go:225] Creating ssh key for kic: /home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819-m02/id_rsa...
I1101 11:09:53.818753 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819-m02/id_rsa.pub -> /home/docker/.ssh/authorized_keys
I1101 11:09:53.818798 564163 kic_runner.go:191] docker (temp): /home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819-m02/id_rsa.pub --> /home/docker/.ssh/authorized_keys (381 bytes)
I1101 11:09:53.843513 564163 cli_runner.go:164] Run: docker container inspect ha-472819-m02 --format={{.State.Status}}
I1101 11:09:53.867129 564163 kic_runner.go:93] Run: chown docker:docker /home/docker/.ssh/authorized_keys
I1101 11:09:53.867148 564163 kic_runner.go:114] Args: [docker exec --privileged ha-472819-m02 chown docker:docker /home/docker/.ssh/authorized_keys]
I1101 11:09:53.915191 564163 cli_runner.go:164] Run: docker container inspect ha-472819-m02 --format={{.State.Status}}
I1101 11:09:53.944431 564163 machine.go:94] provisionDockerMachine start ...
I1101 11:09:53.944522 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819-m02
I1101 11:09:53.971708 564163 main.go:143] libmachine: Using SSH client type: native
I1101 11:09:53.972031 564163 main.go:143] libmachine: &{{{<nil> 0 [] [] []} docker [0x3ef110] 0x3f18d0 <nil> [] 0s} 127.0.0.1 33515 <nil> <nil>}
I1101 11:09:53.972040 564163 main.go:143] libmachine: About to run SSH command:
hostname
I1101 11:09:53.972718 564163 main.go:143] libmachine: Error dialing TCP: ssh: handshake failed: EOF
I1101 11:09:57.125375 564163 main.go:143] libmachine: SSH cmd err, output: <nil>: ha-472819-m02
I1101 11:09:57.125400 564163 ubuntu.go:182] provisioning hostname "ha-472819-m02"
I1101 11:09:57.125484 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819-m02
I1101 11:09:57.151306 564163 main.go:143] libmachine: Using SSH client type: native
I1101 11:09:57.151617 564163 main.go:143] libmachine: &{{{<nil> 0 [] [] []} docker [0x3ef110] 0x3f18d0 <nil> [] 0s} 127.0.0.1 33515 <nil> <nil>}
I1101 11:09:57.151628 564163 main.go:143] libmachine: About to run SSH command:
sudo hostname ha-472819-m02 && echo "ha-472819-m02" | sudo tee /etc/hostname
I1101 11:09:57.311009 564163 main.go:143] libmachine: SSH cmd err, output: <nil>: ha-472819-m02
I1101 11:09:57.311162 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819-m02
I1101 11:09:57.329599 564163 main.go:143] libmachine: Using SSH client type: native
I1101 11:09:57.330232 564163 main.go:143] libmachine: &{{{<nil> 0 [] [] []} docker [0x3ef110] 0x3f18d0 <nil> [] 0s} 127.0.0.1 33515 <nil> <nil>}
I1101 11:09:57.330258 564163 main.go:143] libmachine: About to run SSH command:
if ! grep -xq '.*\sha-472819-m02' /etc/hosts; then
if grep -xq '127.0.1.1\s.*' /etc/hosts; then
sudo sed -i 's/^127.0.1.1\s.*/127.0.1.1 ha-472819-m02/g' /etc/hosts;
else
echo '127.0.1.1 ha-472819-m02' | sudo tee -a /etc/hosts;
fi
fi
I1101 11:09:57.478017 564163 main.go:143] libmachine: SSH cmd err, output: <nil>:
I1101 11:09:57.478047 564163 ubuntu.go:188] set auth options {CertDir:/home/jenkins/minikube-integration/21830-532863/.minikube CaCertPath:/home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem CaPrivateKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca-key.pem CaCertRemotePath:/etc/docker/ca.pem ServerCertPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/server.pem ServerKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/server-key.pem ClientKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/certs/key.pem ServerCertRemotePath:/etc/docker/server.pem ServerKeyRemotePath:/etc/docker/server-key.pem ClientCertPath:/home/jenkins/minikube-integration/21830-532863/.minikube/certs/cert.pem ServerCertSANs:[] StorePath:/home/jenkins/minikube-integration/21830-532863/.minikube}
I1101 11:09:57.478067 564163 ubuntu.go:190] setting up certificates
I1101 11:09:57.478078 564163 provision.go:84] configureAuth start
I1101 11:09:57.478139 564163 cli_runner.go:164] Run: docker container inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}" ha-472819-m02
I1101 11:09:57.496426 564163 provision.go:143] copyHostCerts
I1101 11:09:57.496480 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem -> /home/jenkins/minikube-integration/21830-532863/.minikube/ca.pem
I1101 11:09:57.496514 564163 exec_runner.go:144] found /home/jenkins/minikube-integration/21830-532863/.minikube/ca.pem, removing ...
I1101 11:09:57.496527 564163 exec_runner.go:203] rm: /home/jenkins/minikube-integration/21830-532863/.minikube/ca.pem
I1101 11:09:57.496611 564163 exec_runner.go:151] cp: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem --> /home/jenkins/minikube-integration/21830-532863/.minikube/ca.pem (1078 bytes)
I1101 11:09:57.496704 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/cert.pem -> /home/jenkins/minikube-integration/21830-532863/.minikube/cert.pem
I1101 11:09:57.496727 564163 exec_runner.go:144] found /home/jenkins/minikube-integration/21830-532863/.minikube/cert.pem, removing ...
I1101 11:09:57.496735 564163 exec_runner.go:203] rm: /home/jenkins/minikube-integration/21830-532863/.minikube/cert.pem
I1101 11:09:57.496763 564163 exec_runner.go:151] cp: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/cert.pem --> /home/jenkins/minikube-integration/21830-532863/.minikube/cert.pem (1123 bytes)
I1101 11:09:57.496816 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/key.pem -> /home/jenkins/minikube-integration/21830-532863/.minikube/key.pem
I1101 11:09:57.496837 564163 exec_runner.go:144] found /home/jenkins/minikube-integration/21830-532863/.minikube/key.pem, removing ...
I1101 11:09:57.496845 564163 exec_runner.go:203] rm: /home/jenkins/minikube-integration/21830-532863/.minikube/key.pem
I1101 11:09:57.496872 564163 exec_runner.go:151] cp: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/key.pem --> /home/jenkins/minikube-integration/21830-532863/.minikube/key.pem (1675 bytes)
I1101 11:09:57.496927 564163 provision.go:117] generating server cert: /home/jenkins/minikube-integration/21830-532863/.minikube/machines/server.pem ca-key=/home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem private-key=/home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca-key.pem org=jenkins.ha-472819-m02 san=[127.0.0.1 192.168.49.3 ha-472819-m02 localhost minikube]
I1101 11:09:58.109118 564163 provision.go:177] copyRemoteCerts
I1101 11:09:58.109211 564163 ssh_runner.go:195] Run: sudo mkdir -p /etc/docker /etc/docker /etc/docker
I1101 11:09:58.109257 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819-m02
I1101 11:09:58.129970 564163 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33515 SSHKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819-m02/id_rsa Username:docker}
I1101 11:09:58.239761 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/machines/server-key.pem -> /etc/docker/server-key.pem
I1101 11:09:58.239822 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/machines/server-key.pem --> /etc/docker/server-key.pem (1675 bytes)
I1101 11:09:58.258364 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem -> /etc/docker/ca.pem
I1101 11:09:58.258429 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem --> /etc/docker/ca.pem (1078 bytes)
I1101 11:09:58.277456 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/machines/server.pem -> /etc/docker/server.pem
I1101 11:09:58.277565 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/machines/server.pem --> /etc/docker/server.pem (1208 bytes)
I1101 11:09:58.296719 564163 provision.go:87] duration metric: took 818.627177ms to configureAuth
I1101 11:09:58.296743 564163 ubuntu.go:206] setting minikube options for container-runtime
I1101 11:09:58.296931 564163 config.go:182] Loaded profile config "ha-472819": Driver=docker, ContainerRuntime=crio, KubernetesVersion=v1.34.1
I1101 11:09:58.297053 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819-m02
I1101 11:09:58.314394 564163 main.go:143] libmachine: Using SSH client type: native
I1101 11:09:58.314702 564163 main.go:143] libmachine: &{{{<nil> 0 [] [] []} docker [0x3ef110] 0x3f18d0 <nil> [] 0s} 127.0.0.1 33515 <nil> <nil>}
I1101 11:09:58.314723 564163 main.go:143] libmachine: About to run SSH command:
sudo mkdir -p /etc/sysconfig && printf %s "
CRIO_MINIKUBE_OPTIONS='--insecure-registry 10.96.0.0/12 '
" | sudo tee /etc/sysconfig/crio.minikube && sudo systemctl restart crio
I1101 11:09:58.587462 564163 main.go:143] libmachine: SSH cmd err, output: <nil>:
CRIO_MINIKUBE_OPTIONS='--insecure-registry 10.96.0.0/12 '
I1101 11:09:58.587485 564163 machine.go:97] duration metric: took 4.643032549s to provisionDockerMachine
I1101 11:09:58.587494 564163 client.go:176] duration metric: took 10.55977608s to LocalClient.Create
I1101 11:09:58.587508 564163 start.go:167] duration metric: took 10.559829168s to libmachine.API.Create "ha-472819"
I1101 11:09:58.587515 564163 start.go:293] postStartSetup for "ha-472819-m02" (driver="docker")
I1101 11:09:58.587525 564163 start.go:322] creating required directories: [/etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs]
I1101 11:09:58.587591 564163 ssh_runner.go:195] Run: sudo mkdir -p /etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs
I1101 11:09:58.587640 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819-m02
I1101 11:09:58.607665 564163 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33515 SSHKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819-m02/id_rsa Username:docker}
I1101 11:09:58.713788 564163 ssh_runner.go:195] Run: cat /etc/os-release
I1101 11:09:58.716917 564163 main.go:143] libmachine: Couldn't set key VERSION_CODENAME, no corresponding struct field found
I1101 11:09:58.716946 564163 info.go:137] Remote host: Debian GNU/Linux 12 (bookworm)
I1101 11:09:58.716959 564163 filesync.go:126] Scanning /home/jenkins/minikube-integration/21830-532863/.minikube/addons for local assets ...
I1101 11:09:58.717015 564163 filesync.go:126] Scanning /home/jenkins/minikube-integration/21830-532863/.minikube/files for local assets ...
I1101 11:09:58.717094 564163 filesync.go:149] local asset: /home/jenkins/minikube-integration/21830-532863/.minikube/files/etc/ssl/certs/5347202.pem -> 5347202.pem in /etc/ssl/certs
I1101 11:09:58.717106 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/files/etc/ssl/certs/5347202.pem -> /etc/ssl/certs/5347202.pem
I1101 11:09:58.717202 564163 ssh_runner.go:195] Run: sudo mkdir -p /etc/ssl/certs
I1101 11:09:58.725530 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/files/etc/ssl/certs/5347202.pem --> /etc/ssl/certs/5347202.pem (1708 bytes)
I1101 11:09:58.744321 564163 start.go:296] duration metric: took 156.786029ms for postStartSetup
I1101 11:09:58.744753 564163 cli_runner.go:164] Run: docker container inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}" ha-472819-m02
I1101 11:09:58.763769 564163 profile.go:143] Saving config to /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/config.json ...
I1101 11:09:58.764152 564163 ssh_runner.go:195] Run: sh -c "df -h /var | awk 'NR==2{print $5}'"
I1101 11:09:58.764219 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819-m02
I1101 11:09:58.781753 564163 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33515 SSHKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819-m02/id_rsa Username:docker}
I1101 11:09:58.882710 564163 ssh_runner.go:195] Run: sh -c "df -BG /var | awk 'NR==2{print $4}'"
I1101 11:09:58.887489 564163 start.go:128] duration metric: took 10.863262048s to createHost
I1101 11:09:58.887514 564163 start.go:83] releasing machines lock for "ha-472819-m02", held for 10.863404934s
I1101 11:09:58.887586 564163 cli_runner.go:164] Run: docker container inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}" ha-472819-m02
I1101 11:09:58.910138 564163 out.go:179] * Found network options:
I1101 11:09:58.913143 564163 out.go:179] - NO_PROXY=192.168.49.2
W1101 11:09:58.916072 564163 proxy.go:120] fail to check proxy env: Error ip not in block
W1101 11:09:58.916119 564163 proxy.go:120] fail to check proxy env: Error ip not in block
I1101 11:09:58.916188 564163 ssh_runner.go:195] Run: sudo sh -c "podman version >/dev/null"
I1101 11:09:58.916238 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819-m02
I1101 11:09:58.916502 564163 ssh_runner.go:195] Run: curl -sS -m 2 https://registry.k8s.io/
I1101 11:09:58.916556 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819-m02
I1101 11:09:58.940313 564163 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33515 SSHKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819-m02/id_rsa Username:docker}
I1101 11:09:58.958881 564163 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33515 SSHKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819-m02/id_rsa Username:docker}
I1101 11:09:59.095364 564163 ssh_runner.go:195] Run: sh -c "stat /etc/cni/net.d/*loopback.conf*"
W1101 11:09:59.155074 564163 cni.go:209] loopback cni configuration skipped: "/etc/cni/net.d/*loopback.conf*" not found
I1101 11:09:59.155153 564163 ssh_runner.go:195] Run: sudo find /etc/cni/net.d -maxdepth 1 -type f ( ( -name *bridge* -or -name *podman* ) -and -not -name *.mk_disabled ) -printf "%p, " -exec sh -c "sudo mv {} {}.mk_disabled" ;
I1101 11:09:59.189592 564163 cni.go:262] disabled [/etc/cni/net.d/87-podman-bridge.conflist, /etc/cni/net.d/10-crio-bridge.conflist.disabled] bridge cni config(s)
I1101 11:09:59.189654 564163 start.go:496] detecting cgroup driver to use...
I1101 11:09:59.189741 564163 detect.go:187] detected "cgroupfs" cgroup driver on host os
I1101 11:09:59.189821 564163 ssh_runner.go:195] Run: sudo systemctl stop -f containerd
I1101 11:09:59.208490 564163 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I1101 11:09:59.221089 564163 docker.go:218] disabling cri-docker service (if available) ...
I1101 11:09:59.221155 564163 ssh_runner.go:195] Run: sudo systemctl stop -f cri-docker.socket
I1101 11:09:59.239119 564163 ssh_runner.go:195] Run: sudo systemctl stop -f cri-docker.service
I1101 11:09:59.259726 564163 ssh_runner.go:195] Run: sudo systemctl disable cri-docker.socket
I1101 11:09:59.391794 564163 ssh_runner.go:195] Run: sudo systemctl mask cri-docker.service
I1101 11:09:59.526999 564163 docker.go:234] disabling docker service ...
I1101 11:09:59.527094 564163 ssh_runner.go:195] Run: sudo systemctl stop -f docker.socket
I1101 11:09:59.551165 564163 ssh_runner.go:195] Run: sudo systemctl stop -f docker.service
I1101 11:09:59.564683 564163 ssh_runner.go:195] Run: sudo systemctl disable docker.socket
I1101 11:09:59.689243 564163 ssh_runner.go:195] Run: sudo systemctl mask docker.service
I1101 11:09:59.814478 564163 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service docker
I1101 11:09:59.830425 564163 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///var/run/crio/crio.sock
" | sudo tee /etc/crictl.yaml"
I1101 11:09:59.846686 564163 crio.go:59] configure cri-o to use "registry.k8s.io/pause:3.10.1" pause image...
I1101 11:09:59.846772 564163 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|^.*pause_image = .*$|pause_image = "registry.k8s.io/pause:3.10.1"|' /etc/crio/crio.conf.d/02-crio.conf"
I1101 11:09:59.858345 564163 crio.go:70] configuring cri-o to use "cgroupfs" as cgroup driver...
I1101 11:09:59.858425 564163 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|^.*cgroup_manager = .*$|cgroup_manager = "cgroupfs"|' /etc/crio/crio.conf.d/02-crio.conf"
I1101 11:09:59.867679 564163 ssh_runner.go:195] Run: sh -c "sudo sed -i '/conmon_cgroup = .*/d' /etc/crio/crio.conf.d/02-crio.conf"
I1101 11:09:59.876755 564163 ssh_runner.go:195] Run: sh -c "sudo sed -i '/cgroup_manager = .*/a conmon_cgroup = "pod"' /etc/crio/crio.conf.d/02-crio.conf"
I1101 11:09:59.885935 564163 ssh_runner.go:195] Run: sh -c "sudo rm -rf /etc/cni/net.mk"
I1101 11:09:59.894400 564163 ssh_runner.go:195] Run: sh -c "sudo sed -i '/^ *"net.ipv4.ip_unprivileged_port_start=.*"/d' /etc/crio/crio.conf.d/02-crio.conf"
I1101 11:09:59.903514 564163 ssh_runner.go:195] Run: sh -c "sudo grep -q "^ *default_sysctls" /etc/crio/crio.conf.d/02-crio.conf || sudo sed -i '/conmon_cgroup = .*/a default_sysctls = \[\n\]' /etc/crio/crio.conf.d/02-crio.conf"
I1101 11:09:59.917030 564163 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^default_sysctls *= *\[|&\n "net.ipv4.ip_unprivileged_port_start=0",|' /etc/crio/crio.conf.d/02-crio.conf"
I1101 11:09:59.932785 564163 ssh_runner.go:195] Run: sudo sysctl net.bridge.bridge-nf-call-iptables
I1101 11:09:59.941248 564163 ssh_runner.go:195] Run: sudo sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"
I1101 11:09:59.948963 564163 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1101 11:10:00.218274 564163 ssh_runner.go:195] Run: sudo systemctl restart crio
I1101 11:10:00.467897 564163 start.go:543] Will wait 60s for socket path /var/run/crio/crio.sock
I1101 11:10:00.468023 564163 ssh_runner.go:195] Run: stat /var/run/crio/crio.sock
I1101 11:10:00.473180 564163 start.go:564] Will wait 60s for crictl version
I1101 11:10:00.473312 564163 ssh_runner.go:195] Run: which crictl
I1101 11:10:00.478332 564163 ssh_runner.go:195] Run: sudo /usr/local/bin/crictl version
I1101 11:10:00.530922 564163 start.go:580] Version: 0.1.0
RuntimeName: cri-o
RuntimeVersion: 1.34.1
RuntimeApiVersion: v1
I1101 11:10:00.531117 564163 ssh_runner.go:195] Run: crio --version
I1101 11:10:00.572083 564163 ssh_runner.go:195] Run: crio --version
I1101 11:10:00.626266 564163 out.go:179] * Preparing Kubernetes v1.34.1 on CRI-O 1.34.1 ...
I1101 11:10:00.630488 564163 out.go:179] - env NO_PROXY=192.168.49.2
I1101 11:10:00.640119 564163 cli_runner.go:164] Run: docker network inspect ha-472819 --format "{"Name": "{{.Name}}","Driver": "{{.Driver}}","Subnet": "{{range .IPAM.Config}}{{.Subnet}}{{end}}","Gateway": "{{range .IPAM.Config}}{{.Gateway}}{{end}}","MTU": {{if (index .Options "com.docker.network.driver.mtu")}}{{(index .Options "com.docker.network.driver.mtu")}}{{else}}0{{end}}, "ContainerIPs": [{{range $k,$v := .Containers }}"{{$v.IPv4Address}}",{{end}}]}"
I1101 11:10:00.660609 564163 ssh_runner.go:195] Run: grep 192.168.49.1 host.minikube.internal$ /etc/hosts
I1101 11:10:00.667050 564163 ssh_runner.go:195] Run: /bin/bash -c "{ grep -v $'\thost.minikube.internal$' "/etc/hosts"; echo "192.168.49.1 host.minikube.internal"; } > /tmp/h.$$; sudo cp /tmp/h.$$ "/etc/hosts""
I1101 11:10:00.680675 564163 mustload.go:66] Loading cluster: ha-472819
I1101 11:10:00.680904 564163 config.go:182] Loaded profile config "ha-472819": Driver=docker, ContainerRuntime=crio, KubernetesVersion=v1.34.1
I1101 11:10:00.681185 564163 cli_runner.go:164] Run: docker container inspect ha-472819 --format={{.State.Status}}
I1101 11:10:00.703209 564163 host.go:66] Checking if "ha-472819" exists ...
I1101 11:10:00.703529 564163 certs.go:69] Setting up /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819 for IP: 192.168.49.3
I1101 11:10:00.703549 564163 certs.go:195] generating shared ca certs ...
I1101 11:10:00.703567 564163 certs.go:227] acquiring lock for ca certs: {Name:mkf1eb1b0a157a52860366e1243b59ec23d70467 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1101 11:10:00.703707 564163 certs.go:236] skipping valid "minikubeCA" ca cert: /home/jenkins/minikube-integration/21830-532863/.minikube/ca.key
I1101 11:10:00.703893 564163 certs.go:236] skipping valid "proxyClientCA" ca cert: /home/jenkins/minikube-integration/21830-532863/.minikube/proxy-client-ca.key
I1101 11:10:00.703917 564163 certs.go:257] generating profile certs ...
I1101 11:10:00.704037 564163 certs.go:360] skipping valid signed profile cert regeneration for "minikube-user": /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/client.key
I1101 11:10:00.704075 564163 certs.go:364] generating signed profile cert for "minikube": /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.key.4c464717
I1101 11:10:00.704096 564163 crypto.go:68] Generating cert /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.crt.4c464717 with IP's: [10.96.0.1 127.0.0.1 10.0.0.1 192.168.49.2 192.168.49.3 192.168.49.254]
I1101 11:10:00.826368 564163 crypto.go:156] Writing cert to /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.crt.4c464717 ...
I1101 11:10:00.826415 564163 lock.go:35] WriteFile acquiring /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.crt.4c464717: {Name:mk86b52ad2762405e19fd51a0df3aa2cea75b088 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1101 11:10:00.826658 564163 crypto.go:164] Writing key to /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.key.4c464717 ...
I1101 11:10:00.826682 564163 lock.go:35] WriteFile acquiring /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.key.4c464717: {Name:mk7b72151895b70df48e1e5a1aaae8ffe13ae0ab Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1101 11:10:00.826797 564163 certs.go:382] copying /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.crt.4c464717 -> /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.crt
I1101 11:10:00.826971 564163 certs.go:386] copying /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.key.4c464717 -> /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.key
I1101 11:10:00.827168 564163 certs.go:360] skipping valid signed profile cert regeneration for "aggregator": /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/proxy-client.key
I1101 11:10:00.827191 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/ca.crt -> /var/lib/minikube/certs/ca.crt
I1101 11:10:00.827207 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/ca.key -> /var/lib/minikube/certs/ca.key
I1101 11:10:00.827220 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/proxy-client-ca.crt -> /var/lib/minikube/certs/proxy-client-ca.crt
I1101 11:10:00.827235 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/proxy-client-ca.key -> /var/lib/minikube/certs/proxy-client-ca.key
I1101 11:10:00.827247 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.crt -> /var/lib/minikube/certs/apiserver.crt
I1101 11:10:00.827260 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.key -> /var/lib/minikube/certs/apiserver.key
I1101 11:10:00.827272 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/proxy-client.crt -> /var/lib/minikube/certs/proxy-client.crt
I1101 11:10:00.827284 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/proxy-client.key -> /var/lib/minikube/certs/proxy-client.key
I1101 11:10:00.827342 564163 certs.go:484] found cert: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/534720.pem (1338 bytes)
W1101 11:10:00.827371 564163 certs.go:480] ignoring /home/jenkins/minikube-integration/21830-532863/.minikube/certs/534720_empty.pem, impossibly tiny 0 bytes
I1101 11:10:00.827380 564163 certs.go:484] found cert: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca-key.pem (1679 bytes)
I1101 11:10:00.827406 564163 certs.go:484] found cert: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem (1078 bytes)
I1101 11:10:00.827430 564163 certs.go:484] found cert: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/cert.pem (1123 bytes)
I1101 11:10:00.827453 564163 certs.go:484] found cert: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/key.pem (1675 bytes)
I1101 11:10:00.827502 564163 certs.go:484] found cert: /home/jenkins/minikube-integration/21830-532863/.minikube/files/etc/ssl/certs/5347202.pem (1708 bytes)
I1101 11:10:00.827540 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/ca.crt -> /usr/share/ca-certificates/minikubeCA.pem
I1101 11:10:00.827559 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/534720.pem -> /usr/share/ca-certificates/534720.pem
I1101 11:10:00.827577 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/files/etc/ssl/certs/5347202.pem -> /usr/share/ca-certificates/5347202.pem
I1101 11:10:00.827662 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819
I1101 11:10:00.847637 564163 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33510 SSHKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819/id_rsa Username:docker}
I1101 11:10:00.950109 564163 ssh_runner.go:195] Run: stat -c %s /var/lib/minikube/certs/sa.pub
I1101 11:10:00.954551 564163 ssh_runner.go:447] scp /var/lib/minikube/certs/sa.pub --> memory (451 bytes)
I1101 11:10:00.963598 564163 ssh_runner.go:195] Run: stat -c %s /var/lib/minikube/certs/sa.key
I1101 11:10:00.967474 564163 ssh_runner.go:447] scp /var/lib/minikube/certs/sa.key --> memory (1675 bytes)
I1101 11:10:00.977011 564163 ssh_runner.go:195] Run: stat -c %s /var/lib/minikube/certs/front-proxy-ca.crt
I1101 11:10:00.981033 564163 ssh_runner.go:447] scp /var/lib/minikube/certs/front-proxy-ca.crt --> memory (1123 bytes)
I1101 11:10:00.990888 564163 ssh_runner.go:195] Run: stat -c %s /var/lib/minikube/certs/front-proxy-ca.key
I1101 11:10:00.995065 564163 ssh_runner.go:447] scp /var/lib/minikube/certs/front-proxy-ca.key --> memory (1675 bytes)
I1101 11:10:01.006340 564163 ssh_runner.go:195] Run: stat -c %s /var/lib/minikube/certs/etcd/ca.crt
I1101 11:10:01.011555 564163 ssh_runner.go:447] scp /var/lib/minikube/certs/etcd/ca.crt --> memory (1094 bytes)
I1101 11:10:01.021211 564163 ssh_runner.go:195] Run: stat -c %s /var/lib/minikube/certs/etcd/ca.key
I1101 11:10:01.025375 564163 ssh_runner.go:447] scp /var/lib/minikube/certs/etcd/ca.key --> memory (1675 bytes)
I1101 11:10:01.035197 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/ca.crt --> /var/lib/minikube/certs/ca.crt (1111 bytes)
I1101 11:10:01.059524 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/ca.key --> /var/lib/minikube/certs/ca.key (1679 bytes)
I1101 11:10:01.081340 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/proxy-client-ca.crt --> /var/lib/minikube/certs/proxy-client-ca.crt (1119 bytes)
I1101 11:10:01.103476 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/proxy-client-ca.key --> /var/lib/minikube/certs/proxy-client-ca.key (1679 bytes)
I1101 11:10:01.127247 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.crt --> /var/lib/minikube/certs/apiserver.crt (1436 bytes)
I1101 11:10:01.151362 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.key --> /var/lib/minikube/certs/apiserver.key (1679 bytes)
I1101 11:10:01.172198 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/proxy-client.crt --> /var/lib/minikube/certs/proxy-client.crt (1147 bytes)
I1101 11:10:01.192991 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/proxy-client.key --> /var/lib/minikube/certs/proxy-client.key (1675 bytes)
I1101 11:10:01.213776 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/ca.crt --> /usr/share/ca-certificates/minikubeCA.pem (1111 bytes)
I1101 11:10:01.236349 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/certs/534720.pem --> /usr/share/ca-certificates/534720.pem (1338 bytes)
I1101 11:10:01.258068 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/files/etc/ssl/certs/5347202.pem --> /usr/share/ca-certificates/5347202.pem (1708 bytes)
I1101 11:10:01.278528 564163 ssh_runner.go:362] scp memory --> /var/lib/minikube/certs/sa.pub (451 bytes)
I1101 11:10:01.293835 564163 ssh_runner.go:362] scp memory --> /var/lib/minikube/certs/sa.key (1675 bytes)
I1101 11:10:01.309512 564163 ssh_runner.go:362] scp memory --> /var/lib/minikube/certs/front-proxy-ca.crt (1123 bytes)
I1101 11:10:01.326059 564163 ssh_runner.go:362] scp memory --> /var/lib/minikube/certs/front-proxy-ca.key (1675 bytes)
I1101 11:10:01.340597 564163 ssh_runner.go:362] scp memory --> /var/lib/minikube/certs/etcd/ca.crt (1094 bytes)
I1101 11:10:01.356408 564163 ssh_runner.go:362] scp memory --> /var/lib/minikube/certs/etcd/ca.key (1675 bytes)
I1101 11:10:01.370667 564163 ssh_runner.go:362] scp memory --> /var/lib/minikube/kubeconfig (744 bytes)
I1101 11:10:01.385223 564163 ssh_runner.go:195] Run: openssl version
I1101 11:10:01.391999 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/minikubeCA.pem && ln -fs /usr/share/ca-certificates/minikubeCA.pem /etc/ssl/certs/minikubeCA.pem"
I1101 11:10:01.401466 564163 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/minikubeCA.pem
I1101 11:10:01.405725 564163 certs.go:528] hashing: -rw-r--r-- 1 root root 1111 Nov 1 10:49 /usr/share/ca-certificates/minikubeCA.pem
I1101 11:10:01.405836 564163 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/minikubeCA.pem
I1101 11:10:01.448102 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/b5213941.0 || ln -fs /etc/ssl/certs/minikubeCA.pem /etc/ssl/certs/b5213941.0"
I1101 11:10:01.458363 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/534720.pem && ln -fs /usr/share/ca-certificates/534720.pem /etc/ssl/certs/534720.pem"
I1101 11:10:01.467703 564163 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/534720.pem
I1101 11:10:01.471965 564163 certs.go:528] hashing: -rw-r--r-- 1 root root 1338 Nov 1 10:55 /usr/share/ca-certificates/534720.pem
I1101 11:10:01.472078 564163 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/534720.pem
I1101 11:10:01.516292 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/51391683.0 || ln -fs /etc/ssl/certs/534720.pem /etc/ssl/certs/51391683.0"
I1101 11:10:01.525777 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/5347202.pem && ln -fs /usr/share/ca-certificates/5347202.pem /etc/ssl/certs/5347202.pem"
I1101 11:10:01.535004 564163 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/5347202.pem
I1101 11:10:01.539426 564163 certs.go:528] hashing: -rw-r--r-- 1 root root 1708 Nov 1 10:55 /usr/share/ca-certificates/5347202.pem
I1101 11:10:01.539524 564163 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/5347202.pem
I1101 11:10:01.581188 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/3ec20f2e.0 || ln -fs /etc/ssl/certs/5347202.pem /etc/ssl/certs/3ec20f2e.0"
I1101 11:10:01.590430 564163 ssh_runner.go:195] Run: stat /var/lib/minikube/certs/apiserver-kubelet-client.crt
I1101 11:10:01.594810 564163 certs.go:400] 'apiserver-kubelet-client' cert doesn't exist, likely first start: stat /var/lib/minikube/certs/apiserver-kubelet-client.crt: Process exited with status 1
stdout:
stderr:
stat: cannot statx '/var/lib/minikube/certs/apiserver-kubelet-client.crt': No such file or directory
I1101 11:10:01.594902 564163 kubeadm.go:935] updating node {m02 192.168.49.3 8443 v1.34.1 crio true true} ...
I1101 11:10:01.595058 564163 kubeadm.go:947] kubelet [Unit]
Wants=crio.service
[Service]
ExecStart=
ExecStart=/var/lib/minikube/binaries/v1.34.1/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroups-per-qos=false --config=/var/lib/kubelet/config.yaml --enforce-node-allocatable= --hostname-override=ha-472819-m02 --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.168.49.3
[Install]
config:
{KubernetesVersion:v1.34.1 ClusterName:ha-472819 Namespace:default APIServerHAVIP:192.168.49.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:crio CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:}
I1101 11:10:01.595090 564163 kube-vip.go:115] generating kube-vip config ...
I1101 11:10:01.595158 564163 ssh_runner.go:195] Run: sudo sh -c "lsmod | grep ip_vs"
I1101 11:10:01.610580 564163 kube-vip.go:163] giving up enabling control-plane load-balancing as ipvs kernel modules appears not to be available: sudo sh -c "lsmod | grep ip_vs": Process exited with status 1
stdout:
stderr:
I1101 11:10:01.610640 564163 kube-vip.go:137] kube-vip config:
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
name: kube-vip
namespace: kube-system
spec:
containers:
- args:
- manager
env:
- name: vip_arp
value: "true"
- name: port
value: "8443"
- name: vip_nodename
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: vip_interface
value: eth0
- name: vip_cidr
value: "32"
- name: dns_mode
value: first
- name: cp_enable
value: "true"
- name: cp_namespace
value: kube-system
- name: vip_leaderelection
value: "true"
- name: vip_leasename
value: plndr-cp-lock
- name: vip_leaseduration
value: "5"
- name: vip_renewdeadline
value: "3"
- name: vip_retryperiod
value: "1"
- name: address
value: 192.168.49.254
- name: prometheus_server
value: :2112
image: ghcr.io/kube-vip/kube-vip:v1.0.1
imagePullPolicy: IfNotPresent
name: kube-vip
resources: {}
securityContext:
capabilities:
add:
- NET_ADMIN
- NET_RAW
volumeMounts:
- mountPath: /etc/kubernetes/admin.conf
name: kubeconfig
hostAliases:
- hostnames:
- kubernetes
ip: 127.0.0.1
hostNetwork: true
volumes:
- hostPath:
path: "/etc/kubernetes/admin.conf"
name: kubeconfig
status: {}
I1101 11:10:01.610708 564163 ssh_runner.go:195] Run: sudo ls /var/lib/minikube/binaries/v1.34.1
I1101 11:10:01.619440 564163 binaries.go:44] Found k8s binaries, skipping transfer
I1101 11:10:01.619523 564163 ssh_runner.go:195] Run: sudo mkdir -p /etc/systemd/system/kubelet.service.d /lib/systemd/system /etc/kubernetes/manifests
I1101 11:10:01.628852 564163 ssh_runner.go:362] scp memory --> /etc/systemd/system/kubelet.service.d/10-kubeadm.conf (363 bytes)
I1101 11:10:01.645438 564163 ssh_runner.go:362] scp memory --> /lib/systemd/system/kubelet.service (352 bytes)
I1101 11:10:01.660320 564163 ssh_runner.go:362] scp memory --> /etc/kubernetes/manifests/kube-vip.yaml (1358 bytes)
I1101 11:10:01.674752 564163 ssh_runner.go:195] Run: grep 192.168.49.254 control-plane.minikube.internal$ /etc/hosts
I1101 11:10:01.678687 564163 ssh_runner.go:195] Run: /bin/bash -c "{ grep -v $'\tcontrol-plane.minikube.internal$' "/etc/hosts"; echo "192.168.49.254 control-plane.minikube.internal"; } > /tmp/h.$$; sudo cp /tmp/h.$$ "/etc/hosts""
I1101 11:10:01.689054 564163 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1101 11:10:01.817597 564163 ssh_runner.go:195] Run: sudo systemctl start kubelet
I1101 11:10:01.837667 564163 host.go:66] Checking if "ha-472819" exists ...
I1101 11:10:01.838014 564163 start.go:318] joinCluster: &{Name:ha-472819 KeepContext:false EmbedCerts:false MinikubeISO: KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 Memory:3072 CPUs:2 DiskSize:20000 Driver:docker HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.34.1 ClusterName:ha-472819 Namespace:default APIServerHAVIP:192.168.49.254 APIServerName:minikubeCA APIServerN
ames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:crio CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.168.49.2 Port:8443 KubernetesVersion:v1.34.1 ContainerRuntime:crio ControlPlane:true Worker:true} {Name:m02 IP:192.168.49.3 Port:8443 KubernetesVersion:v1.34.1 ContainerRuntime:crio ControlPlane:true Worker:true}] Addons:map[default-storageclass:true storage-provisioner:true] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s MountString: Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOpti
ons:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:false DisableMetrics:false DisableCoreDNSLog:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s}
I1101 11:10:01.838163 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "env PATH="/var/lib/minikube/binaries/v1.34.1:$PATH" kubeadm token create --print-join-command --ttl=0"
I1101 11:10:01.838226 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819
I1101 11:10:01.859436 564163 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33510 SSHKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819/id_rsa Username:docker}
I1101 11:10:02.042488 564163 start.go:344] trying to join control-plane node "m02" to cluster: &{Name:m02 IP:192.168.49.3 Port:8443 KubernetesVersion:v1.34.1 ContainerRuntime:crio ControlPlane:true Worker:true}
I1101 11:10:02.042580 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "env PATH="/var/lib/minikube/binaries/v1.34.1:$PATH" kubeadm join control-plane.minikube.internal:8443 --token nj59cf.7iwf9knb5mhb6h6v --discovery-token-ca-cert-hash sha256:6154fe00c4b3b6d1ce4f3500ef815797b79de90371950bebbded24106e2601a8 --ignore-preflight-errors=all --cri-socket unix:///var/run/crio/crio.sock --node-name=ha-472819-m02 --control-plane --apiserver-advertise-address=192.168.49.3 --apiserver-bind-port=8443"
I1101 11:10:18.171255 564163 ssh_runner.go:235] Completed: sudo /bin/bash -c "env PATH="/var/lib/minikube/binaries/v1.34.1:$PATH" kubeadm join control-plane.minikube.internal:8443 --token nj59cf.7iwf9knb5mhb6h6v --discovery-token-ca-cert-hash sha256:6154fe00c4b3b6d1ce4f3500ef815797b79de90371950bebbded24106e2601a8 --ignore-preflight-errors=all --cri-socket unix:///var/run/crio/crio.sock --node-name=ha-472819-m02 --control-plane --apiserver-advertise-address=192.168.49.3 --apiserver-bind-port=8443": (16.128647108s)
I1101 11:10:18.171325 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "systemctl daemon-reload && systemctl enable kubelet && systemctl start kubelet"
I1101 11:10:18.661883 564163 ssh_runner.go:195] Run: sudo /var/lib/minikube/binaries/v1.34.1/kubectl --kubeconfig=/var/lib/minikube/kubeconfig label --overwrite nodes ha-472819-m02 minikube.k8s.io/updated_at=2025_11_01T11_10_18_0700 minikube.k8s.io/version=v1.37.0 minikube.k8s.io/commit=8d0f47abe6720ae55a5722df67bba0ddd12c8845 minikube.k8s.io/name=ha-472819 minikube.k8s.io/primary=false
I1101 11:10:18.772634 564163 ssh_runner.go:195] Run: sudo /var/lib/minikube/binaries/v1.34.1/kubectl --kubeconfig=/var/lib/minikube/kubeconfig taint nodes ha-472819-m02 node-role.kubernetes.io/control-plane:NoSchedule-
I1101 11:10:18.882683 564163 start.go:320] duration metric: took 17.044664183s to joinCluster
I1101 11:10:18.882755 564163 start.go:236] Will wait 6m0s for node &{Name:m02 IP:192.168.49.3 Port:8443 KubernetesVersion:v1.34.1 ContainerRuntime:crio ControlPlane:true Worker:true}
I1101 11:10:18.883009 564163 config.go:182] Loaded profile config "ha-472819": Driver=docker, ContainerRuntime=crio, KubernetesVersion=v1.34.1
I1101 11:10:18.885639 564163 out.go:179] * Verifying Kubernetes components...
I1101 11:10:18.888568 564163 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1101 11:10:19.048903 564163 ssh_runner.go:195] Run: sudo systemctl start kubelet
I1101 11:10:19.065355 564163 kapi.go:59] client config for ha-472819: &rest.Config{Host:"https://192.168.49.254:8443", APIPath:"", ContentConfig:rest.ContentConfig{AcceptContentTypes:"", ContentType:"", GroupVersion:(*schema.GroupVersion)(nil), NegotiatedSerializer:runtime.NegotiatedSerializer(nil)}, Username:"", Password:"", BearerToken:"", BearerTokenFile:"", Impersonate:rest.ImpersonationConfig{UserName:"", UID:"", Groups:[]string(nil), Extra:map[string][]string(nil)}, AuthProvider:<nil>, AuthConfigPersister:rest.AuthProviderConfigPersister(nil), ExecProvider:<nil>, TLSClientConfig:rest.sanitizedTLSClientConfig{Insecure:false, ServerName:"", CertFile:"/home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/client.crt", KeyFile:"/home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/client.key", CAFile:"/home/jenkins/minikube-integration/21830-532863/.minikube/ca.crt", CertData:[]uint8(nil), KeyData:[]uint8(nil), CAData:[]uint8(nil), NextProtos:[]string(n
il)}, UserAgent:"", DisableCompression:false, Transport:http.RoundTripper(nil), WrapTransport:(transport.WrapperFunc)(0x21203d0), QPS:0, Burst:0, RateLimiter:flowcontrol.RateLimiter(nil), WarningHandler:rest.WarningHandler(nil), WarningHandlerWithContext:rest.WarningHandlerWithContext(nil), Timeout:0, Dial:(func(context.Context, string, string) (net.Conn, error))(nil), Proxy:(func(*http.Request) (*url.URL, error))(nil)}
W1101 11:10:19.065459 564163 kubeadm.go:492] Overriding stale ClientConfig host https://192.168.49.254:8443 with https://192.168.49.2:8443
I1101 11:10:19.065823 564163 node_ready.go:35] waiting up to 6m0s for node "ha-472819-m02" to be "Ready" ...
W1101 11:10:21.069930 564163 node_ready.go:57] node "ha-472819-m02" has "Ready":"False" status (will retry)
W1101 11:10:23.070173 564163 node_ready.go:57] node "ha-472819-m02" has "Ready":"False" status (will retry)
W1101 11:10:25.070653 564163 node_ready.go:57] node "ha-472819-m02" has "Ready":"False" status (will retry)
W1101 11:10:27.070920 564163 node_ready.go:57] node "ha-472819-m02" has "Ready":"False" status (will retry)
W1101 11:10:29.570450 564163 node_ready.go:57] node "ha-472819-m02" has "Ready":"False" status (will retry)
W1101 11:10:32.070088 564163 node_ready.go:57] node "ha-472819-m02" has "Ready":"False" status (will retry)
W1101 11:10:34.570636 564163 node_ready.go:57] node "ha-472819-m02" has "Ready":"False" status (will retry)
W1101 11:10:37.069136 564163 node_ready.go:57] node "ha-472819-m02" has "Ready":"False" status (will retry)
W1101 11:10:39.069864 564163 node_ready.go:57] node "ha-472819-m02" has "Ready":"False" status (will retry)
W1101 11:10:41.070271 564163 node_ready.go:57] node "ha-472819-m02" has "Ready":"False" status (will retry)
W1101 11:10:43.570410 564163 node_ready.go:57] node "ha-472819-m02" has "Ready":"False" status (will retry)
W1101 11:10:46.070044 564163 node_ready.go:57] node "ha-472819-m02" has "Ready":"False" status (will retry)
W1101 11:10:48.570002 564163 node_ready.go:57] node "ha-472819-m02" has "Ready":"False" status (will retry)
W1101 11:10:51.069255 564163 node_ready.go:57] node "ha-472819-m02" has "Ready":"False" status (will retry)
W1101 11:10:53.069352 564163 node_ready.go:57] node "ha-472819-m02" has "Ready":"False" status (will retry)
W1101 11:10:55.071799 564163 node_ready.go:57] node "ha-472819-m02" has "Ready":"False" status (will retry)
W1101 11:10:57.570240 564163 node_ready.go:57] node "ha-472819-m02" has "Ready":"False" status (will retry)
W1101 11:10:59.570964 564163 node_ready.go:57] node "ha-472819-m02" has "Ready":"False" status (will retry)
W1101 11:11:02.069796 564163 node_ready.go:57] node "ha-472819-m02" has "Ready":"False" status (will retry)
I1101 11:11:02.573138 564163 node_ready.go:49] node "ha-472819-m02" is "Ready"
I1101 11:11:02.573173 564163 node_ready.go:38] duration metric: took 43.507323629s for node "ha-472819-m02" to be "Ready" ...
I1101 11:11:02.573188 564163 api_server.go:52] waiting for apiserver process to appear ...
I1101 11:11:02.573247 564163 ssh_runner.go:195] Run: sudo pgrep -xnf kube-apiserver.*minikube.*
I1101 11:11:02.592905 564163 api_server.go:72] duration metric: took 43.710116122s to wait for apiserver process to appear ...
I1101 11:11:02.592930 564163 api_server.go:88] waiting for apiserver healthz status ...
I1101 11:11:02.592950 564163 api_server.go:253] Checking apiserver healthz at https://192.168.49.2:8443/healthz ...
I1101 11:11:02.601671 564163 api_server.go:279] https://192.168.49.2:8443/healthz returned 200:
ok
I1101 11:11:02.602744 564163 api_server.go:141] control plane version: v1.34.1
I1101 11:11:02.602768 564163 api_server.go:131] duration metric: took 9.830793ms to wait for apiserver health ...
I1101 11:11:02.602776 564163 system_pods.go:43] waiting for kube-system pods to appear ...
I1101 11:11:02.608318 564163 system_pods.go:59] 17 kube-system pods found
I1101 11:11:02.608348 564163 system_pods.go:61] "coredns-66bc5c9577-bntfw" [17503733-2ab6-460c-aa3f-21d031c70abd] Running
I1101 11:11:02.608355 564163 system_pods.go:61] "coredns-66bc5c9577-n2tp2" [4b6711b0-f71a-421e-922d-eb44266c95a4] Running
I1101 11:11:02.608360 564163 system_pods.go:61] "etcd-ha-472819" [6807b695-9ca8-4691-8aac-87ff5cdaca11] Running
I1101 11:11:02.608364 564163 system_pods.go:61] "etcd-ha-472819-m02" [3cef3cc2-cf4e-4445-a55c-ce64fd2279ff] Running
I1101 11:11:02.608368 564163 system_pods.go:61] "kindnet-cw2kt" [70effae0-c034-4a35-b3d9-3e092c079100] Running
I1101 11:11:02.608372 564163 system_pods.go:61] "kindnet-dkhrw" [abb3d05e-e447-4fe5-8996-26e79d7e2b4d] Running
I1101 11:11:02.608376 564163 system_pods.go:61] "kube-apiserver-ha-472819" [a65e9eca-1f17-4ff9-b4d0-2b26612bc846] Running
I1101 11:11:02.608380 564163 system_pods.go:61] "kube-apiserver-ha-472819-m02" [c94a478e-4714-4590-8c91-17468898125c] Running
I1101 11:11:02.608385 564163 system_pods.go:61] "kube-controller-manager-ha-472819" [e6236069-2227-4783-b8e3-6df90e52e82c] Running
I1101 11:11:02.608389 564163 system_pods.go:61] "kube-controller-manager-ha-472819-m02" [f5e22b4d-d7c1-47b0-a044-4007e77d6ebc] Running
I1101 11:11:02.608398 564163 system_pods.go:61] "kube-proxy-47prj" [16f8f4f3-8267-4ce3-997b-1f4afb0f5104] Running
I1101 11:11:02.608402 564163 system_pods.go:61] "kube-proxy-djfvb" [2c010b85-48bd-4004-886f-fbe4e03884a9] Running
I1101 11:11:02.608407 564163 system_pods.go:61] "kube-scheduler-ha-472819" [78ac9fa6-2686-404f-a977-d7710745150b] Running
I1101 11:11:02.608411 564163 system_pods.go:61] "kube-scheduler-ha-472819-m02" [31b58b00-ca07-42ad-a9a7-20da16f0a251] Running
I1101 11:11:02.608415 564163 system_pods.go:61] "kube-vip-ha-472819" [0e1f82b1-9039-49f8-b83f-8c40ab9ec44f] Running
I1101 11:11:02.608419 564163 system_pods.go:61] "kube-vip-ha-472819-m02" [8964dc5d-7184-43bf-a1bd-0f9b261bb9df] Running
I1101 11:11:02.608424 564163 system_pods.go:61] "storage-provisioner" [18119b45-4932-4521-b0e9-e3a73bc6d3b1] Running
I1101 11:11:02.608433 564163 system_pods.go:74] duration metric: took 5.651361ms to wait for pod list to return data ...
I1101 11:11:02.608449 564163 default_sa.go:34] waiting for default service account to be created ...
I1101 11:11:02.615780 564163 default_sa.go:45] found service account: "default"
I1101 11:11:02.615809 564163 default_sa.go:55] duration metric: took 7.352591ms for default service account to be created ...
I1101 11:11:02.615820 564163 system_pods.go:116] waiting for k8s-apps to be running ...
I1101 11:11:02.620640 564163 system_pods.go:86] 17 kube-system pods found
I1101 11:11:02.620675 564163 system_pods.go:89] "coredns-66bc5c9577-bntfw" [17503733-2ab6-460c-aa3f-21d031c70abd] Running
I1101 11:11:02.620683 564163 system_pods.go:89] "coredns-66bc5c9577-n2tp2" [4b6711b0-f71a-421e-922d-eb44266c95a4] Running
I1101 11:11:02.620687 564163 system_pods.go:89] "etcd-ha-472819" [6807b695-9ca8-4691-8aac-87ff5cdaca11] Running
I1101 11:11:02.620691 564163 system_pods.go:89] "etcd-ha-472819-m02" [3cef3cc2-cf4e-4445-a55c-ce64fd2279ff] Running
I1101 11:11:02.620695 564163 system_pods.go:89] "kindnet-cw2kt" [70effae0-c034-4a35-b3d9-3e092c079100] Running
I1101 11:11:02.620698 564163 system_pods.go:89] "kindnet-dkhrw" [abb3d05e-e447-4fe5-8996-26e79d7e2b4d] Running
I1101 11:11:02.620705 564163 system_pods.go:89] "kube-apiserver-ha-472819" [a65e9eca-1f17-4ff9-b4d0-2b26612bc846] Running
I1101 11:11:02.620710 564163 system_pods.go:89] "kube-apiserver-ha-472819-m02" [c94a478e-4714-4590-8c91-17468898125c] Running
I1101 11:11:02.620714 564163 system_pods.go:89] "kube-controller-manager-ha-472819" [e6236069-2227-4783-b8e3-6df90e52e82c] Running
I1101 11:11:02.620718 564163 system_pods.go:89] "kube-controller-manager-ha-472819-m02" [f5e22b4d-d7c1-47b0-a044-4007e77d6ebc] Running
I1101 11:11:02.620722 564163 system_pods.go:89] "kube-proxy-47prj" [16f8f4f3-8267-4ce3-997b-1f4afb0f5104] Running
I1101 11:11:02.620726 564163 system_pods.go:89] "kube-proxy-djfvb" [2c010b85-48bd-4004-886f-fbe4e03884a9] Running
I1101 11:11:02.620731 564163 system_pods.go:89] "kube-scheduler-ha-472819" [78ac9fa6-2686-404f-a977-d7710745150b] Running
I1101 11:11:02.620740 564163 system_pods.go:89] "kube-scheduler-ha-472819-m02" [31b58b00-ca07-42ad-a9a7-20da16f0a251] Running
I1101 11:11:02.620745 564163 system_pods.go:89] "kube-vip-ha-472819" [0e1f82b1-9039-49f8-b83f-8c40ab9ec44f] Running
I1101 11:11:02.620779 564163 system_pods.go:89] "kube-vip-ha-472819-m02" [8964dc5d-7184-43bf-a1bd-0f9b261bb9df] Running
I1101 11:11:02.620787 564163 system_pods.go:89] "storage-provisioner" [18119b45-4932-4521-b0e9-e3a73bc6d3b1] Running
I1101 11:11:02.620796 564163 system_pods.go:126] duration metric: took 4.970499ms to wait for k8s-apps to be running ...
I1101 11:11:02.620805 564163 system_svc.go:44] waiting for kubelet service to be running ....
I1101 11:11:02.620889 564163 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service kubelet
I1101 11:11:02.642752 564163 system_svc.go:56] duration metric: took 21.936584ms WaitForService to wait for kubelet
I1101 11:11:02.642777 564163 kubeadm.go:587] duration metric: took 43.759995961s to wait for: map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true]
I1101 11:11:02.642797 564163 node_conditions.go:102] verifying NodePressure condition ...
I1101 11:11:02.646454 564163 node_conditions.go:122] node storage ephemeral capacity is 203034800Ki
I1101 11:11:02.646483 564163 node_conditions.go:123] node cpu capacity is 2
I1101 11:11:02.646495 564163 node_conditions.go:122] node storage ephemeral capacity is 203034800Ki
I1101 11:11:02.646499 564163 node_conditions.go:123] node cpu capacity is 2
I1101 11:11:02.646505 564163 node_conditions.go:105] duration metric: took 3.701806ms to run NodePressure ...
I1101 11:11:02.646517 564163 start.go:242] waiting for startup goroutines ...
I1101 11:11:02.646543 564163 start.go:256] writing updated cluster config ...
I1101 11:11:02.649940 564163 out.go:203]
I1101 11:11:02.652977 564163 config.go:182] Loaded profile config "ha-472819": Driver=docker, ContainerRuntime=crio, KubernetesVersion=v1.34.1
I1101 11:11:02.653122 564163 profile.go:143] Saving config to /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/config.json ...
I1101 11:11:02.656351 564163 out.go:179] * Starting "ha-472819-m03" control-plane node in "ha-472819" cluster
I1101 11:11:02.659046 564163 cache.go:124] Beginning downloading kic base image for docker with crio
I1101 11:11:02.661940 564163 out.go:179] * Pulling base image v0.0.48-1760939008-21773 ...
I1101 11:11:02.664802 564163 preload.go:183] Checking if preload exists for k8s version v1.34.1 and runtime crio
I1101 11:11:02.664836 564163 cache.go:59] Caching tarball of preloaded images
I1101 11:11:02.664905 564163 image.go:81] Checking for gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 in local docker daemon
I1101 11:11:02.664972 564163 preload.go:233] Found /home/jenkins/minikube-integration/21830-532863/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.34.1-cri-o-overlay-arm64.tar.lz4 in cache, skipping download
I1101 11:11:02.664989 564163 cache.go:62] Finished verifying existence of preloaded tar for v1.34.1 on crio
I1101 11:11:02.665111 564163 profile.go:143] Saving config to /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/config.json ...
I1101 11:11:02.684654 564163 image.go:100] Found gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 in local docker daemon, skipping pull
I1101 11:11:02.684679 564163 cache.go:148] gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 exists in daemon, skipping load
I1101 11:11:02.684692 564163 cache.go:233] Successfully downloaded all kic artifacts
I1101 11:11:02.684714 564163 start.go:360] acquireMachinesLock for ha-472819-m03: {Name:mk3b84885ff8ece87965a525482df80362a95518 Clock:{} Delay:500ms Timeout:10m0s Cancel:<nil>}
I1101 11:11:02.684837 564163 start.go:364] duration metric: took 95.632µs to acquireMachinesLock for "ha-472819-m03"
I1101 11:11:02.684871 564163 start.go:93] Provisioning new machine with config: &{Name:ha-472819 KeepContext:false EmbedCerts:false MinikubeISO: KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 Memory:3072 CPUs:2 DiskSize:20000 Driver:docker HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.34.1 ClusterName:ha-472819 Namespace:default APIServerHAVIP:192.168.49.254 APIServerNa
me:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:crio CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.168.49.2 Port:8443 KubernetesVersion:v1.34.1 ContainerRuntime:crio ControlPlane:true Worker:true} {Name:m02 IP:192.168.49.3 Port:8443 KubernetesVersion:v1.34.1 ContainerRuntime:crio ControlPlane:true Worker:true} {Name:m03 IP: Port:8443 KubernetesVersion:v1.34.1 ContainerRuntime:crio ControlPlane:true Worker:true}] Addons:map[ambassador:false amd-gpu-device-plugin:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false inaccel:false ingress:false ingress-dns:false inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow:fal
se kubetail:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s MountString: Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:false DisableMetrics:false DisableCoreDNSLog:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSH
AuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s} &{Name:m03 IP: Port:8443 KubernetesVersion:v1.34.1 ContainerRuntime:crio ControlPlane:true Worker:true}
I1101 11:11:02.684979 564163 start.go:125] createHost starting for "m03" (driver="docker")
I1101 11:11:02.688507 564163 out.go:252] * Creating docker container (CPUs=2, Memory=3072MB) ...
I1101 11:11:02.688640 564163 start.go:159] libmachine.API.Create for "ha-472819" (driver="docker")
I1101 11:11:02.688672 564163 client.go:173] LocalClient.Create starting
I1101 11:11:02.688778 564163 main.go:143] libmachine: Reading certificate data from /home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem
I1101 11:11:02.688818 564163 main.go:143] libmachine: Decoding PEM data...
I1101 11:11:02.688837 564163 main.go:143] libmachine: Parsing certificate...
I1101 11:11:02.688891 564163 main.go:143] libmachine: Reading certificate data from /home/jenkins/minikube-integration/21830-532863/.minikube/certs/cert.pem
I1101 11:11:02.688915 564163 main.go:143] libmachine: Decoding PEM data...
I1101 11:11:02.688930 564163 main.go:143] libmachine: Parsing certificate...
I1101 11:11:02.689181 564163 cli_runner.go:164] Run: docker network inspect ha-472819 --format "{"Name": "{{.Name}}","Driver": "{{.Driver}}","Subnet": "{{range .IPAM.Config}}{{.Subnet}}{{end}}","Gateway": "{{range .IPAM.Config}}{{.Gateway}}{{end}}","MTU": {{if (index .Options "com.docker.network.driver.mtu")}}{{(index .Options "com.docker.network.driver.mtu")}}{{else}}0{{end}}, "ContainerIPs": [{{range $k,$v := .Containers }}"{{$v.IPv4Address}}",{{end}}]}"
I1101 11:11:02.707661 564163 network_create.go:77] Found existing network {name:ha-472819 subnet:0x400141a090 gateway:[0 0 0 0 0 0 0 0 0 0 255 255 192 168 49 1] mtu:1500}
I1101 11:11:02.707696 564163 kic.go:121] calculated static IP "192.168.49.4" for the "ha-472819-m03" container
I1101 11:11:02.707771 564163 cli_runner.go:164] Run: docker ps -a --format {{.Names}}
I1101 11:11:02.725657 564163 cli_runner.go:164] Run: docker volume create ha-472819-m03 --label name.minikube.sigs.k8s.io=ha-472819-m03 --label created_by.minikube.sigs.k8s.io=true
I1101 11:11:02.747975 564163 oci.go:103] Successfully created a docker volume ha-472819-m03
I1101 11:11:02.748068 564163 cli_runner.go:164] Run: docker run --rm --name ha-472819-m03-preload-sidecar --label created_by.minikube.sigs.k8s.io=true --label name.minikube.sigs.k8s.io=ha-472819-m03 --entrypoint /usr/bin/test -v ha-472819-m03:/var gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 -d /var/lib
I1101 11:11:03.319169 564163 oci.go:107] Successfully prepared a docker volume ha-472819-m03
I1101 11:11:03.319219 564163 preload.go:183] Checking if preload exists for k8s version v1.34.1 and runtime crio
I1101 11:11:03.319239 564163 kic.go:194] Starting extracting preloaded images to volume ...
I1101 11:11:03.319307 564163 cli_runner.go:164] Run: docker run --rm --entrypoint /usr/bin/tar -v /home/jenkins/minikube-integration/21830-532863/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.34.1-cri-o-overlay-arm64.tar.lz4:/preloaded.tar:ro -v ha-472819-m03:/extractDir gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 -I lz4 -xf /preloaded.tar -C /extractDir
I1101 11:11:07.770669 564163 cli_runner.go:217] Completed: docker run --rm --entrypoint /usr/bin/tar -v /home/jenkins/minikube-integration/21830-532863/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.34.1-cri-o-overlay-arm64.tar.lz4:/preloaded.tar:ro -v ha-472819-m03:/extractDir gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 -I lz4 -xf /preloaded.tar -C /extractDir: (4.451319736s)
I1101 11:11:07.770702 564163 kic.go:203] duration metric: took 4.451458674s to extract preloaded images to volume ...
W1101 11:11:07.770834 564163 cgroups_linux.go:77] Your kernel does not support swap limit capabilities or the cgroup is not mounted.
I1101 11:11:07.770945 564163 cli_runner.go:164] Run: docker info --format "'{{json .SecurityOptions}}'"
I1101 11:11:07.830823 564163 cli_runner.go:164] Run: docker run -d -t --privileged --security-opt seccomp=unconfined --tmpfs /tmp --tmpfs /run -v /lib/modules:/lib/modules:ro --hostname ha-472819-m03 --name ha-472819-m03 --label created_by.minikube.sigs.k8s.io=true --label name.minikube.sigs.k8s.io=ha-472819-m03 --label role.minikube.sigs.k8s.io= --label mode.minikube.sigs.k8s.io=ha-472819-m03 --network ha-472819 --ip 192.168.49.4 --volume ha-472819-m03:/var --security-opt apparmor=unconfined --memory=3072mb --cpus=2 -e container=docker --expose 8443 --publish=127.0.0.1::8443 --publish=127.0.0.1::22 --publish=127.0.0.1::2376 --publish=127.0.0.1::5000 --publish=127.0.0.1::32443 gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8
I1101 11:11:08.209425 564163 cli_runner.go:164] Run: docker container inspect ha-472819-m03 --format={{.State.Running}}
I1101 11:11:08.233583 564163 cli_runner.go:164] Run: docker container inspect ha-472819-m03 --format={{.State.Status}}
I1101 11:11:08.259513 564163 cli_runner.go:164] Run: docker exec ha-472819-m03 stat /var/lib/dpkg/alternatives/iptables
I1101 11:11:08.318997 564163 oci.go:144] the created container "ha-472819-m03" has a running status.
I1101 11:11:08.319026 564163 kic.go:225] Creating ssh key for kic: /home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819-m03/id_rsa...
I1101 11:11:08.824570 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819-m03/id_rsa.pub -> /home/docker/.ssh/authorized_keys
I1101 11:11:08.824677 564163 kic_runner.go:191] docker (temp): /home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819-m03/id_rsa.pub --> /home/docker/.ssh/authorized_keys (381 bytes)
I1101 11:11:08.851513 564163 cli_runner.go:164] Run: docker container inspect ha-472819-m03 --format={{.State.Status}}
I1101 11:11:08.872714 564163 kic_runner.go:93] Run: chown docker:docker /home/docker/.ssh/authorized_keys
I1101 11:11:08.872734 564163 kic_runner.go:114] Args: [docker exec --privileged ha-472819-m03 chown docker:docker /home/docker/.ssh/authorized_keys]
I1101 11:11:08.929245 564163 cli_runner.go:164] Run: docker container inspect ha-472819-m03 --format={{.State.Status}}
I1101 11:11:08.951783 564163 machine.go:94] provisionDockerMachine start ...
I1101 11:11:08.951884 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819-m03
I1101 11:11:08.971759 564163 main.go:143] libmachine: Using SSH client type: native
I1101 11:11:08.972075 564163 main.go:143] libmachine: &{{{<nil> 0 [] [] []} docker [0x3ef110] 0x3f18d0 <nil> [] 0s} 127.0.0.1 33520 <nil> <nil>}
I1101 11:11:08.972084 564163 main.go:143] libmachine: About to run SSH command:
hostname
I1101 11:11:08.972780 564163 main.go:143] libmachine: Error dialing TCP: ssh: handshake failed: EOF
I1101 11:11:12.129991 564163 main.go:143] libmachine: SSH cmd err, output: <nil>: ha-472819-m03
I1101 11:11:12.130016 564163 ubuntu.go:182] provisioning hostname "ha-472819-m03"
I1101 11:11:12.130084 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819-m03
I1101 11:11:12.158484 564163 main.go:143] libmachine: Using SSH client type: native
I1101 11:11:12.158792 564163 main.go:143] libmachine: &{{{<nil> 0 [] [] []} docker [0x3ef110] 0x3f18d0 <nil> [] 0s} 127.0.0.1 33520 <nil> <nil>}
I1101 11:11:12.158807 564163 main.go:143] libmachine: About to run SSH command:
sudo hostname ha-472819-m03 && echo "ha-472819-m03" | sudo tee /etc/hostname
I1101 11:11:12.332540 564163 main.go:143] libmachine: SSH cmd err, output: <nil>: ha-472819-m03
I1101 11:11:12.332619 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819-m03
I1101 11:11:12.351114 564163 main.go:143] libmachine: Using SSH client type: native
I1101 11:11:12.351425 564163 main.go:143] libmachine: &{{{<nil> 0 [] [] []} docker [0x3ef110] 0x3f18d0 <nil> [] 0s} 127.0.0.1 33520 <nil> <nil>}
I1101 11:11:12.351444 564163 main.go:143] libmachine: About to run SSH command:
if ! grep -xq '.*\sha-472819-m03' /etc/hosts; then
if grep -xq '127.0.1.1\s.*' /etc/hosts; then
sudo sed -i 's/^127.0.1.1\s.*/127.0.1.1 ha-472819-m03/g' /etc/hosts;
else
echo '127.0.1.1 ha-472819-m03' | sudo tee -a /etc/hosts;
fi
fi
I1101 11:11:12.502378 564163 main.go:143] libmachine: SSH cmd err, output: <nil>:
I1101 11:11:12.502450 564163 ubuntu.go:188] set auth options {CertDir:/home/jenkins/minikube-integration/21830-532863/.minikube CaCertPath:/home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem CaPrivateKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca-key.pem CaCertRemotePath:/etc/docker/ca.pem ServerCertPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/server.pem ServerKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/server-key.pem ClientKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/certs/key.pem ServerCertRemotePath:/etc/docker/server.pem ServerKeyRemotePath:/etc/docker/server-key.pem ClientCertPath:/home/jenkins/minikube-integration/21830-532863/.minikube/certs/cert.pem ServerCertSANs:[] StorePath:/home/jenkins/minikube-integration/21830-532863/.minikube}
I1101 11:11:12.502486 564163 ubuntu.go:190] setting up certificates
I1101 11:11:12.502527 564163 provision.go:84] configureAuth start
I1101 11:11:12.502632 564163 cli_runner.go:164] Run: docker container inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}" ha-472819-m03
I1101 11:11:12.519713 564163 provision.go:143] copyHostCerts
I1101 11:11:12.519756 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem -> /home/jenkins/minikube-integration/21830-532863/.minikube/ca.pem
I1101 11:11:12.519789 564163 exec_runner.go:144] found /home/jenkins/minikube-integration/21830-532863/.minikube/ca.pem, removing ...
I1101 11:11:12.519796 564163 exec_runner.go:203] rm: /home/jenkins/minikube-integration/21830-532863/.minikube/ca.pem
I1101 11:11:12.519876 564163 exec_runner.go:151] cp: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem --> /home/jenkins/minikube-integration/21830-532863/.minikube/ca.pem (1078 bytes)
I1101 11:11:12.519955 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/cert.pem -> /home/jenkins/minikube-integration/21830-532863/.minikube/cert.pem
I1101 11:11:12.519972 564163 exec_runner.go:144] found /home/jenkins/minikube-integration/21830-532863/.minikube/cert.pem, removing ...
I1101 11:11:12.519977 564163 exec_runner.go:203] rm: /home/jenkins/minikube-integration/21830-532863/.minikube/cert.pem
I1101 11:11:12.520002 564163 exec_runner.go:151] cp: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/cert.pem --> /home/jenkins/minikube-integration/21830-532863/.minikube/cert.pem (1123 bytes)
I1101 11:11:12.520040 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/key.pem -> /home/jenkins/minikube-integration/21830-532863/.minikube/key.pem
I1101 11:11:12.520056 564163 exec_runner.go:144] found /home/jenkins/minikube-integration/21830-532863/.minikube/key.pem, removing ...
I1101 11:11:12.520060 564163 exec_runner.go:203] rm: /home/jenkins/minikube-integration/21830-532863/.minikube/key.pem
I1101 11:11:12.520083 564163 exec_runner.go:151] cp: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/key.pem --> /home/jenkins/minikube-integration/21830-532863/.minikube/key.pem (1675 bytes)
I1101 11:11:12.520129 564163 provision.go:117] generating server cert: /home/jenkins/minikube-integration/21830-532863/.minikube/machines/server.pem ca-key=/home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem private-key=/home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca-key.pem org=jenkins.ha-472819-m03 san=[127.0.0.1 192.168.49.4 ha-472819-m03 localhost minikube]
I1101 11:11:13.612826 564163 provision.go:177] copyRemoteCerts
I1101 11:11:13.612953 564163 ssh_runner.go:195] Run: sudo mkdir -p /etc/docker /etc/docker /etc/docker
I1101 11:11:13.613033 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819-m03
I1101 11:11:13.637098 564163 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33520 SSHKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819-m03/id_rsa Username:docker}
I1101 11:11:13.745943 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem -> /etc/docker/ca.pem
I1101 11:11:13.746008 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem --> /etc/docker/ca.pem (1078 bytes)
I1101 11:11:13.765430 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/machines/server.pem -> /etc/docker/server.pem
I1101 11:11:13.765498 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/machines/server.pem --> /etc/docker/server.pem (1208 bytes)
I1101 11:11:13.784115 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/machines/server-key.pem -> /etc/docker/server-key.pem
I1101 11:11:13.784225 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/machines/server-key.pem --> /etc/docker/server-key.pem (1675 bytes)
I1101 11:11:13.804599 564163 provision.go:87] duration metric: took 1.3020372s to configureAuth
I1101 11:11:13.804630 564163 ubuntu.go:206] setting minikube options for container-runtime
I1101 11:11:13.804902 564163 config.go:182] Loaded profile config "ha-472819": Driver=docker, ContainerRuntime=crio, KubernetesVersion=v1.34.1
I1101 11:11:13.805040 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819-m03
I1101 11:11:13.824430 564163 main.go:143] libmachine: Using SSH client type: native
I1101 11:11:13.824739 564163 main.go:143] libmachine: &{{{<nil> 0 [] [] []} docker [0x3ef110] 0x3f18d0 <nil> [] 0s} 127.0.0.1 33520 <nil> <nil>}
I1101 11:11:13.824766 564163 main.go:143] libmachine: About to run SSH command:
sudo mkdir -p /etc/sysconfig && printf %s "
CRIO_MINIKUBE_OPTIONS='--insecure-registry 10.96.0.0/12 '
" | sudo tee /etc/sysconfig/crio.minikube && sudo systemctl restart crio
I1101 11:11:14.162205 564163 main.go:143] libmachine: SSH cmd err, output: <nil>:
CRIO_MINIKUBE_OPTIONS='--insecure-registry 10.96.0.0/12 '
I1101 11:11:14.162231 564163 machine.go:97] duration metric: took 5.210427743s to provisionDockerMachine
I1101 11:11:14.162240 564163 client.go:176] duration metric: took 11.473529387s to LocalClient.Create
I1101 11:11:14.162254 564163 start.go:167] duration metric: took 11.473618142s to libmachine.API.Create "ha-472819"
I1101 11:11:14.162261 564163 start.go:293] postStartSetup for "ha-472819-m03" (driver="docker")
I1101 11:11:14.162271 564163 start.go:322] creating required directories: [/etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs]
I1101 11:11:14.162342 564163 ssh_runner.go:195] Run: sudo mkdir -p /etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs
I1101 11:11:14.162391 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819-m03
I1101 11:11:14.180648 564163 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33520 SSHKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819-m03/id_rsa Username:docker}
I1101 11:11:14.290174 564163 ssh_runner.go:195] Run: cat /etc/os-release
I1101 11:11:14.293569 564163 main.go:143] libmachine: Couldn't set key VERSION_CODENAME, no corresponding struct field found
I1101 11:11:14.293597 564163 info.go:137] Remote host: Debian GNU/Linux 12 (bookworm)
I1101 11:11:14.293610 564163 filesync.go:126] Scanning /home/jenkins/minikube-integration/21830-532863/.minikube/addons for local assets ...
I1101 11:11:14.293673 564163 filesync.go:126] Scanning /home/jenkins/minikube-integration/21830-532863/.minikube/files for local assets ...
I1101 11:11:14.293794 564163 filesync.go:149] local asset: /home/jenkins/minikube-integration/21830-532863/.minikube/files/etc/ssl/certs/5347202.pem -> 5347202.pem in /etc/ssl/certs
I1101 11:11:14.293802 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/files/etc/ssl/certs/5347202.pem -> /etc/ssl/certs/5347202.pem
I1101 11:11:14.293910 564163 ssh_runner.go:195] Run: sudo mkdir -p /etc/ssl/certs
I1101 11:11:14.301571 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/files/etc/ssl/certs/5347202.pem --> /etc/ssl/certs/5347202.pem (1708 bytes)
I1101 11:11:14.319949 564163 start.go:296] duration metric: took 157.672021ms for postStartSetup
I1101 11:11:14.320309 564163 cli_runner.go:164] Run: docker container inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}" ha-472819-m03
I1101 11:11:14.339186 564163 profile.go:143] Saving config to /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/config.json ...
I1101 11:11:14.339488 564163 ssh_runner.go:195] Run: sh -c "df -h /var | awk 'NR==2{print $5}'"
I1101 11:11:14.339544 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819-m03
I1101 11:11:14.356118 564163 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33520 SSHKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819-m03/id_rsa Username:docker}
I1101 11:11:14.462914 564163 ssh_runner.go:195] Run: sh -c "df -BG /var | awk 'NR==2{print $4}'"
I1101 11:11:14.467880 564163 start.go:128] duration metric: took 11.782886335s to createHost
I1101 11:11:14.467906 564163 start.go:83] releasing machines lock for "ha-472819-m03", held for 11.783053073s
I1101 11:11:14.467977 564163 cli_runner.go:164] Run: docker container inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}" ha-472819-m03
I1101 11:11:14.491429 564163 out.go:179] * Found network options:
I1101 11:11:14.494078 564163 out.go:179] - NO_PROXY=192.168.49.2,192.168.49.3
W1101 11:11:14.497086 564163 proxy.go:120] fail to check proxy env: Error ip not in block
W1101 11:11:14.497117 564163 proxy.go:120] fail to check proxy env: Error ip not in block
W1101 11:11:14.497140 564163 proxy.go:120] fail to check proxy env: Error ip not in block
W1101 11:11:14.497150 564163 proxy.go:120] fail to check proxy env: Error ip not in block
I1101 11:11:14.497218 564163 ssh_runner.go:195] Run: sudo sh -c "podman version >/dev/null"
I1101 11:11:14.497268 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819-m03
I1101 11:11:14.497522 564163 ssh_runner.go:195] Run: curl -sS -m 2 https://registry.k8s.io/
I1101 11:11:14.497567 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819-m03
I1101 11:11:14.527486 564163 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33520 SSHKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819-m03/id_rsa Username:docker}
I1101 11:11:14.535318 564163 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33520 SSHKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819-m03/id_rsa Username:docker}
I1101 11:11:14.691428 564163 ssh_runner.go:195] Run: sh -c "stat /etc/cni/net.d/*loopback.conf*"
W1101 11:11:14.751657 564163 cni.go:209] loopback cni configuration skipped: "/etc/cni/net.d/*loopback.conf*" not found
I1101 11:11:14.751737 564163 ssh_runner.go:195] Run: sudo find /etc/cni/net.d -maxdepth 1 -type f ( ( -name *bridge* -or -name *podman* ) -and -not -name *.mk_disabled ) -printf "%p, " -exec sh -c "sudo mv {} {}.mk_disabled" ;
I1101 11:11:14.784282 564163 cni.go:262] disabled [/etc/cni/net.d/87-podman-bridge.conflist, /etc/cni/net.d/10-crio-bridge.conflist.disabled] bridge cni config(s)
I1101 11:11:14.784309 564163 start.go:496] detecting cgroup driver to use...
I1101 11:11:14.784342 564163 detect.go:187] detected "cgroupfs" cgroup driver on host os
I1101 11:11:14.784395 564163 ssh_runner.go:195] Run: sudo systemctl stop -f containerd
I1101 11:11:14.804100 564163 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I1101 11:11:14.817895 564163 docker.go:218] disabling cri-docker service (if available) ...
I1101 11:11:14.817997 564163 ssh_runner.go:195] Run: sudo systemctl stop -f cri-docker.socket
I1101 11:11:14.836631 564163 ssh_runner.go:195] Run: sudo systemctl stop -f cri-docker.service
I1101 11:11:14.858424 564163 ssh_runner.go:195] Run: sudo systemctl disable cri-docker.socket
I1101 11:11:14.995659 564163 ssh_runner.go:195] Run: sudo systemctl mask cri-docker.service
I1101 11:11:15.154010 564163 docker.go:234] disabling docker service ...
I1101 11:11:15.154132 564163 ssh_runner.go:195] Run: sudo systemctl stop -f docker.socket
I1101 11:11:15.178485 564163 ssh_runner.go:195] Run: sudo systemctl stop -f docker.service
I1101 11:11:15.193239 564163 ssh_runner.go:195] Run: sudo systemctl disable docker.socket
I1101 11:11:15.327679 564163 ssh_runner.go:195] Run: sudo systemctl mask docker.service
I1101 11:11:15.454029 564163 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service docker
I1101 11:11:15.467802 564163 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///var/run/crio/crio.sock
" | sudo tee /etc/crictl.yaml"
I1101 11:11:15.484429 564163 crio.go:59] configure cri-o to use "registry.k8s.io/pause:3.10.1" pause image...
I1101 11:11:15.484525 564163 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|^.*pause_image = .*$|pause_image = "registry.k8s.io/pause:3.10.1"|' /etc/crio/crio.conf.d/02-crio.conf"
I1101 11:11:15.494973 564163 crio.go:70] configuring cri-o to use "cgroupfs" as cgroup driver...
I1101 11:11:15.495089 564163 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|^.*cgroup_manager = .*$|cgroup_manager = "cgroupfs"|' /etc/crio/crio.conf.d/02-crio.conf"
I1101 11:11:15.505092 564163 ssh_runner.go:195] Run: sh -c "sudo sed -i '/conmon_cgroup = .*/d' /etc/crio/crio.conf.d/02-crio.conf"
I1101 11:11:15.515189 564163 ssh_runner.go:195] Run: sh -c "sudo sed -i '/cgroup_manager = .*/a conmon_cgroup = "pod"' /etc/crio/crio.conf.d/02-crio.conf"
I1101 11:11:15.525494 564163 ssh_runner.go:195] Run: sh -c "sudo rm -rf /etc/cni/net.mk"
I1101 11:11:15.535841 564163 ssh_runner.go:195] Run: sh -c "sudo sed -i '/^ *"net.ipv4.ip_unprivileged_port_start=.*"/d' /etc/crio/crio.conf.d/02-crio.conf"
I1101 11:11:15.545915 564163 ssh_runner.go:195] Run: sh -c "sudo grep -q "^ *default_sysctls" /etc/crio/crio.conf.d/02-crio.conf || sudo sed -i '/conmon_cgroup = .*/a default_sysctls = \[\n\]' /etc/crio/crio.conf.d/02-crio.conf"
I1101 11:11:15.567914 564163 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^default_sysctls *= *\[|&\n "net.ipv4.ip_unprivileged_port_start=0",|' /etc/crio/crio.conf.d/02-crio.conf"
I1101 11:11:15.577406 564163 ssh_runner.go:195] Run: sudo sysctl net.bridge.bridge-nf-call-iptables
I1101 11:11:15.585734 564163 ssh_runner.go:195] Run: sudo sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"
I1101 11:11:15.594497 564163 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1101 11:11:15.717478 564163 ssh_runner.go:195] Run: sudo systemctl restart crio
I1101 11:11:15.855286 564163 start.go:543] Will wait 60s for socket path /var/run/crio/crio.sock
I1101 11:11:15.855416 564163 ssh_runner.go:195] Run: stat /var/run/crio/crio.sock
I1101 11:11:15.860080 564163 start.go:564] Will wait 60s for crictl version
I1101 11:11:15.860199 564163 ssh_runner.go:195] Run: which crictl
I1101 11:11:15.864416 564163 ssh_runner.go:195] Run: sudo /usr/local/bin/crictl version
I1101 11:11:15.901128 564163 start.go:580] Version: 0.1.0
RuntimeName: cri-o
RuntimeVersion: 1.34.1
RuntimeApiVersion: v1
I1101 11:11:15.901276 564163 ssh_runner.go:195] Run: crio --version
I1101 11:11:15.939191 564163 ssh_runner.go:195] Run: crio --version
I1101 11:11:15.976924 564163 out.go:179] * Preparing Kubernetes v1.34.1 on CRI-O 1.34.1 ...
I1101 11:11:15.979768 564163 out.go:179] - env NO_PROXY=192.168.49.2
I1101 11:11:15.982623 564163 out.go:179] - env NO_PROXY=192.168.49.2,192.168.49.3
I1101 11:11:15.985569 564163 cli_runner.go:164] Run: docker network inspect ha-472819 --format "{"Name": "{{.Name}}","Driver": "{{.Driver}}","Subnet": "{{range .IPAM.Config}}{{.Subnet}}{{end}}","Gateway": "{{range .IPAM.Config}}{{.Gateway}}{{end}}","MTU": {{if (index .Options "com.docker.network.driver.mtu")}}{{(index .Options "com.docker.network.driver.mtu")}}{{else}}0{{end}}, "ContainerIPs": [{{range $k,$v := .Containers }}"{{$v.IPv4Address}}",{{end}}]}"
I1101 11:11:16.003209 564163 ssh_runner.go:195] Run: grep 192.168.49.1 host.minikube.internal$ /etc/hosts
I1101 11:11:16.009752 564163 ssh_runner.go:195] Run: /bin/bash -c "{ grep -v $'\thost.minikube.internal$' "/etc/hosts"; echo "192.168.49.1 host.minikube.internal"; } > /tmp/h.$$; sudo cp /tmp/h.$$ "/etc/hosts""
I1101 11:11:16.022020 564163 mustload.go:66] Loading cluster: ha-472819
I1101 11:11:16.022297 564163 config.go:182] Loaded profile config "ha-472819": Driver=docker, ContainerRuntime=crio, KubernetesVersion=v1.34.1
I1101 11:11:16.022563 564163 cli_runner.go:164] Run: docker container inspect ha-472819 --format={{.State.Status}}
I1101 11:11:16.041810 564163 host.go:66] Checking if "ha-472819" exists ...
I1101 11:11:16.042214 564163 certs.go:69] Setting up /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819 for IP: 192.168.49.4
I1101 11:11:16.042226 564163 certs.go:195] generating shared ca certs ...
I1101 11:11:16.042242 564163 certs.go:227] acquiring lock for ca certs: {Name:mkf1eb1b0a157a52860366e1243b59ec23d70467 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1101 11:11:16.042364 564163 certs.go:236] skipping valid "minikubeCA" ca cert: /home/jenkins/minikube-integration/21830-532863/.minikube/ca.key
I1101 11:11:16.042403 564163 certs.go:236] skipping valid "proxyClientCA" ca cert: /home/jenkins/minikube-integration/21830-532863/.minikube/proxy-client-ca.key
I1101 11:11:16.042420 564163 certs.go:257] generating profile certs ...
I1101 11:11:16.042507 564163 certs.go:360] skipping valid signed profile cert regeneration for "minikube-user": /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/client.key
I1101 11:11:16.042544 564163 certs.go:364] generating signed profile cert for "minikube": /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.key.b77bbb0d
I1101 11:11:16.042559 564163 crypto.go:68] Generating cert /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.crt.b77bbb0d with IP's: [10.96.0.1 127.0.0.1 10.0.0.1 192.168.49.2 192.168.49.3 192.168.49.4 192.168.49.254]
I1101 11:11:17.419467 564163 crypto.go:156] Writing cert to /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.crt.b77bbb0d ...
I1101 11:11:17.419504 564163 lock.go:35] WriteFile acquiring /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.crt.b77bbb0d: {Name:mke3ca75daab1021e235325f0aa6ae3fdb3aebaa Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1101 11:11:17.419709 564163 crypto.go:164] Writing key to /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.key.b77bbb0d ...
I1101 11:11:17.419723 564163 lock.go:35] WriteFile acquiring /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.key.b77bbb0d: {Name:mk05b86323e75bb15d0b4b2c07a8199585004a27 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1101 11:11:17.419819 564163 certs.go:382] copying /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.crt.b77bbb0d -> /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.crt
I1101 11:11:17.419955 564163 certs.go:386] copying /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.key.b77bbb0d -> /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.key
I1101 11:11:17.420103 564163 certs.go:360] skipping valid signed profile cert regeneration for "aggregator": /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/proxy-client.key
I1101 11:11:17.420121 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/ca.crt -> /var/lib/minikube/certs/ca.crt
I1101 11:11:17.420136 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/ca.key -> /var/lib/minikube/certs/ca.key
I1101 11:11:17.420154 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/proxy-client-ca.crt -> /var/lib/minikube/certs/proxy-client-ca.crt
I1101 11:11:17.420170 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/proxy-client-ca.key -> /var/lib/minikube/certs/proxy-client-ca.key
I1101 11:11:17.420183 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.crt -> /var/lib/minikube/certs/apiserver.crt
I1101 11:11:17.420205 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.key -> /var/lib/minikube/certs/apiserver.key
I1101 11:11:17.420223 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/proxy-client.crt -> /var/lib/minikube/certs/proxy-client.crt
I1101 11:11:17.420239 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/proxy-client.key -> /var/lib/minikube/certs/proxy-client.key
I1101 11:11:17.420291 564163 certs.go:484] found cert: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/534720.pem (1338 bytes)
W1101 11:11:17.420324 564163 certs.go:480] ignoring /home/jenkins/minikube-integration/21830-532863/.minikube/certs/534720_empty.pem, impossibly tiny 0 bytes
I1101 11:11:17.420336 564163 certs.go:484] found cert: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca-key.pem (1679 bytes)
I1101 11:11:17.420360 564163 certs.go:484] found cert: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/ca.pem (1078 bytes)
I1101 11:11:17.420385 564163 certs.go:484] found cert: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/cert.pem (1123 bytes)
I1101 11:11:17.420410 564163 certs.go:484] found cert: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/key.pem (1675 bytes)
I1101 11:11:17.420457 564163 certs.go:484] found cert: /home/jenkins/minikube-integration/21830-532863/.minikube/files/etc/ssl/certs/5347202.pem (1708 bytes)
I1101 11:11:17.420490 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/ca.crt -> /usr/share/ca-certificates/minikubeCA.pem
I1101 11:11:17.420505 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/certs/534720.pem -> /usr/share/ca-certificates/534720.pem
I1101 11:11:17.420518 564163 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/21830-532863/.minikube/files/etc/ssl/certs/5347202.pem -> /usr/share/ca-certificates/5347202.pem
I1101 11:11:17.420579 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819
I1101 11:11:17.446123 564163 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33510 SSHKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819/id_rsa Username:docker}
I1101 11:11:17.550099 564163 ssh_runner.go:195] Run: stat -c %s /var/lib/minikube/certs/sa.pub
I1101 11:11:17.554459 564163 ssh_runner.go:447] scp /var/lib/minikube/certs/sa.pub --> memory (451 bytes)
I1101 11:11:17.564628 564163 ssh_runner.go:195] Run: stat -c %s /var/lib/minikube/certs/sa.key
I1101 11:11:17.568775 564163 ssh_runner.go:447] scp /var/lib/minikube/certs/sa.key --> memory (1675 bytes)
I1101 11:11:17.578449 564163 ssh_runner.go:195] Run: stat -c %s /var/lib/minikube/certs/front-proxy-ca.crt
I1101 11:11:17.582314 564163 ssh_runner.go:447] scp /var/lib/minikube/certs/front-proxy-ca.crt --> memory (1123 bytes)
I1101 11:11:17.596987 564163 ssh_runner.go:195] Run: stat -c %s /var/lib/minikube/certs/front-proxy-ca.key
I1101 11:11:17.602143 564163 ssh_runner.go:447] scp /var/lib/minikube/certs/front-proxy-ca.key --> memory (1675 bytes)
I1101 11:11:17.612550 564163 ssh_runner.go:195] Run: stat -c %s /var/lib/minikube/certs/etcd/ca.crt
I1101 11:11:17.616760 564163 ssh_runner.go:447] scp /var/lib/minikube/certs/etcd/ca.crt --> memory (1094 bytes)
I1101 11:11:17.625280 564163 ssh_runner.go:195] Run: stat -c %s /var/lib/minikube/certs/etcd/ca.key
I1101 11:11:17.629323 564163 ssh_runner.go:447] scp /var/lib/minikube/certs/etcd/ca.key --> memory (1675 bytes)
I1101 11:11:17.637956 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/ca.crt --> /var/lib/minikube/certs/ca.crt (1111 bytes)
I1101 11:11:17.670252 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/ca.key --> /var/lib/minikube/certs/ca.key (1679 bytes)
I1101 11:11:17.690944 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/proxy-client-ca.crt --> /var/lib/minikube/certs/proxy-client-ca.crt (1119 bytes)
I1101 11:11:17.712898 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/proxy-client-ca.key --> /var/lib/minikube/certs/proxy-client-ca.key (1679 bytes)
I1101 11:11:17.732957 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.crt --> /var/lib/minikube/certs/apiserver.crt (1444 bytes)
I1101 11:11:17.754424 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/apiserver.key --> /var/lib/minikube/certs/apiserver.key (1675 bytes)
I1101 11:11:17.774366 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/proxy-client.crt --> /var/lib/minikube/certs/proxy-client.crt (1147 bytes)
I1101 11:11:17.794672 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/proxy-client.key --> /var/lib/minikube/certs/proxy-client.key (1675 bytes)
I1101 11:11:17.813962 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/ca.crt --> /usr/share/ca-certificates/minikubeCA.pem (1111 bytes)
I1101 11:11:17.832048 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/certs/534720.pem --> /usr/share/ca-certificates/534720.pem (1338 bytes)
I1101 11:11:17.851761 564163 ssh_runner.go:362] scp /home/jenkins/minikube-integration/21830-532863/.minikube/files/etc/ssl/certs/5347202.pem --> /usr/share/ca-certificates/5347202.pem (1708 bytes)
I1101 11:11:17.870024 564163 ssh_runner.go:362] scp memory --> /var/lib/minikube/certs/sa.pub (451 bytes)
I1101 11:11:17.882634 564163 ssh_runner.go:362] scp memory --> /var/lib/minikube/certs/sa.key (1675 bytes)
I1101 11:11:17.899889 564163 ssh_runner.go:362] scp memory --> /var/lib/minikube/certs/front-proxy-ca.crt (1123 bytes)
I1101 11:11:17.913845 564163 ssh_runner.go:362] scp memory --> /var/lib/minikube/certs/front-proxy-ca.key (1675 bytes)
I1101 11:11:17.938379 564163 ssh_runner.go:362] scp memory --> /var/lib/minikube/certs/etcd/ca.crt (1094 bytes)
I1101 11:11:17.953783 564163 ssh_runner.go:362] scp memory --> /var/lib/minikube/certs/etcd/ca.key (1675 bytes)
I1101 11:11:17.969134 564163 ssh_runner.go:362] scp memory --> /var/lib/minikube/kubeconfig (744 bytes)
I1101 11:11:17.984029 564163 ssh_runner.go:195] Run: openssl version
I1101 11:11:17.990588 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/5347202.pem && ln -fs /usr/share/ca-certificates/5347202.pem /etc/ssl/certs/5347202.pem"
I1101 11:11:17.998983 564163 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/5347202.pem
I1101 11:11:18.003748 564163 certs.go:528] hashing: -rw-r--r-- 1 root root 1708 Nov 1 10:55 /usr/share/ca-certificates/5347202.pem
I1101 11:11:18.003824 564163 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/5347202.pem
I1101 11:11:18.047711 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/3ec20f2e.0 || ln -fs /etc/ssl/certs/5347202.pem /etc/ssl/certs/3ec20f2e.0"
I1101 11:11:18.056769 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/minikubeCA.pem && ln -fs /usr/share/ca-certificates/minikubeCA.pem /etc/ssl/certs/minikubeCA.pem"
I1101 11:11:18.066764 564163 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/minikubeCA.pem
I1101 11:11:18.071057 564163 certs.go:528] hashing: -rw-r--r-- 1 root root 1111 Nov 1 10:49 /usr/share/ca-certificates/minikubeCA.pem
I1101 11:11:18.071127 564163 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/minikubeCA.pem
I1101 11:11:18.115164 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/b5213941.0 || ln -fs /etc/ssl/certs/minikubeCA.pem /etc/ssl/certs/b5213941.0"
I1101 11:11:18.125731 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/534720.pem && ln -fs /usr/share/ca-certificates/534720.pem /etc/ssl/certs/534720.pem"
I1101 11:11:18.136765 564163 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/534720.pem
I1101 11:11:18.143282 564163 certs.go:528] hashing: -rw-r--r-- 1 root root 1338 Nov 1 10:55 /usr/share/ca-certificates/534720.pem
I1101 11:11:18.143350 564163 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/534720.pem
I1101 11:11:18.186179 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/51391683.0 || ln -fs /etc/ssl/certs/534720.pem /etc/ssl/certs/51391683.0"
I1101 11:11:18.195359 564163 ssh_runner.go:195] Run: stat /var/lib/minikube/certs/apiserver-kubelet-client.crt
I1101 11:11:18.199380 564163 certs.go:400] 'apiserver-kubelet-client' cert doesn't exist, likely first start: stat /var/lib/minikube/certs/apiserver-kubelet-client.crt: Process exited with status 1
stdout:
stderr:
stat: cannot statx '/var/lib/minikube/certs/apiserver-kubelet-client.crt': No such file or directory
I1101 11:11:18.199489 564163 kubeadm.go:935] updating node {m03 192.168.49.4 8443 v1.34.1 crio true true} ...
I1101 11:11:18.199610 564163 kubeadm.go:947] kubelet [Unit]
Wants=crio.service
[Service]
ExecStart=
ExecStart=/var/lib/minikube/binaries/v1.34.1/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroups-per-qos=false --config=/var/lib/kubelet/config.yaml --enforce-node-allocatable= --hostname-override=ha-472819-m03 --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.168.49.4
[Install]
config:
{KubernetesVersion:v1.34.1 ClusterName:ha-472819 Namespace:default APIServerHAVIP:192.168.49.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:crio CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:}
I1101 11:11:18.199643 564163 kube-vip.go:115] generating kube-vip config ...
I1101 11:11:18.199705 564163 ssh_runner.go:195] Run: sudo sh -c "lsmod | grep ip_vs"
I1101 11:11:18.212173 564163 kube-vip.go:163] giving up enabling control-plane load-balancing as ipvs kernel modules appears not to be available: sudo sh -c "lsmod | grep ip_vs": Process exited with status 1
stdout:
stderr:
I1101 11:11:18.212238 564163 kube-vip.go:137] kube-vip config:
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
name: kube-vip
namespace: kube-system
spec:
containers:
- args:
- manager
env:
- name: vip_arp
value: "true"
- name: port
value: "8443"
- name: vip_nodename
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: vip_interface
value: eth0
- name: vip_cidr
value: "32"
- name: dns_mode
value: first
- name: cp_enable
value: "true"
- name: cp_namespace
value: kube-system
- name: vip_leaderelection
value: "true"
- name: vip_leasename
value: plndr-cp-lock
- name: vip_leaseduration
value: "5"
- name: vip_renewdeadline
value: "3"
- name: vip_retryperiod
value: "1"
- name: address
value: 192.168.49.254
- name: prometheus_server
value: :2112
image: ghcr.io/kube-vip/kube-vip:v1.0.1
imagePullPolicy: IfNotPresent
name: kube-vip
resources: {}
securityContext:
capabilities:
add:
- NET_ADMIN
- NET_RAW
volumeMounts:
- mountPath: /etc/kubernetes/admin.conf
name: kubeconfig
hostAliases:
- hostnames:
- kubernetes
ip: 127.0.0.1
hostNetwork: true
volumes:
- hostPath:
path: "/etc/kubernetes/admin.conf"
name: kubeconfig
status: {}
I1101 11:11:18.212301 564163 ssh_runner.go:195] Run: sudo ls /var/lib/minikube/binaries/v1.34.1
I1101 11:11:18.220267 564163 binaries.go:44] Found k8s binaries, skipping transfer
I1101 11:11:18.220347 564163 ssh_runner.go:195] Run: sudo mkdir -p /etc/systemd/system/kubelet.service.d /lib/systemd/system /etc/kubernetes/manifests
I1101 11:11:18.228465 564163 ssh_runner.go:362] scp memory --> /etc/systemd/system/kubelet.service.d/10-kubeadm.conf (363 bytes)
I1101 11:11:18.244016 564163 ssh_runner.go:362] scp memory --> /lib/systemd/system/kubelet.service (352 bytes)
I1101 11:11:18.258096 564163 ssh_runner.go:362] scp memory --> /etc/kubernetes/manifests/kube-vip.yaml (1358 bytes)
I1101 11:11:18.272629 564163 ssh_runner.go:195] Run: grep 192.168.49.254 control-plane.minikube.internal$ /etc/hosts
I1101 11:11:18.276508 564163 ssh_runner.go:195] Run: /bin/bash -c "{ grep -v $'\tcontrol-plane.minikube.internal$' "/etc/hosts"; echo "192.168.49.254 control-plane.minikube.internal"; } > /tmp/h.$$; sudo cp /tmp/h.$$ "/etc/hosts""
I1101 11:11:18.287804 564163 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1101 11:11:18.413977 564163 ssh_runner.go:195] Run: sudo systemctl start kubelet
I1101 11:11:18.431959 564163 host.go:66] Checking if "ha-472819" exists ...
I1101 11:11:18.432256 564163 start.go:318] joinCluster: &{Name:ha-472819 KeepContext:false EmbedCerts:false MinikubeISO: KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1760939008-21773@sha256:d8d8a3f29f027433bea12764bddd1aa26c7ad9bb912e016c1bc51278db1343d8 Memory:3072 CPUs:2 DiskSize:20000 Driver:docker HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.34.1 ClusterName:ha-472819 Namespace:default APIServerHAVIP:192.168.49.254 APIServerName:minikubeCA APIServerN
ames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:crio CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.168.49.2 Port:8443 KubernetesVersion:v1.34.1 ContainerRuntime:crio ControlPlane:true Worker:true} {Name:m02 IP:192.168.49.3 Port:8443 KubernetesVersion:v1.34.1 ContainerRuntime:crio ControlPlane:true Worker:true} {Name:m03 IP:192.168.49.4 Port:8443 KubernetesVersion:v1.34.1 ContainerRuntime:crio ControlPlane:true Worker:true}] Addons:map[ambassador:false amd-gpu-device-plugin:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false inaccel:false ingress:false ingress-dns:false inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow:false kubetail:
false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s MountString: Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:false DisableMetrics:false DisableCoreDNSLog:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SS
HAgentPID:0 GPUs: AutoPauseInterval:1m0s}
I1101 11:11:18.432437 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "env PATH="/var/lib/minikube/binaries/v1.34.1:$PATH" kubeadm token create --print-join-command --ttl=0"
I1101 11:11:18.432483 564163 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-472819
I1101 11:11:18.451746 564163 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33510 SSHKeyPath:/home/jenkins/minikube-integration/21830-532863/.minikube/machines/ha-472819/id_rsa Username:docker}
I1101 11:11:18.642595 564163 start.go:344] trying to join control-plane node "m03" to cluster: &{Name:m03 IP:192.168.49.4 Port:8443 KubernetesVersion:v1.34.1 ContainerRuntime:crio ControlPlane:true Worker:true}
I1101 11:11:18.642681 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "env PATH="/var/lib/minikube/binaries/v1.34.1:$PATH" kubeadm join control-plane.minikube.internal:8443 --token hw2jul.ax6v1umh51v4f6c5 --discovery-token-ca-cert-hash sha256:6154fe00c4b3b6d1ce4f3500ef815797b79de90371950bebbded24106e2601a8 --ignore-preflight-errors=all --cri-socket unix:///var/run/crio/crio.sock --node-name=ha-472819-m03 --control-plane --apiserver-advertise-address=192.168.49.4 --apiserver-bind-port=8443"
I1101 11:11:43.319187 564163 ssh_runner.go:235] Completed: sudo /bin/bash -c "env PATH="/var/lib/minikube/binaries/v1.34.1:$PATH" kubeadm join control-plane.minikube.internal:8443 --token hw2jul.ax6v1umh51v4f6c5 --discovery-token-ca-cert-hash sha256:6154fe00c4b3b6d1ce4f3500ef815797b79de90371950bebbded24106e2601a8 --ignore-preflight-errors=all --cri-socket unix:///var/run/crio/crio.sock --node-name=ha-472819-m03 --control-plane --apiserver-advertise-address=192.168.49.4 --apiserver-bind-port=8443": (24.676482939s)
I1101 11:11:43.319257 564163 ssh_runner.go:195] Run: sudo /bin/bash -c "systemctl daemon-reload && systemctl enable kubelet && systemctl start kubelet"
I1101 11:11:43.769284 564163 ssh_runner.go:195] Run: sudo /var/lib/minikube/binaries/v1.34.1/kubectl --kubeconfig=/var/lib/minikube/kubeconfig label --overwrite nodes ha-472819-m03 minikube.k8s.io/updated_at=2025_11_01T11_11_43_0700 minikube.k8s.io/version=v1.37.0 minikube.k8s.io/commit=8d0f47abe6720ae55a5722df67bba0ddd12c8845 minikube.k8s.io/name=ha-472819 minikube.k8s.io/primary=false
I1101 11:11:43.908814 564163 ssh_runner.go:195] Run: sudo /var/lib/minikube/binaries/v1.34.1/kubectl --kubeconfig=/var/lib/minikube/kubeconfig taint nodes ha-472819-m03 node-role.kubernetes.io/control-plane:NoSchedule-
I1101 11:11:44.056105 564163 start.go:320] duration metric: took 25.623843323s to joinCluster
I1101 11:11:44.056180 564163 start.go:236] Will wait 6m0s for node &{Name:m03 IP:192.168.49.4 Port:8443 KubernetesVersion:v1.34.1 ContainerRuntime:crio ControlPlane:true Worker:true}
I1101 11:11:44.056483 564163 config.go:182] Loaded profile config "ha-472819": Driver=docker, ContainerRuntime=crio, KubernetesVersion=v1.34.1
I1101 11:11:44.059138 564163 out.go:179] * Verifying Kubernetes components...
I1101 11:11:44.062045 564163 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1101 11:11:44.226994 564163 ssh_runner.go:195] Run: sudo systemctl start kubelet
I1101 11:11:44.241935 564163 kapi.go:59] client config for ha-472819: &rest.Config{Host:"https://192.168.49.254:8443", APIPath:"", ContentConfig:rest.ContentConfig{AcceptContentTypes:"", ContentType:"", GroupVersion:(*schema.GroupVersion)(nil), NegotiatedSerializer:runtime.NegotiatedSerializer(nil)}, Username:"", Password:"", BearerToken:"", BearerTokenFile:"", Impersonate:rest.ImpersonationConfig{UserName:"", UID:"", Groups:[]string(nil), Extra:map[string][]string(nil)}, AuthProvider:<nil>, AuthConfigPersister:rest.AuthProviderConfigPersister(nil), ExecProvider:<nil>, TLSClientConfig:rest.sanitizedTLSClientConfig{Insecure:false, ServerName:"", CertFile:"/home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/client.crt", KeyFile:"/home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/client.key", CAFile:"/home/jenkins/minikube-integration/21830-532863/.minikube/ca.crt", CertData:[]uint8(nil), KeyData:[]uint8(nil), CAData:[]uint8(nil), NextProtos:[]string(n
il)}, UserAgent:"", DisableCompression:false, Transport:http.RoundTripper(nil), WrapTransport:(transport.WrapperFunc)(0x21203d0), QPS:0, Burst:0, RateLimiter:flowcontrol.RateLimiter(nil), WarningHandler:rest.WarningHandler(nil), WarningHandlerWithContext:rest.WarningHandlerWithContext(nil), Timeout:0, Dial:(func(context.Context, string, string) (net.Conn, error))(nil), Proxy:(func(*http.Request) (*url.URL, error))(nil)}
W1101 11:11:44.242072 564163 kubeadm.go:492] Overriding stale ClientConfig host https://192.168.49.254:8443 with https://192.168.49.2:8443
I1101 11:11:44.243546 564163 node_ready.go:35] waiting up to 6m0s for node "ha-472819-m03" to be "Ready" ...
W1101 11:11:46.248078 564163 node_ready.go:57] node "ha-472819-m03" has "Ready":"False" status (will retry)
W1101 11:11:48.747123 564163 node_ready.go:57] node "ha-472819-m03" has "Ready":"False" status (will retry)
W1101 11:11:50.747643 564163 node_ready.go:57] node "ha-472819-m03" has "Ready":"False" status (will retry)
W1101 11:11:52.747859 564163 node_ready.go:57] node "ha-472819-m03" has "Ready":"False" status (will retry)
W1101 11:11:55.248106 564163 node_ready.go:57] node "ha-472819-m03" has "Ready":"False" status (will retry)
W1101 11:11:57.248199 564163 node_ready.go:57] node "ha-472819-m03" has "Ready":"False" status (will retry)
W1101 11:11:59.747904 564163 node_ready.go:57] node "ha-472819-m03" has "Ready":"False" status (will retry)
W1101 11:12:02.247750 564163 node_ready.go:57] node "ha-472819-m03" has "Ready":"False" status (will retry)
W1101 11:12:04.747562 564163 node_ready.go:57] node "ha-472819-m03" has "Ready":"False" status (will retry)
W1101 11:12:07.246739 564163 node_ready.go:57] node "ha-472819-m03" has "Ready":"False" status (will retry)
W1101 11:12:09.246975 564163 node_ready.go:57] node "ha-472819-m03" has "Ready":"False" status (will retry)
W1101 11:12:11.248196 564163 node_ready.go:57] node "ha-472819-m03" has "Ready":"False" status (will retry)
W1101 11:12:13.747822 564163 node_ready.go:57] node "ha-472819-m03" has "Ready":"False" status (will retry)
W1101 11:12:16.249127 564163 node_ready.go:57] node "ha-472819-m03" has "Ready":"False" status (will retry)
W1101 11:12:18.747882 564163 node_ready.go:57] node "ha-472819-m03" has "Ready":"False" status (will retry)
W1101 11:12:20.749775 564163 node_ready.go:57] node "ha-472819-m03" has "Ready":"False" status (will retry)
W1101 11:12:23.247174 564163 node_ready.go:57] node "ha-472819-m03" has "Ready":"False" status (will retry)
W1101 11:12:25.248192 564163 node_ready.go:57] node "ha-472819-m03" has "Ready":"False" status (will retry)
I1101 11:12:26.249644 564163 node_ready.go:49] node "ha-472819-m03" is "Ready"
I1101 11:12:26.249669 564163 node_ready.go:38] duration metric: took 42.006098949s for node "ha-472819-m03" to be "Ready" ...
I1101 11:12:26.249682 564163 api_server.go:52] waiting for apiserver process to appear ...
I1101 11:12:26.249800 564163 ssh_runner.go:195] Run: sudo pgrep -xnf kube-apiserver.*minikube.*
I1101 11:12:26.262481 564163 api_server.go:72] duration metric: took 42.206265905s to wait for apiserver process to appear ...
I1101 11:12:26.262504 564163 api_server.go:88] waiting for apiserver healthz status ...
I1101 11:12:26.262523 564163 api_server.go:253] Checking apiserver healthz at https://192.168.49.2:8443/healthz ...
I1101 11:12:26.271275 564163 api_server.go:279] https://192.168.49.2:8443/healthz returned 200:
ok
I1101 11:12:26.272261 564163 api_server.go:141] control plane version: v1.34.1
I1101 11:12:26.272284 564163 api_server.go:131] duration metric: took 9.773431ms to wait for apiserver health ...
I1101 11:12:26.272294 564163 system_pods.go:43] waiting for kube-system pods to appear ...
I1101 11:12:26.280500 564163 system_pods.go:59] 24 kube-system pods found
I1101 11:12:26.280537 564163 system_pods.go:61] "coredns-66bc5c9577-bntfw" [17503733-2ab6-460c-aa3f-21d031c70abd] Running
I1101 11:12:26.280544 564163 system_pods.go:61] "coredns-66bc5c9577-n2tp2" [4b6711b0-f71a-421e-922d-eb44266c95a4] Running
I1101 11:12:26.280549 564163 system_pods.go:61] "etcd-ha-472819" [6807b695-9ca8-4691-8aac-87ff5cdaca11] Running
I1101 11:12:26.280553 564163 system_pods.go:61] "etcd-ha-472819-m02" [3cef3cc2-cf4e-4445-a55c-ce64fd2279ff] Running
I1101 11:12:26.280558 564163 system_pods.go:61] "etcd-ha-472819-m03" [80e840dc-9437-4351-967c-2a400d35dc89] Running
I1101 11:12:26.280563 564163 system_pods.go:61] "kindnet-cw2kt" [70effae0-c034-4a35-b3d9-3e092c079100] Running
I1101 11:12:26.280567 564163 system_pods.go:61] "kindnet-dkhrw" [abb3d05e-e447-4fe5-8996-26e79d7e2b4d] Running
I1101 11:12:26.280572 564163 system_pods.go:61] "kindnet-mz6bw" [217b3b0a-0680-4a26-98ee-04dd92e1b732] Running
I1101 11:12:26.280576 564163 system_pods.go:61] "kube-apiserver-ha-472819" [a65e9eca-1f17-4ff9-b4d0-2b26612bc846] Running
I1101 11:12:26.280580 564163 system_pods.go:61] "kube-apiserver-ha-472819-m02" [c94a478e-4714-4590-8c91-17468898125c] Running
I1101 11:12:26.280585 564163 system_pods.go:61] "kube-apiserver-ha-472819-m03" [4dd6c2e8-c1fd-4a41-b208-b227db99ef54] Running
I1101 11:12:26.280595 564163 system_pods.go:61] "kube-controller-manager-ha-472819" [e6236069-2227-4783-b8e3-6df90e52e82c] Running
I1101 11:12:26.280600 564163 system_pods.go:61] "kube-controller-manager-ha-472819-m02" [f5e22b4d-d7c1-47b0-a044-4007e77d6ebc] Running
I1101 11:12:26.280607 564163 system_pods.go:61] "kube-controller-manager-ha-472819-m03" [a67b5941-388f-48a8-b452-ff50be57ca66] Running
I1101 11:12:26.280613 564163 system_pods.go:61] "kube-proxy-47prj" [16f8f4f3-8267-4ce3-997b-1f4afb0f5104] Running
I1101 11:12:26.280624 564163 system_pods.go:61] "kube-proxy-djfvb" [2c010b85-48bd-4004-886f-fbe4e03884a9] Running
I1101 11:12:26.280628 564163 system_pods.go:61] "kube-proxy-gc4g4" [2289bf2a-0371-4bad-8440-6e299ce1e8a9] Running
I1101 11:12:26.280632 564163 system_pods.go:61] "kube-scheduler-ha-472819" [78ac9fa6-2686-404f-a977-d7710745150b] Running
I1101 11:12:26.280644 564163 system_pods.go:61] "kube-scheduler-ha-472819-m02" [31b58b00-ca07-42ad-a9a7-20da16f0a251] Running
I1101 11:12:26.280648 564163 system_pods.go:61] "kube-scheduler-ha-472819-m03" [2b72cc38-a219-4fcc-8a1e-977391aee0b1] Running
I1101 11:12:26.280652 564163 system_pods.go:61] "kube-vip-ha-472819" [0e1f82b1-9039-49f8-b83f-8c40ab9ec44f] Running
I1101 11:12:26.280657 564163 system_pods.go:61] "kube-vip-ha-472819-m02" [8964dc5d-7184-43bf-a1bd-0f9b261bb9df] Running
I1101 11:12:26.280666 564163 system_pods.go:61] "kube-vip-ha-472819-m03" [a3e5599c-b0a5-4792-9192-397f763006fc] Running
I1101 11:12:26.280671 564163 system_pods.go:61] "storage-provisioner" [18119b45-4932-4521-b0e9-e3a73bc6d3b1] Running
I1101 11:12:26.280676 564163 system_pods.go:74] duration metric: took 8.377598ms to wait for pod list to return data ...
I1101 11:12:26.280686 564163 default_sa.go:34] waiting for default service account to be created ...
I1101 11:12:26.284133 564163 default_sa.go:45] found service account: "default"
I1101 11:12:26.284159 564163 default_sa.go:55] duration metric: took 3.464322ms for default service account to be created ...
I1101 11:12:26.284168 564163 system_pods.go:116] waiting for k8s-apps to be running ...
I1101 11:12:26.289312 564163 system_pods.go:86] 24 kube-system pods found
I1101 11:12:26.289355 564163 system_pods.go:89] "coredns-66bc5c9577-bntfw" [17503733-2ab6-460c-aa3f-21d031c70abd] Running
I1101 11:12:26.289363 564163 system_pods.go:89] "coredns-66bc5c9577-n2tp2" [4b6711b0-f71a-421e-922d-eb44266c95a4] Running
I1101 11:12:26.289367 564163 system_pods.go:89] "etcd-ha-472819" [6807b695-9ca8-4691-8aac-87ff5cdaca11] Running
I1101 11:12:26.289372 564163 system_pods.go:89] "etcd-ha-472819-m02" [3cef3cc2-cf4e-4445-a55c-ce64fd2279ff] Running
I1101 11:12:26.289378 564163 system_pods.go:89] "etcd-ha-472819-m03" [80e840dc-9437-4351-967c-2a400d35dc89] Running
I1101 11:12:26.289397 564163 system_pods.go:89] "kindnet-cw2kt" [70effae0-c034-4a35-b3d9-3e092c079100] Running
I1101 11:12:26.289411 564163 system_pods.go:89] "kindnet-dkhrw" [abb3d05e-e447-4fe5-8996-26e79d7e2b4d] Running
I1101 11:12:26.289416 564163 system_pods.go:89] "kindnet-mz6bw" [217b3b0a-0680-4a26-98ee-04dd92e1b732] Running
I1101 11:12:26.289421 564163 system_pods.go:89] "kube-apiserver-ha-472819" [a65e9eca-1f17-4ff9-b4d0-2b26612bc846] Running
I1101 11:12:26.289429 564163 system_pods.go:89] "kube-apiserver-ha-472819-m02" [c94a478e-4714-4590-8c91-17468898125c] Running
I1101 11:12:26.289433 564163 system_pods.go:89] "kube-apiserver-ha-472819-m03" [4dd6c2e8-c1fd-4a41-b208-b227db99ef54] Running
I1101 11:12:26.289438 564163 system_pods.go:89] "kube-controller-manager-ha-472819" [e6236069-2227-4783-b8e3-6df90e52e82c] Running
I1101 11:12:26.289442 564163 system_pods.go:89] "kube-controller-manager-ha-472819-m02" [f5e22b4d-d7c1-47b0-a044-4007e77d6ebc] Running
I1101 11:12:26.289454 564163 system_pods.go:89] "kube-controller-manager-ha-472819-m03" [a67b5941-388f-48a8-b452-ff50be57ca66] Running
I1101 11:12:26.289458 564163 system_pods.go:89] "kube-proxy-47prj" [16f8f4f3-8267-4ce3-997b-1f4afb0f5104] Running
I1101 11:12:26.289461 564163 system_pods.go:89] "kube-proxy-djfvb" [2c010b85-48bd-4004-886f-fbe4e03884a9] Running
I1101 11:12:26.289467 564163 system_pods.go:89] "kube-proxy-gc4g4" [2289bf2a-0371-4bad-8440-6e299ce1e8a9] Running
I1101 11:12:26.289471 564163 system_pods.go:89] "kube-scheduler-ha-472819" [78ac9fa6-2686-404f-a977-d7710745150b] Running
I1101 11:12:26.289475 564163 system_pods.go:89] "kube-scheduler-ha-472819-m02" [31b58b00-ca07-42ad-a9a7-20da16f0a251] Running
I1101 11:12:26.289479 564163 system_pods.go:89] "kube-scheduler-ha-472819-m03" [2b72cc38-a219-4fcc-8a1e-977391aee0b1] Running
I1101 11:12:26.289483 564163 system_pods.go:89] "kube-vip-ha-472819" [0e1f82b1-9039-49f8-b83f-8c40ab9ec44f] Running
I1101 11:12:26.289491 564163 system_pods.go:89] "kube-vip-ha-472819-m02" [8964dc5d-7184-43bf-a1bd-0f9b261bb9df] Running
I1101 11:12:26.289495 564163 system_pods.go:89] "kube-vip-ha-472819-m03" [a3e5599c-b0a5-4792-9192-397f763006fc] Running
I1101 11:12:26.289499 564163 system_pods.go:89] "storage-provisioner" [18119b45-4932-4521-b0e9-e3a73bc6d3b1] Running
I1101 11:12:26.289507 564163 system_pods.go:126] duration metric: took 5.334417ms to wait for k8s-apps to be running ...
I1101 11:12:26.289518 564163 system_svc.go:44] waiting for kubelet service to be running ....
I1101 11:12:26.289578 564163 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service kubelet
I1101 11:12:26.305948 564163 system_svc.go:56] duration metric: took 16.419873ms WaitForService to wait for kubelet
I1101 11:12:26.305975 564163 kubeadm.go:587] duration metric: took 42.249764697s to wait for: map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true]
I1101 11:12:26.305994 564163 node_conditions.go:102] verifying NodePressure condition ...
I1101 11:12:26.309253 564163 node_conditions.go:122] node storage ephemeral capacity is 203034800Ki
I1101 11:12:26.309279 564163 node_conditions.go:123] node cpu capacity is 2
I1101 11:12:26.309289 564163 node_conditions.go:122] node storage ephemeral capacity is 203034800Ki
I1101 11:12:26.309294 564163 node_conditions.go:123] node cpu capacity is 2
I1101 11:12:26.309298 564163 node_conditions.go:122] node storage ephemeral capacity is 203034800Ki
I1101 11:12:26.309303 564163 node_conditions.go:123] node cpu capacity is 2
I1101 11:12:26.309308 564163 node_conditions.go:105] duration metric: took 3.308316ms to run NodePressure ...
I1101 11:12:26.309331 564163 start.go:242] waiting for startup goroutines ...
I1101 11:12:26.309355 564163 start.go:256] writing updated cluster config ...
I1101 11:12:26.309669 564163 ssh_runner.go:195] Run: rm -f paused
I1101 11:12:26.313122 564163 pod_ready.go:37] extra waiting up to 4m0s for all "kube-system" pods having one of [k8s-app=kube-dns component=etcd component=kube-apiserver component=kube-controller-manager k8s-app=kube-proxy component=kube-scheduler] labels to be "Ready" ...
I1101 11:12:26.313679 564163 kapi.go:59] client config for ha-472819: &rest.Config{Host:"https://192.168.49.254:8443", APIPath:"", ContentConfig:rest.ContentConfig{AcceptContentTypes:"", ContentType:"", GroupVersion:(*schema.GroupVersion)(nil), NegotiatedSerializer:runtime.NegotiatedSerializer(nil)}, Username:"", Password:"", BearerToken:"", BearerTokenFile:"", Impersonate:rest.ImpersonationConfig{UserName:"", UID:"", Groups:[]string(nil), Extra:map[string][]string(nil)}, AuthProvider:<nil>, AuthConfigPersister:rest.AuthProviderConfigPersister(nil), ExecProvider:<nil>, TLSClientConfig:rest.sanitizedTLSClientConfig{Insecure:false, ServerName:"", CertFile:"/home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/client.crt", KeyFile:"/home/jenkins/minikube-integration/21830-532863/.minikube/profiles/ha-472819/client.key", CAFile:"/home/jenkins/minikube-integration/21830-532863/.minikube/ca.crt", CertData:[]uint8(nil), KeyData:[]uint8(nil), CAData:[]uint8(nil), NextProtos:[]string(n
il)}, UserAgent:"", DisableCompression:false, Transport:http.RoundTripper(nil), WrapTransport:(transport.WrapperFunc)(0x21203d0), QPS:0, Burst:0, RateLimiter:flowcontrol.RateLimiter(nil), WarningHandler:rest.WarningHandler(nil), WarningHandlerWithContext:rest.WarningHandlerWithContext(nil), Timeout:0, Dial:(func(context.Context, string, string) (net.Conn, error))(nil), Proxy:(func(*http.Request) (*url.URL, error))(nil)}
I1101 11:12:26.330719 564163 pod_ready.go:83] waiting for pod "coredns-66bc5c9577-bntfw" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:26.336573 564163 pod_ready.go:94] pod "coredns-66bc5c9577-bntfw" is "Ready"
I1101 11:12:26.336600 564163 pod_ready.go:86] duration metric: took 5.853087ms for pod "coredns-66bc5c9577-bntfw" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:26.336611 564163 pod_ready.go:83] waiting for pod "coredns-66bc5c9577-n2tp2" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:26.343901 564163 pod_ready.go:94] pod "coredns-66bc5c9577-n2tp2" is "Ready"
I1101 11:12:26.343929 564163 pod_ready.go:86] duration metric: took 7.293605ms for pod "coredns-66bc5c9577-n2tp2" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:26.346848 564163 pod_ready.go:83] waiting for pod "etcd-ha-472819" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:26.352072 564163 pod_ready.go:94] pod "etcd-ha-472819" is "Ready"
I1101 11:12:26.352102 564163 pod_ready.go:86] duration metric: took 5.227692ms for pod "etcd-ha-472819" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:26.352112 564163 pod_ready.go:83] waiting for pod "etcd-ha-472819-m02" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:26.358541 564163 pod_ready.go:94] pod "etcd-ha-472819-m02" is "Ready"
I1101 11:12:26.358573 564163 pod_ready.go:86] duration metric: took 6.453734ms for pod "etcd-ha-472819-m02" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:26.358588 564163 pod_ready.go:83] waiting for pod "etcd-ha-472819-m03" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:26.515018 564163 request.go:683] "Waited before sending request" delay="156.271647ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/namespaces/kube-system/pods/etcd-ha-472819-m03"
I1101 11:12:26.714759 564163 request.go:683] "Waited before sending request" delay="196.328427ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/nodes/ha-472819-m03"
I1101 11:12:26.718151 564163 pod_ready.go:94] pod "etcd-ha-472819-m03" is "Ready"
I1101 11:12:26.718181 564163 pod_ready.go:86] duration metric: took 359.546189ms for pod "etcd-ha-472819-m03" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:26.914587 564163 request.go:683] "Waited before sending request" delay="196.293202ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/namespaces/kube-system/pods?labelSelector=component%3Dkube-apiserver"
I1101 11:12:26.918462 564163 pod_ready.go:83] waiting for pod "kube-apiserver-ha-472819" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:27.115021 564163 request.go:683] "Waited before sending request" delay="196.450874ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/namespaces/kube-system/pods/kube-apiserver-ha-472819"
I1101 11:12:27.314790 564163 request.go:683] "Waited before sending request" delay="196.347915ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/nodes/ha-472819"
I1101 11:12:27.318165 564163 pod_ready.go:94] pod "kube-apiserver-ha-472819" is "Ready"
I1101 11:12:27.318193 564163 pod_ready.go:86] duration metric: took 399.695688ms for pod "kube-apiserver-ha-472819" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:27.318203 564163 pod_ready.go:83] waiting for pod "kube-apiserver-ha-472819-m02" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:27.514681 564163 request.go:683] "Waited before sending request" delay="196.361741ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/namespaces/kube-system/pods/kube-apiserver-ha-472819-m02"
I1101 11:12:27.714676 564163 request.go:683] "Waited before sending request" delay="196.346856ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/nodes/ha-472819-m02"
I1101 11:12:27.718653 564163 pod_ready.go:94] pod "kube-apiserver-ha-472819-m02" is "Ready"
I1101 11:12:27.718733 564163 pod_ready.go:86] duration metric: took 400.522554ms for pod "kube-apiserver-ha-472819-m02" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:27.718769 564163 pod_ready.go:83] waiting for pod "kube-apiserver-ha-472819-m03" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:27.915203 564163 request.go:683] "Waited before sending request" delay="196.340399ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/namespaces/kube-system/pods/kube-apiserver-ha-472819-m03"
I1101 11:12:28.114358 564163 request.go:683] "Waited before sending request" delay="195.249259ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/nodes/ha-472819-m03"
I1101 11:12:28.117775 564163 pod_ready.go:94] pod "kube-apiserver-ha-472819-m03" is "Ready"
I1101 11:12:28.117807 564163 pod_ready.go:86] duration metric: took 399.013365ms for pod "kube-apiserver-ha-472819-m03" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:28.314349 564163 request.go:683] "Waited before sending request" delay="196.417856ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/namespaces/kube-system/pods?labelSelector=component%3Dkube-controller-manager"
I1101 11:12:28.318465 564163 pod_ready.go:83] waiting for pod "kube-controller-manager-ha-472819" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:28.514898 564163 request.go:683] "Waited before sending request" delay="196.319665ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/namespaces/kube-system/pods/kube-controller-manager-ha-472819"
I1101 11:12:28.714748 564163 request.go:683] "Waited before sending request" delay="196.35411ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/nodes/ha-472819"
I1101 11:12:28.718210 564163 pod_ready.go:94] pod "kube-controller-manager-ha-472819" is "Ready"
I1101 11:12:28.718240 564163 pod_ready.go:86] duration metric: took 399.743024ms for pod "kube-controller-manager-ha-472819" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:28.718250 564163 pod_ready.go:83] waiting for pod "kube-controller-manager-ha-472819-m02" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:28.914684 564163 request.go:683] "Waited before sending request" delay="196.336731ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/namespaces/kube-system/pods/kube-controller-manager-ha-472819-m02"
I1101 11:12:29.114836 564163 request.go:683] "Waited before sending request" delay="196.362029ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/nodes/ha-472819-m02"
I1101 11:12:29.118262 564163 pod_ready.go:94] pod "kube-controller-manager-ha-472819-m02" is "Ready"
I1101 11:12:29.118307 564163 pod_ready.go:86] duration metric: took 400.051245ms for pod "kube-controller-manager-ha-472819-m02" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:29.118318 564163 pod_ready.go:83] waiting for pod "kube-controller-manager-ha-472819-m03" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:29.314606 564163 request.go:683] "Waited before sending request" delay="196.212242ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/namespaces/kube-system/pods/kube-controller-manager-ha-472819-m03"
I1101 11:12:29.514310 564163 request.go:683] "Waited before sending request" delay="196.164808ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/nodes/ha-472819-m03"
I1101 11:12:29.519832 564163 pod_ready.go:94] pod "kube-controller-manager-ha-472819-m03" is "Ready"
I1101 11:12:29.519865 564163 pod_ready.go:86] duration metric: took 401.539002ms for pod "kube-controller-manager-ha-472819-m03" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:29.715277 564163 request.go:683] "Waited before sending request" delay="195.313572ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/namespaces/kube-system/pods?labelSelector=k8s-app%3Dkube-proxy"
I1101 11:12:29.719110 564163 pod_ready.go:83] waiting for pod "kube-proxy-47prj" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:29.914444 564163 request.go:683] "Waited before sending request" delay="195.229952ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/namespaces/kube-system/pods/kube-proxy-47prj"
I1101 11:12:30.115044 564163 request.go:683] "Waited before sending request" delay="197.189419ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/nodes/ha-472819-m02"
I1101 11:12:30.122807 564163 pod_ready.go:94] pod "kube-proxy-47prj" is "Ready"
I1101 11:12:30.122901 564163 pod_ready.go:86] duration metric: took 403.759526ms for pod "kube-proxy-47prj" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:30.122920 564163 pod_ready.go:83] waiting for pod "kube-proxy-djfvb" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:30.314268 564163 request.go:683] "Waited before sending request" delay="191.266408ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/namespaces/kube-system/pods/kube-proxy-djfvb"
I1101 11:12:30.515306 564163 request.go:683] "Waited before sending request" delay="197.533768ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/nodes/ha-472819"
I1101 11:12:30.518543 564163 pod_ready.go:94] pod "kube-proxy-djfvb" is "Ready"
I1101 11:12:30.518576 564163 pod_ready.go:86] duration metric: took 395.647433ms for pod "kube-proxy-djfvb" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:30.518587 564163 pod_ready.go:83] waiting for pod "kube-proxy-gc4g4" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:30.715050 564163 request.go:683] "Waited before sending request" delay="196.355957ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/namespaces/kube-system/pods/kube-proxy-gc4g4"
I1101 11:12:30.915104 564163 request.go:683] "Waited before sending request" delay="194.316785ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/nodes/ha-472819-m03"
I1101 11:12:30.920032 564163 pod_ready.go:94] pod "kube-proxy-gc4g4" is "Ready"
I1101 11:12:30.920064 564163 pod_ready.go:86] duration metric: took 401.469274ms for pod "kube-proxy-gc4g4" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:31.114457 564163 request.go:683] "Waited before sending request" delay="194.275438ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/namespaces/kube-system/pods?labelSelector=component%3Dkube-scheduler"
I1101 11:12:31.118943 564163 pod_ready.go:83] waiting for pod "kube-scheduler-ha-472819" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:31.314321 564163 request.go:683] "Waited before sending request" delay="195.278536ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/namespaces/kube-system/pods/kube-scheduler-ha-472819"
I1101 11:12:31.515230 564163 request.go:683] "Waited before sending request" delay="197.303661ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/nodes/ha-472819"
I1101 11:12:31.518454 564163 pod_ready.go:94] pod "kube-scheduler-ha-472819" is "Ready"
I1101 11:12:31.518487 564163 pod_ready.go:86] duration metric: took 399.509488ms for pod "kube-scheduler-ha-472819" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:31.518498 564163 pod_ready.go:83] waiting for pod "kube-scheduler-ha-472819-m02" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:31.714950 564163 request.go:683] "Waited before sending request" delay="196.3489ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/namespaces/kube-system/pods/kube-scheduler-ha-472819-m02"
I1101 11:12:31.914922 564163 request.go:683] "Waited before sending request" delay="196.326566ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/nodes/ha-472819-m02"
I1101 11:12:31.918119 564163 pod_ready.go:94] pod "kube-scheduler-ha-472819-m02" is "Ready"
I1101 11:12:31.918150 564163 pod_ready.go:86] duration metric: took 399.645153ms for pod "kube-scheduler-ha-472819-m02" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:31.918159 564163 pod_ready.go:83] waiting for pod "kube-scheduler-ha-472819-m03" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:32.114581 564163 request.go:683] "Waited before sending request" delay="196.334475ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/namespaces/kube-system/pods/kube-scheduler-ha-472819-m03"
I1101 11:12:32.315116 564163 request.go:683] "Waited before sending request" delay="196.314915ms" reason="client-side throttling, not priority and fairness" verb="GET" URL="https://192.168.49.254:8443/api/v1/nodes/ha-472819-m03"
I1101 11:12:32.318291 564163 pod_ready.go:94] pod "kube-scheduler-ha-472819-m03" is "Ready"
I1101 11:12:32.318319 564163 pod_ready.go:86] duration metric: took 400.143654ms for pod "kube-scheduler-ha-472819-m03" in "kube-system" namespace to be "Ready" or be gone ...
I1101 11:12:32.318333 564163 pod_ready.go:40] duration metric: took 6.005166383s for extra waiting for all "kube-system" pods having one of [k8s-app=kube-dns component=etcd component=kube-apiserver component=kube-controller-manager k8s-app=kube-proxy component=kube-scheduler] labels to be "Ready" ...
I1101 11:12:32.374771 564163 start.go:628] kubectl: 1.33.2, cluster: 1.34.1 (minor skew: 1)
I1101 11:12:32.379957 564163 out.go:179] * Done! kubectl is now configured to use "ha-472819" cluster and "default" namespace by default
==> CRI-O <==
Nov 01 11:10:29 ha-472819 crio[835]: time="2025-11-01T11:10:29.724673509Z" level=info msg="Created container b91918178a88a5685429c28d6c36fba100356470fd0f83517aa7e116b189eb4a: kube-system/coredns-66bc5c9577-n2tp2/coredns" id=4a2a84c1-22ae-4f46-8149-9f44dec0e1df name=/runtime.v1.RuntimeService/CreateContainer
Nov 01 11:10:29 ha-472819 crio[835]: time="2025-11-01T11:10:29.726091618Z" level=info msg="Starting container: b91918178a88a5685429c28d6c36fba100356470fd0f83517aa7e116b189eb4a" id=182c8112-171d-4979-810a-ec966f9ee5bd name=/runtime.v1.RuntimeService/StartContainer
Nov 01 11:10:29 ha-472819 crio[835]: time="2025-11-01T11:10:29.728389345Z" level=info msg="Started container" PID=1830 containerID=b91918178a88a5685429c28d6c36fba100356470fd0f83517aa7e116b189eb4a description=kube-system/coredns-66bc5c9577-n2tp2/coredns id=182c8112-171d-4979-810a-ec966f9ee5bd name=/runtime.v1.RuntimeService/StartContainer sandboxID=2c45f2568b0e8e33cb1da636920d9b841b29c754a967265ee7a2ff1803ba718d
Nov 01 11:12:34 ha-472819 crio[835]: time="2025-11-01T11:12:34.120480779Z" level=info msg="Running pod sandbox: default/busybox-7b57f96db7-lm6r8/POD" id=ae159a9e-0cdc-4387-8101-f3339714c067 name=/runtime.v1.RuntimeService/RunPodSandbox
Nov 01 11:12:34 ha-472819 crio[835]: time="2025-11-01T11:12:34.120560313Z" level=info msg="Allowed annotations are specified for workload [io.containers.trace-syscall]"
Nov 01 11:12:34 ha-472819 crio[835]: time="2025-11-01T11:12:34.151990285Z" level=info msg="Got pod network &{Name:busybox-7b57f96db7-lm6r8 Namespace:default ID:1d1abc560619e7aa1a8b60798b93f19527128629e10f8828a25552f3c73770b6 UID:3faf7e64-22cf-4338-92ef-39a2978dacb5 NetNS:/var/run/netns/273515f3-fe48-4c9a-a5a0-ca5b0e3ab433 Networks:[{Name:kindnet Ifname:eth0}] RuntimeConfig:map[kindnet:{IP: MAC: PortMappings:[] Bandwidth:<nil> IpRanges:[] CgroupPath: PodAnnotations:0x4001d90530}] Aliases:map[]}"
Nov 01 11:12:34 ha-472819 crio[835]: time="2025-11-01T11:12:34.152188171Z" level=info msg="Adding pod default_busybox-7b57f96db7-lm6r8 to CNI network \"kindnet\" (type=ptp)"
Nov 01 11:12:34 ha-472819 crio[835]: time="2025-11-01T11:12:34.174049137Z" level=info msg="Got pod network &{Name:busybox-7b57f96db7-lm6r8 Namespace:default ID:1d1abc560619e7aa1a8b60798b93f19527128629e10f8828a25552f3c73770b6 UID:3faf7e64-22cf-4338-92ef-39a2978dacb5 NetNS:/var/run/netns/273515f3-fe48-4c9a-a5a0-ca5b0e3ab433 Networks:[{Name:kindnet Ifname:eth0}] RuntimeConfig:map[kindnet:{IP: MAC: PortMappings:[] Bandwidth:<nil> IpRanges:[] CgroupPath: PodAnnotations:0x4001d90530}] Aliases:map[]}"
Nov 01 11:12:34 ha-472819 crio[835]: time="2025-11-01T11:12:34.174388982Z" level=info msg="Checking pod default_busybox-7b57f96db7-lm6r8 for CNI network kindnet (type=ptp)"
Nov 01 11:12:34 ha-472819 crio[835]: time="2025-11-01T11:12:34.18392826Z" level=info msg="Ran pod sandbox 1d1abc560619e7aa1a8b60798b93f19527128629e10f8828a25552f3c73770b6 with infra container: default/busybox-7b57f96db7-lm6r8/POD" id=ae159a9e-0cdc-4387-8101-f3339714c067 name=/runtime.v1.RuntimeService/RunPodSandbox
Nov 01 11:12:34 ha-472819 crio[835]: time="2025-11-01T11:12:34.18737883Z" level=info msg="Checking image status: gcr.io/k8s-minikube/busybox:1.28" id=f0278987-4284-4ba6-99d3-3e1b3bcbe42b name=/runtime.v1.ImageService/ImageStatus
Nov 01 11:12:34 ha-472819 crio[835]: time="2025-11-01T11:12:34.187693714Z" level=info msg="Image gcr.io/k8s-minikube/busybox:1.28 not found" id=f0278987-4284-4ba6-99d3-3e1b3bcbe42b name=/runtime.v1.ImageService/ImageStatus
Nov 01 11:12:34 ha-472819 crio[835]: time="2025-11-01T11:12:34.187803098Z" level=info msg="Neither image nor artfiact gcr.io/k8s-minikube/busybox:1.28 found" id=f0278987-4284-4ba6-99d3-3e1b3bcbe42b name=/runtime.v1.ImageService/ImageStatus
Nov 01 11:12:34 ha-472819 crio[835]: time="2025-11-01T11:12:34.189366827Z" level=info msg="Pulling image: gcr.io/k8s-minikube/busybox:1.28" id=8af5159f-b7ff-433f-9810-f5aaf54d8516 name=/runtime.v1.ImageService/PullImage
Nov 01 11:12:34 ha-472819 crio[835]: time="2025-11-01T11:12:34.19276737Z" level=info msg="Trying to access \"gcr.io/k8s-minikube/busybox:1.28\""
Nov 01 11:12:36 ha-472819 crio[835]: time="2025-11-01T11:12:36.210173626Z" level=info msg="Pulled image: gcr.io/k8s-minikube/busybox@sha256:859d41e4316c182cb559f9ae3c5ffcac8602ee1179794a1707c06cd092a008d3" id=8af5159f-b7ff-433f-9810-f5aaf54d8516 name=/runtime.v1.ImageService/PullImage
Nov 01 11:12:36 ha-472819 crio[835]: time="2025-11-01T11:12:36.211378047Z" level=info msg="Checking image status: gcr.io/k8s-minikube/busybox:1.28" id=62e5c4c6-2347-4768-b7b2-e7d361a90c66 name=/runtime.v1.ImageService/ImageStatus
Nov 01 11:12:36 ha-472819 crio[835]: time="2025-11-01T11:12:36.213299064Z" level=info msg="Checking image status: gcr.io/k8s-minikube/busybox:1.28" id=9d165b72-72dd-44dd-baee-e3b9079ec16f name=/runtime.v1.ImageService/ImageStatus
Nov 01 11:12:36 ha-472819 crio[835]: time="2025-11-01T11:12:36.224587271Z" level=info msg="Creating container: default/busybox-7b57f96db7-lm6r8/busybox" id=f3cbc5a1-6aaf-468c-ad23-e310e4e6c169 name=/runtime.v1.RuntimeService/CreateContainer
Nov 01 11:12:36 ha-472819 crio[835]: time="2025-11-01T11:12:36.224848058Z" level=info msg="Allowed annotations are specified for workload [io.containers.trace-syscall]"
Nov 01 11:12:36 ha-472819 crio[835]: time="2025-11-01T11:12:36.249270156Z" level=info msg="Allowed annotations are specified for workload [io.containers.trace-syscall]"
Nov 01 11:12:36 ha-472819 crio[835]: time="2025-11-01T11:12:36.250049973Z" level=info msg="Allowed annotations are specified for workload [io.containers.trace-syscall]"
Nov 01 11:12:36 ha-472819 crio[835]: time="2025-11-01T11:12:36.282130596Z" level=info msg="Created container dff6a4a869cee8df9dc4d3d269f3081a5f7b6994fbe3813528d07d7a06f03fb6: default/busybox-7b57f96db7-lm6r8/busybox" id=f3cbc5a1-6aaf-468c-ad23-e310e4e6c169 name=/runtime.v1.RuntimeService/CreateContainer
Nov 01 11:12:36 ha-472819 crio[835]: time="2025-11-01T11:12:36.288280744Z" level=info msg="Starting container: dff6a4a869cee8df9dc4d3d269f3081a5f7b6994fbe3813528d07d7a06f03fb6" id=0733d7c4-1f36-4706-b6d2-98a90d511fb9 name=/runtime.v1.RuntimeService/StartContainer
Nov 01 11:12:36 ha-472819 crio[835]: time="2025-11-01T11:12:36.300369166Z" level=info msg="Started container" PID=1986 containerID=dff6a4a869cee8df9dc4d3d269f3081a5f7b6994fbe3813528d07d7a06f03fb6 description=default/busybox-7b57f96db7-lm6r8/busybox id=0733d7c4-1f36-4706-b6d2-98a90d511fb9 name=/runtime.v1.RuntimeService/StartContainer sandboxID=1d1abc560619e7aa1a8b60798b93f19527128629e10f8828a25552f3c73770b6
==> container status <==
CONTAINER IMAGE CREATED STATE NAME ATTEMPT POD ID POD NAMESPACE
dff6a4a869cee gcr.io/k8s-minikube/busybox@sha256:859d41e4316c182cb559f9ae3c5ffcac8602ee1179794a1707c06cd092a008d3 10 minutes ago Running busybox 0 1d1abc560619e busybox-7b57f96db7-lm6r8 default
b91918178a88a 138784d87c9c50f8e59412544da4cf4928d61ccbaf93b9f5898a3ba406871bfc 12 minutes ago Running coredns 0 2c45f2568b0e8 coredns-66bc5c9577-n2tp2 kube-system
c8ab7117746d2 138784d87c9c50f8e59412544da4cf4928d61ccbaf93b9f5898a3ba406871bfc 12 minutes ago Running coredns 0 f161ed77d0204 coredns-66bc5c9577-bntfw kube-system
f3816faa8e434 ba04bb24b95753201135cbc420b233c1b0b9fa2e1fd21d28319c348c33fbcde6 12 minutes ago Running storage-provisioner 0 48ac5b7666614 storage-provisioner kube-system
7078104c50ff2 05baa95f5142d87797a2bc1d3d11edfb0bf0a9236d436243d15061fae8b58cb9 12 minutes ago Running kube-proxy 0 cbb2812743bd4 kube-proxy-djfvb kube-system
6af4febe46d8a b1a8c6f707935fd5f346ce5846d21ff8dd65e14c15406a14dbd16b9b897b9b4c 12 minutes ago Running kindnet-cni 0 45d9b924aafb5 kindnet-dkhrw kube-system
58f10619def7f ghcr.io/kube-vip/kube-vip@sha256:a9c131fb1bd4690cd4563761c2f545eb89b92cc8ea19aec96c833d1b4b0211eb 13 minutes ago Running kube-vip 0 1eb623b05a53f kube-vip-ha-472819 kube-system
91af80c077c55 43911e833d64d4f30460862fc0c54bb61999d60bc7d063feca71e9fc610d5196 13 minutes ago Running kube-apiserver 0 86e0901f54771 kube-apiserver-ha-472819 kube-system
f940f08b4a7e5 b5f57ec6b98676d815366685a0422bd164ecf0732540b79ac51b1186cef97ff0 13 minutes ago Running kube-scheduler 0 956e3189233cf kube-scheduler-ha-472819 kube-system
640585dbb86b9 a1894772a478e07c67a56e8bf32335fdbe1dd4ec96976a5987083164bd00bc0e 13 minutes ago Running etcd 0 b159389b39c8d etcd-ha-472819 kube-system
6bf6ea4411cda 7eb2c6ff0c5a768fd309321bc2ade0e4e11afcf4f2017ef1d0ff00d91fdf992a 13 minutes ago Running kube-controller-manager 0 feba7cf49ce4a kube-controller-manager-ha-472819 kube-system
==> coredns [b91918178a88a5685429c28d6c36fba100356470fd0f83517aa7e116b189eb4a] <==
[INFO] 10.244.2.2:44007 - 5 "PTR IN 90.167.197.15.in-addr.arpa. udp 44 false 512" NOERROR qr,rd,ra 126 0.115679076s
[INFO] 10.244.0.4:48262 - 5 "PTR IN 90.167.197.15.in-addr.arpa. udp 44 false 512" NOERROR qr,aa,rd,ra 126 0.000087016s
[INFO] 10.244.1.2:44384 - 2 "PTR IN 10.0.96.10.in-addr.arpa. udp 41 false 512" NOERROR qr,aa,rd 116 0.000134082s
[INFO] 10.244.1.2:56915 - 3 "AAAA IN kubernetes.io. udp 31 false 512" NOERROR qr,aa,rd,ra 31 0.0000838s
[INFO] 10.244.1.2:41130 - 5 "PTR IN 90.167.197.15.in-addr.arpa. udp 44 false 512" NOERROR qr,aa,rd,ra 126 0.00008604s
[INFO] 10.244.2.2:49290 - 6 "A IN kubernetes.default. udp 36 false 512" NXDOMAIN qr,rd,ra 36 0.001659803s
[INFO] 10.244.2.2:38229 - 8 "A IN kubernetes.default.svc.cluster.local. udp 54 false 512" NOERROR qr,aa,rd 106 0.000154456s
[INFO] 10.244.0.4:54270 - 6 "A IN kubernetes.default. udp 36 false 512" NXDOMAIN qr,rd,ra 36 0.004350553s
[INFO] 10.244.0.4:50580 - 7 "A IN kubernetes.default.default.svc.cluster.local. udp 62 false 512" NXDOMAIN qr,aa,rd 155 0.000195005s
[INFO] 10.244.1.2:42868 - 2 "PTR IN 10.0.96.10.in-addr.arpa. udp 41 false 512" NOERROR qr,aa,rd 116 0.00013002s
[INFO] 10.244.1.2:36862 - 3 "AAAA IN kubernetes.default. udp 36 false 512" NXDOMAIN qr,rd,ra 36 0.001492458s
[INFO] 10.244.1.2:48136 - 4 "AAAA IN kubernetes.default.default.svc.cluster.local. udp 62 false 512" NXDOMAIN qr,aa,rd 155 0.000177652s
[INFO] 10.244.1.2:50876 - 5 "AAAA IN kubernetes.default.svc.cluster.local. udp 54 false 512" NOERROR qr,aa,rd 147 0.000135198s
[INFO] 10.244.1.2:44573 - 6 "A IN kubernetes.default. udp 36 false 512" NXDOMAIN qr,rd,ra 36 0.001077618s
[INFO] 10.244.1.2:38478 - 9 "PTR IN 1.0.96.10.in-addr.arpa. udp 40 false 512" NOERROR qr,aa,rd 112 0.000152281s
[INFO] 10.244.2.2:52114 - 2 "PTR IN 10.0.96.10.in-addr.arpa. udp 41 false 512" NOERROR qr,aa,rd 116 0.000135033s
[INFO] 10.244.2.2:49246 - 3 "AAAA IN kubernetes.default.svc.cluster.local. udp 54 false 512" NOERROR qr,aa,rd 147 0.000184634s
[INFO] 10.244.2.2:58049 - 4 "A IN kubernetes.default.svc.cluster.local. udp 54 false 512" NOERROR qr,aa,rd 106 0.00018548s
[INFO] 10.244.2.2:35795 - 5 "PTR IN 1.0.96.10.in-addr.arpa. udp 40 false 512" NOERROR qr,aa,rd 112 0.000086098s
[INFO] 10.244.0.4:60969 - 5 "PTR IN 1.0.96.10.in-addr.arpa. udp 40 false 512" NOERROR qr,aa,rd 112 0.000090381s
[INFO] 10.244.1.2:54184 - 4 "A IN kubernetes.default.svc.cluster.local. udp 54 false 512" NOERROR qr,aa,rd 106 0.000173106s
[INFO] 10.244.1.2:37354 - 5 "PTR IN 1.0.96.10.in-addr.arpa. udp 40 false 512" NOERROR qr,aa,rd 112 0.000067488s
[INFO] 10.244.2.2:38119 - 2 "PTR IN 10.0.96.10.in-addr.arpa. udp 41 false 512" NOERROR qr,aa,rd 116 0.000161463s
[INFO] 10.244.2.2:47922 - 3 "AAAA IN host.minikube.internal. udp 40 false 512" NOERROR qr,aa,rd 40 0.000124909s
[INFO] 10.244.0.4:45686 - 2 "PTR IN 10.0.96.10.in-addr.arpa. udp 41 false 512" NOERROR qr,aa,rd 116 0.000119181s
==> coredns [c8ab7117746d22a14339221aee8d8b6add959c38472cacd236bfc7b815920794] <==
[INFO] 10.244.2.2:46792 - 5 "AAAA IN kubernetes.default.svc.cluster.local. udp 54 false 512" NOERROR qr,aa,rd 147 0.000103722s
[INFO] 10.244.2.2:60032 - 7 "A IN kubernetes.default.default.svc.cluster.local. udp 62 false 512" NXDOMAIN qr,aa,rd 155 0.000116769s
[INFO] 10.244.2.2:32816 - 9 "PTR IN 1.0.96.10.in-addr.arpa. udp 40 false 512" NOERROR qr,aa,rd 112 0.000157507s
[INFO] 10.244.0.4:34068 - 2 "PTR IN 10.0.96.10.in-addr.arpa. udp 41 false 512" NOERROR qr,aa,rd 116 0.000119369s
[INFO] 10.244.0.4:37836 - 3 "AAAA IN kubernetes.default. udp 36 false 512" NXDOMAIN qr,rd,ra 36 0.003113598s
[INFO] 10.244.0.4:33426 - 4 "AAAA IN kubernetes.default.default.svc.cluster.local. udp 62 false 512" NXDOMAIN qr,aa,rd 155 0.00023439s
[INFO] 10.244.0.4:48388 - 5 "AAAA IN kubernetes.default.svc.cluster.local. udp 54 false 512" NOERROR qr,aa,rd 147 0.000131604s
[INFO] 10.244.0.4:39230 - 8 "A IN kubernetes.default.svc.cluster.local. udp 54 false 512" NOERROR qr,aa,rd 106 0.00017862s
[INFO] 10.244.0.4:50943 - 9 "PTR IN 1.0.96.10.in-addr.arpa. udp 40 false 512" NOERROR qr,aa,rd 112 0.000083883s
[INFO] 10.244.1.2:46996 - 7 "A IN kubernetes.default.default.svc.cluster.local. udp 62 false 512" NXDOMAIN qr,aa,rd 155 0.00014355s
[INFO] 10.244.1.2:58261 - 8 "A IN kubernetes.default.svc.cluster.local. udp 54 false 512" NOERROR qr,aa,rd 106 0.000099439s
[INFO] 10.244.0.4:58425 - 2 "PTR IN 10.0.96.10.in-addr.arpa. udp 41 false 512" NOERROR qr,aa,rd 116 0.00010762s
[INFO] 10.244.0.4:55611 - 3 "AAAA IN kubernetes.default.svc.cluster.local. udp 54 false 512" NOERROR qr,aa,rd 147 0.000159009s
[INFO] 10.244.0.4:48378 - 4 "A IN kubernetes.default.svc.cluster.local. udp 54 false 512" NOERROR qr,aa,rd 106 0.000115505s
[INFO] 10.244.1.2:38348 - 2 "PTR IN 10.0.96.10.in-addr.arpa. udp 41 false 512" NOERROR qr,aa,rd 116 0.000116711s
[INFO] 10.244.1.2:59106 - 3 "AAAA IN kubernetes.default.svc.cluster.local. udp 54 false 512" NOERROR qr,aa,rd 147 0.000220499s
[INFO] 10.244.2.2:37813 - 4 "A IN host.minikube.internal. udp 40 false 512" NOERROR qr,aa,rd 78 0.000152002s
[INFO] 10.244.2.2:56106 - 5 "PTR IN 1.49.168.192.in-addr.arpa. udp 43 false 512" NOERROR qr,aa,rd 104 0.00019215s
[INFO] 10.244.0.4:41265 - 3 "AAAA IN host.minikube.internal. udp 40 false 512" NOERROR qr,aa,rd 40 0.000094089s
[INFO] 10.244.0.4:47425 - 4 "A IN host.minikube.internal. udp 40 false 512" NOERROR qr,aa,rd 78 0.000059792s
[INFO] 10.244.0.4:33602 - 5 "PTR IN 1.49.168.192.in-addr.arpa. udp 43 false 512" NOERROR qr,aa,rd 104 0.000068554s
[INFO] 10.244.1.2:35104 - 2 "PTR IN 10.0.96.10.in-addr.arpa. udp 41 false 512" NOERROR qr,aa,rd 116 0.000143871s
[INFO] 10.244.1.2:59782 - 3 "AAAA IN host.minikube.internal. udp 40 false 512" NOERROR qr,aa,rd 40 0.000081625s
[INFO] 10.244.1.2:39995 - 4 "A IN host.minikube.internal. udp 40 false 512" NOERROR qr,aa,rd 78 0.000080247s
[INFO] 10.244.1.2:46827 - 5 "PTR IN 1.49.168.192.in-addr.arpa. udp 43 false 512" NOERROR qr,aa,rd 104 0.000061088s
==> describe nodes <==
Name: ha-472819
Roles: control-plane
Labels: beta.kubernetes.io/arch=arm64
beta.kubernetes.io/os=linux
kubernetes.io/arch=arm64
kubernetes.io/hostname=ha-472819
kubernetes.io/os=linux
minikube.k8s.io/commit=8d0f47abe6720ae55a5722df67bba0ddd12c8845
minikube.k8s.io/name=ha-472819
minikube.k8s.io/primary=true
minikube.k8s.io/updated_at=2025_11_01T11_09_42_0700
minikube.k8s.io/version=v1.37.0
node-role.kubernetes.io/control-plane=
node.kubernetes.io/exclude-from-external-load-balancers=
Annotations: node.alpha.kubernetes.io/ttl: 0
volumes.kubernetes.io/controller-managed-attach-detach: true
CreationTimestamp: Sat, 01 Nov 2025 11:09:38 +0000
Taints: <none>
Unschedulable: false
Lease:
HolderIdentity: ha-472819
AcquireTime: <unset>
RenewTime: Sat, 01 Nov 2025 11:22:37 +0000
Conditions:
Type Status LastHeartbeatTime LastTransitionTime Reason Message
---- ------ ----------------- ------------------ ------ -------
MemoryPressure False Sat, 01 Nov 2025 11:22:37 +0000 Sat, 01 Nov 2025 11:09:35 +0000 KubeletHasSufficientMemory kubelet has sufficient memory available
DiskPressure False Sat, 01 Nov 2025 11:22:37 +0000 Sat, 01 Nov 2025 11:09:35 +0000 KubeletHasNoDiskPressure kubelet has no disk pressure
PIDPressure False Sat, 01 Nov 2025 11:22:37 +0000 Sat, 01 Nov 2025 11:09:35 +0000 KubeletHasSufficientPID kubelet has sufficient PID available
Ready True Sat, 01 Nov 2025 11:22:37 +0000 Sat, 01 Nov 2025 11:10:29 +0000 KubeletReady kubelet is posting ready status
Addresses:
InternalIP: 192.168.49.2
Hostname: ha-472819
Capacity:
cpu: 2
ephemeral-storage: 203034800Ki
hugepages-1Gi: 0
hugepages-2Mi: 0
hugepages-32Mi: 0
hugepages-64Ki: 0
memory: 8022296Ki
pods: 110
Allocatable:
cpu: 2
ephemeral-storage: 203034800Ki
hugepages-1Gi: 0
hugepages-2Mi: 0
hugepages-32Mi: 0
hugepages-64Ki: 0
memory: 8022296Ki
pods: 110
System Info:
Machine ID: ef38fbc8889a0e5f09e9dc0868f5cd19
System UUID: 60304d9d-d149-4b0e-8acf-98dc18a25376
Boot ID: a8ac8503-6b7a-4208-b896-162cdcafe81c
Kernel Version: 5.15.0-1084-aws
OS Image: Debian GNU/Linux 12 (bookworm)
Operating System: linux
Architecture: arm64
Container Runtime Version: cri-o://1.34.1
Kubelet Version: v1.34.1
Kube-Proxy Version:
PodCIDR: 10.244.0.0/24
PodCIDRs: 10.244.0.0/24
Non-terminated Pods: (11 in total)
Namespace Name CPU Requests CPU Limits Memory Requests Memory Limits Age
--------- ---- ------------ ---------- --------------- ------------- ---
default busybox-7b57f96db7-lm6r8 0 (0%) 0 (0%) 0 (0%) 0 (0%) 10m
kube-system coredns-66bc5c9577-bntfw 100m (5%) 0 (0%) 70Mi (0%) 170Mi (2%) 13m
kube-system coredns-66bc5c9577-n2tp2 100m (5%) 0 (0%) 70Mi (0%) 170Mi (2%) 13m
kube-system etcd-ha-472819 100m (5%) 0 (0%) 100Mi (1%) 0 (0%) 13m
kube-system kindnet-dkhrw 100m (5%) 100m (5%) 50Mi (0%) 50Mi (0%) 13m
kube-system kube-apiserver-ha-472819 250m (12%) 0 (0%) 0 (0%) 0 (0%) 13m
kube-system kube-controller-manager-ha-472819 200m (10%) 0 (0%) 0 (0%) 0 (0%) 13m
kube-system kube-proxy-djfvb 0 (0%) 0 (0%) 0 (0%) 0 (0%) 13m
kube-system kube-scheduler-ha-472819 100m (5%) 0 (0%) 0 (0%) 0 (0%) 13m
kube-system kube-vip-ha-472819 0 (0%) 0 (0%) 0 (0%) 0 (0%) 13m
kube-system storage-provisioner 0 (0%) 0 (0%) 0 (0%) 0 (0%) 13m
Allocated resources:
(Total limits may be over 100 percent, i.e., overcommitted.)
Resource Requests Limits
-------- -------- ------
cpu 950m (47%) 100m (5%)
memory 290Mi (3%) 390Mi (4%)
ephemeral-storage 0 (0%) 0 (0%)
hugepages-1Gi 0 (0%) 0 (0%)
hugepages-2Mi 0 (0%) 0 (0%)
hugepages-32Mi 0 (0%) 0 (0%)
hugepages-64Ki 0 (0%) 0 (0%)
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Starting 12m kube-proxy
Normal Starting 13m kubelet Starting kubelet.
Warning CgroupV1 13m kubelet cgroup v1 support is in maintenance mode, please migrate to cgroup v2
Normal NodeHasSufficientMemory 13m kubelet Node ha-472819 status is now: NodeHasSufficientMemory
Normal NodeHasNoDiskPressure 13m kubelet Node ha-472819 status is now: NodeHasNoDiskPressure
Normal NodeHasSufficientPID 13m kubelet Node ha-472819 status is now: NodeHasSufficientPID
Normal RegisteredNode 13m node-controller Node ha-472819 event: Registered Node ha-472819 in Controller
Normal RegisteredNode 12m node-controller Node ha-472819 event: Registered Node ha-472819 in Controller
Normal NodeReady 12m kubelet Node ha-472819 status is now: NodeReady
Normal RegisteredNode 11m node-controller Node ha-472819 event: Registered Node ha-472819 in Controller
Name: ha-472819-m02
Roles: control-plane
Labels: beta.kubernetes.io/arch=arm64
beta.kubernetes.io/os=linux
kubernetes.io/arch=arm64
kubernetes.io/hostname=ha-472819-m02
kubernetes.io/os=linux
minikube.k8s.io/commit=8d0f47abe6720ae55a5722df67bba0ddd12c8845
minikube.k8s.io/name=ha-472819
minikube.k8s.io/primary=false
minikube.k8s.io/updated_at=2025_11_01T11_10_18_0700
minikube.k8s.io/version=v1.37.0
node-role.kubernetes.io/control-plane=
node.kubernetes.io/exclude-from-external-load-balancers=
Annotations: node.alpha.kubernetes.io/ttl: 0
volumes.kubernetes.io/controller-managed-attach-detach: true
CreationTimestamp: Sat, 01 Nov 2025 11:10:18 +0000
Taints: node.kubernetes.io/unreachable:NoExecute
node.kubernetes.io/unreachable:NoSchedule
Unschedulable: false
Lease:
HolderIdentity: ha-472819-m02
AcquireTime: <unset>
RenewTime: Sat, 01 Nov 2025 11:14:03 +0000
Conditions:
Type Status LastHeartbeatTime LastTransitionTime Reason Message
---- ------ ----------------- ------------------ ------ -------
MemoryPressure Unknown Sat, 01 Nov 2025 11:12:41 +0000 Sat, 01 Nov 2025 11:14:56 +0000 NodeStatusUnknown Kubelet stopped posting node status.
DiskPressure Unknown Sat, 01 Nov 2025 11:12:41 +0000 Sat, 01 Nov 2025 11:14:56 +0000 NodeStatusUnknown Kubelet stopped posting node status.
PIDPressure Unknown Sat, 01 Nov 2025 11:12:41 +0000 Sat, 01 Nov 2025 11:14:56 +0000 NodeStatusUnknown Kubelet stopped posting node status.
Ready Unknown Sat, 01 Nov 2025 11:12:41 +0000 Sat, 01 Nov 2025 11:14:56 +0000 NodeStatusUnknown Kubelet stopped posting node status.
Addresses:
InternalIP: 192.168.49.3
Hostname: ha-472819-m02
Capacity:
cpu: 2
ephemeral-storage: 203034800Ki
hugepages-1Gi: 0
hugepages-2Mi: 0
hugepages-32Mi: 0
hugepages-64Ki: 0
memory: 8022296Ki
pods: 110
Allocatable:
cpu: 2
ephemeral-storage: 203034800Ki
hugepages-1Gi: 0
hugepages-2Mi: 0
hugepages-32Mi: 0
hugepages-64Ki: 0
memory: 8022296Ki
pods: 110
System Info:
Machine ID: ef38fbc8889a0e5f09e9dc0868f5cd19
System UUID: c598c781-8aa3-4c9a-acbe-21bfb38aa260
Boot ID: a8ac8503-6b7a-4208-b896-162cdcafe81c
Kernel Version: 5.15.0-1084-aws
OS Image: Debian GNU/Linux 12 (bookworm)
Operating System: linux
Architecture: arm64
Container Runtime Version: cri-o://1.34.1
Kubelet Version: v1.34.1
Kube-Proxy Version:
PodCIDR: 10.244.1.0/24
PodCIDRs: 10.244.1.0/24
Non-terminated Pods: (8 in total)
Namespace Name CPU Requests CPU Limits Memory Requests Memory Limits Age
--------- ---- ------------ ---------- --------------- ------------- ---
default busybox-7b57f96db7-x679v 0 (0%) 0 (0%) 0 (0%) 0 (0%) 10m
kube-system etcd-ha-472819-m02 100m (5%) 0 (0%) 100Mi (1%) 0 (0%) 12m
kube-system kindnet-cw2kt 100m (5%) 100m (5%) 50Mi (0%) 50Mi (0%) 12m
kube-system kube-apiserver-ha-472819-m02 250m (12%) 0 (0%) 0 (0%) 0 (0%) 12m
kube-system kube-controller-manager-ha-472819-m02 200m (10%) 0 (0%) 0 (0%) 0 (0%) 12m
kube-system kube-proxy-47prj 0 (0%) 0 (0%) 0 (0%) 0 (0%) 12m
kube-system kube-scheduler-ha-472819-m02 100m (5%) 0 (0%) 0 (0%) 0 (0%) 12m
kube-system kube-vip-ha-472819-m02 0 (0%) 0 (0%) 0 (0%) 0 (0%) 12m
Allocated resources:
(Total limits may be over 100 percent, i.e., overcommitted.)
Resource Requests Limits
-------- -------- ------
cpu 750m (37%) 100m (5%)
memory 150Mi (1%) 50Mi (0%)
ephemeral-storage 0 (0%) 0 (0%)
hugepages-1Gi 0 (0%) 0 (0%)
hugepages-2Mi 0 (0%) 0 (0%)
hugepages-32Mi 0 (0%) 0 (0%)
hugepages-64Ki 0 (0%) 0 (0%)
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Starting 12m kube-proxy
Normal RegisteredNode 12m node-controller Node ha-472819-m02 event: Registered Node ha-472819-m02 in Controller
Normal RegisteredNode 12m node-controller Node ha-472819-m02 event: Registered Node ha-472819-m02 in Controller
Normal RegisteredNode 11m node-controller Node ha-472819-m02 event: Registered Node ha-472819-m02 in Controller
Normal NodeNotReady 7m51s node-controller Node ha-472819-m02 status is now: NodeNotReady
Name: ha-472819-m03
Roles: control-plane
Labels: beta.kubernetes.io/arch=arm64
beta.kubernetes.io/os=linux
kubernetes.io/arch=arm64
kubernetes.io/hostname=ha-472819-m03
kubernetes.io/os=linux
minikube.k8s.io/commit=8d0f47abe6720ae55a5722df67bba0ddd12c8845
minikube.k8s.io/name=ha-472819
minikube.k8s.io/primary=false
minikube.k8s.io/updated_at=2025_11_01T11_11_43_0700
minikube.k8s.io/version=v1.37.0
node-role.kubernetes.io/control-plane=
node.kubernetes.io/exclude-from-external-load-balancers=
Annotations: node.alpha.kubernetes.io/ttl: 0
volumes.kubernetes.io/controller-managed-attach-detach: true
CreationTimestamp: Sat, 01 Nov 2025 11:11:42 +0000
Taints: <none>
Unschedulable: false
Lease:
HolderIdentity: ha-472819-m03
AcquireTime: <unset>
RenewTime: Sat, 01 Nov 2025 11:22:47 +0000
Conditions:
Type Status LastHeartbeatTime LastTransitionTime Reason Message
---- ------ ----------------- ------------------ ------ -------
MemoryPressure False Sat, 01 Nov 2025 11:18:01 +0000 Sat, 01 Nov 2025 11:11:42 +0000 KubeletHasSufficientMemory kubelet has sufficient memory available
DiskPressure False Sat, 01 Nov 2025 11:18:01 +0000 Sat, 01 Nov 2025 11:11:42 +0000 KubeletHasNoDiskPressure kubelet has no disk pressure
PIDPressure False Sat, 01 Nov 2025 11:18:01 +0000 Sat, 01 Nov 2025 11:11:42 +0000 KubeletHasSufficientPID kubelet has sufficient PID available
Ready True Sat, 01 Nov 2025 11:18:01 +0000 Sat, 01 Nov 2025 11:12:25 +0000 KubeletReady kubelet is posting ready status
Addresses:
InternalIP: 192.168.49.4
Hostname: ha-472819-m03
Capacity:
cpu: 2
ephemeral-storage: 203034800Ki
hugepages-1Gi: 0
hugepages-2Mi: 0
hugepages-32Mi: 0
hugepages-64Ki: 0
memory: 8022296Ki
pods: 110
Allocatable:
cpu: 2
ephemeral-storage: 203034800Ki
hugepages-1Gi: 0
hugepages-2Mi: 0
hugepages-32Mi: 0
hugepages-64Ki: 0
memory: 8022296Ki
pods: 110
System Info:
Machine ID: ef38fbc8889a0e5f09e9dc0868f5cd19
System UUID: dab78d92-59b1-457d-81c0-7efcc6e5bf35
Boot ID: a8ac8503-6b7a-4208-b896-162cdcafe81c
Kernel Version: 5.15.0-1084-aws
OS Image: Debian GNU/Linux 12 (bookworm)
Operating System: linux
Architecture: arm64
Container Runtime Version: cri-o://1.34.1
Kubelet Version: v1.34.1
Kube-Proxy Version:
PodCIDR: 10.244.2.0/24
PodCIDRs: 10.244.2.0/24
Non-terminated Pods: (8 in total)
Namespace Name CPU Requests CPU Limits Memory Requests Memory Limits Age
--------- ---- ------------ ---------- --------------- ------------- ---
default busybox-7b57f96db7-7m8cp 0 (0%) 0 (0%) 0 (0%) 0 (0%) 10m
kube-system etcd-ha-472819-m03 100m (5%) 0 (0%) 100Mi (1%) 0 (0%) 11m
kube-system kindnet-mz6bw 100m (5%) 100m (5%) 50Mi (0%) 50Mi (0%) 11m
kube-system kube-apiserver-ha-472819-m03 250m (12%) 0 (0%) 0 (0%) 0 (0%) 11m
kube-system kube-controller-manager-ha-472819-m03 200m (10%) 0 (0%) 0 (0%) 0 (0%) 11m
kube-system kube-proxy-gc4g4 0 (0%) 0 (0%) 0 (0%) 0 (0%) 11m
kube-system kube-scheduler-ha-472819-m03 100m (5%) 0 (0%) 0 (0%) 0 (0%) 11m
kube-system kube-vip-ha-472819-m03 0 (0%) 0 (0%) 0 (0%) 0 (0%) 11m
Allocated resources:
(Total limits may be over 100 percent, i.e., overcommitted.)
Resource Requests Limits
-------- -------- ------
cpu 750m (37%) 100m (5%)
memory 150Mi (1%) 50Mi (0%)
ephemeral-storage 0 (0%) 0 (0%)
hugepages-1Gi 0 (0%) 0 (0%)
hugepages-2Mi 0 (0%) 0 (0%)
hugepages-32Mi 0 (0%) 0 (0%)
hugepages-64Ki 0 (0%) 0 (0%)
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Starting 11m kube-proxy
Normal CIDRAssignmentFailed 11m cidrAllocator Node ha-472819-m03 status is now: CIDRAssignmentFailed
Normal RegisteredNode 11m node-controller Node ha-472819-m03 event: Registered Node ha-472819-m03 in Controller
Normal RegisteredNode 11m node-controller Node ha-472819-m03 event: Registered Node ha-472819-m03 in Controller
Normal RegisteredNode 11m node-controller Node ha-472819-m03 event: Registered Node ha-472819-m03 in Controller
Name: ha-472819-m04
Roles: <none>
Labels: beta.kubernetes.io/arch=arm64
beta.kubernetes.io/os=linux
kubernetes.io/arch=arm64
kubernetes.io/hostname=ha-472819-m04
kubernetes.io/os=linux
minikube.k8s.io/commit=8d0f47abe6720ae55a5722df67bba0ddd12c8845
minikube.k8s.io/name=ha-472819
minikube.k8s.io/primary=false
minikube.k8s.io/updated_at=2025_11_01T11_13_00_0700
minikube.k8s.io/version=v1.37.0
Annotations: node.alpha.kubernetes.io/ttl: 0
volumes.kubernetes.io/controller-managed-attach-detach: true
CreationTimestamp: Sat, 01 Nov 2025 11:12:59 +0000
Taints: <none>
Unschedulable: false
Lease:
HolderIdentity: ha-472819-m04
AcquireTime: <unset>
RenewTime: Sat, 01 Nov 2025 11:22:40 +0000
Conditions:
Type Status LastHeartbeatTime LastTransitionTime Reason Message
---- ------ ----------------- ------------------ ------ -------
MemoryPressure False Sat, 01 Nov 2025 11:20:48 +0000 Sat, 01 Nov 2025 11:12:59 +0000 KubeletHasSufficientMemory kubelet has sufficient memory available
DiskPressure False Sat, 01 Nov 2025 11:20:48 +0000 Sat, 01 Nov 2025 11:12:59 +0000 KubeletHasNoDiskPressure kubelet has no disk pressure
PIDPressure False Sat, 01 Nov 2025 11:20:48 +0000 Sat, 01 Nov 2025 11:12:59 +0000 KubeletHasSufficientPID kubelet has sufficient PID available
Ready True Sat, 01 Nov 2025 11:20:48 +0000 Sat, 01 Nov 2025 11:13:42 +0000 KubeletReady kubelet is posting ready status
Addresses:
InternalIP: 192.168.49.5
Hostname: ha-472819-m04
Capacity:
cpu: 2
ephemeral-storage: 203034800Ki
hugepages-1Gi: 0
hugepages-2Mi: 0
hugepages-32Mi: 0
hugepages-64Ki: 0
memory: 8022296Ki
pods: 110
Allocatable:
cpu: 2
ephemeral-storage: 203034800Ki
hugepages-1Gi: 0
hugepages-2Mi: 0
hugepages-32Mi: 0
hugepages-64Ki: 0
memory: 8022296Ki
pods: 110
System Info:
Machine ID: ef38fbc8889a0e5f09e9dc0868f5cd19
System UUID: e495f925-4ef2-41a0-86db-65c0daddf116
Boot ID: a8ac8503-6b7a-4208-b896-162cdcafe81c
Kernel Version: 5.15.0-1084-aws
OS Image: Debian GNU/Linux 12 (bookworm)
Operating System: linux
Architecture: arm64
Container Runtime Version: cri-o://1.34.1
Kubelet Version: v1.34.1
Kube-Proxy Version:
PodCIDR: 10.244.4.0/24
PodCIDRs: 10.244.4.0/24
Non-terminated Pods: (3 in total)
Namespace Name CPU Requests CPU Limits Memory Requests Memory Limits Age
--------- ---- ------------ ---------- --------------- ------------- ---
default busybox-7b57f96db7-x67zv 0 (0%) 0 (0%) 0 (0%) 0 (0%) 2m51s
kube-system kindnet-88sf2 100m (5%) 100m (5%) 50Mi (0%) 50Mi (0%) 9m48s
kube-system kube-proxy-79nw9 0 (0%) 0 (0%) 0 (0%) 0 (0%) 9m48s
Allocated resources:
(Total limits may be over 100 percent, i.e., overcommitted.)
Resource Requests Limits
-------- -------- ------
cpu 100m (5%) 100m (5%)
memory 50Mi (0%) 50Mi (0%)
ephemeral-storage 0 (0%) 0 (0%)
hugepages-1Gi 0 (0%) 0 (0%)
hugepages-2Mi 0 (0%) 0 (0%)
hugepages-32Mi 0 (0%) 0 (0%)
hugepages-64Ki 0 (0%) 0 (0%)
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Normal Starting 9m45s kube-proxy
Normal CIDRAssignmentFailed 9m48s cidrAllocator Node ha-472819-m04 status is now: CIDRAssignmentFailed
Normal CIDRAssignmentFailed 9m48s cidrAllocator Node ha-472819-m04 status is now: CIDRAssignmentFailed
Normal NodeHasSufficientMemory 9m48s (x3 over 9m48s) kubelet Node ha-472819-m04 status is now: NodeHasSufficientMemory
Normal NodeHasNoDiskPressure 9m48s (x3 over 9m48s) kubelet Node ha-472819-m04 status is now: NodeHasNoDiskPressure
Normal NodeHasSufficientPID 9m48s (x3 over 9m48s) kubelet Node ha-472819-m04 status is now: NodeHasSufficientPID
Normal RegisteredNode 9m46s node-controller Node ha-472819-m04 event: Registered Node ha-472819-m04 in Controller
Normal RegisteredNode 9m46s node-controller Node ha-472819-m04 event: Registered Node ha-472819-m04 in Controller
Normal RegisteredNode 9m46s node-controller Node ha-472819-m04 event: Registered Node ha-472819-m04 in Controller
Normal NodeReady 9m5s kubelet Node ha-472819-m04 status is now: NodeReady
==> dmesg <==
[Nov 1 10:33] overlayfs: idmapped layers are currently not supported
[Nov 1 10:34] overlayfs: idmapped layers are currently not supported
[Nov 1 10:35] overlayfs: idmapped layers are currently not supported
[Nov 1 10:36] overlayfs: idmapped layers are currently not supported
[ +20.644099] overlayfs: idmapped layers are currently not supported
[Nov 1 10:37] overlayfs: idmapped layers are currently not supported
[Nov 1 10:38] overlayfs: idmapped layers are currently not supported
[ +26.122524] overlayfs: idmapped layers are currently not supported
[Nov 1 10:39] overlayfs: idmapped layers are currently not supported
[ +9.289237] overlayfs: idmapped layers are currently not supported
[Nov 1 10:40] overlayfs: idmapped layers are currently not supported
[Nov 1 10:41] overlayfs: idmapped layers are currently not supported
[Nov 1 10:43] overlayfs: idmapped layers are currently not supported
[ +12.370416] overlayfs: idmapped layers are currently not supported
[Nov 1 10:44] overlayfs: idmapped layers are currently not supported
[Nov 1 10:45] overlayfs: idmapped layers are currently not supported
[Nov 1 10:47] kauditd_printk_skb: 8 callbacks suppressed
[Nov 1 10:49] overlayfs: idmapped layers are currently not supported
[Nov 1 10:55] overlayfs: idmapped layers are currently not supported
[Nov 1 10:56] overlayfs: idmapped layers are currently not supported
[Nov 1 11:09] overlayfs: idmapped layers are currently not supported
[Nov 1 11:10] overlayfs: idmapped layers are currently not supported
[Nov 1 11:11] overlayfs: idmapped layers are currently not supported
[Nov 1 11:13] overlayfs: idmapped layers are currently not supported
[Nov 1 11:14] overlayfs: idmapped layers are currently not supported
==> etcd [640585dbb86b99532c1a5c54e4cb7548846d3ee044b85ae39e75a467ff5a3081] <==
{"level":"warn","ts":"2025-11-01T11:22:20.328463Z","caller":"etcdserver/cluster_util.go:160","msg":"failed to get version","remote-member-id":"99ad86fd494346b","error":"Get \"https://192.168.49.3:2380/version\": dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:21.723300Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_SNAPSHOT","remote-peer-id":"99ad86fd494346b","rtt":"31.357601ms","error":"dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:21.723316Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_RAFT_MESSAGE","remote-peer-id":"99ad86fd494346b","rtt":"44.334372ms","error":"dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:24.330041Z","caller":"etcdserver/cluster_util.go:259","msg":"failed to reach the peer URL","address":"https://192.168.49.3:2380/version","remote-member-id":"99ad86fd494346b","error":"Get \"https://192.168.49.3:2380/version\": dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:24.330107Z","caller":"etcdserver/cluster_util.go:160","msg":"failed to get version","remote-member-id":"99ad86fd494346b","error":"Get \"https://192.168.49.3:2380/version\": dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:26.724057Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_SNAPSHOT","remote-peer-id":"99ad86fd494346b","rtt":"31.357601ms","error":"dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:26.724046Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_RAFT_MESSAGE","remote-peer-id":"99ad86fd494346b","rtt":"44.334372ms","error":"dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:28.331190Z","caller":"etcdserver/cluster_util.go:259","msg":"failed to reach the peer URL","address":"https://192.168.49.3:2380/version","remote-member-id":"99ad86fd494346b","error":"Get \"https://192.168.49.3:2380/version\": dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:28.331378Z","caller":"etcdserver/cluster_util.go:160","msg":"failed to get version","remote-member-id":"99ad86fd494346b","error":"Get \"https://192.168.49.3:2380/version\": dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:31.725003Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_RAFT_MESSAGE","remote-peer-id":"99ad86fd494346b","rtt":"44.334372ms","error":"dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:31.725022Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_SNAPSHOT","remote-peer-id":"99ad86fd494346b","rtt":"31.357601ms","error":"dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:32.332651Z","caller":"etcdserver/cluster_util.go:259","msg":"failed to reach the peer URL","address":"https://192.168.49.3:2380/version","remote-member-id":"99ad86fd494346b","error":"Get \"https://192.168.49.3:2380/version\": dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:32.332707Z","caller":"etcdserver/cluster_util.go:160","msg":"failed to get version","remote-member-id":"99ad86fd494346b","error":"Get \"https://192.168.49.3:2380/version\": dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:36.334390Z","caller":"etcdserver/cluster_util.go:259","msg":"failed to reach the peer URL","address":"https://192.168.49.3:2380/version","remote-member-id":"99ad86fd494346b","error":"Get \"https://192.168.49.3:2380/version\": dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:36.334512Z","caller":"etcdserver/cluster_util.go:160","msg":"failed to get version","remote-member-id":"99ad86fd494346b","error":"Get \"https://192.168.49.3:2380/version\": dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:36.725833Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_SNAPSHOT","remote-peer-id":"99ad86fd494346b","rtt":"31.357601ms","error":"dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:36.725839Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_RAFT_MESSAGE","remote-peer-id":"99ad86fd494346b","rtt":"44.334372ms","error":"dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:40.335613Z","caller":"etcdserver/cluster_util.go:259","msg":"failed to reach the peer URL","address":"https://192.168.49.3:2380/version","remote-member-id":"99ad86fd494346b","error":"Get \"https://192.168.49.3:2380/version\": dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:40.335670Z","caller":"etcdserver/cluster_util.go:160","msg":"failed to get version","remote-member-id":"99ad86fd494346b","error":"Get \"https://192.168.49.3:2380/version\": dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:41.726016Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_RAFT_MESSAGE","remote-peer-id":"99ad86fd494346b","rtt":"44.334372ms","error":"dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:41.726102Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_SNAPSHOT","remote-peer-id":"99ad86fd494346b","rtt":"31.357601ms","error":"dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:44.337230Z","caller":"etcdserver/cluster_util.go:259","msg":"failed to reach the peer URL","address":"https://192.168.49.3:2380/version","remote-member-id":"99ad86fd494346b","error":"Get \"https://192.168.49.3:2380/version\": dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:44.337315Z","caller":"etcdserver/cluster_util.go:160","msg":"failed to get version","remote-member-id":"99ad86fd494346b","error":"Get \"https://192.168.49.3:2380/version\": dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:46.726611Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_SNAPSHOT","remote-peer-id":"99ad86fd494346b","rtt":"31.357601ms","error":"dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-11-01T11:22:46.726745Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_RAFT_MESSAGE","remote-peer-id":"99ad86fd494346b","rtt":"44.334372ms","error":"dial tcp 192.168.49.3:2380: connect: connection refused"}
==> kernel <==
11:22:47 up 3:05, 0 user, load average: 0.99, 0.97, 1.37
Linux ha-472819 5.15.0-1084-aws #91~20.04.1-Ubuntu SMP Fri May 2 07:00:04 UTC 2025 aarch64 GNU/Linux
PRETTY_NAME="Debian GNU/Linux 12 (bookworm)"
==> kindnet [6af4febe46d8a121ee2c8a9dbe81d96bb1173a205d2aadbbf0c7fd9d38d70f1b] <==
I1101 11:22:08.933679 1 main.go:324] Node ha-472819-m02 has CIDR [10.244.1.0/24]
I1101 11:22:18.941992 1 main.go:297] Handling node with IPs: map[192.168.49.2:{}]
I1101 11:22:18.942027 1 main.go:301] handling current node
I1101 11:22:18.942044 1 main.go:297] Handling node with IPs: map[192.168.49.3:{}]
I1101 11:22:18.942051 1 main.go:324] Node ha-472819-m02 has CIDR [10.244.1.0/24]
I1101 11:22:18.942222 1 main.go:297] Handling node with IPs: map[192.168.49.4:{}]
I1101 11:22:18.942231 1 main.go:324] Node ha-472819-m03 has CIDR [10.244.2.0/24]
I1101 11:22:18.942293 1 main.go:297] Handling node with IPs: map[192.168.49.5:{}]
I1101 11:22:18.942304 1 main.go:324] Node ha-472819-m04 has CIDR [10.244.4.0/24]
I1101 11:22:28.940035 1 main.go:297] Handling node with IPs: map[192.168.49.5:{}]
I1101 11:22:28.940091 1 main.go:324] Node ha-472819-m04 has CIDR [10.244.4.0/24]
I1101 11:22:28.940300 1 main.go:297] Handling node with IPs: map[192.168.49.2:{}]
I1101 11:22:28.940311 1 main.go:301] handling current node
I1101 11:22:28.940352 1 main.go:297] Handling node with IPs: map[192.168.49.3:{}]
I1101 11:22:28.940360 1 main.go:324] Node ha-472819-m02 has CIDR [10.244.1.0/24]
I1101 11:22:28.940440 1 main.go:297] Handling node with IPs: map[192.168.49.4:{}]
I1101 11:22:28.940446 1 main.go:324] Node ha-472819-m03 has CIDR [10.244.2.0/24]
I1101 11:22:38.933759 1 main.go:297] Handling node with IPs: map[192.168.49.2:{}]
I1101 11:22:38.933796 1 main.go:301] handling current node
I1101 11:22:38.933812 1 main.go:297] Handling node with IPs: map[192.168.49.3:{}]
I1101 11:22:38.933819 1 main.go:324] Node ha-472819-m02 has CIDR [10.244.1.0/24]
I1101 11:22:38.933954 1 main.go:297] Handling node with IPs: map[192.168.49.4:{}]
I1101 11:22:38.933969 1 main.go:324] Node ha-472819-m03 has CIDR [10.244.2.0/24]
I1101 11:22:38.934025 1 main.go:297] Handling node with IPs: map[192.168.49.5:{}]
I1101 11:22:38.934036 1 main.go:324] Node ha-472819-m04 has CIDR [10.244.4.0/24]
==> kube-apiserver [91af80c077c55f22c55a82cba007fef6ec8fa3f92d010ceb23da188210f136c8] <==
I1101 11:09:40.797825 1 controller.go:667] quota admission added evaluator for: roles.rbac.authorization.k8s.io
I1101 11:09:40.850033 1 controller.go:667] quota admission added evaluator for: rolebindings.rbac.authorization.k8s.io
I1101 11:09:40.915102 1 alloc.go:328] "allocated clusterIPs" service="default/kubernetes" clusterIPs={"IPv4":"10.96.0.1"}
W1101 11:09:40.924959 1 lease.go:265] Resetting endpoints for master service "kubernetes" to [192.168.49.2]
I1101 11:09:40.926249 1 controller.go:667] quota admission added evaluator for: endpoints
I1101 11:09:40.931594 1 controller.go:667] quota admission added evaluator for: endpointslices.discovery.k8s.io
I1101 11:09:41.095631 1 controller.go:667] quota admission added evaluator for: serviceaccounts
I1101 11:09:41.859882 1 controller.go:667] quota admission added evaluator for: deployments.apps
I1101 11:09:41.878887 1 alloc.go:328] "allocated clusterIPs" service="kube-system/kube-dns" clusterIPs={"IPv4":"10.96.0.10"}
I1101 11:09:41.887812 1 controller.go:667] quota admission added evaluator for: daemonsets.apps
I1101 11:09:46.450412 1 controller.go:667] quota admission added evaluator for: replicasets.apps
I1101 11:09:47.188658 1 cidrallocator.go:277] updated ClusterIP allocator for Service CIDR 10.96.0.0/12
I1101 11:09:47.202210 1 cidrallocator.go:277] updated ClusterIP allocator for Service CIDR 10.96.0.0/12
I1101 11:09:47.251429 1 controller.go:667] quota admission added evaluator for: controllerrevisions.apps
E1101 11:12:37.069649 1 watch.go:272] "Unhandled Error" err="http2: stream closed" logger="UnhandledError"
E1101 11:12:37.755999 1 conn.go:339] Error on socket receive: read tcp 192.168.49.254:8443->192.168.49.1:39660: use of closed network connection
E1101 11:12:38.787350 1 conn.go:339] Error on socket receive: read tcp 192.168.49.254:8443->192.168.49.1:39724: use of closed network connection
E1101 11:12:39.032436 1 conn.go:339] Error on socket receive: read tcp 192.168.49.254:8443->192.168.49.1:39744: use of closed network connection
E1101 11:12:39.263792 1 conn.go:339] Error on socket receive: read tcp 192.168.49.254:8443->192.168.49.1:39766: use of closed network connection
E1101 11:12:39.693531 1 conn.go:339] Error on socket receive: read tcp 192.168.49.254:8443->192.168.49.1:39796: use of closed network connection
E1101 11:12:40.115015 1 conn.go:339] Error on socket receive: read tcp 192.168.49.254:8443->192.168.49.1:39810: use of closed network connection
E1101 11:12:40.338287 1 conn.go:339] Error on socket receive: read tcp 192.168.49.254:8443->192.168.49.1:39828: use of closed network connection
E1101 11:12:40.547346 1 conn.go:339] Error on socket receive: read tcp 192.168.49.254:8443->192.168.49.1:39850: use of closed network connection
E1101 11:12:40.760719 1 conn.go:339] Error on socket receive: read tcp 192.168.49.254:8443->192.168.49.1:39870: use of closed network connection
I1101 11:19:38.988023 1 cidrallocator.go:277] updated ClusterIP allocator for Service CIDR 10.96.0.0/12
==> kube-controller-manager [6bf6ea4411cda5dbfef374975a27f08c60164beec1853c8ba8df3c4f23b6c666] <==
I1101 11:09:46.183504 1 range_allocator.go:428] "Set node PodCIDR" logger="node-ipam-controller" node="ha-472819" podCIDRs=["10.244.0.0/24"]
I1101 11:09:46.190773 1 shared_informer.go:356] "Caches are synced" controller="validatingadmissionpolicy-status"
I1101 11:10:18.068385 1 actual_state_of_world.go:541] "Failed to update statusUpdateNeeded field in actual state of world" logger="persistentvolume-attach-detach-controller" err="Failed to set statusUpdateNeeded to needed true, because nodeName=\"ha-472819-m02\" does not exist"
I1101 11:10:18.086957 1 range_allocator.go:428] "Set node PodCIDR" logger="node-ipam-controller" node="ha-472819-m02" podCIDRs=["10.244.1.0/24"]
I1101 11:10:21.154242 1 node_lifecycle_controller.go:873] "Missing timestamp for Node. Assuming now as a timestamp" logger="node-lifecycle-controller" node="ha-472819-m02"
I1101 11:10:30.317774 1 endpointslice_controller.go:344] "Error syncing endpoint slices for service, retrying" logger="endpointslice-controller" key="kube-system/kube-dns" err="failed to update kube-dns-tql2r EndpointSlice for Service kube-system/kube-dns: Operation cannot be fulfilled on endpointslices.discovery.k8s.io \"kube-dns-tql2r\": the object has been modified; please apply your changes to the latest version and try again"
I1101 11:10:30.318055 1 event.go:377] Event(v1.ObjectReference{Kind:"Service", Namespace:"kube-system", Name:"kube-dns", UID:"e71f1e87-b843-4235-9d7d-ceeca6034661", APIVersion:"v1", ResourceVersion:"250", FieldPath:""}): type: 'Warning' reason: 'FailedToUpdateEndpointSlices' Error updating Endpoint Slices for Service kube-system/kube-dns: failed to update kube-dns-tql2r EndpointSlice for Service kube-system/kube-dns: Operation cannot be fulfilled on endpointslices.discovery.k8s.io "kube-dns-tql2r": the object has been modified; please apply your changes to the latest version and try again
I1101 11:10:31.155699 1 node_lifecycle_controller.go:1044] "Controller detected that some Nodes are Ready. Exiting master disruption mode" logger="node-lifecycle-controller"
E1101 11:11:42.459308 1 certificate_controller.go:151] "Unhandled Error" err="Sync csr-drx6q failed with : error updating approval for csr: Operation cannot be fulfilled on certificatesigningrequests.certificates.k8s.io \"csr-drx6q\": the object has been modified; please apply your changes to the latest version and try again" logger="UnhandledError"
E1101 11:11:42.471104 1 certificate_controller.go:151] "Unhandled Error" err="Sync csr-drx6q failed with : error updating signature for csr: Operation cannot be fulfilled on certificatesigningrequests.certificates.k8s.io \"csr-drx6q\": the object has been modified; please apply your changes to the latest version and try again" logger="UnhandledError"
I1101 11:11:42.992046 1 actual_state_of_world.go:541] "Failed to update statusUpdateNeeded field in actual state of world" logger="persistentvolume-attach-detach-controller" err="Failed to set statusUpdateNeeded to needed true, because nodeName=\"ha-472819-m03\" does not exist"
I1101 11:11:43.069592 1 range_allocator.go:428] "Set node PodCIDR" logger="node-ipam-controller" node="ha-472819-m03" podCIDRs=["10.244.2.0/24"]
I1101 11:11:46.211739 1 node_lifecycle_controller.go:873] "Missing timestamp for Node. Assuming now as a timestamp" logger="node-lifecycle-controller" node="ha-472819-m03"
E1101 11:12:59.200871 1 certificate_controller.go:151] "Unhandled Error" err="Sync csr-mdtrv failed with : error updating approval for csr: Operation cannot be fulfilled on certificatesigningrequests.certificates.k8s.io \"csr-mdtrv\": the object has been modified; please apply your changes to the latest version and try again" logger="UnhandledError"
E1101 11:12:59.200944 1 certificate_controller.go:151] "Unhandled Error" err="Sync csr-mdtrv failed with : error updating signature for csr: Operation cannot be fulfilled on certificatesigningrequests.certificates.k8s.io \"csr-mdtrv\": the object has been modified; please apply your changes to the latest version and try again" logger="UnhandledError"
I1101 11:12:59.416895 1 actual_state_of_world.go:541] "Failed to update statusUpdateNeeded field in actual state of world" logger="persistentvolume-attach-detach-controller" err="Failed to set statusUpdateNeeded to needed true, because nodeName=\"ha-472819-m04\" does not exist"
E1101 11:12:59.620317 1 range_allocator.go:433] "Failed to update node PodCIDR after multiple attempts" err="failed to patch node CIDR: Node \"ha-472819-m04\" is invalid: [spec.podCIDRs: Invalid value: [\"10.244.3.0/24\",\"10.244.4.0/24\"]: may specify no more than one CIDR for each IP family, spec.podCIDRs: Forbidden: node updates may not change podCIDR except from \"\" to valid]" logger="node-ipam-controller" node="ha-472819-m04" podCIDRs=["10.244.3.0/24"]
E1101 11:12:59.620451 1 range_allocator.go:439] "CIDR assignment for node failed. Releasing allocated CIDR" err="failed to patch node CIDR: Node \"ha-472819-m04\" is invalid: [spec.podCIDRs: Invalid value: [\"10.244.3.0/24\",\"10.244.4.0/24\"]: may specify no more than one CIDR for each IP family, spec.podCIDRs: Forbidden: node updates may not change podCIDR except from \"\" to valid]" logger="node-ipam-controller" node="ha-472819-m04"
E1101 11:12:59.620535 1 range_allocator.go:252] "Unhandled Error" err="error syncing 'ha-472819-m04': failed to patch node CIDR: Node \"ha-472819-m04\" is invalid: [spec.podCIDRs: Invalid value: [\"10.244.3.0/24\",\"10.244.4.0/24\"]: may specify no more than one CIDR for each IP family, spec.podCIDRs: Forbidden: node updates may not change podCIDR except from \"\" to valid], requeuing" logger="UnhandledError"
E1101 11:12:59.854362 1 daemon_controller.go:346] "Unhandled Error" err="kube-system/kube-proxy failed with : error storing status for daemon set &v1.DaemonSet{TypeMeta:v1.TypeMeta{Kind:\"\", APIVersion:\"\"}, ObjectMeta:v1.ObjectMeta{Name:\"kube-proxy\", GenerateName:\"\", Namespace:\"kube-system\", SelfLink:\"\", UID:\"c68c3d5f-200a-4729-99a6-399d13923da3\", ResourceVersion:\"902\", Generation:1, CreationTimestamp:time.Date(2025, time.November, 1, 11, 9, 41, 0, time.Local), DeletionTimestamp:<nil>, DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string{\"k8s-app\":\"kube-proxy\"}, Annotations:map[string]string{\"deprecated.daemonset.template.generation\":\"1\"}, OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ManagedFields:[]v1.ManagedFieldsEntry(nil)}, Spec:v1.DaemonSetSpec{Selector:(*v1.LabelSelector)(0x40019ffba0), Template:v1.PodTemplateSpec{ObjectMeta:v1.ObjectMeta{Name:\"\", GenerateName:\"\", Namespace:\"\", SelfLink:\"\", UID:\"\", ResourceVersion:\"\",
Generation:0, CreationTimestamp:time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC), DeletionTimestamp:<nil>, DeletionGracePeriodSeconds:(*int64)(nil), Labels:map[string]string{\"k8s-app\":\"kube-proxy\"}, Annotations:map[string]string(nil), OwnerReferences:[]v1.OwnerReference(nil), Finalizers:[]string(nil), ManagedFields:[]v1.ManagedFieldsEntry(nil)}, Spec:v1.PodSpec{Volumes:[]v1.Volume{v1.Volume{Name:\"kube-proxy\", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(nil), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(
nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(0x4002d4d580), VsphereVolume:(*v1.VsphereVirtualDiskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolumeSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil), Image:(*v1.ImageVolumeSource)(nil)}}, v1.Volume{Name:\"xtables-lock\", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(0x4000e12768), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeS
ource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(nil), VsphereVolume:(*v1.VsphereVirtualDiskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolu
meSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil), Image:(*v1.ImageVolumeSource)(nil)}}, v1.Volume{Name:\"lib-modules\", VolumeSource:v1.VolumeSource{HostPath:(*v1.HostPathVolumeSource)(0x4000e12780), EmptyDir:(*v1.EmptyDirVolumeSource)(nil), GCEPersistentDisk:(*v1.GCEPersistentDiskVolumeSource)(nil), AWSElasticBlockStore:(*v1.AWSElasticBlockStoreVolumeSource)(nil), GitRepo:(*v1.GitRepoVolumeSource)(nil), Secret:(*v1.SecretVolumeSource)(nil), NFS:(*v1.NFSVolumeSource)(nil), ISCSI:(*v1.ISCSIVolumeSource)(nil), Glusterfs:(*v1.GlusterfsVolumeSource)(nil), PersistentVolumeClaim:(*v1.PersistentVolumeClaimVolumeSource)(nil), RBD:(*v1.RBDVolumeSource)(nil), FlexVolume:(*v1.FlexVolumeSource)(nil), Cinder:(*v1.CinderVolumeSource)(nil), CephFS:(*v1.CephFSVolumeSource)(nil), Flocker:(*v1.FlockerVolumeSource)(nil), DownwardAPI:(*v1.DownwardAPIVolumeSource)(nil), FC:(*v1.FCVolumeSource)(nil), AzureFile:(*v1.AzureFileVolumeSource)(nil), ConfigMap:(*v1.ConfigMapVolumeSource)(nil), VsphereVolume:(*v1.VsphereVirtualD
iskVolumeSource)(nil), Quobyte:(*v1.QuobyteVolumeSource)(nil), AzureDisk:(*v1.AzureDiskVolumeSource)(nil), PhotonPersistentDisk:(*v1.PhotonPersistentDiskVolumeSource)(nil), Projected:(*v1.ProjectedVolumeSource)(nil), PortworxVolume:(*v1.PortworxVolumeSource)(nil), ScaleIO:(*v1.ScaleIOVolumeSource)(nil), StorageOS:(*v1.StorageOSVolumeSource)(nil), CSI:(*v1.CSIVolumeSource)(nil), Ephemeral:(*v1.EphemeralVolumeSource)(nil), Image:(*v1.ImageVolumeSource)(nil)}}}, InitContainers:[]v1.Container(nil), Containers:[]v1.Container{v1.Container{Name:\"kube-proxy\", Image:\"registry.k8s.io/kube-proxy:v1.34.1\", Command:[]string{\"/usr/local/bin/kube-proxy\", \"--config=/var/lib/kube-proxy/config.conf\", \"--hostname-override=$(NODE_NAME)\"}, Args:[]string(nil), WorkingDir:\"\", Ports:[]v1.ContainerPort(nil), EnvFrom:[]v1.EnvFromSource(nil), Env:[]v1.EnvVar{v1.EnvVar{Name:\"NODE_NAME\", Value:\"\", ValueFrom:(*v1.EnvVarSource)(0x4002c98ab0)}}, Resources:v1.ResourceRequirements{Limits:v1.ResourceList(nil), Requests:v1.Resou
rceList(nil), Claims:[]v1.ResourceClaim(nil)}, ResizePolicy:[]v1.ContainerResizePolicy(nil), RestartPolicy:(*v1.ContainerRestartPolicy)(nil), RestartPolicyRules:[]v1.ContainerRestartRule(nil), VolumeMounts:[]v1.VolumeMount{v1.VolumeMount{Name:\"kube-proxy\", ReadOnly:false, RecursiveReadOnly:(*v1.RecursiveReadOnlyMode)(nil), MountPath:\"/var/lib/kube-proxy\", SubPath:\"\", MountPropagation:(*v1.MountPropagationMode)(nil), SubPathExpr:\"\"}, v1.VolumeMount{Name:\"xtables-lock\", ReadOnly:false, RecursiveReadOnly:(*v1.RecursiveReadOnlyMode)(nil), MountPath:\"/run/xtables.lock\", SubPath:\"\", MountPropagation:(*v1.MountPropagationMode)(nil), SubPathExpr:\"\"}, v1.VolumeMount{Name:\"lib-modules\", ReadOnly:true, RecursiveReadOnly:(*v1.RecursiveReadOnlyMode)(nil), MountPath:\"/lib/modules\", SubPath:\"\", MountPropagation:(*v1.MountPropagationMode)(nil), SubPathExpr:\"\"}}, VolumeDevices:[]v1.VolumeDevice(nil), LivenessProbe:(*v1.Probe)(nil), ReadinessProbe:(*v1.Probe)(nil), StartupProbe:(*v1.Probe)(nil), Lifecyc
le:(*v1.Lifecycle)(nil), TerminationMessagePath:\"/dev/termination-log\", TerminationMessagePolicy:\"File\", ImagePullPolicy:\"IfNotPresent\", SecurityContext:(*v1.SecurityContext)(0x4002a864e0), Stdin:false, StdinOnce:false, TTY:false}}, EphemeralContainers:[]v1.EphemeralContainer(nil), RestartPolicy:\"Always\", TerminationGracePeriodSeconds:(*int64)(0x4002d266f8), ActiveDeadlineSeconds:(*int64)(nil), DNSPolicy:\"ClusterFirst\", NodeSelector:map[string]string{\"kubernetes.io/os\":\"linux\"}, ServiceAccountName:\"kube-proxy\", DeprecatedServiceAccount:\"kube-proxy\", AutomountServiceAccountToken:(*bool)(nil), NodeName:\"\", HostNetwork:true, HostPID:false, HostIPC:false, ShareProcessNamespace:(*bool)(nil), SecurityContext:(*v1.PodSecurityContext)(0x4002f47a70), ImagePullSecrets:[]v1.LocalObjectReference(nil), Hostname:\"\", Subdomain:\"\", Affinity:(*v1.Affinity)(nil), SchedulerName:\"default-scheduler\", Tolerations:[]v1.Toleration{v1.Toleration{Key:\"\", Operator:\"Exists\", Value:\"\", Effect:\"\", Tolerat
ionSeconds:(*int64)(nil)}}, HostAliases:[]v1.HostAlias(nil), PriorityClassName:\"system-node-critical\", Priority:(*int32)(nil), DNSConfig:(*v1.PodDNSConfig)(nil), ReadinessGates:[]v1.PodReadinessGate(nil), RuntimeClassName:(*string)(nil), EnableServiceLinks:(*bool)(nil), PreemptionPolicy:(*v1.PreemptionPolicy)(nil), Overhead:v1.ResourceList(nil), TopologySpreadConstraints:[]v1.TopologySpreadConstraint(nil), SetHostnameAsFQDN:(*bool)(nil), OS:(*v1.PodOS)(nil), HostUsers:(*bool)(nil), SchedulingGates:[]v1.PodSchedulingGate(nil), ResourceClaims:[]v1.PodResourceClaim(nil), Resources:(*v1.ResourceRequirements)(nil), HostnameOverride:(*string)(nil)}}, UpdateStrategy:v1.DaemonSetUpdateStrategy{Type:\"RollingUpdate\", RollingUpdate:(*v1.RollingUpdateDaemonSet)(0x4002d0b960)}, MinReadySeconds:0, RevisionHistoryLimit:(*int32)(0x4002d26750)}, Status:v1.DaemonSetStatus{CurrentNumberScheduled:3, NumberMisscheduled:0, DesiredNumberScheduled:3, NumberReady:3, ObservedGeneration:1, UpdatedNumberScheduled:3, NumberAvailable:
3, NumberUnavailable:0, CollisionCount:(*int32)(nil), Conditions:[]v1.DaemonSetCondition(nil)}}: Operation cannot be fulfilled on daemonsets.apps \"kube-proxy\": the object has been modified; please apply your changes to the latest version and try again" logger="UnhandledError"
I1101 11:13:01.243524 1 node_lifecycle_controller.go:873] "Missing timestamp for Node. Assuming now as a timestamp" logger="node-lifecycle-controller" node="ha-472819-m04"
I1101 11:13:42.484705 1 topologycache.go:237] "Can't get CPU or zone information for node" logger="endpointslice-controller" node="ha-472819-m04"
I1101 11:14:56.316641 1 topologycache.go:237] "Can't get CPU or zone information for node" logger="endpointslice-controller" node="ha-472819-m04"
I1101 11:19:56.579535 1 taint_eviction.go:111] "Deleting pod" logger="taint-eviction-controller" controller="taint-eviction-controller" pod="default/busybox-7b57f96db7-x679v"
==> kube-proxy [7078104c50ff2f92f7e2c1df5b91f0bd0cf730fe4a2b36f8082f1d451dd65225] <==
I1101 11:09:48.742762 1 server_linux.go:53] "Using iptables proxy"
I1101 11:09:48.836468 1 shared_informer.go:349] "Waiting for caches to sync" controller="node informer cache"
I1101 11:09:48.950591 1 shared_informer.go:356] "Caches are synced" controller="node informer cache"
I1101 11:09:48.950696 1 server.go:219] "Successfully retrieved NodeIPs" NodeIPs=["192.168.49.2"]
E1101 11:09:48.950821 1 server.go:256] "Kube-proxy configuration may be incomplete or incorrect" err="nodePortAddresses is unset; NodePort connections will be accepted on all local IPs. Consider using `--nodeport-addresses primary`"
I1101 11:09:49.042998 1 server.go:265] "kube-proxy running in dual-stack mode" primary ipFamily="IPv4"
I1101 11:09:49.043148 1 server_linux.go:132] "Using iptables Proxier"
I1101 11:09:49.123164 1 proxier.go:242] "Setting route_localnet=1 to allow node-ports on localhost; to change this either disable iptables.localhostNodePorts (--iptables-localhost-nodeports) or set nodePortAddresses (--nodeport-addresses) to filter loopback addresses" ipFamily="IPv4"
I1101 11:09:49.123454 1 server.go:527] "Version info" version="v1.34.1"
I1101 11:09:49.123477 1 server.go:529] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I1101 11:09:49.125197 1 config.go:200] "Starting service config controller"
I1101 11:09:49.125224 1 shared_informer.go:349] "Waiting for caches to sync" controller="service config"
I1101 11:09:49.125242 1 config.go:106] "Starting endpoint slice config controller"
I1101 11:09:49.125247 1 shared_informer.go:349] "Waiting for caches to sync" controller="endpoint slice config"
I1101 11:09:49.125257 1 config.go:403] "Starting serviceCIDR config controller"
I1101 11:09:49.125261 1 shared_informer.go:349] "Waiting for caches to sync" controller="serviceCIDR config"
I1101 11:09:49.126013 1 config.go:309] "Starting node config controller"
I1101 11:09:49.126032 1 shared_informer.go:349] "Waiting for caches to sync" controller="node config"
I1101 11:09:49.126039 1 shared_informer.go:356] "Caches are synced" controller="node config"
I1101 11:09:49.225716 1 shared_informer.go:356] "Caches are synced" controller="serviceCIDR config"
I1101 11:09:49.225755 1 shared_informer.go:356] "Caches are synced" controller="service config"
I1101 11:09:49.225779 1 shared_informer.go:356] "Caches are synced" controller="endpoint slice config"
==> kube-scheduler [f940f08b4a7e5f2a89503aec05980619c7af103b702262fe033b3ddbff81a5db] <==
I1101 11:12:33.814471 1 schedule_one.go:1092] "Pod has been assigned to node. Abort adding it back to queue." pod="default/busybox-7b57f96db7-7m8cp" node="ha-472819-m03"
E1101 11:12:59.527016 1 framework.go:1400] "Plugin Failed" err="Operation cannot be fulfilled on pods/binding \"kube-proxy-j9j6f\": pod kube-proxy-j9j6f is already assigned to node \"ha-472819-m04\"" plugin="DefaultBinder" pod="kube-system/kube-proxy-j9j6f" node="ha-472819-m04"
E1101 11:12:59.527158 1 schedule_one.go:379] "scheduler cache ForgetPod failed" err="pod 82874d15-7a7e-4291-bdfe-322ff3beceb7(kube-system/kube-proxy-j9j6f) wasn't assumed so cannot be forgotten" logger="UnhandledError" pod="kube-system/kube-proxy-j9j6f"
E1101 11:12:59.527232 1 schedule_one.go:1079] "Error scheduling pod; retrying" err="running Bind plugin \"DefaultBinder\": Operation cannot be fulfilled on pods/binding \"kube-proxy-j9j6f\": pod kube-proxy-j9j6f is already assigned to node \"ha-472819-m04\"" logger="UnhandledError" pod="kube-system/kube-proxy-j9j6f"
I1101 11:12:59.531550 1 schedule_one.go:1092] "Pod has been assigned to node. Abort adding it back to queue." pod="kube-system/kube-proxy-j9j6f" node="ha-472819-m04"
E1101 11:12:59.593960 1 framework.go:1400] "Plugin Failed" err="Operation cannot be fulfilled on pods/binding \"kindnet-r2qzc\": pod kindnet-r2qzc is already assigned to node \"ha-472819-m04\"" plugin="DefaultBinder" pod="kube-system/kindnet-r2qzc" node="ha-472819-m04"
E1101 11:12:59.594130 1 schedule_one.go:379] "scheduler cache ForgetPod failed" err="pod 5cf8f544-dc95-4924-b3df-2e668d7cd5bd(kube-system/kindnet-r2qzc) wasn't assumed so cannot be forgotten" logger="UnhandledError" pod="kube-system/kindnet-r2qzc"
E1101 11:12:59.594206 1 schedule_one.go:1079] "Error scheduling pod; retrying" err="running Bind plugin \"DefaultBinder\": Operation cannot be fulfilled on pods/binding \"kindnet-r2qzc\": pod kindnet-r2qzc is already assigned to node \"ha-472819-m04\"" logger="UnhandledError" pod="kube-system/kindnet-r2qzc"
I1101 11:12:59.597504 1 schedule_one.go:1092] "Pod has been assigned to node. Abort adding it back to queue." pod="kube-system/kindnet-r2qzc" node="ha-472819-m04"
E1101 11:12:59.662573 1 framework.go:1400] "Plugin Failed" err="Operation cannot be fulfilled on pods/binding \"kindnet-hdmgp\": pod kindnet-hdmgp is already assigned to node \"ha-472819-m04\"" plugin="DefaultBinder" pod="kube-system/kindnet-hdmgp" node="ha-472819-m04"
E1101 11:12:59.662698 1 schedule_one.go:379] "scheduler cache ForgetPod failed" err="pod 5db403fb-b722-43b5-a7f8-72eb2cb15ab8(kube-system/kindnet-hdmgp) wasn't assumed so cannot be forgotten" logger="UnhandledError" pod="kube-system/kindnet-hdmgp"
E1101 11:12:59.662754 1 schedule_one.go:1079] "Error scheduling pod; retrying" err="running Bind plugin \"DefaultBinder\": Operation cannot be fulfilled on pods/binding \"kindnet-hdmgp\": pod kindnet-hdmgp is already assigned to node \"ha-472819-m04\"" logger="UnhandledError" pod="kube-system/kindnet-hdmgp"
I1101 11:12:59.672405 1 schedule_one.go:1092] "Pod has been assigned to node. Abort adding it back to queue." pod="kube-system/kindnet-hdmgp" node="ha-472819-m04"
E1101 11:12:59.723505 1 framework.go:1400] "Plugin Failed" err="Operation cannot be fulfilled on pods/binding \"kindnet-88sf2\": pod kindnet-88sf2 is already assigned to node \"ha-472819-m04\"" plugin="DefaultBinder" pod="kube-system/kindnet-88sf2" node="ha-472819-m04"
E1101 11:12:59.723658 1 schedule_one.go:379] "scheduler cache ForgetPod failed" err="pod 61cdb567-0db6-43a9-b37e-206c4b1e424b(kube-system/kindnet-88sf2) wasn't assumed so cannot be forgotten" logger="UnhandledError" pod="kube-system/kindnet-88sf2"
E1101 11:12:59.723717 1 schedule_one.go:1079] "Error scheduling pod; retrying" err="running Bind plugin \"DefaultBinder\": Operation cannot be fulfilled on pods/binding \"kindnet-88sf2\": pod kindnet-88sf2 is already assigned to node \"ha-472819-m04\"" logger="UnhandledError" pod="kube-system/kindnet-88sf2"
I1101 11:12:59.724870 1 schedule_one.go:1092] "Pod has been assigned to node. Abort adding it back to queue." pod="kube-system/kindnet-88sf2" node="ha-472819-m04"
E1101 11:12:59.725681 1 framework.go:1400] "Plugin Failed" err="Operation cannot be fulfilled on pods/binding \"kube-proxy-chm7z\": pod kube-proxy-chm7z is already assigned to node \"ha-472819-m04\"" plugin="DefaultBinder" pod="kube-system/kube-proxy-chm7z" node="ha-472819-m04"
E1101 11:12:59.725836 1 schedule_one.go:379] "scheduler cache ForgetPod failed" err="pod 91c0e8b9-9d13-45a7-b93c-cbc34b19bbf2(kube-system/kube-proxy-chm7z) wasn't assumed so cannot be forgotten" logger="UnhandledError" pod="kube-system/kube-proxy-chm7z"
E1101 11:12:59.725898 1 schedule_one.go:1079] "Error scheduling pod; retrying" err="running Bind plugin \"DefaultBinder\": Operation cannot be fulfilled on pods/binding \"kube-proxy-chm7z\": pod kube-proxy-chm7z is already assigned to node \"ha-472819-m04\"" logger="UnhandledError" pod="kube-system/kube-proxy-chm7z"
I1101 11:12:59.727068 1 schedule_one.go:1092] "Pod has been assigned to node. Abort adding it back to queue." pod="kube-system/kube-proxy-chm7z" node="ha-472819-m04"
E1101 11:19:56.681886 1 framework.go:1400] "Plugin Failed" err="Operation cannot be fulfilled on pods/binding \"busybox-7b57f96db7-x67zv\": pod busybox-7b57f96db7-x67zv is already assigned to node \"ha-472819-m04\"" plugin="DefaultBinder" pod="default/busybox-7b57f96db7-x67zv" node="ha-472819-m04"
E1101 11:19:56.681946 1 schedule_one.go:379] "scheduler cache ForgetPod failed" err="pod 26e53bec-ba78-49cd-9271-6982e344344b(default/busybox-7b57f96db7-x67zv) wasn't assumed so cannot be forgotten" logger="UnhandledError" pod="default/busybox-7b57f96db7-x67zv"
E1101 11:19:56.681966 1 schedule_one.go:1079] "Error scheduling pod; retrying" err="running Bind plugin \"DefaultBinder\": Operation cannot be fulfilled on pods/binding \"busybox-7b57f96db7-x67zv\": pod busybox-7b57f96db7-x67zv is already assigned to node \"ha-472819-m04\"" logger="UnhandledError" pod="default/busybox-7b57f96db7-x67zv"
I1101 11:19:56.683079 1 schedule_one.go:1092] "Pod has been assigned to node. Abort adding it back to queue." pod="default/busybox-7b57f96db7-x67zv" node="ha-472819-m04"
==> kubelet <==
Nov 01 11:09:47 ha-472819 kubelet[1341]: I1101 11:09:47.376661 1341 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"xtables-lock\" (UniqueName: \"kubernetes.io/host-path/2c010b85-48bd-4004-886f-fbe4e03884a9-xtables-lock\") pod \"kube-proxy-djfvb\" (UID: \"2c010b85-48bd-4004-886f-fbe4e03884a9\") " pod="kube-system/kube-proxy-djfvb"
Nov 01 11:09:47 ha-472819 kubelet[1341]: I1101 11:09:47.376718 1341 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"lib-modules\" (UniqueName: \"kubernetes.io/host-path/2c010b85-48bd-4004-886f-fbe4e03884a9-lib-modules\") pod \"kube-proxy-djfvb\" (UID: \"2c010b85-48bd-4004-886f-fbe4e03884a9\") " pod="kube-system/kube-proxy-djfvb"
Nov 01 11:09:47 ha-472819 kubelet[1341]: I1101 11:09:47.376737 1341 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-zwcg8\" (UniqueName: \"kubernetes.io/projected/2c010b85-48bd-4004-886f-fbe4e03884a9-kube-api-access-zwcg8\") pod \"kube-proxy-djfvb\" (UID: \"2c010b85-48bd-4004-886f-fbe4e03884a9\") " pod="kube-system/kube-proxy-djfvb"
Nov 01 11:09:47 ha-472819 kubelet[1341]: I1101 11:09:47.376798 1341 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"xtables-lock\" (UniqueName: \"kubernetes.io/host-path/abb3d05e-e447-4fe5-8996-26e79d7e2b4d-xtables-lock\") pod \"kindnet-dkhrw\" (UID: \"abb3d05e-e447-4fe5-8996-26e79d7e2b4d\") " pod="kube-system/kindnet-dkhrw"
Nov 01 11:09:47 ha-472819 kubelet[1341]: I1101 11:09:47.376817 1341 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-56qtk\" (UniqueName: \"kubernetes.io/projected/abb3d05e-e447-4fe5-8996-26e79d7e2b4d-kube-api-access-56qtk\") pod \"kindnet-dkhrw\" (UID: \"abb3d05e-e447-4fe5-8996-26e79d7e2b4d\") " pod="kube-system/kindnet-dkhrw"
Nov 01 11:09:47 ha-472819 kubelet[1341]: I1101 11:09:47.376871 1341 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"cni-cfg\" (UniqueName: \"kubernetes.io/host-path/abb3d05e-e447-4fe5-8996-26e79d7e2b4d-cni-cfg\") pod \"kindnet-dkhrw\" (UID: \"abb3d05e-e447-4fe5-8996-26e79d7e2b4d\") " pod="kube-system/kindnet-dkhrw"
Nov 01 11:09:47 ha-472819 kubelet[1341]: I1101 11:09:47.376891 1341 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"lib-modules\" (UniqueName: \"kubernetes.io/host-path/abb3d05e-e447-4fe5-8996-26e79d7e2b4d-lib-modules\") pod \"kindnet-dkhrw\" (UID: \"abb3d05e-e447-4fe5-8996-26e79d7e2b4d\") " pod="kube-system/kindnet-dkhrw"
Nov 01 11:09:48 ha-472819 kubelet[1341]: I1101 11:09:48.435774 1341 swap_util.go:74] "error creating dir to test if tmpfs noswap is enabled. Assuming not supported" mount path="" error="stat /var/lib/kubelet/plugins/kubernetes.io/empty-dir: no such file or directory"
Nov 01 11:09:48 ha-472819 kubelet[1341]: I1101 11:09:48.997603 1341 pod_startup_latency_tracker.go:104] "Observed pod startup duration" pod="kube-system/kindnet-dkhrw" podStartSLOduration=1.997582561 podStartE2EDuration="1.997582561s" podCreationTimestamp="2025-11-01 11:09:47 +0000 UTC" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2025-11-01 11:09:48.987766505 +0000 UTC m=+7.290989791" watchObservedRunningTime="2025-11-01 11:09:48.997582561 +0000 UTC m=+7.300805838"
Nov 01 11:09:50 ha-472819 kubelet[1341]: I1101 11:09:50.808988 1341 pod_startup_latency_tracker.go:104] "Observed pod startup duration" pod="kube-system/kube-proxy-djfvb" podStartSLOduration=3.808968293 podStartE2EDuration="3.808968293s" podCreationTimestamp="2025-11-01 11:09:47 +0000 UTC" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2025-11-01 11:09:49.024949256 +0000 UTC m=+7.328172541" watchObservedRunningTime="2025-11-01 11:09:50.808968293 +0000 UTC m=+9.112191562"
Nov 01 11:10:29 ha-472819 kubelet[1341]: I1101 11:10:29.183523 1341 kubelet_node_status.go:439] "Fast updating node status as it just became ready"
Nov 01 11:10:29 ha-472819 kubelet[1341]: I1101 11:10:29.306931 1341 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-kpbgz\" (UniqueName: \"kubernetes.io/projected/17503733-2ab6-460c-aa3f-21d031c70abd-kube-api-access-kpbgz\") pod \"coredns-66bc5c9577-bntfw\" (UID: \"17503733-2ab6-460c-aa3f-21d031c70abd\") " pod="kube-system/coredns-66bc5c9577-bntfw"
Nov 01 11:10:29 ha-472819 kubelet[1341]: I1101 11:10:29.307142 1341 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"tmp\" (UniqueName: \"kubernetes.io/host-path/18119b45-4932-4521-b0e9-e3a73bc6d3b1-tmp\") pod \"storage-provisioner\" (UID: \"18119b45-4932-4521-b0e9-e3a73bc6d3b1\") " pod="kube-system/storage-provisioner"
Nov 01 11:10:29 ha-472819 kubelet[1341]: I1101 11:10:29.307230 1341 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"config-volume\" (UniqueName: \"kubernetes.io/configmap/17503733-2ab6-460c-aa3f-21d031c70abd-config-volume\") pod \"coredns-66bc5c9577-bntfw\" (UID: \"17503733-2ab6-460c-aa3f-21d031c70abd\") " pod="kube-system/coredns-66bc5c9577-bntfw"
Nov 01 11:10:29 ha-472819 kubelet[1341]: I1101 11:10:29.307322 1341 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-ltf68\" (UniqueName: \"kubernetes.io/projected/18119b45-4932-4521-b0e9-e3a73bc6d3b1-kube-api-access-ltf68\") pod \"storage-provisioner\" (UID: \"18119b45-4932-4521-b0e9-e3a73bc6d3b1\") " pod="kube-system/storage-provisioner"
Nov 01 11:10:29 ha-472819 kubelet[1341]: I1101 11:10:29.408072 1341 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"config-volume\" (UniqueName: \"kubernetes.io/configmap/4b6711b0-f71a-421e-922d-eb44266c95a4-config-volume\") pod \"coredns-66bc5c9577-n2tp2\" (UID: \"4b6711b0-f71a-421e-922d-eb44266c95a4\") " pod="kube-system/coredns-66bc5c9577-n2tp2"
Nov 01 11:10:29 ha-472819 kubelet[1341]: I1101 11:10:29.408314 1341 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-gfm8p\" (UniqueName: \"kubernetes.io/projected/4b6711b0-f71a-421e-922d-eb44266c95a4-kube-api-access-gfm8p\") pod \"coredns-66bc5c9577-n2tp2\" (UID: \"4b6711b0-f71a-421e-922d-eb44266c95a4\") " pod="kube-system/coredns-66bc5c9577-n2tp2"
Nov 01 11:10:29 ha-472819 kubelet[1341]: W1101 11:10:29.609774 1341 manager.go:1169] Failed to process watch event {EventType:0 Name:/docker/66de5fe90fef65b9b7fdcec88f1cb31289b5fd1d95cc881b8beb6ec5f94ceb5c/crio-f161ed77d020465b8012f1a83590dec691bc6100c6055b30c7d61753e2d2be2a WatchSource:0}: Error finding container f161ed77d020465b8012f1a83590dec691bc6100c6055b30c7d61753e2d2be2a: Status 404 returned error can't find the container with id f161ed77d020465b8012f1a83590dec691bc6100c6055b30c7d61753e2d2be2a
Nov 01 11:10:29 ha-472819 kubelet[1341]: W1101 11:10:29.613683 1341 manager.go:1169] Failed to process watch event {EventType:0 Name:/docker/66de5fe90fef65b9b7fdcec88f1cb31289b5fd1d95cc881b8beb6ec5f94ceb5c/crio-2c45f2568b0e8e33cb1da636920d9b841b29c754a967265ee7a2ff1803ba718d WatchSource:0}: Error finding container 2c45f2568b0e8e33cb1da636920d9b841b29c754a967265ee7a2ff1803ba718d: Status 404 returned error can't find the container with id 2c45f2568b0e8e33cb1da636920d9b841b29c754a967265ee7a2ff1803ba718d
Nov 01 11:10:30 ha-472819 kubelet[1341]: I1101 11:10:30.116802 1341 pod_startup_latency_tracker.go:104] "Observed pod startup duration" pod="kube-system/coredns-66bc5c9577-bntfw" podStartSLOduration=43.116782296 podStartE2EDuration="43.116782296s" podCreationTimestamp="2025-11-01 11:09:47 +0000 UTC" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2025-11-01 11:10:30.116630032 +0000 UTC m=+48.419853300" watchObservedRunningTime="2025-11-01 11:10:30.116782296 +0000 UTC m=+48.420005573"
Nov 01 11:10:30 ha-472819 kubelet[1341]: I1101 11:10:30.270805 1341 pod_startup_latency_tracker.go:104] "Observed pod startup duration" pod="kube-system/storage-provisioner" podStartSLOduration=43.270784831 podStartE2EDuration="43.270784831s" podCreationTimestamp="2025-11-01 11:09:47 +0000 UTC" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2025-11-01 11:10:30.247572064 +0000 UTC m=+48.550795357" watchObservedRunningTime="2025-11-01 11:10:30.270784831 +0000 UTC m=+48.574008108"
Nov 01 11:12:33 ha-472819 kubelet[1341]: I1101 11:12:33.798528 1341 pod_startup_latency_tracker.go:104] "Observed pod startup duration" pod="kube-system/coredns-66bc5c9577-n2tp2" podStartSLOduration=166.798500202 podStartE2EDuration="2m46.798500202s" podCreationTimestamp="2025-11-01 11:09:47 +0000 UTC" firstStartedPulling="0001-01-01 00:00:00 +0000 UTC" lastFinishedPulling="0001-01-01 00:00:00 +0000 UTC" observedRunningTime="2025-11-01 11:10:30.272373486 +0000 UTC m=+48.575596780" watchObservedRunningTime="2025-11-01 11:12:33.798500202 +0000 UTC m=+172.101723635"
Nov 01 11:12:33 ha-472819 kubelet[1341]: I1101 11:12:33.919051 1341 reconciler_common.go:251] "operationExecutor.VerifyControllerAttachedVolume started for volume \"kube-api-access-dqgfw\" (UniqueName: \"kubernetes.io/projected/3faf7e64-22cf-4338-92ef-39a2978dacb5-kube-api-access-dqgfw\") pod \"busybox-7b57f96db7-lm6r8\" (UID: \"3faf7e64-22cf-4338-92ef-39a2978dacb5\") " pod="default/busybox-7b57f96db7-lm6r8"
Nov 01 11:12:34 ha-472819 kubelet[1341]: W1101 11:12:34.180348 1341 manager.go:1169] Failed to process watch event {EventType:0 Name:/docker/66de5fe90fef65b9b7fdcec88f1cb31289b5fd1d95cc881b8beb6ec5f94ceb5c/crio-1d1abc560619e7aa1a8b60798b93f19527128629e10f8828a25552f3c73770b6 WatchSource:0}: Error finding container 1d1abc560619e7aa1a8b60798b93f19527128629e10f8828a25552f3c73770b6: Status 404 returned error can't find the container with id 1d1abc560619e7aa1a8b60798b93f19527128629e10f8828a25552f3c73770b6
Nov 01 11:12:36 ha-472819 kubelet[1341]: I1101 11:12:36.528629 1341 pod_startup_latency_tracker.go:104] "Observed pod startup duration" pod="default/busybox-7b57f96db7-lm6r8" podStartSLOduration=1.504563771 podStartE2EDuration="3.528612432s" podCreationTimestamp="2025-11-01 11:12:33 +0000 UTC" firstStartedPulling="2025-11-01 11:12:34.188294132 +0000 UTC m=+172.491517409" lastFinishedPulling="2025-11-01 11:12:36.212342793 +0000 UTC m=+174.515566070" observedRunningTime="2025-11-01 11:12:36.528156057 +0000 UTC m=+174.831379350" watchObservedRunningTime="2025-11-01 11:12:36.528612432 +0000 UTC m=+174.831835701"
-- /stdout --
helpers_test.go:262: (dbg) Run: out/minikube-linux-arm64 status --format={{.APIServer}} -p ha-472819 -n ha-472819
helpers_test.go:269: (dbg) Run: kubectl --context ha-472819 get po -o=jsonpath={.items[*].metadata.name} -A --field-selector=status.phase!=Running
helpers_test.go:293: <<< TestMultiControlPlane/serial/HAppyAfterSecondaryNodeRestart FAILED: end of post-mortem logs <<<
helpers_test.go:294: ---------------------/post-mortem---------------------------------
--- FAIL: TestMultiControlPlane/serial/HAppyAfterSecondaryNodeRestart (3.80s)