=== RUN TestMultiControlPlane/serial/DegradedAfterClusterRestart
ha_test.go:392: (dbg) Run: out/minikube-linux-arm64 profile list --output json
ha_test.go:415: expected profile "ha-513251" in json of 'profile list' to have "Degraded" status but have "Starting" status. got *"{\"invalid\":[],\"valid\":[{\"Name\":\"ha-513251\",\"Status\":\"Starting\",\"Config\":{\"Name\":\"ha-513251\",\"KeepContext\":false,\"EmbedCerts\":false,\"MinikubeISO\":\"\",\"KicBaseImage\":\"gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1766570851-22316@sha256:7975a7a1117280f99ad7696c9c80bdca993064fe9e309e9984685e0ce989758a\",\"Memory\":3072,\"CPUs\":2,\"DiskSize\":20000,\"Driver\":\"docker\",\"HyperkitVpnKitSock\":\"\",\"HyperkitVSockPorts\":[],\"DockerEnv\":null,\"ContainerVolumeMounts\":null,\"InsecureRegistry\":null,\"RegistryMirror\":[],\"HostOnlyCIDR\":\"192.168.59.1/24\",\"HypervVirtualSwitch\":\"\",\"HypervUseExternalSwitch\":false,\"HypervExternalAdapter\":\"\",\"KVMNetwork\":\"default\",\"KVMQemuURI\":\"qemu:///system\",\"KVMGPU\":false,\"KVMHidden\":false,\"KVMNUMACount\":1,\"APIServerPort\":8443,\"DockerOpt\":null,\"DisableDriverMounts\":false,\"NFSShare\":[],\"NFSS
haresRoot\":\"/nfsshares\",\"UUID\":\"\",\"NoVTXCheck\":false,\"DNSProxy\":false,\"HostDNSResolver\":true,\"HostOnlyNicType\":\"virtio\",\"NatNicType\":\"virtio\",\"SSHIPAddress\":\"\",\"SSHUser\":\"root\",\"SSHKey\":\"\",\"SSHPort\":22,\"KubernetesConfig\":{\"KubernetesVersion\":\"v1.35.0\",\"ClusterName\":\"ha-513251\",\"Namespace\":\"default\",\"APIServerHAVIP\":\"192.168.49.254\",\"APIServerName\":\"minikubeCA\",\"APIServerNames\":null,\"APIServerIPs\":null,\"DNSDomain\":\"cluster.local\",\"ContainerRuntime\":\"crio\",\"CRISocket\":\"\",\"NetworkPlugin\":\"cni\",\"FeatureGates\":\"\",\"ServiceCIDR\":\"10.96.0.0/12\",\"ImageRepository\":\"\",\"LoadBalancerStartIP\":\"\",\"LoadBalancerEndIP\":\"\",\"CustomIngressCert\":\"\",\"RegistryAliases\":\"\",\"ExtraOptions\":null,\"ShouldLoadCachedImages\":true,\"EnableDefaultCNI\":false,\"CNI\":\"\"},\"Nodes\":[{\"Name\":\"\",\"IP\":\"192.168.49.2\",\"Port\":8443,\"KubernetesVersion\":\"v1.35.0\",\"ContainerRuntime\":\"crio\",\"ControlPlane\":true,\"Worker\":true},{
\"Name\":\"m02\",\"IP\":\"192.168.49.3\",\"Port\":8443,\"KubernetesVersion\":\"v1.35.0\",\"ContainerRuntime\":\"crio\",\"ControlPlane\":true,\"Worker\":true},{\"Name\":\"m04\",\"IP\":\"192.168.49.5\",\"Port\":0,\"KubernetesVersion\":\"v1.35.0\",\"ContainerRuntime\":\"crio\",\"ControlPlane\":false,\"Worker\":true}],\"Addons\":{\"ambassador\":false,\"amd-gpu-device-plugin\":false,\"auto-pause\":false,\"cloud-spanner\":false,\"csi-hostpath-driver\":false,\"dashboard\":false,\"default-storageclass\":false,\"efk\":false,\"freshpod\":false,\"gcp-auth\":false,\"gvisor\":false,\"headlamp\":false,\"inaccel\":false,\"ingress\":false,\"ingress-dns\":false,\"inspektor-gadget\":false,\"istio\":false,\"istio-provisioner\":false,\"kong\":false,\"kubeflow\":false,\"kubetail\":false,\"kubevirt\":false,\"logviewer\":false,\"metallb\":false,\"metrics-server\":false,\"nvidia-device-plugin\":false,\"nvidia-driver-installer\":false,\"nvidia-gpu-device-plugin\":false,\"olm\":false,\"pod-security-policy\":false,\"portainer\":false,\
"registry\":false,\"registry-aliases\":false,\"registry-creds\":false,\"storage-provisioner\":false,\"storage-provisioner-rancher\":false,\"volcano\":false,\"volumesnapshots\":false,\"yakd\":false},\"CustomAddonImages\":null,\"CustomAddonRegistries\":null,\"VerifyComponents\":{\"apiserver\":true,\"apps_running\":true,\"default_sa\":true,\"extra\":true,\"kubelet\":true,\"node_ready\":true,\"system_pods\":true},\"StartHostTimeout\":360000000000,\"ScheduledStop\":null,\"ExposedPorts\":[],\"ListenAddress\":\"\",\"Network\":\"\",\"Subnet\":\"\",\"MultiNodeRequested\":true,\"ExtraDisks\":0,\"CertExpiration\":94608000000000000,\"MountString\":\"\",\"Mount9PVersion\":\"9p2000.L\",\"MountGID\":\"docker\",\"MountIP\":\"\",\"MountMSize\":262144,\"MountOptions\":[],\"MountPort\":0,\"MountType\":\"9p\",\"MountUID\":\"docker\",\"BinaryMirror\":\"\",\"DisableOptimizations\":false,\"DisableMetrics\":false,\"DisableCoreDNSLog\":false,\"CustomQemuFirmwarePath\":\"\",\"SocketVMnetClientPath\":\"\",\"SocketVMnetPath\":\"\",\"Sta
ticIP\":\"\",\"SSHAuthSock\":\"\",\"SSHAgentPID\":0,\"GPUs\":\"\",\"AutoPauseInterval\":60000000000,\"Rosetta\":false},\"Active\":false,\"ActiveKubeContext\":true}]}"*. args: "out/minikube-linux-arm64 profile list --output json"
helpers_test.go:223: -----------------------post-mortem--------------------------------
helpers_test.go:224: ======> post-mortem[TestMultiControlPlane/serial/DegradedAfterClusterRestart]: network settings <======
helpers_test.go:231: HOST ENV snapshots: PROXY env: HTTP_PROXY="<empty>" HTTPS_PROXY="<empty>" NO_PROXY="<empty>"
helpers_test.go:239: ======> post-mortem[TestMultiControlPlane/serial/DegradedAfterClusterRestart]: docker inspect <======
helpers_test.go:240: (dbg) Run: docker inspect ha-513251
helpers_test.go:244: (dbg) docker inspect ha-513251:
-- stdout --
[
{
"Id": "bb5d0cc0ca44506f8d7080f95a3d22859783bac85870a83a386fe347d2175d13",
"Created": "2025-12-27T09:37:38.963263504Z",
"Path": "/usr/local/bin/entrypoint",
"Args": [
"/sbin/init"
],
"State": {
"Status": "running",
"Running": true,
"Paused": false,
"Restarting": false,
"OOMKilled": false,
"Dead": false,
"Pid": 353813,
"ExitCode": 0,
"Error": "",
"StartedAt": "2025-12-27T09:45:18.061061871Z",
"FinishedAt": "2025-12-27T09:45:17.324877839Z"
},
"Image": "sha256:713d59eed1cb7ea89d31e3cbef8f6274a6ab7509a421e96cf0197c334e76398d",
"ResolvConfPath": "/var/lib/docker/containers/bb5d0cc0ca44506f8d7080f95a3d22859783bac85870a83a386fe347d2175d13/resolv.conf",
"HostnamePath": "/var/lib/docker/containers/bb5d0cc0ca44506f8d7080f95a3d22859783bac85870a83a386fe347d2175d13/hostname",
"HostsPath": "/var/lib/docker/containers/bb5d0cc0ca44506f8d7080f95a3d22859783bac85870a83a386fe347d2175d13/hosts",
"LogPath": "/var/lib/docker/containers/bb5d0cc0ca44506f8d7080f95a3d22859783bac85870a83a386fe347d2175d13/bb5d0cc0ca44506f8d7080f95a3d22859783bac85870a83a386fe347d2175d13-json.log",
"Name": "/ha-513251",
"RestartCount": 0,
"Driver": "overlay2",
"Platform": "linux",
"MountLabel": "",
"ProcessLabel": "",
"AppArmorProfile": "unconfined",
"ExecIDs": null,
"HostConfig": {
"Binds": [
"/lib/modules:/lib/modules:ro",
"ha-513251:/var"
],
"ContainerIDFile": "",
"LogConfig": {
"Type": "json-file",
"Config": {}
},
"NetworkMode": "ha-513251",
"PortBindings": {
"22/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": ""
}
],
"2376/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": ""
}
],
"32443/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": ""
}
],
"5000/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": ""
}
],
"8443/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": ""
}
]
},
"RestartPolicy": {
"Name": "no",
"MaximumRetryCount": 0
},
"AutoRemove": false,
"VolumeDriver": "",
"VolumesFrom": null,
"ConsoleSize": [
0,
0
],
"CapAdd": null,
"CapDrop": null,
"CgroupnsMode": "host",
"Dns": [],
"DnsOptions": [],
"DnsSearch": [],
"ExtraHosts": null,
"GroupAdd": null,
"IpcMode": "private",
"Cgroup": "",
"Links": null,
"OomScoreAdj": 0,
"PidMode": "",
"Privileged": true,
"PublishAllPorts": false,
"ReadonlyRootfs": false,
"SecurityOpt": [
"seccomp=unconfined",
"apparmor=unconfined",
"label=disable"
],
"Tmpfs": {
"/run": "",
"/tmp": ""
},
"UTSMode": "",
"UsernsMode": "",
"ShmSize": 67108864,
"Runtime": "runc",
"Isolation": "",
"CpuShares": 0,
"Memory": 3221225472,
"NanoCpus": 2000000000,
"CgroupParent": "",
"BlkioWeight": 0,
"BlkioWeightDevice": [],
"BlkioDeviceReadBps": [],
"BlkioDeviceWriteBps": [],
"BlkioDeviceReadIOps": [],
"BlkioDeviceWriteIOps": [],
"CpuPeriod": 0,
"CpuQuota": 0,
"CpuRealtimePeriod": 0,
"CpuRealtimeRuntime": 0,
"CpusetCpus": "",
"CpusetMems": "",
"Devices": [],
"DeviceCgroupRules": null,
"DeviceRequests": null,
"MemoryReservation": 0,
"MemorySwap": 6442450944,
"MemorySwappiness": null,
"OomKillDisable": false,
"PidsLimit": null,
"Ulimits": [],
"CpuCount": 0,
"CpuPercent": 0,
"IOMaximumIOps": 0,
"IOMaximumBandwidth": 0,
"MaskedPaths": null,
"ReadonlyPaths": null
},
"GraphDriver": {
"Data": {
"ID": "bb5d0cc0ca44506f8d7080f95a3d22859783bac85870a83a386fe347d2175d13",
"LowerDir": "/var/lib/docker/overlay2/029bd90e651b8421f72d6d16dd18adc04535f447d42c210939d6a126c2033a6f-init/diff:/var/lib/docker/overlay2/497217ac9ef1c1f1b82ea8d7581e8803e8a30cce98305363d99b1ecb1ccc4cac/diff",
"MergedDir": "/var/lib/docker/overlay2/029bd90e651b8421f72d6d16dd18adc04535f447d42c210939d6a126c2033a6f/merged",
"UpperDir": "/var/lib/docker/overlay2/029bd90e651b8421f72d6d16dd18adc04535f447d42c210939d6a126c2033a6f/diff",
"WorkDir": "/var/lib/docker/overlay2/029bd90e651b8421f72d6d16dd18adc04535f447d42c210939d6a126c2033a6f/work"
},
"Name": "overlay2"
},
"Mounts": [
{
"Type": "bind",
"Source": "/lib/modules",
"Destination": "/lib/modules",
"Mode": "ro",
"RW": false,
"Propagation": "rprivate"
},
{
"Type": "volume",
"Name": "ha-513251",
"Source": "/var/lib/docker/volumes/ha-513251/_data",
"Destination": "/var",
"Driver": "local",
"Mode": "z",
"RW": true,
"Propagation": ""
}
],
"Config": {
"Hostname": "ha-513251",
"Domainname": "",
"User": "",
"AttachStdin": false,
"AttachStdout": false,
"AttachStderr": false,
"ExposedPorts": {
"22/tcp": {},
"2376/tcp": {},
"32443/tcp": {},
"5000/tcp": {},
"8443/tcp": {}
},
"Tty": true,
"OpenStdin": false,
"StdinOnce": false,
"Env": [
"container=docker",
"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
],
"Cmd": null,
"Image": "gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1766570851-22316@sha256:7975a7a1117280f99ad7696c9c80bdca993064fe9e309e9984685e0ce989758a",
"Volumes": null,
"WorkingDir": "/",
"Entrypoint": [
"/usr/local/bin/entrypoint",
"/sbin/init"
],
"OnBuild": null,
"Labels": {
"created_by.minikube.sigs.k8s.io": "true",
"mode.minikube.sigs.k8s.io": "ha-513251",
"name.minikube.sigs.k8s.io": "ha-513251",
"role.minikube.sigs.k8s.io": ""
},
"StopSignal": "SIGRTMIN+3"
},
"NetworkSettings": {
"Bridge": "",
"SandboxID": "a36bf48a852f2142e03dad97328b97c989e14e43fba2676424d26ea683f38f8a",
"SandboxKey": "/var/run/docker/netns/a36bf48a852f",
"Ports": {
"22/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "33198"
}
],
"2376/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "33199"
}
],
"32443/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "33202"
}
],
"5000/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "33200"
}
],
"8443/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "33201"
}
]
},
"HairpinMode": false,
"LinkLocalIPv6Address": "",
"LinkLocalIPv6PrefixLen": 0,
"SecondaryIPAddresses": null,
"SecondaryIPv6Addresses": null,
"EndpointID": "",
"Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"IPAddress": "",
"IPPrefixLen": 0,
"IPv6Gateway": "",
"MacAddress": "",
"Networks": {
"ha-513251": {
"IPAMConfig": {
"IPv4Address": "192.168.49.2"
},
"Links": null,
"Aliases": null,
"MacAddress": "d2:f9:a2:53:37:e6",
"DriverOpts": null,
"GwPriority": 0,
"NetworkID": "1b4d8553c414af9c151cf56182ba5e11cb773bee9162fafd694324331063b48e",
"EndpointID": "076755f827ee23e4371e7e48c17c1b2920cab289dad51349a1a50ffb80554b20",
"Gateway": "192.168.49.1",
"IPAddress": "192.168.49.2",
"IPPrefixLen": 24,
"IPv6Gateway": "",
"GlobalIPv6Address": "",
"GlobalIPv6PrefixLen": 0,
"DNSNames": [
"ha-513251",
"bb5d0cc0ca44"
]
}
}
}
}
]
-- /stdout --
helpers_test.go:248: (dbg) Run: out/minikube-linux-arm64 status --format={{.Host}} -p ha-513251 -n ha-513251
helpers_test.go:248: (dbg) Non-zero exit: out/minikube-linux-arm64 status --format={{.Host}} -p ha-513251 -n ha-513251: exit status 2 (330.772766ms)
-- stdout --
Running
-- /stdout --
helpers_test.go:248: status error: exit status 2 (may be ok)
helpers_test.go:253: <<< TestMultiControlPlane/serial/DegradedAfterClusterRestart FAILED: start of post-mortem logs <<<
helpers_test.go:254: ======> post-mortem[TestMultiControlPlane/serial/DegradedAfterClusterRestart]: minikube logs <======
helpers_test.go:256: (dbg) Run: out/minikube-linux-arm64 -p ha-513251 logs -n 25
helpers_test.go:261: TestMultiControlPlane/serial/DegradedAfterClusterRestart logs:
-- stdout --
==> Audit <==
┌─────────┬──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┬───────────┬─────────┬─────────┬─────────────────────┬─────────────────────┐
│ COMMAND │ ARGS │ PROFILE │ USER │ VERSION │ START TIME │ END TIME │
├─────────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┼───────────┼─────────┼─────────┼─────────────────────┼─────────────────────┤
│ cp │ ha-513251 cp ha-513251-m03:/home/docker/cp-test.txt ha-513251-m04:/home/docker/cp-test_ha-513251-m03_ha-513251-m04.txt │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:41 UTC │ 27 Dec 25 09:41 UTC │
│ ssh │ ha-513251 ssh -n ha-513251-m03 sudo cat /home/docker/cp-test.txt │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:41 UTC │ 27 Dec 25 09:41 UTC │
│ ssh │ ha-513251 ssh -n ha-513251-m04 sudo cat /home/docker/cp-test_ha-513251-m03_ha-513251-m04.txt │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:41 UTC │ 27 Dec 25 09:41 UTC │
│ cp │ ha-513251 cp testdata/cp-test.txt ha-513251-m04:/home/docker/cp-test.txt │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:41 UTC │ 27 Dec 25 09:41 UTC │
│ ssh │ ha-513251 ssh -n ha-513251-m04 sudo cat /home/docker/cp-test.txt │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:41 UTC │ 27 Dec 25 09:41 UTC │
│ cp │ ha-513251 cp ha-513251-m04:/home/docker/cp-test.txt /tmp/TestMultiControlPlaneserialCopyFile4265014863/001/cp-test_ha-513251-m04.txt │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:41 UTC │ 27 Dec 25 09:41 UTC │
│ ssh │ ha-513251 ssh -n ha-513251-m04 sudo cat /home/docker/cp-test.txt │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:41 UTC │ 27 Dec 25 09:41 UTC │
│ cp │ ha-513251 cp ha-513251-m04:/home/docker/cp-test.txt ha-513251:/home/docker/cp-test_ha-513251-m04_ha-513251.txt │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:41 UTC │ 27 Dec 25 09:41 UTC │
│ ssh │ ha-513251 ssh -n ha-513251-m04 sudo cat /home/docker/cp-test.txt │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:41 UTC │ 27 Dec 25 09:41 UTC │
│ ssh │ ha-513251 ssh -n ha-513251 sudo cat /home/docker/cp-test_ha-513251-m04_ha-513251.txt │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:41 UTC │ 27 Dec 25 09:41 UTC │
│ cp │ ha-513251 cp ha-513251-m04:/home/docker/cp-test.txt ha-513251-m02:/home/docker/cp-test_ha-513251-m04_ha-513251-m02.txt │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:41 UTC │ 27 Dec 25 09:41 UTC │
│ ssh │ ha-513251 ssh -n ha-513251-m04 sudo cat /home/docker/cp-test.txt │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:41 UTC │ 27 Dec 25 09:41 UTC │
│ ssh │ ha-513251 ssh -n ha-513251-m02 sudo cat /home/docker/cp-test_ha-513251-m04_ha-513251-m02.txt │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:41 UTC │ 27 Dec 25 09:41 UTC │
│ cp │ ha-513251 cp ha-513251-m04:/home/docker/cp-test.txt ha-513251-m03:/home/docker/cp-test_ha-513251-m04_ha-513251-m03.txt │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:41 UTC │ 27 Dec 25 09:41 UTC │
│ ssh │ ha-513251 ssh -n ha-513251-m04 sudo cat /home/docker/cp-test.txt │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:41 UTC │ 27 Dec 25 09:41 UTC │
│ ssh │ ha-513251 ssh -n ha-513251-m03 sudo cat /home/docker/cp-test_ha-513251-m04_ha-513251-m03.txt │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:41 UTC │ 27 Dec 25 09:41 UTC │
│ node │ ha-513251 node stop m02 --alsologtostderr -v 5 │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:41 UTC │ 27 Dec 25 09:41 UTC │
│ node │ ha-513251 node start m02 --alsologtostderr -v 5 │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:42 UTC │ 27 Dec 25 09:42 UTC │
│ node │ ha-513251 node list --alsologtostderr -v 5 │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:42 UTC │ │
│ stop │ ha-513251 stop --alsologtostderr -v 5 │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:42 UTC │ 27 Dec 25 09:43 UTC │
│ start │ ha-513251 start --wait true --alsologtostderr -v 5 │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:43 UTC │ 27 Dec 25 09:44 UTC │
│ node │ ha-513251 node list --alsologtostderr -v 5 │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:44 UTC │ │
│ node │ ha-513251 node delete m03 --alsologtostderr -v 5 │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:44 UTC │ 27 Dec 25 09:44 UTC │
│ stop │ ha-513251 stop --alsologtostderr -v 5 │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:44 UTC │ 27 Dec 25 09:45 UTC │
│ start │ ha-513251 start --wait true --alsologtostderr -v 5 --driver=docker --container-runtime=crio │ ha-513251 │ jenkins │ v1.37.0 │ 27 Dec 25 09:45 UTC │ │
└─────────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┴───────────┴─────────┴─────────┴─────────────────────┴─────────────────────┘
==> Last Start <==
Log file created at: 2025/12/27 09:45:17
Running on machine: ip-172-31-24-2
Binary: Built with gc go1.25.5 for linux/arm64
Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg
I1227 09:45:17.780858 353683 out.go:360] Setting OutFile to fd 1 ...
I1227 09:45:17.781066 353683 out.go:408] TERM=,COLORTERM=, which probably does not support color
I1227 09:45:17.781099 353683 out.go:374] Setting ErrFile to fd 2...
I1227 09:45:17.781121 353683 out.go:408] TERM=,COLORTERM=, which probably does not support color
I1227 09:45:17.781427 353683 root.go:338] Updating PATH: /home/jenkins/minikube-integration/22343-297941/.minikube/bin
I1227 09:45:17.781839 353683 out.go:368] Setting JSON to false
I1227 09:45:17.782724 353683 start.go:133] hostinfo: {"hostname":"ip-172-31-24-2","uptime":5271,"bootTime":1766823447,"procs":157,"os":"linux","platform":"ubuntu","platformFamily":"debian","platformVersion":"20.04","kernelVersion":"5.15.0-1084-aws","kernelArch":"aarch64","virtualizationSystem":"","virtualizationRole":"","hostId":"6d436adf-771e-4269-b9a3-c25fd4fca4f5"}
I1227 09:45:17.782828 353683 start.go:143] virtualization:
I1227 09:45:17.786847 353683 out.go:179] * [ha-513251] minikube v1.37.0 on Ubuntu 20.04 (arm64)
I1227 09:45:17.789790 353683 out.go:179] - MINIKUBE_LOCATION=22343
I1227 09:45:17.789897 353683 notify.go:221] Checking for updates...
I1227 09:45:17.795846 353683 out.go:179] - MINIKUBE_SUPPRESS_DOCKER_PERFORMANCE=true
I1227 09:45:17.798784 353683 out.go:179] - KUBECONFIG=/home/jenkins/minikube-integration/22343-297941/kubeconfig
I1227 09:45:17.801736 353683 out.go:179] - MINIKUBE_HOME=/home/jenkins/minikube-integration/22343-297941/.minikube
I1227 09:45:17.804638 353683 out.go:179] - MINIKUBE_BIN=out/minikube-linux-arm64
I1227 09:45:17.807626 353683 out.go:179] - MINIKUBE_FORCE_SYSTEMD=
I1227 09:45:17.811252 353683 config.go:182] Loaded profile config "ha-513251": Driver=docker, ContainerRuntime=crio, KubernetesVersion=v1.35.0
I1227 09:45:17.811891 353683 driver.go:422] Setting default libvirt URI to qemu:///system
I1227 09:45:17.840112 353683 docker.go:124] docker version: linux-28.1.1:Docker Engine - Community
I1227 09:45:17.840288 353683 cli_runner.go:164] Run: docker system info --format "{{json .}}"
I1227 09:45:17.900770 353683 info.go:266] docker info: {ID:J4M5:W6MX:GOX4:4LAQ:VI7E:VJNF:J3OP:OPBH:GF7G:PPY4:WQWD:7N4L Containers:3 ContainersRunning:0 ContainersPaused:0 ContainersStopped:3 Images:3 Driver:overlay2 DriverStatus:[[Backing Filesystem extfs] [Supports d_type true] [Using metacopy false] [Native Overlay Diff true] [userxattr false]] SystemStatus:<nil> Plugins:{Volume:[local] Network:[bridge host ipvlan macvlan null overlay] Authorization:<nil> Log:[awslogs fluentd gcplogs gelf journald json-file local splunk syslog]} MemoryLimit:true SwapLimit:true KernelMemory:false KernelMemoryTCP:true CPUCfsPeriod:true CPUCfsQuota:true CPUShares:true CPUSet:true PidsLimit:true IPv4Forwarding:true BridgeNfIptables:false BridgeNfIP6Tables:false Debug:false NFd:26 OomKillDisable:true NGoroutines:42 SystemTime:2025-12-27 09:45:17.89071505 +0000 UTC LoggingDriver:json-file CgroupDriver:cgroupfs NEventsListener:0 KernelVersion:5.15.0-1084-aws OperatingSystem:Ubuntu 20.04.6 LTS OSType:linux Architecture:aa
rch64 IndexServerAddress:https://index.docker.io/v1/ RegistryConfig:{AllowNondistributableArtifactsCIDRs:[] AllowNondistributableArtifactsHostnames:[] InsecureRegistryCIDRs:[::1/128 127.0.0.0/8] IndexConfigs:{DockerIo:{Name:docker.io Mirrors:[] Secure:true Official:true}} Mirrors:[]} NCPU:2 MemTotal:8214831104 GenericResources:<nil> DockerRootDir:/var/lib/docker HTTPProxy: HTTPSProxy: NoProxy: Name:ip-172-31-24-2 Labels:[] ExperimentalBuild:false ServerVersion:28.1.1 ClusterStore: ClusterAdvertise: Runtimes:{Runc:{Path:runc}} DefaultRuntime:runc Swarm:{NodeID: NodeAddr: LocalNodeState:inactive ControlAvailable:false Error: RemoteManagers:<nil>} LiveRestoreEnabled:false Isolation: InitBinary:docker-init ContainerdCommit:{ID:05044ec0a9a75232cad458027ca83437aae3f4da Expected:} RuncCommit:{ID:v1.2.5-0-g59923ef Expected:} InitCommit:{ID:de40ad0 Expected:} SecurityOptions:[name=apparmor name=seccomp,profile=builtin] ProductLicense: Warnings:<nil> ServerErrors:[] ClientInfo:{Debug:false Plugins:[map[Name:buildx Path
:/usr/libexec/docker/cli-plugins/docker-buildx SchemaVersion:0.1.0 ShortDescription:Docker Buildx Vendor:Docker Inc. Version:v0.23.0] map[Name:compose Path:/usr/libexec/docker/cli-plugins/docker-compose SchemaVersion:0.1.0 ShortDescription:Docker Compose Vendor:Docker Inc. Version:v2.35.1]] Warnings:<nil>}}
I1227 09:45:17.900884 353683 docker.go:319] overlay module found
I1227 09:45:17.905637 353683 out.go:179] * Using the docker driver based on existing profile
I1227 09:45:17.908470 353683 start.go:309] selected driver: docker
I1227 09:45:17.908492 353683 start.go:928] validating driver "docker" against &{Name:ha-513251 KeepContext:false EmbedCerts:false MinikubeISO: KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1766570851-22316@sha256:7975a7a1117280f99ad7696c9c80bdca993064fe9e309e9984685e0ce989758a Memory:3072 CPUs:2 DiskSize:20000 Driver:docker HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.35.0 ClusterName:ha-513251 Namespace:default APIServerHAVIP:192.168.49.254 APIServerName
:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:crio CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.168.49.2 Port:8443 KubernetesVersion:v1.35.0 ContainerRuntime:crio ControlPlane:true Worker:true} {Name:m02 IP:192.168.49.3 Port:8443 KubernetesVersion:v1.35.0 ContainerRuntime:crio ControlPlane:true Worker:true} {Name:m04 IP:192.168.49.5 Port:0 KubernetesVersion:v1.35.0 ContainerRuntime: ControlPlane:false Worker:true}] Addons:map[ambassador:false amd-gpu-device-plugin:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false inaccel:false ingress:false ingress-dns:false inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow
:false kubetail:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s MountString: Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:false DisableMetrics:false DisableCoreDNSLog:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP:
SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s Rosetta:false}
I1227 09:45:17.908638 353683 start.go:939] status for docker: {Installed:true Healthy:true Running:false NeedsImprovement:false Error:<nil> Reason: Fix: Doc: Version:}
I1227 09:45:17.908737 353683 cli_runner.go:164] Run: docker system info --format "{{json .}}"
I1227 09:45:17.967550 353683 info.go:266] docker info: {ID:J4M5:W6MX:GOX4:4LAQ:VI7E:VJNF:J3OP:OPBH:GF7G:PPY4:WQWD:7N4L Containers:3 ContainersRunning:0 ContainersPaused:0 ContainersStopped:3 Images:3 Driver:overlay2 DriverStatus:[[Backing Filesystem extfs] [Supports d_type true] [Using metacopy false] [Native Overlay Diff true] [userxattr false]] SystemStatus:<nil> Plugins:{Volume:[local] Network:[bridge host ipvlan macvlan null overlay] Authorization:<nil> Log:[awslogs fluentd gcplogs gelf journald json-file local splunk syslog]} MemoryLimit:true SwapLimit:true KernelMemory:false KernelMemoryTCP:true CPUCfsPeriod:true CPUCfsQuota:true CPUShares:true CPUSet:true PidsLimit:true IPv4Forwarding:true BridgeNfIptables:false BridgeNfIP6Tables:false Debug:false NFd:26 OomKillDisable:true NGoroutines:42 SystemTime:2025-12-27 09:45:17.958343241 +0000 UTC LoggingDriver:json-file CgroupDriver:cgroupfs NEventsListener:0 KernelVersion:5.15.0-1084-aws OperatingSystem:Ubuntu 20.04.6 LTS OSType:linux Architecture:a
arch64 IndexServerAddress:https://index.docker.io/v1/ RegistryConfig:{AllowNondistributableArtifactsCIDRs:[] AllowNondistributableArtifactsHostnames:[] InsecureRegistryCIDRs:[::1/128 127.0.0.0/8] IndexConfigs:{DockerIo:{Name:docker.io Mirrors:[] Secure:true Official:true}} Mirrors:[]} NCPU:2 MemTotal:8214831104 GenericResources:<nil> DockerRootDir:/var/lib/docker HTTPProxy: HTTPSProxy: NoProxy: Name:ip-172-31-24-2 Labels:[] ExperimentalBuild:false ServerVersion:28.1.1 ClusterStore: ClusterAdvertise: Runtimes:{Runc:{Path:runc}} DefaultRuntime:runc Swarm:{NodeID: NodeAddr: LocalNodeState:inactive ControlAvailable:false Error: RemoteManagers:<nil>} LiveRestoreEnabled:false Isolation: InitBinary:docker-init ContainerdCommit:{ID:05044ec0a9a75232cad458027ca83437aae3f4da Expected:} RuncCommit:{ID:v1.2.5-0-g59923ef Expected:} InitCommit:{ID:de40ad0 Expected:} SecurityOptions:[name=apparmor name=seccomp,profile=builtin] ProductLicense: Warnings:<nil> ServerErrors:[] ClientInfo:{Debug:false Plugins:[map[Name:buildx Pat
h:/usr/libexec/docker/cli-plugins/docker-buildx SchemaVersion:0.1.0 ShortDescription:Docker Buildx Vendor:Docker Inc. Version:v0.23.0] map[Name:compose Path:/usr/libexec/docker/cli-plugins/docker-compose SchemaVersion:0.1.0 ShortDescription:Docker Compose Vendor:Docker Inc. Version:v2.35.1]] Warnings:<nil>}}
I1227 09:45:17.968010 353683 start_flags.go:1019] Waiting for all components: map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true]
I1227 09:45:17.968048 353683 cni.go:84] Creating CNI manager for ""
I1227 09:45:17.968104 353683 cni.go:136] multinode detected (3 nodes found), recommending kindnet
I1227 09:45:17.968157 353683 start.go:353] cluster config:
{Name:ha-513251 KeepContext:false EmbedCerts:false MinikubeISO: KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1766570851-22316@sha256:7975a7a1117280f99ad7696c9c80bdca993064fe9e309e9984685e0ce989758a Memory:3072 CPUs:2 DiskSize:20000 Driver:docker HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.35.0 ClusterName:ha-513251 Namespace:default APIServerHAVIP:192.168.49.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerR
untime:crio CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.168.49.2 Port:8443 KubernetesVersion:v1.35.0 ContainerRuntime:crio ControlPlane:true Worker:true} {Name:m02 IP:192.168.49.3 Port:8443 KubernetesVersion:v1.35.0 ContainerRuntime:crio ControlPlane:true Worker:true} {Name:m04 IP:192.168.49.5 Port:0 KubernetesVersion:v1.35.0 ContainerRuntime:crio ControlPlane:false Worker:true}] Addons:map[ambassador:false amd-gpu-device-plugin:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false inaccel:false ingress:false ingress-dns:false inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow:false kubetail:false kubevirt:false logviewer:false metallb:false metrics-s
erver:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s MountString: Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:false DisableMetrics:false DisableCoreDNSLog:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s Rosetta:false}
I1227 09:45:17.971557 353683 out.go:179] * Starting "ha-513251" primary control-plane node in "ha-513251" cluster
I1227 09:45:17.974341 353683 cache.go:134] Beginning downloading kic base image for docker with crio
I1227 09:45:17.977308 353683 out.go:179] * Pulling base image v0.0.48-1766570851-22316 ...
I1227 09:45:17.980127 353683 preload.go:188] Checking if preload exists for k8s version v1.35.0 and runtime crio
I1227 09:45:17.980181 353683 preload.go:203] Found local preload: /home/jenkins/minikube-integration/22343-297941/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.35.0-cri-o-overlay-arm64.tar.lz4
I1227 09:45:17.980196 353683 cache.go:65] Caching tarball of preloaded images
I1227 09:45:17.980207 353683 image.go:81] Checking for gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1766570851-22316@sha256:7975a7a1117280f99ad7696c9c80bdca993064fe9e309e9984685e0ce989758a in local docker daemon
I1227 09:45:17.980281 353683 preload.go:251] Found /home/jenkins/minikube-integration/22343-297941/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.35.0-cri-o-overlay-arm64.tar.lz4 in cache, skipping download
I1227 09:45:17.980293 353683 cache.go:68] Finished verifying existence of preloaded tar for v1.35.0 on crio
I1227 09:45:17.980447 353683 profile.go:143] Saving config to /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/config.json ...
I1227 09:45:18.000295 353683 image.go:100] Found gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1766570851-22316@sha256:7975a7a1117280f99ad7696c9c80bdca993064fe9e309e9984685e0ce989758a in local docker daemon, skipping pull
I1227 09:45:18.000319 353683 cache.go:158] gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1766570851-22316@sha256:7975a7a1117280f99ad7696c9c80bdca993064fe9e309e9984685e0ce989758a exists in daemon, skipping load
I1227 09:45:18.000341 353683 cache.go:243] Successfully downloaded all kic artifacts
I1227 09:45:18.000375 353683 start.go:360] acquireMachinesLock for ha-513251: {Name:mka277024f8c2226ae51cd2727a8e25e47e84998 Clock:{} Delay:500ms Timeout:10m0s Cancel:<nil>}
I1227 09:45:18.000447 353683 start.go:364] duration metric: took 46.926µs to acquireMachinesLock for "ha-513251"
I1227 09:45:18.000468 353683 start.go:96] Skipping create...Using existing machine configuration
I1227 09:45:18.000475 353683 fix.go:54] fixHost starting:
I1227 09:45:18.000773 353683 cli_runner.go:164] Run: docker container inspect ha-513251 --format={{.State.Status}}
I1227 09:45:18.022293 353683 fix.go:112] recreateIfNeeded on ha-513251: state=Stopped err=<nil>
W1227 09:45:18.022327 353683 fix.go:138] unexpected machine state, will restart: <nil>
I1227 09:45:18.025796 353683 out.go:252] * Restarting existing docker container for "ha-513251" ...
I1227 09:45:18.025962 353683 cli_runner.go:164] Run: docker start ha-513251
I1227 09:45:18.291407 353683 cli_runner.go:164] Run: docker container inspect ha-513251 --format={{.State.Status}}
I1227 09:45:18.313034 353683 kic.go:430] container "ha-513251" state is running.
I1227 09:45:18.313680 353683 cli_runner.go:164] Run: docker container inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}" ha-513251
I1227 09:45:18.336728 353683 profile.go:143] Saving config to /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/config.json ...
I1227 09:45:18.337162 353683 machine.go:94] provisionDockerMachine start ...
I1227 09:45:18.337228 353683 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-513251
I1227 09:45:18.363888 353683 main.go:144] libmachine: Using SSH client type: native
I1227 09:45:18.364313 353683 main.go:144] libmachine: &{{{<nil> 0 [] [] []} docker [0x3dbbe0] 0x3de0e0 <nil> [] 0s} 127.0.0.1 33198 <nil> <nil>}
I1227 09:45:18.364324 353683 main.go:144] libmachine: About to run SSH command:
hostname
I1227 09:45:18.365396 353683 main.go:144] libmachine: Error dialing TCP: ssh: handshake failed: EOF
I1227 09:45:21.507722 353683 main.go:144] libmachine: SSH cmd err, output: <nil>: ha-513251
I1227 09:45:21.507748 353683 ubuntu.go:182] provisioning hostname "ha-513251"
I1227 09:45:21.507813 353683 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-513251
I1227 09:45:21.525335 353683 main.go:144] libmachine: Using SSH client type: native
I1227 09:45:21.525658 353683 main.go:144] libmachine: &{{{<nil> 0 [] [] []} docker [0x3dbbe0] 0x3de0e0 <nil> [] 0s} 127.0.0.1 33198 <nil> <nil>}
I1227 09:45:21.525674 353683 main.go:144] libmachine: About to run SSH command:
sudo hostname ha-513251 && echo "ha-513251" | sudo tee /etc/hostname
I1227 09:45:21.674143 353683 main.go:144] libmachine: SSH cmd err, output: <nil>: ha-513251
I1227 09:45:21.674300 353683 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-513251
I1227 09:45:21.692486 353683 main.go:144] libmachine: Using SSH client type: native
I1227 09:45:21.692814 353683 main.go:144] libmachine: &{{{<nil> 0 [] [] []} docker [0x3dbbe0] 0x3de0e0 <nil> [] 0s} 127.0.0.1 33198 <nil> <nil>}
I1227 09:45:21.692838 353683 main.go:144] libmachine: About to run SSH command:
if ! grep -xq '.*\sha-513251' /etc/hosts; then
if grep -xq '127.0.1.1\s.*' /etc/hosts; then
sudo sed -i 's/^127.0.1.1\s.*/127.0.1.1 ha-513251/g' /etc/hosts;
else
echo '127.0.1.1 ha-513251' | sudo tee -a /etc/hosts;
fi
fi
I1227 09:45:21.832635 353683 main.go:144] libmachine: SSH cmd err, output: <nil>:
I1227 09:45:21.832681 353683 ubuntu.go:188] set auth options {CertDir:/home/jenkins/minikube-integration/22343-297941/.minikube CaCertPath:/home/jenkins/minikube-integration/22343-297941/.minikube/certs/ca.pem CaPrivateKeyPath:/home/jenkins/minikube-integration/22343-297941/.minikube/certs/ca-key.pem CaCertRemotePath:/etc/docker/ca.pem ServerCertPath:/home/jenkins/minikube-integration/22343-297941/.minikube/machines/server.pem ServerKeyPath:/home/jenkins/minikube-integration/22343-297941/.minikube/machines/server-key.pem ClientKeyPath:/home/jenkins/minikube-integration/22343-297941/.minikube/certs/key.pem ServerCertRemotePath:/etc/docker/server.pem ServerKeyRemotePath:/etc/docker/server-key.pem ClientCertPath:/home/jenkins/minikube-integration/22343-297941/.minikube/certs/cert.pem ServerCertSANs:[] StorePath:/home/jenkins/minikube-integration/22343-297941/.minikube}
I1227 09:45:21.832704 353683 ubuntu.go:190] setting up certificates
I1227 09:45:21.832713 353683 provision.go:84] configureAuth start
I1227 09:45:21.832776 353683 cli_runner.go:164] Run: docker container inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}" ha-513251
I1227 09:45:21.851553 353683 provision.go:143] copyHostCerts
I1227 09:45:21.851617 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/ca.pem -> /home/jenkins/minikube-integration/22343-297941/.minikube/ca.pem
I1227 09:45:21.851676 353683 exec_runner.go:144] found /home/jenkins/minikube-integration/22343-297941/.minikube/ca.pem, removing ...
I1227 09:45:21.851690 353683 exec_runner.go:203] rm: /home/jenkins/minikube-integration/22343-297941/.minikube/ca.pem
I1227 09:45:21.851770 353683 exec_runner.go:151] cp: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/ca.pem --> /home/jenkins/minikube-integration/22343-297941/.minikube/ca.pem (1082 bytes)
I1227 09:45:21.851873 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/cert.pem -> /home/jenkins/minikube-integration/22343-297941/.minikube/cert.pem
I1227 09:45:21.851904 353683 exec_runner.go:144] found /home/jenkins/minikube-integration/22343-297941/.minikube/cert.pem, removing ...
I1227 09:45:21.851923 353683 exec_runner.go:203] rm: /home/jenkins/minikube-integration/22343-297941/.minikube/cert.pem
I1227 09:45:21.851962 353683 exec_runner.go:151] cp: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/cert.pem --> /home/jenkins/minikube-integration/22343-297941/.minikube/cert.pem (1123 bytes)
I1227 09:45:21.852092 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/key.pem -> /home/jenkins/minikube-integration/22343-297941/.minikube/key.pem
I1227 09:45:21.852114 353683 exec_runner.go:144] found /home/jenkins/minikube-integration/22343-297941/.minikube/key.pem, removing ...
I1227 09:45:21.852123 353683 exec_runner.go:203] rm: /home/jenkins/minikube-integration/22343-297941/.minikube/key.pem
I1227 09:45:21.852155 353683 exec_runner.go:151] cp: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/key.pem --> /home/jenkins/minikube-integration/22343-297941/.minikube/key.pem (1675 bytes)
I1227 09:45:21.852214 353683 provision.go:117] generating server cert: /home/jenkins/minikube-integration/22343-297941/.minikube/machines/server.pem ca-key=/home/jenkins/minikube-integration/22343-297941/.minikube/certs/ca.pem private-key=/home/jenkins/minikube-integration/22343-297941/.minikube/certs/ca-key.pem org=jenkins.ha-513251 san=[127.0.0.1 192.168.49.2 ha-513251 localhost minikube]
I1227 09:45:21.903039 353683 provision.go:177] copyRemoteCerts
I1227 09:45:21.903143 353683 ssh_runner.go:195] Run: sudo mkdir -p /etc/docker /etc/docker /etc/docker
I1227 09:45:21.903193 353683 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-513251
I1227 09:45:21.920995 353683 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33198 SSHKeyPath:/home/jenkins/minikube-integration/22343-297941/.minikube/machines/ha-513251/id_rsa Username:docker}
I1227 09:45:22.020706 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/machines/server.pem -> /etc/docker/server.pem
I1227 09:45:22.020772 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/machines/server.pem --> /etc/docker/server.pem (1200 bytes)
I1227 09:45:22.040457 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/machines/server-key.pem -> /etc/docker/server-key.pem
I1227 09:45:22.040545 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/machines/server-key.pem --> /etc/docker/server-key.pem (1679 bytes)
I1227 09:45:22.059426 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/ca.pem -> /etc/docker/ca.pem
I1227 09:45:22.059522 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/certs/ca.pem --> /etc/docker/ca.pem (1082 bytes)
I1227 09:45:22.078437 353683 provision.go:87] duration metric: took 245.707104ms to configureAuth
I1227 09:45:22.078487 353683 ubuntu.go:206] setting minikube options for container-runtime
I1227 09:45:22.078740 353683 config.go:182] Loaded profile config "ha-513251": Driver=docker, ContainerRuntime=crio, KubernetesVersion=v1.35.0
I1227 09:45:22.078852 353683 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-513251
I1227 09:45:22.097273 353683 main.go:144] libmachine: Using SSH client type: native
I1227 09:45:22.097592 353683 main.go:144] libmachine: &{{{<nil> 0 [] [] []} docker [0x3dbbe0] 0x3de0e0 <nil> [] 0s} 127.0.0.1 33198 <nil> <nil>}
I1227 09:45:22.097611 353683 main.go:144] libmachine: About to run SSH command:
sudo mkdir -p /etc/sysconfig && printf %s "
CRIO_MINIKUBE_OPTIONS='--insecure-registry 10.96.0.0/12 '
" | sudo tee /etc/sysconfig/crio.minikube && sudo systemctl restart crio
I1227 09:45:22.461249 353683 main.go:144] libmachine: SSH cmd err, output: <nil>:
CRIO_MINIKUBE_OPTIONS='--insecure-registry 10.96.0.0/12 '
I1227 09:45:22.461332 353683 machine.go:97] duration metric: took 4.124155515s to provisionDockerMachine
I1227 09:45:22.461358 353683 start.go:293] postStartSetup for "ha-513251" (driver="docker")
I1227 09:45:22.461396 353683 start.go:322] creating required directories: [/etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs]
I1227 09:45:22.461505 353683 ssh_runner.go:195] Run: sudo mkdir -p /etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs
I1227 09:45:22.461577 353683 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-513251
I1227 09:45:22.484466 353683 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33198 SSHKeyPath:/home/jenkins/minikube-integration/22343-297941/.minikube/machines/ha-513251/id_rsa Username:docker}
I1227 09:45:22.588039 353683 ssh_runner.go:195] Run: cat /etc/os-release
I1227 09:45:22.591353 353683 main.go:144] libmachine: Couldn't set key VERSION_CODENAME, no corresponding struct field found
I1227 09:45:22.591383 353683 info.go:137] Remote host: Debian GNU/Linux 12 (bookworm)
I1227 09:45:22.591396 353683 filesync.go:126] Scanning /home/jenkins/minikube-integration/22343-297941/.minikube/addons for local assets ...
I1227 09:45:22.591453 353683 filesync.go:126] Scanning /home/jenkins/minikube-integration/22343-297941/.minikube/files for local assets ...
I1227 09:45:22.591540 353683 filesync.go:149] local asset: /home/jenkins/minikube-integration/22343-297941/.minikube/files/etc/ssl/certs/2998112.pem -> 2998112.pem in /etc/ssl/certs
I1227 09:45:22.591553 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/files/etc/ssl/certs/2998112.pem -> /etc/ssl/certs/2998112.pem
I1227 09:45:22.591653 353683 ssh_runner.go:195] Run: sudo mkdir -p /etc/ssl/certs
I1227 09:45:22.599440 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/files/etc/ssl/certs/2998112.pem --> /etc/ssl/certs/2998112.pem (1708 bytes)
I1227 09:45:22.617415 353683 start.go:296] duration metric: took 156.015491ms for postStartSetup
I1227 09:45:22.617497 353683 ssh_runner.go:195] Run: sh -c "df -h /var | awk 'NR==2{print $5}'"
I1227 09:45:22.617543 353683 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-513251
I1227 09:45:22.635627 353683 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33198 SSHKeyPath:/home/jenkins/minikube-integration/22343-297941/.minikube/machines/ha-513251/id_rsa Username:docker}
I1227 09:45:22.733536 353683 ssh_runner.go:195] Run: sh -c "df -BG /var | awk 'NR==2{print $4}'"
I1227 09:45:22.738441 353683 fix.go:56] duration metric: took 4.73795966s for fixHost
I1227 09:45:22.738473 353683 start.go:83] releasing machines lock for "ha-513251", held for 4.738016497s
I1227 09:45:22.738547 353683 cli_runner.go:164] Run: docker container inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}" ha-513251
I1227 09:45:22.756007 353683 ssh_runner.go:195] Run: cat /version.json
I1227 09:45:22.756077 353683 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-513251
I1227 09:45:22.756356 353683 ssh_runner.go:195] Run: curl -sS -m 2 https://registry.k8s.io/
I1227 09:45:22.756411 353683 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-513251
I1227 09:45:22.775684 353683 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33198 SSHKeyPath:/home/jenkins/minikube-integration/22343-297941/.minikube/machines/ha-513251/id_rsa Username:docker}
I1227 09:45:22.784683 353683 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33198 SSHKeyPath:/home/jenkins/minikube-integration/22343-297941/.minikube/machines/ha-513251/id_rsa Username:docker}
I1227 09:45:22.974776 353683 ssh_runner.go:195] Run: systemctl --version
I1227 09:45:22.981407 353683 ssh_runner.go:195] Run: sudo sh -c "podman version >/dev/null"
I1227 09:45:23.019688 353683 ssh_runner.go:195] Run: sh -c "stat /etc/cni/net.d/*loopback.conf*"
W1227 09:45:23.024397 353683 cni.go:209] loopback cni configuration skipped: "/etc/cni/net.d/*loopback.conf*" not found
I1227 09:45:23.024482 353683 ssh_runner.go:195] Run: sudo find /etc/cni/net.d -maxdepth 1 -type f ( ( -name *bridge* -or -name *podman* ) -and -not -name *.mk_disabled ) -printf "%p, " -exec sh -c "sudo mv {} {}.mk_disabled" ;
I1227 09:45:23.033023 353683 cni.go:259] no active bridge cni configs found in "/etc/cni/net.d" - nothing to disable
I1227 09:45:23.033048 353683 start.go:496] detecting cgroup driver to use...
I1227 09:45:23.033080 353683 detect.go:187] detected "cgroupfs" cgroup driver on host os
I1227 09:45:23.033128 353683 ssh_runner.go:195] Run: sudo systemctl stop -f containerd
I1227 09:45:23.048890 353683 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I1227 09:45:23.062391 353683 docker.go:218] disabling cri-docker service (if available) ...
I1227 09:45:23.062461 353683 ssh_runner.go:195] Run: sudo systemctl stop -f cri-docker.socket
I1227 09:45:23.078874 353683 ssh_runner.go:195] Run: sudo systemctl stop -f cri-docker.service
I1227 09:45:23.092641 353683 ssh_runner.go:195] Run: sudo systemctl disable cri-docker.socket
I1227 09:45:23.215628 353683 ssh_runner.go:195] Run: sudo systemctl mask cri-docker.service
I1227 09:45:23.336773 353683 docker.go:234] disabling docker service ...
I1227 09:45:23.336856 353683 ssh_runner.go:195] Run: sudo systemctl stop -f docker.socket
I1227 09:45:23.351993 353683 ssh_runner.go:195] Run: sudo systemctl stop -f docker.service
I1227 09:45:23.365076 353683 ssh_runner.go:195] Run: sudo systemctl disable docker.socket
I1227 09:45:23.486999 353683 ssh_runner.go:195] Run: sudo systemctl mask docker.service
I1227 09:45:23.607630 353683 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service docker
I1227 09:45:23.621666 353683 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///var/run/crio/crio.sock
" | sudo tee /etc/crictl.yaml"
I1227 09:45:23.637617 353683 crio.go:59] configure cri-o to use "registry.k8s.io/pause:3.10.1" pause image...
I1227 09:45:23.637733 353683 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|^.*pause_image = .*$|pause_image = "registry.k8s.io/pause:3.10.1"|' /etc/crio/crio.conf.d/02-crio.conf"
I1227 09:45:23.646729 353683 crio.go:70] configuring cri-o to use "cgroupfs" as cgroup driver...
I1227 09:45:23.646803 353683 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|^.*cgroup_manager = .*$|cgroup_manager = "cgroupfs"|' /etc/crio/crio.conf.d/02-crio.conf"
I1227 09:45:23.656407 353683 ssh_runner.go:195] Run: sh -c "sudo sed -i '/conmon_cgroup = .*/d' /etc/crio/crio.conf.d/02-crio.conf"
I1227 09:45:23.665374 353683 ssh_runner.go:195] Run: sh -c "sudo sed -i '/cgroup_manager = .*/a conmon_cgroup = "pod"' /etc/crio/crio.conf.d/02-crio.conf"
I1227 09:45:23.674513 353683 ssh_runner.go:195] Run: sh -c "sudo rm -rf /etc/cni/net.mk"
I1227 09:45:23.682899 353683 ssh_runner.go:195] Run: sh -c "sudo sed -i '/^ *"net.ipv4.ip_unprivileged_port_start=.*"/d' /etc/crio/crio.conf.d/02-crio.conf"
I1227 09:45:23.692638 353683 ssh_runner.go:195] Run: sh -c "sudo grep -q "^ *default_sysctls" /etc/crio/crio.conf.d/02-crio.conf || sudo sed -i '/conmon_cgroup = .*/a default_sysctls = \[\n\]' /etc/crio/crio.conf.d/02-crio.conf"
I1227 09:45:23.701500 353683 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^default_sysctls *= *\[|&\n "net.ipv4.ip_unprivileged_port_start=0",|' /etc/crio/crio.conf.d/02-crio.conf"
I1227 09:45:23.710461 353683 ssh_runner.go:195] Run: sudo sysctl net.bridge.bridge-nf-call-iptables
I1227 09:45:23.718222 353683 ssh_runner.go:195] Run: sudo sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"
I1227 09:45:23.726035 353683 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1227 09:45:23.837128 353683 ssh_runner.go:195] Run: sudo systemctl restart crio
I1227 09:45:24.007170 353683 start.go:553] Will wait 60s for socket path /var/run/crio/crio.sock
I1227 09:45:24.007319 353683 ssh_runner.go:195] Run: stat /var/run/crio/crio.sock
I1227 09:45:24.014123 353683 start.go:574] Will wait 60s for crictl version
I1227 09:45:24.014245 353683 ssh_runner.go:195] Run: which crictl
I1227 09:45:24.033366 353683 ssh_runner.go:195] Run: sudo /usr/local/bin/crictl version
I1227 09:45:24.058444 353683 start.go:590] Version: 0.1.0
RuntimeName: cri-o
RuntimeVersion: 1.34.3
RuntimeApiVersion: v1
I1227 09:45:24.058524 353683 ssh_runner.go:195] Run: crio --version
I1227 09:45:24.087072 353683 ssh_runner.go:195] Run: crio --version
I1227 09:45:24.118588 353683 out.go:179] * Preparing Kubernetes v1.35.0 on CRI-O 1.34.3 ...
I1227 09:45:24.121527 353683 cli_runner.go:164] Run: docker network inspect ha-513251 --format "{"Name": "{{.Name}}","Driver": "{{.Driver}}","Subnet": "{{range .IPAM.Config}}{{.Subnet}}{{end}}","Gateway": "{{range .IPAM.Config}}{{.Gateway}}{{end}}","MTU": {{if (index .Options "com.docker.network.driver.mtu")}}{{(index .Options "com.docker.network.driver.mtu")}}{{else}}0{{end}}, "ContainerIPs": [{{range $k,$v := .Containers }}"{{$v.IPv4Address}}",{{end}}]}"
I1227 09:45:24.138224 353683 ssh_runner.go:195] Run: grep 192.168.49.1 host.minikube.internal$ /etc/hosts
I1227 09:45:24.142467 353683 ssh_runner.go:195] Run: /bin/bash -c "{ grep -v $'\thost.minikube.internal$' "/etc/hosts"; echo "192.168.49.1 host.minikube.internal"; } > /tmp/h.$$; sudo cp /tmp/h.$$ "/etc/hosts""
I1227 09:45:24.152932 353683 kubeadm.go:884] updating cluster {Name:ha-513251 KeepContext:false EmbedCerts:false MinikubeISO: KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1766570851-22316@sha256:7975a7a1117280f99ad7696c9c80bdca993064fe9e309e9984685e0ce989758a Memory:3072 CPUs:2 DiskSize:20000 Driver:docker HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.35.0 ClusterName:ha-513251 Namespace:default APIServerHAVIP:192.168.49.254 APIServerName:minikubeCA APISe
rverNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:crio CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.168.49.2 Port:8443 KubernetesVersion:v1.35.0 ContainerRuntime:crio ControlPlane:true Worker:true} {Name:m02 IP:192.168.49.3 Port:8443 KubernetesVersion:v1.35.0 ContainerRuntime:crio ControlPlane:true Worker:true} {Name:m04 IP:192.168.49.5 Port:0 KubernetesVersion:v1.35.0 ContainerRuntime:crio ControlPlane:false Worker:true}] Addons:map[ambassador:false amd-gpu-device-plugin:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false inaccel:false ingress:false ingress-dns:false inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow:false kubeta
il:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s MountString: Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:false DisableMetrics:false DisableCoreDNSLog:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock:
SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s Rosetta:false} ...
I1227 09:45:24.153087 353683 preload.go:188] Checking if preload exists for k8s version v1.35.0 and runtime crio
I1227 09:45:24.153163 353683 ssh_runner.go:195] Run: sudo crictl images --output json
I1227 09:45:24.188918 353683 crio.go:561] all images are preloaded for cri-o runtime.
I1227 09:45:24.188945 353683 crio.go:433] Images already preloaded, skipping extraction
I1227 09:45:24.189006 353683 ssh_runner.go:195] Run: sudo crictl images --output json
I1227 09:45:24.216272 353683 crio.go:561] all images are preloaded for cri-o runtime.
I1227 09:45:24.216301 353683 cache_images.go:86] Images are preloaded, skipping loading
I1227 09:45:24.216314 353683 kubeadm.go:935] updating node { 192.168.49.2 8443 v1.35.0 crio true true} ...
I1227 09:45:24.216440 353683 kubeadm.go:947] kubelet [Unit]
Wants=crio.service
[Service]
ExecStart=
ExecStart=/var/lib/minikube/binaries/v1.35.0/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroups-per-qos=false --config=/var/lib/kubelet/config.yaml --enforce-node-allocatable= --hostname-override=ha-513251 --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.168.49.2
[Install]
config:
{KubernetesVersion:v1.35.0 ClusterName:ha-513251 Namespace:default APIServerHAVIP:192.168.49.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:crio CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:}
I1227 09:45:24.216534 353683 ssh_runner.go:195] Run: crio config
I1227 09:45:24.292083 353683 cni.go:84] Creating CNI manager for ""
I1227 09:45:24.292105 353683 cni.go:136] multinode detected (3 nodes found), recommending kindnet
I1227 09:45:24.292144 353683 kubeadm.go:85] Using pod CIDR: 10.244.0.0/16
I1227 09:45:24.292181 353683 kubeadm.go:197] kubeadm options: {CertDir:/var/lib/minikube/certs ServiceCIDR:10.96.0.0/12 PodSubnet:10.244.0.0/16 AdvertiseAddress:192.168.49.2 APIServerPort:8443 KubernetesVersion:v1.35.0 EtcdDataDir:/var/lib/minikube/etcd EtcdExtraArgs:map[] ClusterName:ha-513251 NodeName:ha-513251 DNSDomain:cluster.local CRISocket:/var/run/crio/crio.sock ImageRepository: ComponentOptions:[{Component:apiServer ExtraArgs:map[enable-admission-plugins:NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota] Pairs:map[certSANs:["127.0.0.1", "localhost", "192.168.49.2"]]} {Component:controllerManager ExtraArgs:map[allocate-node-cidrs:true leader-elect:false] Pairs:map[]} {Component:scheduler ExtraArgs:map[leader-elect:false] Pairs:map[]}] FeatureArgs:map[] NodeIP:192.168.49.2 CgroupDriver:cgroupfs ClientCAFile:/var/lib/minikube/certs/ca.crt StaticPodPath:/etc/kubernetes/mani
fests ControlPlaneAddress:control-plane.minikube.internal KubeProxyOptions:map[] ResolvConfSearchRegression:false KubeletConfigOpts:map[containerRuntimeEndpoint:unix:///var/run/crio/crio.sock failCgroupV1:false hairpinMode:hairpin-veth runtimeRequestTimeout:15m] PrependCriSocketUnix:true}
I1227 09:45:24.292330 353683 kubeadm.go:203] kubeadm config:
apiVersion: kubeadm.k8s.io/v1beta4
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 192.168.49.2
bindPort: 8443
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
ttl: 24h0m0s
usages:
- signing
- authentication
nodeRegistration:
criSocket: unix:///var/run/crio/crio.sock
name: "ha-513251"
kubeletExtraArgs:
- name: "node-ip"
value: "192.168.49.2"
taints: []
---
apiVersion: kubeadm.k8s.io/v1beta4
kind: ClusterConfiguration
apiServer:
certSANs: ["127.0.0.1", "localhost", "192.168.49.2"]
extraArgs:
- name: "enable-admission-plugins"
value: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
controllerManager:
extraArgs:
- name: "allocate-node-cidrs"
value: "true"
- name: "leader-elect"
value: "false"
scheduler:
extraArgs:
- name: "leader-elect"
value: "false"
certificatesDir: /var/lib/minikube/certs
clusterName: mk
controlPlaneEndpoint: control-plane.minikube.internal:8443
etcd:
local:
dataDir: /var/lib/minikube/etcd
kubernetesVersion: v1.35.0
networking:
dnsDomain: cluster.local
podSubnet: "10.244.0.0/16"
serviceSubnet: 10.96.0.0/12
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
authentication:
x509:
clientCAFile: /var/lib/minikube/certs/ca.crt
cgroupDriver: cgroupfs
containerRuntimeEndpoint: unix:///var/run/crio/crio.sock
failCgroupV1: false
hairpinMode: hairpin-veth
runtimeRequestTimeout: 15m
clusterDomain: "cluster.local"
# disable disk resource management by default
imageGCHighThresholdPercent: 100
evictionHard:
nodefs.available: "0%"
nodefs.inodesFree: "0%"
imagefs.available: "0%"
failSwapOn: false
staticPodPath: /etc/kubernetes/manifests
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
clusterCIDR: "10.244.0.0/16"
metricsBindAddress: 0.0.0.0:10249
conntrack:
maxPerCore: 0
# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_established"
tcpEstablishedTimeout: 0s
# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_close"
tcpCloseWaitTimeout: 0s
I1227 09:45:24.292352 353683 kube-vip.go:115] generating kube-vip config ...
I1227 09:45:24.292412 353683 ssh_runner.go:195] Run: sudo sh -c "lsmod | grep ip_vs"
I1227 09:45:24.304778 353683 kube-vip.go:163] giving up enabling control-plane load-balancing as ipvs kernel modules appears not to be available: sudo sh -c "lsmod | grep ip_vs": Process exited with status 1
stdout:
stderr:
I1227 09:45:24.304912 353683 kube-vip.go:137] kube-vip config:
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
name: kube-vip
namespace: kube-system
spec:
containers:
- args:
- manager
env:
- name: vip_arp
value: "true"
- name: port
value: "8443"
- name: vip_nodename
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: vip_interface
value: eth0
- name: vip_cidr
value: "32"
- name: dns_mode
value: first
- name: cp_enable
value: "true"
- name: cp_namespace
value: kube-system
- name: vip_leaderelection
value: "true"
- name: vip_leasename
value: plndr-cp-lock
- name: vip_leaseduration
value: "5"
- name: vip_renewdeadline
value: "3"
- name: vip_retryperiod
value: "1"
- name: address
value: 192.168.49.254
- name: prometheus_server
value: :2112
image: ghcr.io/kube-vip/kube-vip:v1.0.3
imagePullPolicy: IfNotPresent
name: kube-vip
resources: {}
securityContext:
capabilities:
add:
- NET_ADMIN
- NET_RAW
volumeMounts:
- mountPath: /etc/kubernetes/admin.conf
name: kubeconfig
hostAliases:
- hostnames:
- kubernetes
ip: 127.0.0.1
hostNetwork: true
volumes:
- hostPath:
path: "/etc/kubernetes/admin.conf"
name: kubeconfig
status: {}
I1227 09:45:24.305012 353683 ssh_runner.go:195] Run: sudo ls /var/lib/minikube/binaries/v1.35.0
I1227 09:45:24.312901 353683 binaries.go:51] Found k8s binaries, skipping transfer
I1227 09:45:24.312976 353683 ssh_runner.go:195] Run: sudo mkdir -p /etc/systemd/system/kubelet.service.d /lib/systemd/system /var/tmp/minikube /etc/kubernetes/manifests
I1227 09:45:24.320559 353683 ssh_runner.go:362] scp memory --> /etc/systemd/system/kubelet.service.d/10-kubeadm.conf (359 bytes)
I1227 09:45:24.334537 353683 ssh_runner.go:362] scp memory --> /lib/systemd/system/kubelet.service (352 bytes)
I1227 09:45:24.347371 353683 ssh_runner.go:362] scp memory --> /var/tmp/minikube/kubeadm.yaml.new (2226 bytes)
I1227 09:45:24.360123 353683 ssh_runner.go:362] scp memory --> /etc/kubernetes/manifests/kube-vip.yaml (1358 bytes)
I1227 09:45:24.373098 353683 ssh_runner.go:195] Run: grep 192.168.49.254 control-plane.minikube.internal$ /etc/hosts
I1227 09:45:24.376820 353683 ssh_runner.go:195] Run: /bin/bash -c "{ grep -v $'\tcontrol-plane.minikube.internal$' "/etc/hosts"; echo "192.168.49.254 control-plane.minikube.internal"; } > /tmp/h.$$; sudo cp /tmp/h.$$ "/etc/hosts""
I1227 09:45:24.387127 353683 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1227 09:45:24.503934 353683 ssh_runner.go:195] Run: sudo systemctl start kubelet
I1227 09:45:24.522185 353683 certs.go:69] Setting up /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251 for IP: 192.168.49.2
I1227 09:45:24.522204 353683 certs.go:195] generating shared ca certs ...
I1227 09:45:24.522219 353683 certs.go:227] acquiring lock for ca certs: {Name:mkf0455840442d82ad2865a090879fc8de65e5e9 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1227 09:45:24.522359 353683 certs.go:236] skipping valid "minikubeCA" ca cert: /home/jenkins/minikube-integration/22343-297941/.minikube/ca.key
I1227 09:45:24.522410 353683 certs.go:236] skipping valid "proxyClientCA" ca cert: /home/jenkins/minikube-integration/22343-297941/.minikube/proxy-client-ca.key
I1227 09:45:24.522417 353683 certs.go:257] generating profile certs ...
I1227 09:45:24.522498 353683 certs.go:360] skipping valid signed profile cert regeneration for "minikube-user": /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/client.key
I1227 09:45:24.522526 353683 certs.go:364] generating signed profile cert for "minikube": /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/apiserver.key.667f4b14
I1227 09:45:24.522540 353683 crypto.go:68] Generating cert /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/apiserver.crt.667f4b14 with IP's: [10.96.0.1 127.0.0.1 10.0.0.1 192.168.49.2 192.168.49.3 192.168.49.254]
I1227 09:45:24.644648 353683 crypto.go:156] Writing cert to /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/apiserver.crt.667f4b14 ...
I1227 09:45:24.648971 353683 lock.go:35] WriteFile acquiring /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/apiserver.crt.667f4b14: {Name:mkb5dff6e9ccf7c0fd52113e0d144d6316de11fd Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1227 09:45:24.649217 353683 crypto.go:164] Writing key to /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/apiserver.key.667f4b14 ...
I1227 09:45:24.649259 353683 lock.go:35] WriteFile acquiring /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/apiserver.key.667f4b14: {Name:mk0fad6909993d85239fadc763725d8b8b7a440c Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1227 09:45:24.649401 353683 certs.go:382] copying /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/apiserver.crt.667f4b14 -> /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/apiserver.crt
I1227 09:45:24.649572 353683 certs.go:386] copying /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/apiserver.key.667f4b14 -> /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/apiserver.key
I1227 09:45:24.649765 353683 certs.go:360] skipping valid signed profile cert regeneration for "aggregator": /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/proxy-client.key
I1227 09:45:24.649810 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/ca.crt -> /var/lib/minikube/certs/ca.crt
I1227 09:45:24.649846 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/ca.key -> /var/lib/minikube/certs/ca.key
I1227 09:45:24.649875 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/proxy-client-ca.crt -> /var/lib/minikube/certs/proxy-client-ca.crt
I1227 09:45:24.649918 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/proxy-client-ca.key -> /var/lib/minikube/certs/proxy-client-ca.key
I1227 09:45:24.649950 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/apiserver.crt -> /var/lib/minikube/certs/apiserver.crt
I1227 09:45:24.649988 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/apiserver.key -> /var/lib/minikube/certs/apiserver.key
I1227 09:45:24.650030 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/proxy-client.crt -> /var/lib/minikube/certs/proxy-client.crt
I1227 09:45:24.650060 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/proxy-client.key -> /var/lib/minikube/certs/proxy-client.key
I1227 09:45:24.650137 353683 certs.go:484] found cert: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/299811.pem (1338 bytes)
W1227 09:45:24.650200 353683 certs.go:480] ignoring /home/jenkins/minikube-integration/22343-297941/.minikube/certs/299811_empty.pem, impossibly tiny 0 bytes
I1227 09:45:24.650235 353683 certs.go:484] found cert: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/ca-key.pem (1675 bytes)
I1227 09:45:24.650297 353683 certs.go:484] found cert: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/ca.pem (1082 bytes)
I1227 09:45:24.650344 353683 certs.go:484] found cert: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/cert.pem (1123 bytes)
I1227 09:45:24.650434 353683 certs.go:484] found cert: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/key.pem (1675 bytes)
I1227 09:45:24.650545 353683 certs.go:484] found cert: /home/jenkins/minikube-integration/22343-297941/.minikube/files/etc/ssl/certs/2998112.pem (1708 bytes)
I1227 09:45:24.650616 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/files/etc/ssl/certs/2998112.pem -> /usr/share/ca-certificates/2998112.pem
I1227 09:45:24.650660 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/ca.crt -> /usr/share/ca-certificates/minikubeCA.pem
I1227 09:45:24.650689 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/299811.pem -> /usr/share/ca-certificates/299811.pem
I1227 09:45:24.651244 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/ca.crt --> /var/lib/minikube/certs/ca.crt (1111 bytes)
I1227 09:45:24.675286 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/ca.key --> /var/lib/minikube/certs/ca.key (1679 bytes)
I1227 09:45:24.694694 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/proxy-client-ca.crt --> /var/lib/minikube/certs/proxy-client-ca.crt (1119 bytes)
I1227 09:45:24.717231 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/proxy-client-ca.key --> /var/lib/minikube/certs/proxy-client-ca.key (1679 bytes)
I1227 09:45:24.749389 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/apiserver.crt --> /var/lib/minikube/certs/apiserver.crt (1432 bytes)
I1227 09:45:24.770851 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/apiserver.key --> /var/lib/minikube/certs/apiserver.key (1679 bytes)
I1227 09:45:24.790309 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/proxy-client.crt --> /var/lib/minikube/certs/proxy-client.crt (1147 bytes)
I1227 09:45:24.811612 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/proxy-client.key --> /var/lib/minikube/certs/proxy-client.key (1675 bytes)
I1227 09:45:24.834366 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/files/etc/ssl/certs/2998112.pem --> /usr/share/ca-certificates/2998112.pem (1708 bytes)
I1227 09:45:24.853802 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/ca.crt --> /usr/share/ca-certificates/minikubeCA.pem (1111 bytes)
I1227 09:45:24.871797 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/certs/299811.pem --> /usr/share/ca-certificates/299811.pem (1338 bytes)
I1227 09:45:24.894130 353683 ssh_runner.go:362] scp memory --> /var/lib/minikube/kubeconfig (722 bytes)
I1227 09:45:24.908139 353683 ssh_runner.go:195] Run: openssl version
I1227 09:45:24.914716 353683 ssh_runner.go:195] Run: sudo test -s /usr/share/ca-certificates/299811.pem
I1227 09:45:24.922797 353683 ssh_runner.go:195] Run: sudo ln -fs /usr/share/ca-certificates/299811.pem /etc/ssl/certs/299811.pem
I1227 09:45:24.930729 353683 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/299811.pem
I1227 09:45:24.934601 353683 certs.go:528] hashing: -rw-r--r-- 1 root root 1338 Dec 27 09:34 /usr/share/ca-certificates/299811.pem
I1227 09:45:24.934686 353683 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/299811.pem
I1227 09:45:24.976521 353683 ssh_runner.go:195] Run: sudo test -L /etc/ssl/certs/51391683.0
I1227 09:45:24.984298 353683 ssh_runner.go:195] Run: sudo test -s /usr/share/ca-certificates/2998112.pem
I1227 09:45:24.991944 353683 ssh_runner.go:195] Run: sudo ln -fs /usr/share/ca-certificates/2998112.pem /etc/ssl/certs/2998112.pem
I1227 09:45:24.999664 353683 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/2998112.pem
I1227 09:45:25.020750 353683 certs.go:528] hashing: -rw-r--r-- 1 root root 1708 Dec 27 09:34 /usr/share/ca-certificates/2998112.pem
I1227 09:45:25.020853 353683 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/2998112.pem
I1227 09:45:25.066886 353683 ssh_runner.go:195] Run: sudo test -L /etc/ssl/certs/3ec20f2e.0
I1227 09:45:25.076628 353683 ssh_runner.go:195] Run: sudo test -s /usr/share/ca-certificates/minikubeCA.pem
I1227 09:45:25.086029 353683 ssh_runner.go:195] Run: sudo ln -fs /usr/share/ca-certificates/minikubeCA.pem /etc/ssl/certs/minikubeCA.pem
I1227 09:45:25.095338 353683 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/minikubeCA.pem
I1227 09:45:25.101041 353683 certs.go:528] hashing: -rw-r--r-- 1 root root 1111 Dec 27 09:30 /usr/share/ca-certificates/minikubeCA.pem
I1227 09:45:25.101118 353683 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/minikubeCA.pem
I1227 09:45:25.145647 353683 ssh_runner.go:195] Run: sudo test -L /etc/ssl/certs/b5213941.0
I1227 09:45:25.156431 353683 ssh_runner.go:195] Run: stat /var/lib/minikube/certs/apiserver-kubelet-client.crt
I1227 09:45:25.165145 353683 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/apiserver-etcd-client.crt -checkend 86400
I1227 09:45:25.214664 353683 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/apiserver-kubelet-client.crt -checkend 86400
I1227 09:45:25.265928 353683 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/etcd/server.crt -checkend 86400
I1227 09:45:25.352085 353683 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/etcd/healthcheck-client.crt -checkend 86400
I1227 09:45:25.431634 353683 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/etcd/peer.crt -checkend 86400
I1227 09:45:25.492845 353683 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/front-proxy-client.crt -checkend 86400
I1227 09:45:25.554400 353683 kubeadm.go:401] StartCluster: {Name:ha-513251 KeepContext:false EmbedCerts:false MinikubeISO: KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1766570851-22316@sha256:7975a7a1117280f99ad7696c9c80bdca993064fe9e309e9984685e0ce989758a Memory:3072 CPUs:2 DiskSize:20000 Driver:docker HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.35.0 ClusterName:ha-513251 Namespace:default APIServerHAVIP:192.168.49.254 APIServerName:minikubeCA APIServe
rNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:crio CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.168.49.2 Port:8443 KubernetesVersion:v1.35.0 ContainerRuntime:crio ControlPlane:true Worker:true} {Name:m02 IP:192.168.49.3 Port:8443 KubernetesVersion:v1.35.0 ContainerRuntime:crio ControlPlane:true Worker:true} {Name:m04 IP:192.168.49.5 Port:0 KubernetesVersion:v1.35.0 ContainerRuntime:crio ControlPlane:false Worker:true}] Addons:map[ambassador:false amd-gpu-device-plugin:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false inaccel:false ingress:false ingress-dns:false inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow:false kubetail:
false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s MountString: Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:false DisableMetrics:false DisableCoreDNSLog:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SS
HAgentPID:0 GPUs: AutoPauseInterval:1m0s Rosetta:false}
I1227 09:45:25.554601 353683 cri.go:61] listing CRI containers in root : {State:paused Name: Namespaces:[kube-system]}
I1227 09:45:25.554705 353683 ssh_runner.go:195] Run: sudo -s eval "crictl --timeout=10s ps -a --quiet --label io.kubernetes.pod.namespace=kube-system"
I1227 09:45:25.597558 353683 cri.go:96] found id: "7b1da10d6de7d31911e815a0a6e65bec0b462f36adac4663bcba270a51072ce3"
I1227 09:45:25.597630 353683 cri.go:96] found id: "f69e010776644f8005f4cd92f4774d5dc92d62b50dadf798020d9d8db93f52a7"
I1227 09:45:25.597649 353683 cri.go:96] found id: "f7e841ab1c87c3a73fb0fa9774a7d5540fae4454f87f94803231876049f07db7"
I1227 09:45:25.597672 353683 cri.go:96] found id: "c8b5eff27c4f32b2e2d3926915d5eef69dcc564f101afeb65284237bedc9de47"
I1227 09:45:25.597710 353683 cri.go:96] found id: "cc9aea908d640c5405a83f2749f502470c2bdf01223971af7da3ebb2588fd6ab"
I1227 09:45:25.597733 353683 cri.go:96] found id: ""
I1227 09:45:25.597819 353683 ssh_runner.go:195] Run: sudo runc list -f json
W1227 09:45:25.609417 353683 kubeadm.go:408] unpause failed: list paused: runc: sudo runc list -f json: Process exited with status 1
stdout:
stderr:
time="2025-12-27T09:45:25Z" level=error msg="open /run/runc: no such file or directory"
I1227 09:45:25.609569 353683 ssh_runner.go:195] Run: sudo ls /var/lib/kubelet/kubeadm-flags.env /var/lib/kubelet/config.yaml /var/lib/minikube/etcd
I1227 09:45:25.618182 353683 kubeadm.go:417] found existing configuration files, will attempt cluster restart
I1227 09:45:25.618252 353683 kubeadm.go:598] restartPrimaryControlPlane start ...
I1227 09:45:25.618336 353683 ssh_runner.go:195] Run: sudo test -d /data/minikube
I1227 09:45:25.632559 353683 kubeadm.go:131] /data/minikube skipping compat symlinks: sudo test -d /data/minikube: Process exited with status 1
stdout:
stderr:
I1227 09:45:25.633086 353683 kubeconfig.go:47] verify endpoint returned: get endpoint: "ha-513251" does not appear in /home/jenkins/minikube-integration/22343-297941/kubeconfig
I1227 09:45:25.633265 353683 kubeconfig.go:62] /home/jenkins/minikube-integration/22343-297941/kubeconfig needs updating (will repair): [kubeconfig missing "ha-513251" cluster setting kubeconfig missing "ha-513251" context setting]
I1227 09:45:25.633617 353683 lock.go:35] WriteFile acquiring /home/jenkins/minikube-integration/22343-297941/kubeconfig: {Name:mk9b0395cf9df161cfb49b66943a0d24161dfe93 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1227 09:45:25.634243 353683 kapi.go:59] client config for ha-513251: &rest.Config{Host:"https://192.168.49.2:8443", APIPath:"", ContentConfig:rest.ContentConfig{AcceptContentTypes:"", ContentType:"", GroupVersion:(*schema.GroupVersion)(nil), NegotiatedSerializer:runtime.NegotiatedSerializer(nil)}, Username:"", Password:"", BearerToken:"", BearerTokenFile:"", Impersonate:rest.ImpersonationConfig{UserName:"", UID:"", Groups:[]string(nil), Extra:map[string][]string(nil)}, AuthProvider:<nil>, AuthConfigPersister:rest.AuthProviderConfigPersister(nil), ExecProvider:<nil>, TLSClientConfig:rest.sanitizedTLSClientConfig{Insecure:false, ServerName:"", CertFile:"/home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/client.crt", KeyFile:"/home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/client.key", CAFile:"/home/jenkins/minikube-integration/22343-297941/.minikube/ca.crt", CertData:[]uint8(nil), KeyData:[]uint8(nil), CAData:[]uint8(nil), NextProtos:[]string(nil
)}, UserAgent:"", DisableCompression:false, Transport:http.RoundTripper(nil), WrapTransport:(transport.WrapperFunc)(0x1f30470), QPS:0, Burst:0, RateLimiter:flowcontrol.RateLimiter(nil), WarningHandler:rest.WarningHandler(nil), WarningHandlerWithContext:rest.WarningHandlerWithContext(nil), Timeout:0, Dial:(func(context.Context, string, string) (net.Conn, error))(nil), Proxy:(func(*http.Request) (*url.URL, error))(nil)}
I1227 09:45:25.635070 353683 envvar.go:172] "Feature gate default state" feature="WatchListClient" enabled=true
I1227 09:45:25.635170 353683 envvar.go:172] "Feature gate default state" feature="ClientsAllowCBOR" enabled=false
I1227 09:45:25.635191 353683 envvar.go:172] "Feature gate default state" feature="ClientsPreferCBOR" enabled=false
I1227 09:45:25.635109 353683 cert_rotation.go:141] "Starting client certificate rotation controller" logger="tls-transport-cache"
I1227 09:45:25.635305 353683 envvar.go:172] "Feature gate default state" feature="InOrderInformers" enabled=true
I1227 09:45:25.635338 353683 envvar.go:172] "Feature gate default state" feature="InOrderInformersBatchProcess" enabled=true
I1227 09:45:25.635362 353683 envvar.go:172] "Feature gate default state" feature="InformerResourceVersion" enabled=true
I1227 09:45:25.635701 353683 ssh_runner.go:195] Run: sudo diff -u /var/tmp/minikube/kubeadm.yaml /var/tmp/minikube/kubeadm.yaml.new
I1227 09:45:25.649140 353683 kubeadm.go:635] The running cluster does not require reconfiguration: 192.168.49.2
I1227 09:45:25.649211 353683 kubeadm.go:602] duration metric: took 30.937903ms to restartPrimaryControlPlane
I1227 09:45:25.649235 353683 kubeadm.go:403] duration metric: took 94.844629ms to StartCluster
I1227 09:45:25.649264 353683 settings.go:142] acquiring lock: {Name:mkd8982e2b055d379b1ed67f6c962a36a3b427c8 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1227 09:45:25.649374 353683 settings.go:150] Updating kubeconfig: /home/jenkins/minikube-integration/22343-297941/kubeconfig
I1227 09:45:25.650129 353683 lock.go:35] WriteFile acquiring /home/jenkins/minikube-integration/22343-297941/kubeconfig: {Name:mk9b0395cf9df161cfb49b66943a0d24161dfe93 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1227 09:45:25.650407 353683 start.go:234] HA (multi-control plane) cluster: will skip waiting for primary control-plane node &{Name: IP:192.168.49.2 Port:8443 KubernetesVersion:v1.35.0 ContainerRuntime:crio ControlPlane:true Worker:true}
I1227 09:45:25.650466 353683 start.go:242] waiting for startup goroutines ...
I1227 09:45:25.650497 353683 addons.go:527] enable addons start: toEnable=map[ambassador:false amd-gpu-device-plugin:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false inaccel:false ingress:false ingress-dns:false inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow:false kubetail:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false]
I1227 09:45:25.651321 353683 config.go:182] Loaded profile config "ha-513251": Driver=docker, ContainerRuntime=crio, KubernetesVersion=v1.35.0
I1227 09:45:25.656567 353683 out.go:179] * Enabled addons:
I1227 09:45:25.659428 353683 addons.go:530] duration metric: took 8.91449ms for enable addons: enabled=[]
I1227 09:45:25.659506 353683 start.go:247] waiting for cluster config update ...
I1227 09:45:25.659529 353683 start.go:256] writing updated cluster config ...
I1227 09:45:25.662807 353683 out.go:203]
I1227 09:45:25.666068 353683 config.go:182] Loaded profile config "ha-513251": Driver=docker, ContainerRuntime=crio, KubernetesVersion=v1.35.0
I1227 09:45:25.666232 353683 profile.go:143] Saving config to /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/config.json ...
I1227 09:45:25.669730 353683 out.go:179] * Starting "ha-513251-m02" control-plane node in "ha-513251" cluster
I1227 09:45:25.672614 353683 cache.go:134] Beginning downloading kic base image for docker with crio
I1227 09:45:25.675545 353683 out.go:179] * Pulling base image v0.0.48-1766570851-22316 ...
I1227 09:45:25.678485 353683 preload.go:188] Checking if preload exists for k8s version v1.35.0 and runtime crio
I1227 09:45:25.678506 353683 cache.go:65] Caching tarball of preloaded images
I1227 09:45:25.678618 353683 preload.go:251] Found /home/jenkins/minikube-integration/22343-297941/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.35.0-cri-o-overlay-arm64.tar.lz4 in cache, skipping download
I1227 09:45:25.678630 353683 cache.go:68] Finished verifying existence of preloaded tar for v1.35.0 on crio
I1227 09:45:25.678752 353683 profile.go:143] Saving config to /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/config.json ...
I1227 09:45:25.678961 353683 image.go:81] Checking for gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1766570851-22316@sha256:7975a7a1117280f99ad7696c9c80bdca993064fe9e309e9984685e0ce989758a in local docker daemon
I1227 09:45:25.700973 353683 image.go:100] Found gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1766570851-22316@sha256:7975a7a1117280f99ad7696c9c80bdca993064fe9e309e9984685e0ce989758a in local docker daemon, skipping pull
I1227 09:45:25.701000 353683 cache.go:158] gcr.io/k8s-minikube/kicbase-builds:v0.0.48-1766570851-22316@sha256:7975a7a1117280f99ad7696c9c80bdca993064fe9e309e9984685e0ce989758a exists in daemon, skipping load
I1227 09:45:25.701015 353683 cache.go:243] Successfully downloaded all kic artifacts
I1227 09:45:25.701040 353683 start.go:360] acquireMachinesLock for ha-513251-m02: {Name:mk859480e290b8b366277aa9ac48e168657809ba Clock:{} Delay:500ms Timeout:10m0s Cancel:<nil>}
I1227 09:45:25.701095 353683 start.go:364] duration metric: took 35.808µs to acquireMachinesLock for "ha-513251-m02"
I1227 09:45:25.701120 353683 start.go:96] Skipping create...Using existing machine configuration
I1227 09:45:25.701128 353683 fix.go:54] fixHost starting: m02
I1227 09:45:25.701383 353683 cli_runner.go:164] Run: docker container inspect ha-513251-m02 --format={{.State.Status}}
I1227 09:45:25.721891 353683 fix.go:112] recreateIfNeeded on ha-513251-m02: state=Stopped err=<nil>
W1227 09:45:25.721916 353683 fix.go:138] unexpected machine state, will restart: <nil>
I1227 09:45:25.725291 353683 out.go:252] * Restarting existing docker container for "ha-513251-m02" ...
I1227 09:45:25.725375 353683 cli_runner.go:164] Run: docker start ha-513251-m02
I1227 09:45:26.149022 353683 cli_runner.go:164] Run: docker container inspect ha-513251-m02 --format={{.State.Status}}
I1227 09:45:26.186961 353683 kic.go:430] container "ha-513251-m02" state is running.
I1227 09:45:26.187328 353683 cli_runner.go:164] Run: docker container inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}" ha-513251-m02
I1227 09:45:26.217667 353683 profile.go:143] Saving config to /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/config.json ...
I1227 09:45:26.217913 353683 machine.go:94] provisionDockerMachine start ...
I1227 09:45:26.217973 353683 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-513251-m02
I1227 09:45:26.245157 353683 main.go:144] libmachine: Using SSH client type: native
I1227 09:45:26.245467 353683 main.go:144] libmachine: &{{{<nil> 0 [] [] []} docker [0x3dbbe0] 0x3de0e0 <nil> [] 0s} 127.0.0.1 33203 <nil> <nil>}
I1227 09:45:26.245482 353683 main.go:144] libmachine: About to run SSH command:
hostname
I1227 09:45:26.246067 353683 main.go:144] libmachine: Error dialing TCP: ssh: handshake failed: read tcp 127.0.0.1:55528->127.0.0.1:33203: read: connection reset by peer
I1227 09:45:29.476637 353683 main.go:144] libmachine: SSH cmd err, output: <nil>: ha-513251-m02
I1227 09:45:29.476662 353683 ubuntu.go:182] provisioning hostname "ha-513251-m02"
I1227 09:45:29.476730 353683 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-513251-m02
I1227 09:45:29.515584 353683 main.go:144] libmachine: Using SSH client type: native
I1227 09:45:29.515885 353683 main.go:144] libmachine: &{{{<nil> 0 [] [] []} docker [0x3dbbe0] 0x3de0e0 <nil> [] 0s} 127.0.0.1 33203 <nil> <nil>}
I1227 09:45:29.515896 353683 main.go:144] libmachine: About to run SSH command:
sudo hostname ha-513251-m02 && echo "ha-513251-m02" | sudo tee /etc/hostname
I1227 09:45:29.753613 353683 main.go:144] libmachine: SSH cmd err, output: <nil>: ha-513251-m02
I1227 09:45:29.753763 353683 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-513251-m02
I1227 09:45:29.802708 353683 main.go:144] libmachine: Using SSH client type: native
I1227 09:45:29.803015 353683 main.go:144] libmachine: &{{{<nil> 0 [] [] []} docker [0x3dbbe0] 0x3de0e0 <nil> [] 0s} 127.0.0.1 33203 <nil> <nil>}
I1227 09:45:29.803031 353683 main.go:144] libmachine: About to run SSH command:
if ! grep -xq '.*\sha-513251-m02' /etc/hosts; then
if grep -xq '127.0.1.1\s.*' /etc/hosts; then
sudo sed -i 's/^127.0.1.1\s.*/127.0.1.1 ha-513251-m02/g' /etc/hosts;
else
echo '127.0.1.1 ha-513251-m02' | sudo tee -a /etc/hosts;
fi
fi
I1227 09:45:30.026916 353683 main.go:144] libmachine: SSH cmd err, output: <nil>:
I1227 09:45:30.027002 353683 ubuntu.go:188] set auth options {CertDir:/home/jenkins/minikube-integration/22343-297941/.minikube CaCertPath:/home/jenkins/minikube-integration/22343-297941/.minikube/certs/ca.pem CaPrivateKeyPath:/home/jenkins/minikube-integration/22343-297941/.minikube/certs/ca-key.pem CaCertRemotePath:/etc/docker/ca.pem ServerCertPath:/home/jenkins/minikube-integration/22343-297941/.minikube/machines/server.pem ServerKeyPath:/home/jenkins/minikube-integration/22343-297941/.minikube/machines/server-key.pem ClientKeyPath:/home/jenkins/minikube-integration/22343-297941/.minikube/certs/key.pem ServerCertRemotePath:/etc/docker/server.pem ServerKeyRemotePath:/etc/docker/server-key.pem ClientCertPath:/home/jenkins/minikube-integration/22343-297941/.minikube/certs/cert.pem ServerCertSANs:[] StorePath:/home/jenkins/minikube-integration/22343-297941/.minikube}
I1227 09:45:30.027040 353683 ubuntu.go:190] setting up certificates
I1227 09:45:30.027088 353683 provision.go:84] configureAuth start
I1227 09:45:30.027213 353683 cli_runner.go:164] Run: docker container inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}" ha-513251-m02
I1227 09:45:30.061355 353683 provision.go:143] copyHostCerts
I1227 09:45:30.061395 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/ca.pem -> /home/jenkins/minikube-integration/22343-297941/.minikube/ca.pem
I1227 09:45:30.061429 353683 exec_runner.go:144] found /home/jenkins/minikube-integration/22343-297941/.minikube/ca.pem, removing ...
I1227 09:45:30.061436 353683 exec_runner.go:203] rm: /home/jenkins/minikube-integration/22343-297941/.minikube/ca.pem
I1227 09:45:30.061516 353683 exec_runner.go:151] cp: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/ca.pem --> /home/jenkins/minikube-integration/22343-297941/.minikube/ca.pem (1082 bytes)
I1227 09:45:30.061646 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/cert.pem -> /home/jenkins/minikube-integration/22343-297941/.minikube/cert.pem
I1227 09:45:30.061664 353683 exec_runner.go:144] found /home/jenkins/minikube-integration/22343-297941/.minikube/cert.pem, removing ...
I1227 09:45:30.061668 353683 exec_runner.go:203] rm: /home/jenkins/minikube-integration/22343-297941/.minikube/cert.pem
I1227 09:45:30.061698 353683 exec_runner.go:151] cp: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/cert.pem --> /home/jenkins/minikube-integration/22343-297941/.minikube/cert.pem (1123 bytes)
I1227 09:45:30.061741 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/key.pem -> /home/jenkins/minikube-integration/22343-297941/.minikube/key.pem
I1227 09:45:30.061761 353683 exec_runner.go:144] found /home/jenkins/minikube-integration/22343-297941/.minikube/key.pem, removing ...
I1227 09:45:30.061766 353683 exec_runner.go:203] rm: /home/jenkins/minikube-integration/22343-297941/.minikube/key.pem
I1227 09:45:30.061789 353683 exec_runner.go:151] cp: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/key.pem --> /home/jenkins/minikube-integration/22343-297941/.minikube/key.pem (1675 bytes)
I1227 09:45:30.061835 353683 provision.go:117] generating server cert: /home/jenkins/minikube-integration/22343-297941/.minikube/machines/server.pem ca-key=/home/jenkins/minikube-integration/22343-297941/.minikube/certs/ca.pem private-key=/home/jenkins/minikube-integration/22343-297941/.minikube/certs/ca-key.pem org=jenkins.ha-513251-m02 san=[127.0.0.1 192.168.49.3 ha-513251-m02 localhost minikube]
I1227 09:45:30.366138 353683 provision.go:177] copyRemoteCerts
I1227 09:45:30.366258 353683 ssh_runner.go:195] Run: sudo mkdir -p /etc/docker /etc/docker /etc/docker
I1227 09:45:30.366380 353683 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-513251-m02
I1227 09:45:30.384700 353683 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33203 SSHKeyPath:/home/jenkins/minikube-integration/22343-297941/.minikube/machines/ha-513251-m02/id_rsa Username:docker}
I1227 09:45:30.494344 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/ca.pem -> /etc/docker/ca.pem
I1227 09:45:30.494406 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/certs/ca.pem --> /etc/docker/ca.pem (1082 bytes)
I1227 09:45:30.530895 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/machines/server.pem -> /etc/docker/server.pem
I1227 09:45:30.530955 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/machines/server.pem --> /etc/docker/server.pem (1208 bytes)
I1227 09:45:30.561682 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/machines/server-key.pem -> /etc/docker/server-key.pem
I1227 09:45:30.561747 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/machines/server-key.pem --> /etc/docker/server-key.pem (1675 bytes)
I1227 09:45:30.591679 353683 provision.go:87] duration metric: took 564.557502ms to configureAuth
I1227 09:45:30.591755 353683 ubuntu.go:206] setting minikube options for container-runtime
I1227 09:45:30.592084 353683 config.go:182] Loaded profile config "ha-513251": Driver=docker, ContainerRuntime=crio, KubernetesVersion=v1.35.0
I1227 09:45:30.592246 353683 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-513251-m02
I1227 09:45:30.621605 353683 main.go:144] libmachine: Using SSH client type: native
I1227 09:45:30.621922 353683 main.go:144] libmachine: &{{{<nil> 0 [] [] []} docker [0x3dbbe0] 0x3de0e0 <nil> [] 0s} 127.0.0.1 33203 <nil> <nil>}
I1227 09:45:30.621937 353683 main.go:144] libmachine: About to run SSH command:
sudo mkdir -p /etc/sysconfig && printf %s "
CRIO_MINIKUBE_OPTIONS='--insecure-registry 10.96.0.0/12 '
" | sudo tee /etc/sysconfig/crio.minikube && sudo systemctl restart crio
I1227 09:45:31.635140 353683 main.go:144] libmachine: SSH cmd err, output: <nil>:
CRIO_MINIKUBE_OPTIONS='--insecure-registry 10.96.0.0/12 '
I1227 09:45:31.635164 353683 machine.go:97] duration metric: took 5.417238886s to provisionDockerMachine
I1227 09:45:31.635176 353683 start.go:293] postStartSetup for "ha-513251-m02" (driver="docker")
I1227 09:45:31.635186 353683 start.go:322] creating required directories: [/etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs]
I1227 09:45:31.635250 353683 ssh_runner.go:195] Run: sudo mkdir -p /etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs
I1227 09:45:31.635298 353683 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-513251-m02
I1227 09:45:31.672186 353683 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33203 SSHKeyPath:/home/jenkins/minikube-integration/22343-297941/.minikube/machines/ha-513251-m02/id_rsa Username:docker}
I1227 09:45:31.803466 353683 ssh_runner.go:195] Run: cat /etc/os-release
I1227 09:45:31.807580 353683 main.go:144] libmachine: Couldn't set key VERSION_CODENAME, no corresponding struct field found
I1227 09:45:31.807606 353683 info.go:137] Remote host: Debian GNU/Linux 12 (bookworm)
I1227 09:45:31.807617 353683 filesync.go:126] Scanning /home/jenkins/minikube-integration/22343-297941/.minikube/addons for local assets ...
I1227 09:45:31.807677 353683 filesync.go:126] Scanning /home/jenkins/minikube-integration/22343-297941/.minikube/files for local assets ...
I1227 09:45:31.807750 353683 filesync.go:149] local asset: /home/jenkins/minikube-integration/22343-297941/.minikube/files/etc/ssl/certs/2998112.pem -> 2998112.pem in /etc/ssl/certs
I1227 09:45:31.807757 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/files/etc/ssl/certs/2998112.pem -> /etc/ssl/certs/2998112.pem
I1227 09:45:31.807862 353683 ssh_runner.go:195] Run: sudo mkdir -p /etc/ssl/certs
I1227 09:45:31.825236 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/files/etc/ssl/certs/2998112.pem --> /etc/ssl/certs/2998112.pem (1708 bytes)
I1227 09:45:31.845471 353683 start.go:296] duration metric: took 210.280443ms for postStartSetup
I1227 09:45:31.845631 353683 ssh_runner.go:195] Run: sh -c "df -h /var | awk 'NR==2{print $5}'"
I1227 09:45:31.845704 353683 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-513251-m02
I1227 09:45:31.863181 353683 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33203 SSHKeyPath:/home/jenkins/minikube-integration/22343-297941/.minikube/machines/ha-513251-m02/id_rsa Username:docker}
I1227 09:45:31.978613 353683 ssh_runner.go:195] Run: sh -c "df -BG /var | awk 'NR==2{print $4}'"
I1227 09:45:31.988190 353683 fix.go:56] duration metric: took 6.287056138s for fixHost
I1227 09:45:31.988218 353683 start.go:83] releasing machines lock for "ha-513251-m02", held for 6.287109349s
I1227 09:45:31.988301 353683 cli_runner.go:164] Run: docker container inspect -f "{{range .NetworkSettings.Networks}}{{.IPAddress}},{{.GlobalIPv6Address}}{{end}}" ha-513251-m02
I1227 09:45:32.022351 353683 out.go:179] * Found network options:
I1227 09:45:32.025233 353683 out.go:179] - NO_PROXY=192.168.49.2
W1227 09:45:32.028060 353683 proxy.go:120] fail to check proxy env: Error ip not in block
W1227 09:45:32.028113 353683 proxy.go:120] fail to check proxy env: Error ip not in block
I1227 09:45:32.028186 353683 ssh_runner.go:195] Run: sudo sh -c "podman version >/dev/null"
I1227 09:45:32.028235 353683 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-513251-m02
I1227 09:45:32.028260 353683 ssh_runner.go:195] Run: curl -sS -m 2 https://registry.k8s.io/
I1227 09:45:32.028315 353683 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-513251-m02
I1227 09:45:32.062562 353683 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33203 SSHKeyPath:/home/jenkins/minikube-integration/22343-297941/.minikube/machines/ha-513251-m02/id_rsa Username:docker}
I1227 09:45:32.071385 353683 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33203 SSHKeyPath:/home/jenkins/minikube-integration/22343-297941/.minikube/machines/ha-513251-m02/id_rsa Username:docker}
I1227 09:45:32.418806 353683 ssh_runner.go:195] Run: sh -c "stat /etc/cni/net.d/*loopback.conf*"
W1227 09:45:32.560316 353683 cni.go:209] loopback cni configuration skipped: "/etc/cni/net.d/*loopback.conf*" not found
I1227 09:45:32.560399 353683 ssh_runner.go:195] Run: sudo find /etc/cni/net.d -maxdepth 1 -type f ( ( -name *bridge* -or -name *podman* ) -and -not -name *.mk_disabled ) -printf "%p, " -exec sh -c "sudo mv {} {}.mk_disabled" ;
I1227 09:45:32.576611 353683 cni.go:259] no active bridge cni configs found in "/etc/cni/net.d" - nothing to disable
I1227 09:45:32.576635 353683 start.go:496] detecting cgroup driver to use...
I1227 09:45:32.576667 353683 detect.go:187] detected "cgroupfs" cgroup driver on host os
I1227 09:45:32.576717 353683 ssh_runner.go:195] Run: sudo systemctl stop -f containerd
I1227 09:45:32.603470 353683 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I1227 09:45:32.627343 353683 docker.go:218] disabling cri-docker service (if available) ...
I1227 09:45:32.627407 353683 ssh_runner.go:195] Run: sudo systemctl stop -f cri-docker.socket
I1227 09:45:32.650889 353683 ssh_runner.go:195] Run: sudo systemctl stop -f cri-docker.service
I1227 09:45:32.671280 353683 ssh_runner.go:195] Run: sudo systemctl disable cri-docker.socket
I1227 09:45:32.901177 353683 ssh_runner.go:195] Run: sudo systemctl mask cri-docker.service
I1227 09:45:33.083402 353683 docker.go:234] disabling docker service ...
I1227 09:45:33.083516 353683 ssh_runner.go:195] Run: sudo systemctl stop -f docker.socket
I1227 09:45:33.102162 353683 ssh_runner.go:195] Run: sudo systemctl stop -f docker.service
I1227 09:45:33.117631 353683 ssh_runner.go:195] Run: sudo systemctl disable docker.socket
I1227 09:45:33.330335 353683 ssh_runner.go:195] Run: sudo systemctl mask docker.service
I1227 09:45:33.571932 353683 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service docker
I1227 09:45:33.588507 353683 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///var/run/crio/crio.sock
" | sudo tee /etc/crictl.yaml"
I1227 09:45:33.603417 353683 crio.go:59] configure cri-o to use "registry.k8s.io/pause:3.10.1" pause image...
I1227 09:45:33.603487 353683 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|^.*pause_image = .*$|pause_image = "registry.k8s.io/pause:3.10.1"|' /etc/crio/crio.conf.d/02-crio.conf"
I1227 09:45:33.613092 353683 crio.go:70] configuring cri-o to use "cgroupfs" as cgroup driver...
I1227 09:45:33.613161 353683 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|^.*cgroup_manager = .*$|cgroup_manager = "cgroupfs"|' /etc/crio/crio.conf.d/02-crio.conf"
I1227 09:45:33.622600 353683 ssh_runner.go:195] Run: sh -c "sudo sed -i '/conmon_cgroup = .*/d' /etc/crio/crio.conf.d/02-crio.conf"
I1227 09:45:33.632017 353683 ssh_runner.go:195] Run: sh -c "sudo sed -i '/cgroup_manager = .*/a conmon_cgroup = "pod"' /etc/crio/crio.conf.d/02-crio.conf"
I1227 09:45:33.641471 353683 ssh_runner.go:195] Run: sh -c "sudo rm -rf /etc/cni/net.mk"
I1227 09:45:33.650218 353683 ssh_runner.go:195] Run: sh -c "sudo sed -i '/^ *"net.ipv4.ip_unprivileged_port_start=.*"/d' /etc/crio/crio.conf.d/02-crio.conf"
I1227 09:45:33.659580 353683 ssh_runner.go:195] Run: sh -c "sudo grep -q "^ *default_sysctls" /etc/crio/crio.conf.d/02-crio.conf || sudo sed -i '/conmon_cgroup = .*/a default_sysctls = \[\n\]' /etc/crio/crio.conf.d/02-crio.conf"
I1227 09:45:33.675788 353683 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^default_sysctls *= *\[|&\n "net.ipv4.ip_unprivileged_port_start=0",|' /etc/crio/crio.conf.d/02-crio.conf"
I1227 09:45:33.690916 353683 ssh_runner.go:195] Run: sudo sysctl net.bridge.bridge-nf-call-iptables
I1227 09:45:33.699830 353683 ssh_runner.go:195] Run: sudo sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"
I1227 09:45:33.710022 353683 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1227 09:45:33.856695 353683 ssh_runner.go:195] Run: sudo systemctl restart crio
I1227 09:47:04.177050 353683 ssh_runner.go:235] Completed: sudo systemctl restart crio: (1m30.320263495s)
I1227 09:47:04.177079 353683 start.go:553] Will wait 60s for socket path /var/run/crio/crio.sock
I1227 09:47:04.177137 353683 ssh_runner.go:195] Run: stat /var/run/crio/crio.sock
I1227 09:47:04.181790 353683 start.go:574] Will wait 60s for crictl version
I1227 09:47:04.181861 353683 ssh_runner.go:195] Run: which crictl
I1227 09:47:04.185784 353683 ssh_runner.go:195] Run: sudo /usr/local/bin/crictl version
I1227 09:47:04.214501 353683 start.go:590] Version: 0.1.0
RuntimeName: cri-o
RuntimeVersion: 1.34.3
RuntimeApiVersion: v1
I1227 09:47:04.214588 353683 ssh_runner.go:195] Run: crio --version
I1227 09:47:04.244971 353683 ssh_runner.go:195] Run: crio --version
I1227 09:47:04.277197 353683 out.go:179] * Preparing Kubernetes v1.35.0 on CRI-O 1.34.3 ...
I1227 09:47:04.280209 353683 out.go:179] - env NO_PROXY=192.168.49.2
I1227 09:47:04.283165 353683 cli_runner.go:164] Run: docker network inspect ha-513251 --format "{"Name": "{{.Name}}","Driver": "{{.Driver}}","Subnet": "{{range .IPAM.Config}}{{.Subnet}}{{end}}","Gateway": "{{range .IPAM.Config}}{{.Gateway}}{{end}}","MTU": {{if (index .Options "com.docker.network.driver.mtu")}}{{(index .Options "com.docker.network.driver.mtu")}}{{else}}0{{end}}, "ContainerIPs": [{{range $k,$v := .Containers }}"{{$v.IPv4Address}}",{{end}}]}"
I1227 09:47:04.300447 353683 ssh_runner.go:195] Run: grep 192.168.49.1 host.minikube.internal$ /etc/hosts
I1227 09:47:04.304396 353683 ssh_runner.go:195] Run: /bin/bash -c "{ grep -v $'\thost.minikube.internal$' "/etc/hosts"; echo "192.168.49.1 host.minikube.internal"; } > /tmp/h.$$; sudo cp /tmp/h.$$ "/etc/hosts""
I1227 09:47:04.314914 353683 mustload.go:66] Loading cluster: ha-513251
I1227 09:47:04.315173 353683 config.go:182] Loaded profile config "ha-513251": Driver=docker, ContainerRuntime=crio, KubernetesVersion=v1.35.0
I1227 09:47:04.315461 353683 cli_runner.go:164] Run: docker container inspect ha-513251 --format={{.State.Status}}
I1227 09:47:04.333467 353683 host.go:66] Checking if "ha-513251" exists ...
I1227 09:47:04.333753 353683 certs.go:69] Setting up /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251 for IP: 192.168.49.3
I1227 09:47:04.333767 353683 certs.go:195] generating shared ca certs ...
I1227 09:47:04.333782 353683 certs.go:227] acquiring lock for ca certs: {Name:mkf0455840442d82ad2865a090879fc8de65e5e9 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1227 09:47:04.333906 353683 certs.go:236] skipping valid "minikubeCA" ca cert: /home/jenkins/minikube-integration/22343-297941/.minikube/ca.key
I1227 09:47:04.333952 353683 certs.go:236] skipping valid "proxyClientCA" ca cert: /home/jenkins/minikube-integration/22343-297941/.minikube/proxy-client-ca.key
I1227 09:47:04.333962 353683 certs.go:257] generating profile certs ...
I1227 09:47:04.334040 353683 certs.go:360] skipping valid signed profile cert regeneration for "minikube-user": /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/client.key
I1227 09:47:04.334105 353683 certs.go:360] skipping valid signed profile cert regeneration for "minikube": /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/apiserver.key.2d598068
I1227 09:47:04.334153 353683 certs.go:360] skipping valid signed profile cert regeneration for "aggregator": /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/proxy-client.key
I1227 09:47:04.334168 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/ca.crt -> /var/lib/minikube/certs/ca.crt
I1227 09:47:04.334198 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/ca.key -> /var/lib/minikube/certs/ca.key
I1227 09:47:04.334237 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/proxy-client-ca.crt -> /var/lib/minikube/certs/proxy-client-ca.crt
I1227 09:47:04.334248 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/proxy-client-ca.key -> /var/lib/minikube/certs/proxy-client-ca.key
I1227 09:47:04.334259 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/apiserver.crt -> /var/lib/minikube/certs/apiserver.crt
I1227 09:47:04.334275 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/apiserver.key -> /var/lib/minikube/certs/apiserver.key
I1227 09:47:04.334287 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/proxy-client.crt -> /var/lib/minikube/certs/proxy-client.crt
I1227 09:47:04.334306 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/proxy-client.key -> /var/lib/minikube/certs/proxy-client.key
I1227 09:47:04.334366 353683 certs.go:484] found cert: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/299811.pem (1338 bytes)
W1227 09:47:04.334408 353683 certs.go:480] ignoring /home/jenkins/minikube-integration/22343-297941/.minikube/certs/299811_empty.pem, impossibly tiny 0 bytes
I1227 09:47:04.334421 353683 certs.go:484] found cert: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/ca-key.pem (1675 bytes)
I1227 09:47:04.334448 353683 certs.go:484] found cert: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/ca.pem (1082 bytes)
I1227 09:47:04.334582 353683 certs.go:484] found cert: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/cert.pem (1123 bytes)
I1227 09:47:04.334618 353683 certs.go:484] found cert: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/key.pem (1675 bytes)
I1227 09:47:04.334672 353683 certs.go:484] found cert: /home/jenkins/minikube-integration/22343-297941/.minikube/files/etc/ssl/certs/2998112.pem (1708 bytes)
I1227 09:47:04.334711 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/files/etc/ssl/certs/2998112.pem -> /usr/share/ca-certificates/2998112.pem
I1227 09:47:04.334729 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/ca.crt -> /usr/share/ca-certificates/minikubeCA.pem
I1227 09:47:04.334741 353683 vm_assets.go:164] NewFileAsset: /home/jenkins/minikube-integration/22343-297941/.minikube/certs/299811.pem -> /usr/share/ca-certificates/299811.pem
I1227 09:47:04.334806 353683 cli_runner.go:164] Run: docker container inspect -f "'{{(index (index .NetworkSettings.Ports "22/tcp") 0).HostPort}}'" ha-513251
I1227 09:47:04.352745 353683 sshutil.go:53] new ssh client: &{IP:127.0.0.1 Port:33198 SSHKeyPath:/home/jenkins/minikube-integration/22343-297941/.minikube/machines/ha-513251/id_rsa Username:docker}
I1227 09:47:04.444298 353683 ssh_runner.go:195] Run: stat -c %s /var/lib/minikube/certs/sa.pub
I1227 09:47:04.448354 353683 ssh_runner.go:448] scp /var/lib/minikube/certs/sa.pub --> memory (451 bytes)
I1227 09:47:04.456699 353683 ssh_runner.go:195] Run: stat -c %s /var/lib/minikube/certs/sa.key
I1227 09:47:04.460541 353683 ssh_runner.go:448] scp /var/lib/minikube/certs/sa.key --> memory (1675 bytes)
I1227 09:47:04.469121 353683 ssh_runner.go:195] Run: stat -c %s /var/lib/minikube/certs/front-proxy-ca.crt
I1227 09:47:04.472996 353683 ssh_runner.go:448] scp /var/lib/minikube/certs/front-proxy-ca.crt --> memory (1123 bytes)
I1227 09:47:04.481446 353683 ssh_runner.go:195] Run: stat -c %s /var/lib/minikube/certs/front-proxy-ca.key
I1227 09:47:04.484933 353683 ssh_runner.go:448] scp /var/lib/minikube/certs/front-proxy-ca.key --> memory (1679 bytes)
I1227 09:47:04.493259 353683 ssh_runner.go:195] Run: stat -c %s /var/lib/minikube/certs/etcd/ca.crt
I1227 09:47:04.497027 353683 ssh_runner.go:448] scp /var/lib/minikube/certs/etcd/ca.crt --> memory (1094 bytes)
I1227 09:47:04.505596 353683 ssh_runner.go:195] Run: stat -c %s /var/lib/minikube/certs/etcd/ca.key
I1227 09:47:04.509294 353683 ssh_runner.go:448] scp /var/lib/minikube/certs/etcd/ca.key --> memory (1675 bytes)
I1227 09:47:04.517713 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/ca.crt --> /var/lib/minikube/certs/ca.crt (1111 bytes)
I1227 09:47:04.537012 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/ca.key --> /var/lib/minikube/certs/ca.key (1679 bytes)
I1227 09:47:04.556494 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/proxy-client-ca.crt --> /var/lib/minikube/certs/proxy-client-ca.crt (1119 bytes)
I1227 09:47:04.576418 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/proxy-client-ca.key --> /var/lib/minikube/certs/proxy-client-ca.key (1679 bytes)
I1227 09:47:04.597182 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/apiserver.crt --> /var/lib/minikube/certs/apiserver.crt (1432 bytes)
I1227 09:47:04.618229 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/apiserver.key --> /var/lib/minikube/certs/apiserver.key (1679 bytes)
I1227 09:47:04.641696 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/proxy-client.crt --> /var/lib/minikube/certs/proxy-client.crt (1147 bytes)
I1227 09:47:04.663252 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/proxy-client.key --> /var/lib/minikube/certs/proxy-client.key (1675 bytes)
I1227 09:47:04.684934 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/files/etc/ssl/certs/2998112.pem --> /usr/share/ca-certificates/2998112.pem (1708 bytes)
I1227 09:47:04.716644 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/ca.crt --> /usr/share/ca-certificates/minikubeCA.pem (1111 bytes)
I1227 09:47:04.737307 353683 ssh_runner.go:362] scp /home/jenkins/minikube-integration/22343-297941/.minikube/certs/299811.pem --> /usr/share/ca-certificates/299811.pem (1338 bytes)
I1227 09:47:04.758667 353683 ssh_runner.go:362] scp memory --> /var/lib/minikube/certs/sa.pub (451 bytes)
I1227 09:47:04.773792 353683 ssh_runner.go:362] scp memory --> /var/lib/minikube/certs/sa.key (1675 bytes)
I1227 09:47:04.788292 353683 ssh_runner.go:362] scp memory --> /var/lib/minikube/certs/front-proxy-ca.crt (1123 bytes)
I1227 09:47:04.802374 353683 ssh_runner.go:362] scp memory --> /var/lib/minikube/certs/front-proxy-ca.key (1679 bytes)
I1227 09:47:04.817583 353683 ssh_runner.go:362] scp memory --> /var/lib/minikube/certs/etcd/ca.crt (1094 bytes)
I1227 09:47:04.831128 353683 ssh_runner.go:362] scp memory --> /var/lib/minikube/certs/etcd/ca.key (1675 bytes)
I1227 09:47:04.845769 353683 ssh_runner.go:362] scp memory --> /var/lib/minikube/kubeconfig (728 bytes)
I1227 09:47:04.860041 353683 ssh_runner.go:195] Run: openssl version
I1227 09:47:04.866442 353683 ssh_runner.go:195] Run: sudo test -s /usr/share/ca-certificates/2998112.pem
I1227 09:47:04.874396 353683 ssh_runner.go:195] Run: sudo ln -fs /usr/share/ca-certificates/2998112.pem /etc/ssl/certs/2998112.pem
I1227 09:47:04.882193 353683 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/2998112.pem
I1227 09:47:04.886310 353683 certs.go:528] hashing: -rw-r--r-- 1 root root 1708 Dec 27 09:34 /usr/share/ca-certificates/2998112.pem
I1227 09:47:04.886373 353683 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/2998112.pem
I1227 09:47:04.928354 353683 ssh_runner.go:195] Run: sudo test -L /etc/ssl/certs/3ec20f2e.0
I1227 09:47:04.936052 353683 ssh_runner.go:195] Run: sudo test -s /usr/share/ca-certificates/minikubeCA.pem
I1227 09:47:04.943752 353683 ssh_runner.go:195] Run: sudo ln -fs /usr/share/ca-certificates/minikubeCA.pem /etc/ssl/certs/minikubeCA.pem
I1227 09:47:04.952048 353683 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/minikubeCA.pem
I1227 09:47:04.956067 353683 certs.go:528] hashing: -rw-r--r-- 1 root root 1111 Dec 27 09:30 /usr/share/ca-certificates/minikubeCA.pem
I1227 09:47:04.956176 353683 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/minikubeCA.pem
I1227 09:47:04.997608 353683 ssh_runner.go:195] Run: sudo test -L /etc/ssl/certs/b5213941.0
I1227 09:47:05.007408 353683 ssh_runner.go:195] Run: sudo test -s /usr/share/ca-certificates/299811.pem
I1227 09:47:05.017602 353683 ssh_runner.go:195] Run: sudo ln -fs /usr/share/ca-certificates/299811.pem /etc/ssl/certs/299811.pem
I1227 09:47:05.026017 353683 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/299811.pem
I1227 09:47:05.030271 353683 certs.go:528] hashing: -rw-r--r-- 1 root root 1338 Dec 27 09:34 /usr/share/ca-certificates/299811.pem
I1227 09:47:05.030427 353683 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/299811.pem
I1227 09:47:05.074213 353683 ssh_runner.go:195] Run: sudo test -L /etc/ssl/certs/51391683.0
I1227 09:47:05.082090 353683 ssh_runner.go:195] Run: stat /var/lib/minikube/certs/apiserver-kubelet-client.crt
I1227 09:47:05.086100 353683 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/apiserver-etcd-client.crt -checkend 86400
I1227 09:47:05.128461 353683 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/apiserver-kubelet-client.crt -checkend 86400
I1227 09:47:05.172974 353683 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/etcd/server.crt -checkend 86400
I1227 09:47:05.215663 353683 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/etcd/healthcheck-client.crt -checkend 86400
I1227 09:47:05.263541 353683 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/etcd/peer.crt -checkend 86400
I1227 09:47:05.307445 353683 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/front-proxy-client.crt -checkend 86400
I1227 09:47:05.354461 353683 kubeadm.go:935] updating node {m02 192.168.49.3 8443 v1.35.0 crio true true} ...
I1227 09:47:05.354578 353683 kubeadm.go:947] kubelet [Unit]
Wants=crio.service
[Service]
ExecStart=
ExecStart=/var/lib/minikube/binaries/v1.35.0/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --cgroups-per-qos=false --config=/var/lib/kubelet/config.yaml --enforce-node-allocatable= --hostname-override=ha-513251-m02 --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.168.49.3
[Install]
config:
{KubernetesVersion:v1.35.0 ClusterName:ha-513251 Namespace:default APIServerHAVIP:192.168.49.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:crio CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:}
I1227 09:47:05.354622 353683 kube-vip.go:115] generating kube-vip config ...
I1227 09:47:05.354681 353683 ssh_runner.go:195] Run: sudo sh -c "lsmod | grep ip_vs"
I1227 09:47:05.367621 353683 kube-vip.go:163] giving up enabling control-plane load-balancing as ipvs kernel modules appears not to be available: sudo sh -c "lsmod | grep ip_vs": Process exited with status 1
stdout:
stderr:
I1227 09:47:05.367701 353683 kube-vip.go:137] kube-vip config:
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
name: kube-vip
namespace: kube-system
spec:
containers:
- args:
- manager
env:
- name: vip_arp
value: "true"
- name: port
value: "8443"
- name: vip_nodename
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: vip_interface
value: eth0
- name: vip_cidr
value: "32"
- name: dns_mode
value: first
- name: cp_enable
value: "true"
- name: cp_namespace
value: kube-system
- name: vip_leaderelection
value: "true"
- name: vip_leasename
value: plndr-cp-lock
- name: vip_leaseduration
value: "5"
- name: vip_renewdeadline
value: "3"
- name: vip_retryperiod
value: "1"
- name: address
value: 192.168.49.254
- name: prometheus_server
value: :2112
image: ghcr.io/kube-vip/kube-vip:v1.0.3
imagePullPolicy: IfNotPresent
name: kube-vip
resources: {}
securityContext:
capabilities:
add:
- NET_ADMIN
- NET_RAW
volumeMounts:
- mountPath: /etc/kubernetes/admin.conf
name: kubeconfig
hostAliases:
- hostnames:
- kubernetes
ip: 127.0.0.1
hostNetwork: true
volumes:
- hostPath:
path: "/etc/kubernetes/admin.conf"
name: kubeconfig
status: {}
I1227 09:47:05.367789 353683 ssh_runner.go:195] Run: sudo ls /var/lib/minikube/binaries/v1.35.0
I1227 09:47:05.376110 353683 binaries.go:51] Found k8s binaries, skipping transfer
I1227 09:47:05.376225 353683 ssh_runner.go:195] Run: sudo mkdir -p /etc/systemd/system/kubelet.service.d /lib/systemd/system /etc/kubernetes/manifests
I1227 09:47:05.385227 353683 ssh_runner.go:362] scp memory --> /etc/systemd/system/kubelet.service.d/10-kubeadm.conf (363 bytes)
I1227 09:47:05.399058 353683 ssh_runner.go:362] scp memory --> /lib/systemd/system/kubelet.service (352 bytes)
I1227 09:47:05.412225 353683 ssh_runner.go:362] scp memory --> /etc/kubernetes/manifests/kube-vip.yaml (1358 bytes)
I1227 09:47:05.433740 353683 ssh_runner.go:195] Run: grep 192.168.49.254 control-plane.minikube.internal$ /etc/hosts
I1227 09:47:05.438137 353683 ssh_runner.go:195] Run: /bin/bash -c "{ grep -v $'\tcontrol-plane.minikube.internal$' "/etc/hosts"; echo "192.168.49.254 control-plane.minikube.internal"; } > /tmp/h.$$; sudo cp /tmp/h.$$ "/etc/hosts""
I1227 09:47:05.449160 353683 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1227 09:47:05.584548 353683 ssh_runner.go:195] Run: sudo systemctl start kubelet
I1227 09:47:05.598901 353683 start.go:236] Will wait 6m0s for node &{Name:m02 IP:192.168.49.3 Port:8443 KubernetesVersion:v1.35.0 ContainerRuntime:crio ControlPlane:true Worker:true}
I1227 09:47:05.599307 353683 config.go:182] Loaded profile config "ha-513251": Driver=docker, ContainerRuntime=crio, KubernetesVersion=v1.35.0
I1227 09:47:05.602962 353683 out.go:179] * Verifying Kubernetes components...
I1227 09:47:05.605544 353683 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1227 09:47:05.743183 353683 ssh_runner.go:195] Run: sudo systemctl start kubelet
I1227 09:47:05.759331 353683 kapi.go:59] client config for ha-513251: &rest.Config{Host:"https://192.168.49.254:8443", APIPath:"", ContentConfig:rest.ContentConfig{AcceptContentTypes:"", ContentType:"", GroupVersion:(*schema.GroupVersion)(nil), NegotiatedSerializer:runtime.NegotiatedSerializer(nil)}, Username:"", Password:"", BearerToken:"", BearerTokenFile:"", Impersonate:rest.ImpersonationConfig{UserName:"", UID:"", Groups:[]string(nil), Extra:map[string][]string(nil)}, AuthProvider:<nil>, AuthConfigPersister:rest.AuthProviderConfigPersister(nil), ExecProvider:<nil>, TLSClientConfig:rest.sanitizedTLSClientConfig{Insecure:false, ServerName:"", CertFile:"/home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/client.crt", KeyFile:"/home/jenkins/minikube-integration/22343-297941/.minikube/profiles/ha-513251/client.key", CAFile:"/home/jenkins/minikube-integration/22343-297941/.minikube/ca.crt", CertData:[]uint8(nil), KeyData:[]uint8(nil), CAData:[]uint8(nil), NextProtos:[]string(n
il)}, UserAgent:"", DisableCompression:false, Transport:http.RoundTripper(nil), WrapTransport:(transport.WrapperFunc)(0x1f30470), QPS:0, Burst:0, RateLimiter:flowcontrol.RateLimiter(nil), WarningHandler:rest.WarningHandler(nil), WarningHandlerWithContext:rest.WarningHandlerWithContext(nil), Timeout:0, Dial:(func(context.Context, string, string) (net.Conn, error))(nil), Proxy:(func(*http.Request) (*url.URL, error))(nil)}
W1227 09:47:05.759399 353683 kubeadm.go:492] Overriding stale ClientConfig host https://192.168.49.254:8443 with https://192.168.49.2:8443
I1227 09:47:05.759628 353683 node_ready.go:35] waiting up to 6m0s for node "ha-513251-m02" to be "Ready" ...
I1227 09:47:36.941690 353683 with_retry.go:234] "Got a Retry-After response" delay="1s" attempt=1 url="https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02"
W1227 09:47:36.942141 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused - error from a previous attempt: unexpected EOF
W1227 09:47:39.260337 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:47:41.261089 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:47:43.760342 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:47:45.760773 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
I1227 09:48:49.689213 353683 with_retry.go:234] "Got a Retry-After response" delay="1s" attempt=1 url="https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02"
W1227 09:48:49.689567 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused - error from a previous attempt: read tcp 192.168.49.1:59702->192.168.49.2:8443: read: connection reset by peer
W1227 09:48:51.761173 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:48:54.260275 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:48:56.260764 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:48:58.260950 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:49:00.261274 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:49:02.761180 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:49:05.261164 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:49:07.760850 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:49:09.761126 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:49:12.261097 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
I1227 09:50:17.401158 353683 with_retry.go:234] "Got a Retry-After response" delay="1s" attempt=1 url="https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02"
W1227 09:50:17.401610 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused - error from a previous attempt: unexpected EOF
W1227 09:50:19.760255 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:50:21.761012 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:50:23.761193 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:50:26.260515 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:50:28.760208 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:50:30.760293 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:50:33.261011 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:50:35.760559 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:50:38.260275 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:50:40.760183 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:50:42.761015 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:50:45.260386 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:50:47.760256 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:50:50.260156 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:50:52.261185 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:50:54.760529 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:50:56.760914 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:50:58.761079 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:51:01.260894 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:51:03.261105 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:51:05.760186 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:51:07.761034 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:51:09.761091 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:51:21.261754 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": net/http: TLS handshake timeout
W1227 09:51:31.267176 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": net/http: TLS handshake timeout
W1227 09:51:33.760222 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:51:35.760997 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:51:37.761041 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:51:40.260968 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:51:42.261084 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:51:44.761080 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:51:47.260390 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:51:49.760248 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:51:51.760405 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:51:54.260216 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:51:56.260474 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:51:58.261041 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:00.760814 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:03.260223 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:05.261042 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:07.760972 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:09.761080 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:12.261019 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:14.760290 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:17.260953 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:19.261221 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:21.760250 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:23.760454 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:25.760687 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:28.260326 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:30.260374 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:32.760183 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:34.761068 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:37.261218 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:39.760434 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:41.760931 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:44.260297 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:46.260721 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:48.261137 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:50.760243 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:53.260234 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:52:55.261149 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": dial tcp 192.168.49.2:8443: connect: connection refused
W1227 09:53:05.759756 353683 node_ready.go:55] error getting node "ha-513251-m02" condition "Ready" status (will retry): Get "https://192.168.49.2:8443/api/v1/nodes/ha-513251-m02": context deadline exceeded
I1227 09:53:05.759808 353683 node_ready.go:38] duration metric: took 6m0.000151574s for node "ha-513251-m02" to be "Ready" ...
I1227 09:53:05.763182 353683 out.go:203]
W1227 09:53:05.766205 353683 out.go:285] X Exiting due to GUEST_START: failed to start node: adding node: wait 6m0s for node: waiting for node to be ready: WaitNodeCondition: context deadline exceeded
W1227 09:53:05.766232 353683 out.go:285] *
W1227 09:53:05.766486 353683 out.go:308] ╭─────────────────────────────────────────────────────────────────────────────────────────────╮
│ │
│ * If the above advice does not help, please let us know: │
│ https://github.com/kubernetes/minikube/issues/new/choose │
│ │
│ * Please run `minikube logs --file=logs.txt` and attach logs.txt to the GitHub issue. │
│ │
╰─────────────────────────────────────────────────────────────────────────────────────────────╯
I1227 09:53:05.771303 353683 out.go:203]
==> CRI-O <==
Dec 27 09:52:16 ha-513251 crio[669]: time="2025-12-27T09:52:16.704047446Z" level=info msg="Checking image status: registry.k8s.io/kube-controller-manager:v1.35.0" id=df594719-7494-4e4b-8b96-ff6b50da7943 name=/runtime.v1.ImageService/ImageStatus
Dec 27 09:52:16 ha-513251 crio[669]: time="2025-12-27T09:52:16.705214137Z" level=info msg="Creating container: kube-system/kube-controller-manager-ha-513251/kube-controller-manager" id=1db5f92f-e3de-4051-b1e8-f4a521df221b name=/runtime.v1.RuntimeService/CreateContainer
Dec 27 09:52:16 ha-513251 crio[669]: time="2025-12-27T09:52:16.705367008Z" level=info msg="Allowed annotations are specified for workload [io.containers.trace-syscall]"
Dec 27 09:52:16 ha-513251 crio[669]: time="2025-12-27T09:52:16.714658246Z" level=info msg="Allowed annotations are specified for workload [io.containers.trace-syscall]"
Dec 27 09:52:16 ha-513251 crio[669]: time="2025-12-27T09:52:16.715313645Z" level=info msg="Allowed annotations are specified for workload [io.containers.trace-syscall]"
Dec 27 09:52:16 ha-513251 crio[669]: time="2025-12-27T09:52:16.733436178Z" level=info msg="Created container 4694ec899710cc574db82e0ddd1b1b6e98a94664666cb5377ea1dbb9d5c5b2d3: kube-system/kube-controller-manager-ha-513251/kube-controller-manager" id=1db5f92f-e3de-4051-b1e8-f4a521df221b name=/runtime.v1.RuntimeService/CreateContainer
Dec 27 09:52:16 ha-513251 crio[669]: time="2025-12-27T09:52:16.734077735Z" level=info msg="Starting container: 4694ec899710cc574db82e0ddd1b1b6e98a94664666cb5377ea1dbb9d5c5b2d3" id=eeb97769-30ec-478a-bc87-4f69060f31cf name=/runtime.v1.RuntimeService/StartContainer
Dec 27 09:52:16 ha-513251 crio[669]: time="2025-12-27T09:52:16.736017513Z" level=info msg="Started container" PID=1255 containerID=4694ec899710cc574db82e0ddd1b1b6e98a94664666cb5377ea1dbb9d5c5b2d3 description=kube-system/kube-controller-manager-ha-513251/kube-controller-manager id=eeb97769-30ec-478a-bc87-4f69060f31cf name=/runtime.v1.RuntimeService/StartContainer sandboxID=77c125920c3f982d94f3bc7831d664d32af1a76fa71da885b790a865c893eed1
Dec 27 09:52:27 ha-513251 conmon[1253]: conmon 4694ec899710cc574db8 <ninfo>: container 1255 exited with status 1
Dec 27 09:52:27 ha-513251 crio[669]: time="2025-12-27T09:52:27.770243941Z" level=info msg="Removing container: 2d96035cdd3ce31e663f85efbc2212452112dbdba91bb658842c231359c318e6" id=da55eb3c-7976-48ba-a75f-a39739218412 name=/runtime.v1.RuntimeService/RemoveContainer
Dec 27 09:52:27 ha-513251 crio[669]: time="2025-12-27T09:52:27.777709971Z" level=info msg="Error loading conmon cgroup of container 2d96035cdd3ce31e663f85efbc2212452112dbdba91bb658842c231359c318e6: cgroup deleted" id=da55eb3c-7976-48ba-a75f-a39739218412 name=/runtime.v1.RuntimeService/RemoveContainer
Dec 27 09:52:27 ha-513251 crio[669]: time="2025-12-27T09:52:27.780784669Z" level=info msg="Removed container 2d96035cdd3ce31e663f85efbc2212452112dbdba91bb658842c231359c318e6: kube-system/kube-controller-manager-ha-513251/kube-controller-manager" id=da55eb3c-7976-48ba-a75f-a39739218412 name=/runtime.v1.RuntimeService/RemoveContainer
Dec 27 09:52:56 ha-513251 crio[669]: time="2025-12-27T09:52:56.701490281Z" level=info msg="Checking image status: registry.k8s.io/kube-apiserver:v1.35.0" id=079299f4-9d89-491a-8d17-2a3678443aa9 name=/runtime.v1.ImageService/ImageStatus
Dec 27 09:52:56 ha-513251 crio[669]: time="2025-12-27T09:52:56.702675032Z" level=info msg="Checking image status: registry.k8s.io/kube-apiserver:v1.35.0" id=8c967218-255a-4dbf-a2a1-3e466c02b6e8 name=/runtime.v1.ImageService/ImageStatus
Dec 27 09:52:56 ha-513251 crio[669]: time="2025-12-27T09:52:56.703767818Z" level=info msg="Creating container: kube-system/kube-apiserver-ha-513251/kube-apiserver" id=0fd3892f-ad02-44ff-b1fb-2d96da8680c0 name=/runtime.v1.RuntimeService/CreateContainer
Dec 27 09:52:56 ha-513251 crio[669]: time="2025-12-27T09:52:56.70386432Z" level=info msg="Allowed annotations are specified for workload [io.containers.trace-syscall]"
Dec 27 09:52:56 ha-513251 crio[669]: time="2025-12-27T09:52:56.712337252Z" level=info msg="Allowed annotations are specified for workload [io.containers.trace-syscall]"
Dec 27 09:52:56 ha-513251 crio[669]: time="2025-12-27T09:52:56.712883974Z" level=info msg="Allowed annotations are specified for workload [io.containers.trace-syscall]"
Dec 27 09:52:56 ha-513251 crio[669]: time="2025-12-27T09:52:56.73007927Z" level=info msg="Created container 7e32d77299b93ef151c5217dafe3ce3478c4c71878af32039cb46a1c45e07908: kube-system/kube-apiserver-ha-513251/kube-apiserver" id=0fd3892f-ad02-44ff-b1fb-2d96da8680c0 name=/runtime.v1.RuntimeService/CreateContainer
Dec 27 09:52:56 ha-513251 crio[669]: time="2025-12-27T09:52:56.730833829Z" level=info msg="Starting container: 7e32d77299b93ef151c5217dafe3ce3478c4c71878af32039cb46a1c45e07908" id=0cf41e99-8376-4017-8d87-0efd593514d8 name=/runtime.v1.RuntimeService/StartContainer
Dec 27 09:52:56 ha-513251 crio[669]: time="2025-12-27T09:52:56.740489445Z" level=info msg="Started container" PID=1272 containerID=7e32d77299b93ef151c5217dafe3ce3478c4c71878af32039cb46a1c45e07908 description=kube-system/kube-apiserver-ha-513251/kube-apiserver id=0cf41e99-8376-4017-8d87-0efd593514d8 name=/runtime.v1.RuntimeService/StartContainer sandboxID=a678fda46be5a152fa8932be97637587d68f62be01ebcbef8a2cc06dc92777be
Dec 27 09:53:17 ha-513251 conmon[1269]: conmon 7e32d77299b93ef151c5 <ninfo>: container 1272 exited with status 255
Dec 27 09:53:17 ha-513251 crio[669]: time="2025-12-27T09:53:17.881086782Z" level=info msg="Removing container: 1ec411df6464eb13f470690685876070ae1d07d5525d5abf026a035ab3f6cf53" id=3eb0280a-0821-455b-a788-d923172551a2 name=/runtime.v1.RuntimeService/RemoveContainer
Dec 27 09:53:17 ha-513251 crio[669]: time="2025-12-27T09:53:17.888327431Z" level=info msg="Error loading conmon cgroup of container 1ec411df6464eb13f470690685876070ae1d07d5525d5abf026a035ab3f6cf53: cgroup deleted" id=3eb0280a-0821-455b-a788-d923172551a2 name=/runtime.v1.RuntimeService/RemoveContainer
Dec 27 09:53:17 ha-513251 crio[669]: time="2025-12-27T09:53:17.891415742Z" level=info msg="Removed container 1ec411df6464eb13f470690685876070ae1d07d5525d5abf026a035ab3f6cf53: kube-system/kube-apiserver-ha-513251/kube-apiserver" id=3eb0280a-0821-455b-a788-d923172551a2 name=/runtime.v1.RuntimeService/RemoveContainer
==> container status <==
CONTAINER IMAGE CREATED STATE NAME ATTEMPT POD ID POD NAMESPACE
7e32d77299b93 c3fcf259c473a57a5d7da116e29161904491091743512d27467c907c5516f856 29 seconds ago Exited kube-apiserver 7 a678fda46be5a kube-apiserver-ha-513251 kube-system
4694ec899710c 88898f1d1a62a3ea9db5d4d099dee7aa52ebe8191016c5b3c721388a309983e0 About a minute ago Exited kube-controller-manager 9 77c125920c3f9 kube-controller-manager-ha-513251 kube-system
3e2f79bfcc297 271e49a0ebc56647476845128fcd2a73bb138beeca3878cc3bf52b4ff1172a57 2 minutes ago Running etcd 3 0b4fdbfc50d52 etcd-ha-513251 kube-system
f69e010776644 ddc8422d4d35a6fc66c34be61e24df795e5cebf197eb546f62740d0bafef874f 8 minutes ago Running kube-scheduler 2 8f6686604e637 kube-scheduler-ha-513251 kube-system
f7e841ab1c87c 271e49a0ebc56647476845128fcd2a73bb138beeca3878cc3bf52b4ff1172a57 8 minutes ago Exited etcd 2 0b4fdbfc50d52 etcd-ha-513251 kube-system
cc9aea908d640 28c5662932f6032ee4faba083d9c2af90232797e1d4f89d9892cb92b26fec299 8 minutes ago Running kube-vip 2 9c394d0758080 kube-vip-ha-513251 kube-system
==> describe nodes <==
command /bin/bash -c "sudo /var/lib/minikube/binaries/v1.35.0/kubectl describe nodes --kubeconfig=/var/lib/minikube/kubeconfig" failed with error: /bin/bash -c "sudo /var/lib/minikube/binaries/v1.35.0/kubectl describe nodes --kubeconfig=/var/lib/minikube/kubeconfig": Process exited with status 1
stdout:
stderr:
The connection to the server localhost:8443 was refused - did you specify the right host or port?
==> dmesg <==
[Dec27 08:17] ACPI: SRAT not present
[ +0.000000] ACPI: SRAT not present
[ +0.000000] SPI driver altr_a10sr has no spi_device_id for altr,a10sr
[ +0.015479] device-mapper: core: CONFIG_IMA_DISABLE_HTABLE is disabled. Duplicate IMA measurements will not be recorded in the IMA log.
[ +0.516409] systemd[1]: Configuration file /run/systemd/system/netplan-ovs-cleanup.service is marked world-inaccessible. This has no effect as configuration data is accessible via APIs without restrictions. Proceeding anyway.
[ +0.034238] systemd[1]: /lib/systemd/system/snapd.service:23: Unknown key name 'RestartMode' in section 'Service', ignoring.
[ +0.771451] ena 0000:00:05.0: LLQ is not supported Fallback to host mode policy.
[ +6.481009] kauditd_printk_skb: 39 callbacks suppressed
[Dec27 08:29] hrtimer: interrupt took 43410871 ns
[Dec27 09:29] kauditd_printk_skb: 8 callbacks suppressed
[Dec27 09:30] overlayfs: idmapped layers are currently not supported
[ +0.068519] kmem.limit_in_bytes is deprecated and will be removed. Please report your usecase to linux-mm@kvack.org if you depend on this functionality.
[Dec27 09:34] overlayfs: idmapped layers are currently not supported
[ +46.937326] overlayfs: idmapped layers are currently not supported
[Dec27 09:37] overlayfs: idmapped layers are currently not supported
[Dec27 09:38] overlayfs: idmapped layers are currently not supported
[Dec27 09:39] overlayfs: idmapped layers are currently not supported
[Dec27 09:41] overlayfs: idmapped layers are currently not supported
[Dec27 09:42] overlayfs: idmapped layers are currently not supported
[Dec27 09:43] overlayfs: idmapped layers are currently not supported
[ +3.379616] overlayfs: idmapped layers are currently not supported
[ +26.881821] overlayfs: idmapped layers are currently not supported
[Dec27 09:44] overlayfs: idmapped layers are currently not supported
[Dec27 09:45] overlayfs: idmapped layers are currently not supported
[ +3.382865] overlayfs: idmapped layers are currently not supported
==> etcd [3e2f79bfcc29755ed4c6ee91cec29fd05896c608e4d72883a5b019d5f8609903] <==
{"level":"info","ts":"2025-12-27T09:53:23.261658Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"aec36adc501070cc became pre-candidate at term 4"}
{"level":"info","ts":"2025-12-27T09:53:23.261681Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"aec36adc501070cc [logterm: 4, index: 2866] sent MsgPreVote request to 8e7fd81d8c1de671 at term 4"}
{"level":"info","ts":"2025-12-27T09:53:23.261727Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"aec36adc501070cc received MsgPreVoteResp from aec36adc501070cc at term 4"}
{"level":"info","ts":"2025-12-27T09:53:23.261746Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"aec36adc501070cc has received 1 MsgPreVoteResp votes and 0 vote rejections"}
{"level":"warn","ts":"2025-12-27T09:53:23.341131Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8128042260754447398,"retry-timeout":"500ms"}
{"level":"warn","ts":"2025-12-27T09:53:23.842273Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8128042260754447398,"retry-timeout":"500ms"}
{"level":"warn","ts":"2025-12-27T09:53:24.343090Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8128042260754447398,"retry-timeout":"500ms"}
{"level":"warn","ts":"2025-12-27T09:53:24.843240Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8128042260754447398,"retry-timeout":"500ms"}
{"level":"info","ts":"2025-12-27T09:53:24.861510Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"aec36adc501070cc is starting a new election at term 4"}
{"level":"info","ts":"2025-12-27T09:53:24.861557Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"aec36adc501070cc became pre-candidate at term 4"}
{"level":"info","ts":"2025-12-27T09:53:24.861577Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"aec36adc501070cc [logterm: 4, index: 2866] sent MsgPreVote request to 8e7fd81d8c1de671 at term 4"}
{"level":"info","ts":"2025-12-27T09:53:24.861606Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"aec36adc501070cc received MsgPreVoteResp from aec36adc501070cc at term 4"}
{"level":"info","ts":"2025-12-27T09:53:24.861616Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"aec36adc501070cc has received 1 MsgPreVoteResp votes and 0 vote rejections"}
{"level":"warn","ts":"2025-12-27T09:53:25.105493Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_RAFT_MESSAGE","remote-peer-id":"8e7fd81d8c1de671","rtt":"0s","error":"dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-12-27T09:53:25.105566Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_SNAPSHOT","remote-peer-id":"8e7fd81d8c1de671","rtt":"0s","error":"dial tcp 192.168.49.3:2380: connect: connection refused"}
{"level":"warn","ts":"2025-12-27T09:53:25.343930Z","caller":"etcdserver/v3_server.go:911","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":8128042260754447398,"retry-timeout":"500ms"}
{"level":"warn","ts":"2025-12-27T09:53:25.834493Z","caller":"etcdserver/v3_server.go:923","msg":"timed out waiting for read index response (local node might have slow network)","timeout":"7s"}
{"level":"warn","ts":"2025-12-27T09:53:25.834596Z","caller":"txn/util.go:93","msg":"apply request took too long","took":"7.000333125s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"","error":"etcdserver: request timed out"}
{"level":"info","ts":"2025-12-27T09:53:25.834623Z","caller":"traceutil/trace.go:172","msg":"trace[2092953664] range","detail":"{range_begin:; range_end:; }","duration":"7.000376874s","start":"2025-12-27T09:53:18.834234Z","end":"2025-12-27T09:53:25.834611Z","steps":["trace[2092953664] 'agreement among raft nodes before linearized reading' (duration: 7.000330728s)"],"step_count":1}
{"level":"error","ts":"2025-12-27T09:53:25.834682Z","caller":"etcdhttp/health.go:345","msg":"Health check error","path":"/readyz","reason":"[+]data_corruption ok\n[+]serializable_read ok\n[-]linearizable_read failed: etcdserver: request timed out\n[+]non_learner ok\n","status-code":503,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp.(*CheckRegistry).installRootHTTPEndpoint.newHealthHandler.func2\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/health.go:345\nnet/http.HandlerFunc.ServeHTTP\n\tnet/http/server.go:2294\nnet/http.(*ServeMux).ServeHTTP\n\tnet/http/server.go:2822\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:3301\nnet/http.(*conn).serve\n\tnet/http/server.go:2102"}
{"level":"info","ts":"2025-12-27T09:53:26.461387Z","logger":"raft","caller":"v3@v3.6.0/raft.go:988","msg":"aec36adc501070cc is starting a new election at term 4"}
{"level":"info","ts":"2025-12-27T09:53:26.461441Z","logger":"raft","caller":"v3@v3.6.0/raft.go:930","msg":"aec36adc501070cc became pre-candidate at term 4"}
{"level":"info","ts":"2025-12-27T09:53:26.461464Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1064","msg":"aec36adc501070cc [logterm: 4, index: 2866] sent MsgPreVote request to 8e7fd81d8c1de671 at term 4"}
{"level":"info","ts":"2025-12-27T09:53:26.461493Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1077","msg":"aec36adc501070cc received MsgPreVoteResp from aec36adc501070cc at term 4"}
{"level":"info","ts":"2025-12-27T09:53:26.461504Z","logger":"raft","caller":"v3@v3.6.0/raft.go:1693","msg":"aec36adc501070cc has received 1 MsgPreVoteResp votes and 0 vote rejections"}
==> etcd [f7e841ab1c87c3a73fb0fa9774a7d5540fae4454f87f94803231876049f07db7] <==
{"level":"info","ts":"2025-12-27T09:50:39.850450Z","caller":"osutil/interrupt_unix.go:65","msg":"received signal; shutting down","signal":"terminated"}
{"level":"info","ts":"2025-12-27T09:50:39.850492Z","caller":"embed/etcd.go:426","msg":"closing etcd server","name":"ha-513251","data-dir":"/var/lib/minikube/etcd","advertise-peer-urls":["https://192.168.49.2:2380"],"advertise-client-urls":["https://192.168.49.2:2379"]}
{"level":"error","ts":"2025-12-27T09:50:39.850587Z","caller":"embed/etcd.go:912","msg":"setting up serving from embedded etcd failed.","error":"http: Server closed","stacktrace":"go.etcd.io/etcd/server/v3/embed.(*Etcd).errHandler\n\tgo.etcd.io/etcd/server/v3/embed/etcd.go:912\ngo.etcd.io/etcd/server/v3/embed.(*serveCtx).startHandler.func1\n\tgo.etcd.io/etcd/server/v3/embed/serve.go:90"}
{"level":"error","ts":"2025-12-27T09:50:39.852081Z","caller":"embed/etcd.go:912","msg":"setting up serving from embedded etcd failed.","error":"http: Server closed","stacktrace":"go.etcd.io/etcd/server/v3/embed.(*Etcd).errHandler\n\tgo.etcd.io/etcd/server/v3/embed/etcd.go:912\ngo.etcd.io/etcd/server/v3/embed.(*serveCtx).startHandler.func1\n\tgo.etcd.io/etcd/server/v3/embed/serve.go:90"}
{"level":"error","ts":"2025-12-27T09:50:39.853588Z","caller":"embed/etcd.go:912","msg":"setting up serving from embedded etcd failed.","error":"accept tcp 127.0.0.1:2381: use of closed network connection","stacktrace":"go.etcd.io/etcd/server/v3/embed.(*Etcd).errHandler\n\tgo.etcd.io/etcd/server/v3/embed/etcd.go:912\ngo.etcd.io/etcd/server/v3/embed.(*Etcd).startHandler.func1\n\tgo.etcd.io/etcd/server/v3/embed/etcd.go:906"}
{"level":"info","ts":"2025-12-27T09:50:39.853628Z","caller":"etcdserver/server.go:1288","msg":"skipped leadership transfer; local server is not leader","local-member-id":"aec36adc501070cc","current-leader-member-id":"0"}
{"level":"info","ts":"2025-12-27T09:50:39.853662Z","caller":"etcdserver/server.go:2335","msg":"server has stopped; stopping cluster version's monitor"}
{"level":"info","ts":"2025-12-27T09:50:39.853729Z","caller":"etcdserver/server.go:2358","msg":"server has stopped; stopping storage version's monitor"}
{"level":"info","ts":"2025-12-27T09:50:39.853751Z","caller":"rafthttp/peer.go:316","msg":"stopping remote peer","remote-peer-id":"8e7fd81d8c1de671"}
{"level":"info","ts":"2025-12-27T09:50:39.853765Z","caller":"rafthttp/stream.go:293","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","remote-peer-id":"8e7fd81d8c1de671"}
{"level":"warn","ts":"2025-12-27T09:50:39.853767Z","caller":"embed/serve.go:245","msg":"stopping secure grpc server due to error","error":"accept tcp 127.0.0.1:2379: use of closed network connection"}
{"level":"info","ts":"2025-12-27T09:50:39.853782Z","caller":"rafthttp/stream.go:293","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream Message","remote-peer-id":"8e7fd81d8c1de671"}
{"level":"warn","ts":"2025-12-27T09:50:39.853783Z","caller":"embed/serve.go:247","msg":"stopped secure grpc server due to error","error":"accept tcp 127.0.0.1:2379: use of closed network connection"}
{"level":"error","ts":"2025-12-27T09:50:39.853792Z","caller":"embed/etcd.go:912","msg":"setting up serving from embedded etcd failed.","error":"accept tcp 127.0.0.1:2379: use of closed network connection","stacktrace":"go.etcd.io/etcd/server/v3/embed.(*Etcd).errHandler\n\tgo.etcd.io/etcd/server/v3/embed/etcd.go:912\ngo.etcd.io/etcd/server/v3/embed.(*Etcd).startHandler.func1\n\tgo.etcd.io/etcd/server/v3/embed/etcd.go:906"}
{"level":"info","ts":"2025-12-27T09:50:39.853813Z","caller":"rafthttp/pipeline.go:85","msg":"stopped HTTP pipelining with remote peer","local-member-id":"aec36adc501070cc","remote-peer-id":"8e7fd81d8c1de671"}
{"level":"warn","ts":"2025-12-27T09:50:39.853823Z","caller":"embed/serve.go:245","msg":"stopping secure grpc server due to error","error":"accept tcp 192.168.49.2:2379: use of closed network connection"}
{"level":"info","ts":"2025-12-27T09:50:39.853829Z","caller":"rafthttp/stream.go:441","msg":"stopped stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"aec36adc501070cc","remote-peer-id":"8e7fd81d8c1de671"}
{"level":"warn","ts":"2025-12-27T09:50:39.853834Z","caller":"embed/serve.go:247","msg":"stopped secure grpc server due to error","error":"accept tcp 192.168.49.2:2379: use of closed network connection"}
{"level":"info","ts":"2025-12-27T09:50:39.853843Z","caller":"rafthttp/stream.go:441","msg":"stopped stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"aec36adc501070cc","remote-peer-id":"8e7fd81d8c1de671"}
{"level":"error","ts":"2025-12-27T09:50:39.853842Z","caller":"embed/etcd.go:912","msg":"setting up serving from embedded etcd failed.","error":"accept tcp 192.168.49.2:2379: use of closed network connection","stacktrace":"go.etcd.io/etcd/server/v3/embed.(*Etcd).errHandler\n\tgo.etcd.io/etcd/server/v3/embed/etcd.go:912\ngo.etcd.io/etcd/server/v3/embed.(*Etcd).startHandler.func1\n\tgo.etcd.io/etcd/server/v3/embed/etcd.go:906"}
{"level":"info","ts":"2025-12-27T09:50:39.853851Z","caller":"rafthttp/peer.go:321","msg":"stopped remote peer","remote-peer-id":"8e7fd81d8c1de671"}
{"level":"info","ts":"2025-12-27T09:50:39.860164Z","caller":"embed/etcd.go:621","msg":"stopping serving peer traffic","address":"192.168.49.2:2380"}
{"level":"error","ts":"2025-12-27T09:50:39.860448Z","caller":"embed/etcd.go:912","msg":"setting up serving from embedded etcd failed.","error":"accept tcp 192.168.49.2:2380: use of closed network connection","stacktrace":"go.etcd.io/etcd/server/v3/embed.(*Etcd).errHandler\n\tgo.etcd.io/etcd/server/v3/embed/etcd.go:912\ngo.etcd.io/etcd/server/v3/embed.(*Etcd).startHandler.func1\n\tgo.etcd.io/etcd/server/v3/embed/etcd.go:906"}
{"level":"info","ts":"2025-12-27T09:50:39.860488Z","caller":"embed/etcd.go:626","msg":"stopped serving peer traffic","address":"192.168.49.2:2380"}
{"level":"info","ts":"2025-12-27T09:50:39.860499Z","caller":"embed/etcd.go:428","msg":"closed etcd server","name":"ha-513251","data-dir":"/var/lib/minikube/etcd","advertise-peer-urls":["https://192.168.49.2:2380"],"advertise-client-urls":["https://192.168.49.2:2379"]}
==> kernel <==
09:53:26 up 1:35, 0 user, load average: 0.30, 0.80, 1.66
Linux ha-513251 5.15.0-1084-aws #91~20.04.1-Ubuntu SMP Fri May 2 07:00:04 UTC 2025 aarch64 GNU/Linux
PRETTY_NAME="Debian GNU/Linux 12 (bookworm)"
==> kube-apiserver [7e32d77299b93ef151c5217dafe3ce3478c4c71878af32039cb46a1c45e07908] <==
I1227 09:52:56.791839 1 options.go:263] external host was not specified, using 192.168.49.2
I1227 09:52:56.794835 1 server.go:150] Version: v1.35.0
I1227 09:52:56.794953 1 server.go:152] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
W1227 09:52:57.278394 1 logging.go:55] [core] [Channel #1 SubChannel #2]grpc: addrConn.createTransport failed to connect to {Addr: "127.0.0.1:2379", ServerName: "127.0.0.1:2379", BalancerAttributes: {"<%!p(pickfirstleaf.managedByPickfirstKeyType={})>": "<%!p(bool=true)>" }}. Err: connection error: desc = "transport: authentication handshake failed: context canceled"
W1227 09:52:57.279880 1 logging.go:55] [core] [Channel #4 SubChannel #5]grpc: addrConn.createTransport failed to connect to {Addr: "127.0.0.1:2379", ServerName: "127.0.0.1:2379", BalancerAttributes: {"<%!p(pickfirstleaf.managedByPickfirstKeyType={})>": "<%!p(bool=true)>" }}. Err: connection error: desc = "transport: Error while dialing: dial tcp 127.0.0.1:2379: operation was canceled"
I1227 09:52:57.280532 1 shared_informer.go:349] "Waiting for caches to sync" controller="node_authorizer"
I1227 09:52:57.284066 1 shared_informer.go:370] "Waiting for caches to sync"
I1227 09:52:57.287400 1 plugins.go:157] Loaded 14 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,PodTopologyLabels,MutatingAdmissionPolicy,MutatingAdmissionWebhook.
I1227 09:52:57.287488 1 plugins.go:160] Loaded 14 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,PodSecurity,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,ClusterTrustBundleAttest,CertificateSubjectRestriction,NodeDeclaredFeatureValidator,ValidatingAdmissionPolicy,ValidatingAdmissionWebhook,ResourceQuota.
I1227 09:52:57.287738 1 instance.go:240] Using reconciler: lease
W1227 09:52:57.289397 1 logging.go:55] [core] [Channel #7 SubChannel #8]grpc: addrConn.createTransport failed to connect to {Addr: "127.0.0.1:2379", ServerName: "127.0.0.1:2379", BalancerAttributes: {"<%!p(pickfirstleaf.managedByPickfirstKeyType={})>": "<%!p(bool=true)>" }}. Err: connection error: desc = "transport: authentication handshake failed: context canceled"
W1227 09:53:17.278022 1 logging.go:55] [core] [Channel #1 SubChannel #3]grpc: addrConn.createTransport failed to connect to {Addr: "127.0.0.1:2379", ServerName: "127.0.0.1:2379", BalancerAttributes: {"<%!p(pickfirstleaf.managedByPickfirstKeyType={})>": "<%!p(bool=true)>" }}. Err: connection error: desc = "transport: authentication handshake failed: context canceled"
W1227 09:53:17.280084 1 logging.go:55] [core] [Channel #4 SubChannel #6]grpc: addrConn.createTransport failed to connect to {Addr: "127.0.0.1:2379", ServerName: "127.0.0.1:2379", BalancerAttributes: {"<%!p(pickfirstleaf.managedByPickfirstKeyType={})>": "<%!p(bool=true)>" }}. Err: connection error: desc = "transport: authentication handshake failed: context deadline exceeded"
F1227 09:53:17.288757 1 instance.go:233] Error creating leases: error creating storage factory: context deadline exceeded
W1227 09:53:17.288844 1 logging.go:55] [core] [Channel #7 SubChannel #9]grpc: addrConn.createTransport failed to connect to {Addr: "127.0.0.1:2379", ServerName: "127.0.0.1:2379", BalancerAttributes: {"<%!p(pickfirstleaf.managedByPickfirstKeyType={})>": "<%!p(bool=true)>" }}. Err: connection error: desc = "transport: authentication handshake failed: context canceled"
==> kube-controller-manager [4694ec899710cc574db82e0ddd1b1b6e98a94664666cb5377ea1dbb9d5c5b2d3] <==
I1227 09:52:17.366708 1 serving.go:386] Generated self-signed cert in-memory
I1227 09:52:17.376666 1 controllermanager.go:189] "Starting" version="v1.35.0"
I1227 09:52:17.376702 1 controllermanager.go:191] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I1227 09:52:17.378190 1 dynamic_cafile_content.go:161] "Starting controller" name="request-header::/var/lib/minikube/certs/front-proxy-ca.crt"
I1227 09:52:17.378332 1 dynamic_cafile_content.go:161] "Starting controller" name="client-ca-bundle::/var/lib/minikube/certs/ca.crt"
I1227 09:52:17.378381 1 secure_serving.go:211] Serving securely on 127.0.0.1:10257
I1227 09:52:17.378538 1 tlsconfig.go:243] "Starting DynamicServingCertificateController"
E1227 09:52:27.380746 1 controllermanager.go:250] "Error building controller context" err="failed to wait for apiserver being healthy: timed out waiting for the condition: failed to get apiserver /healthz status: Get \"https://192.168.49.2:8443/healthz\": dial tcp 192.168.49.2:8443: connect: connection refused"
==> kube-scheduler [f69e010776644f8005f4cd92f4774d5dc92d62b50dadf798020d9d8db93f52a7] <==
E1227 09:49:23.577558 1 reflector.go:204] "Failed to watch" err="failed to list *v1.DeviceClass: deviceclasses.resource.k8s.io is forbidden: User \"system:kube-scheduler\" cannot list resource \"deviceclasses\" in API group \"resource.k8s.io\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.DeviceClass"
E1227 09:49:25.962683 1 reflector.go:204] "Failed to watch" err="failed to list *v1.CSINode: csinodes.storage.k8s.io is forbidden: User \"system:kube-scheduler\" cannot list resource \"csinodes\" in API group \"storage.k8s.io\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.CSINode"
E1227 09:49:27.718552 1 reflector.go:204] "Failed to watch" err="failed to list *v1.ResourceSlice: resourceslices.resource.k8s.io is forbidden: User \"system:kube-scheduler\" cannot list resource \"resourceslices\" in API group \"resource.k8s.io\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.ResourceSlice"
E1227 09:49:28.729793 1 reflector.go:204] "Failed to watch" err="failed to list *v1.PersistentVolumeClaim: persistentvolumeclaims is forbidden: User \"system:kube-scheduler\" cannot list resource \"persistentvolumeclaims\" in API group \"\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.PersistentVolumeClaim"
E1227 09:49:30.037535 1 reflector.go:204] "Failed to watch" err="failed to list *v1.CSIStorageCapacity: csistoragecapacities.storage.k8s.io is forbidden: User \"system:kube-scheduler\" cannot list resource \"csistoragecapacities\" in API group \"storage.k8s.io\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.CSIStorageCapacity"
E1227 09:49:34.733607 1 reflector.go:204] "Failed to watch" err="failed to list *v1.StorageClass: storageclasses.storage.k8s.io is forbidden: User \"system:kube-scheduler\" cannot list resource \"storageclasses\" in API group \"storage.k8s.io\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.StorageClass"
E1227 09:49:34.929599 1 reflector.go:204] "Failed to watch" err="failed to list *v1.CSIDriver: csidrivers.storage.k8s.io is forbidden: User \"system:kube-scheduler\" cannot list resource \"csidrivers\" in API group \"storage.k8s.io\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.CSIDriver"
E1227 09:49:35.092788 1 reflector.go:204] "Failed to watch" err="failed to list *v1.VolumeAttachment: volumeattachments.storage.k8s.io is forbidden: User \"system:kube-scheduler\" cannot list resource \"volumeattachments\" in API group \"storage.k8s.io\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.VolumeAttachment"
E1227 09:49:35.988125 1 reflector.go:204] "Failed to watch" err="failed to list *v1.Node: nodes is forbidden: User \"system:kube-scheduler\" cannot list resource \"nodes\" in API group \"\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Node"
E1227 09:49:38.452688 1 reflector.go:204] "Failed to watch" err="failed to list *v1.PersistentVolume: persistentvolumes is forbidden: User \"system:kube-scheduler\" cannot list resource \"persistentvolumes\" in API group \"\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.PersistentVolume"
E1227 09:49:38.595135 1 reflector.go:204] "Failed to watch" err="failed to list *v1.PodDisruptionBudget: poddisruptionbudgets.policy is forbidden: User \"system:kube-scheduler\" cannot list resource \"poddisruptionbudgets\" in API group \"policy\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.PodDisruptionBudget"
E1227 09:49:44.386717 1 reflector.go:204] "Failed to watch" err="failed to list *v1.Namespace: namespaces is forbidden: User \"system:kube-scheduler\" cannot list resource \"namespaces\" in API group \"\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Namespace"
E1227 09:49:45.790610 1 reflector.go:204] "Failed to watch" err="failed to list *v1.Pod: pods is forbidden: User \"system:kube-scheduler\" cannot list resource \"pods\" in API group \"\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Pod"
E1227 09:49:49.151819 1 reflector.go:204] "Failed to watch" err="failed to list *v1.ReplicaSet: replicasets.apps is forbidden: User \"system:kube-scheduler\" cannot list resource \"replicasets\" in API group \"apps\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.ReplicaSet"
E1227 09:50:03.444648 1 reflector.go:204] "Failed to watch" err="failed to list *v1.ConfigMap: configmaps \"extension-apiserver-authentication\" is forbidden: User \"system:kube-scheduler\" cannot list resource \"configmaps\" in API group \"\" in the namespace \"kube-system\"" logger="UnhandledError" reflector="runtime/asm_arm64.s:1268" type="*v1.ConfigMap"
E1227 09:50:03.690487 1 reflector.go:204] "Failed to watch" err="failed to list *v1.Service: services is forbidden: User \"system:kube-scheduler\" cannot list resource \"services\" in API group \"\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.Service"
E1227 09:50:03.834739 1 reflector.go:204] "Failed to watch" err="failed to list *v1.StatefulSet: statefulsets.apps is forbidden: User \"system:kube-scheduler\" cannot list resource \"statefulsets\" in API group \"apps\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.StatefulSet"
E1227 09:50:04.045150 1 reflector.go:204] "Failed to watch" err="failed to list *v1.ReplicationController: replicationcontrollers is forbidden: User \"system:kube-scheduler\" cannot list resource \"replicationcontrollers\" in API group \"\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.ReplicationController"
E1227 09:50:07.144662 1 reflector.go:204] "Failed to watch" err="failed to list *v1.ResourceClaim: resourceclaims.resource.k8s.io is forbidden: User \"system:kube-scheduler\" cannot list resource \"resourceclaims\" in API group \"resource.k8s.io\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.ResourceClaim"
E1227 09:50:07.401271 1 reflector.go:204] "Failed to watch" err="failed to list *v1.CSINode: csinodes.storage.k8s.io is forbidden: User \"system:kube-scheduler\" cannot list resource \"csinodes\" in API group \"storage.k8s.io\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.CSINode"
E1227 09:50:07.608201 1 reflector.go:204] "Failed to watch" err="failed to list *v1.CSIDriver: csidrivers.storage.k8s.io is forbidden: User \"system:kube-scheduler\" cannot list resource \"csidrivers\" in API group \"storage.k8s.io\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.CSIDriver"
E1227 09:50:10.033692 1 reflector.go:204] "Failed to watch" err="failed to list *v1.CSIStorageCapacity: csistoragecapacities.storage.k8s.io is forbidden: User \"system:kube-scheduler\" cannot list resource \"csistoragecapacities\" in API group \"storage.k8s.io\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.CSIStorageCapacity"
E1227 09:50:13.073103 1 reflector.go:204] "Failed to watch" err="failed to list *v1.PersistentVolumeClaim: persistentvolumeclaims is forbidden: User \"system:kube-scheduler\" cannot list resource \"persistentvolumeclaims\" in API group \"\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.PersistentVolumeClaim"
E1227 09:50:14.539398 1 reflector.go:204] "Failed to watch" err="failed to list *v1.StorageClass: storageclasses.storage.k8s.io is forbidden: User \"system:kube-scheduler\" cannot list resource \"storageclasses\" in API group \"storage.k8s.io\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.StorageClass"
E1227 09:50:15.853200 1 reflector.go:204] "Failed to watch" err="failed to list *v1.VolumeAttachment: volumeattachments.storage.k8s.io is forbidden: User \"system:kube-scheduler\" cannot list resource \"volumeattachments\" in API group \"storage.k8s.io\" at the cluster scope" logger="UnhandledError" reflector="k8s.io/client-go/informers/factory.go:161" type="*v1.VolumeAttachment"
==> kubelet <==
Dec 27 09:53:25 ha-513251 kubelet[805]: E1227 09:53:25.069823 805 reconstruct.go:188] "Failed to get Node status to reconstruct device paths" err="Get \"https://192.168.49.2:8443/api/v1/nodes/ha-513251\": dial tcp 192.168.49.2:8443: connect: connection refused"
Dec 27 09:53:25 ha-513251 kubelet[805]: E1227 09:53:25.170366 805 reconstruct.go:188] "Failed to get Node status to reconstruct device paths" err="Get \"https://192.168.49.2:8443/api/v1/nodes/ha-513251\": dial tcp 192.168.49.2:8443: connect: connection refused"
Dec 27 09:53:25 ha-513251 kubelet[805]: E1227 09:53:25.270911 805 reconstruct.go:188] "Failed to get Node status to reconstruct device paths" err="Get \"https://192.168.49.2:8443/api/v1/nodes/ha-513251\": dial tcp 192.168.49.2:8443: connect: connection refused"
Dec 27 09:53:25 ha-513251 kubelet[805]: E1227 09:53:25.372344 805 reconstruct.go:188] "Failed to get Node status to reconstruct device paths" err="Get \"https://192.168.49.2:8443/api/v1/nodes/ha-513251\": dial tcp 192.168.49.2:8443: connect: connection refused"
Dec 27 09:53:25 ha-513251 kubelet[805]: E1227 09:53:25.473214 805 reconstruct.go:188] "Failed to get Node status to reconstruct device paths" err="Get \"https://192.168.49.2:8443/api/v1/nodes/ha-513251\": dial tcp 192.168.49.2:8443: connect: connection refused"
Dec 27 09:53:25 ha-513251 kubelet[805]: E1227 09:53:25.574787 805 reconstruct.go:188] "Failed to get Node status to reconstruct device paths" err="Get \"https://192.168.49.2:8443/api/v1/nodes/ha-513251\": dial tcp 192.168.49.2:8443: connect: connection refused"
Dec 27 09:53:25 ha-513251 kubelet[805]: E1227 09:53:25.675821 805 reconstruct.go:188] "Failed to get Node status to reconstruct device paths" err="Get \"https://192.168.49.2:8443/api/v1/nodes/ha-513251\": dial tcp 192.168.49.2:8443: connect: connection refused"
Dec 27 09:53:25 ha-513251 kubelet[805]: E1227 09:53:25.776812 805 reconstruct.go:188] "Failed to get Node status to reconstruct device paths" err="Get \"https://192.168.49.2:8443/api/v1/nodes/ha-513251\": dial tcp 192.168.49.2:8443: connect: connection refused"
Dec 27 09:53:25 ha-513251 kubelet[805]: E1227 09:53:25.878178 805 reconstruct.go:188] "Failed to get Node status to reconstruct device paths" err="Get \"https://192.168.49.2:8443/api/v1/nodes/ha-513251\": dial tcp 192.168.49.2:8443: connect: connection refused"
Dec 27 09:53:25 ha-513251 kubelet[805]: E1227 09:53:25.979423 805 reconstruct.go:188] "Failed to get Node status to reconstruct device paths" err="Get \"https://192.168.49.2:8443/api/v1/nodes/ha-513251\": dial tcp 192.168.49.2:8443: connect: connection refused"
Dec 27 09:53:26 ha-513251 kubelet[805]: E1227 09:53:26.006248 805 kubelet.go:3336] "No need to create a mirror pod, since failed to get node info from the cluster" err="node \"ha-513251\" not found" node="ha-513251"
Dec 27 09:53:26 ha-513251 kubelet[805]: E1227 09:53:26.006693 805 prober_manager.go:197] "Startup probe already exists for container" pod="kube-system/kube-apiserver-ha-513251" containerName="kube-apiserver"
Dec 27 09:53:26 ha-513251 kubelet[805]: I1227 09:53:26.006907 805 scope.go:122] "RemoveContainer" containerID="7e32d77299b93ef151c5217dafe3ce3478c4c71878af32039cb46a1c45e07908"
Dec 27 09:53:26 ha-513251 kubelet[805]: E1227 09:53:26.007251 805 pod_workers.go:1324] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-apiserver\" with CrashLoopBackOff: \"back-off 2m40s restarting failed container=kube-apiserver pod=kube-apiserver-ha-513251_kube-system(0da9dd3b4fd74f83fca53d342cc4832b)\"" pod="kube-system/kube-apiserver-ha-513251" podUID="0da9dd3b4fd74f83fca53d342cc4832b"
Dec 27 09:53:26 ha-513251 kubelet[805]: E1227 09:53:26.080581 805 reconstruct.go:188] "Failed to get Node status to reconstruct device paths" err="Get \"https://192.168.49.2:8443/api/v1/nodes/ha-513251\": dial tcp 192.168.49.2:8443: connect: connection refused"
Dec 27 09:53:26 ha-513251 kubelet[805]: E1227 09:53:26.182086 805 reconstruct.go:188] "Failed to get Node status to reconstruct device paths" err="Get \"https://192.168.49.2:8443/api/v1/nodes/ha-513251\": dial tcp 192.168.49.2:8443: connect: connection refused"
Dec 27 09:53:26 ha-513251 kubelet[805]: E1227 09:53:26.283078 805 reconstruct.go:188] "Failed to get Node status to reconstruct device paths" err="Get \"https://192.168.49.2:8443/api/v1/nodes/ha-513251\": dial tcp 192.168.49.2:8443: connect: connection refused"
Dec 27 09:53:26 ha-513251 kubelet[805]: I1227 09:53:26.299132 805 kubelet_node_status.go:74] "Attempting to register node" node="ha-513251"
Dec 27 09:53:26 ha-513251 kubelet[805]: E1227 09:53:26.299522 805 kubelet_node_status.go:106] "Unable to register node with API server" err="Post \"https://192.168.49.2:8443/api/v1/nodes\": dial tcp 192.168.49.2:8443: connect: connection refused" node="ha-513251"
Dec 27 09:53:26 ha-513251 kubelet[805]: E1227 09:53:26.384422 805 reconstruct.go:188] "Failed to get Node status to reconstruct device paths" err="Get \"https://192.168.49.2:8443/api/v1/nodes/ha-513251\": dial tcp 192.168.49.2:8443: connect: connection refused"
Dec 27 09:53:26 ha-513251 kubelet[805]: E1227 09:53:26.485387 805 reconstruct.go:188] "Failed to get Node status to reconstruct device paths" err="Get \"https://192.168.49.2:8443/api/v1/nodes/ha-513251\": dial tcp 192.168.49.2:8443: connect: connection refused"
Dec 27 09:53:26 ha-513251 kubelet[805]: E1227 09:53:26.586078 805 reconstruct.go:188] "Failed to get Node status to reconstruct device paths" err="Get \"https://192.168.49.2:8443/api/v1/nodes/ha-513251\": dial tcp 192.168.49.2:8443: connect: connection refused"
Dec 27 09:53:26 ha-513251 kubelet[805]: E1227 09:53:26.687021 805 reconstruct.go:188] "Failed to get Node status to reconstruct device paths" err="Get \"https://192.168.49.2:8443/api/v1/nodes/ha-513251\": dial tcp 192.168.49.2:8443: connect: connection refused"
Dec 27 09:53:26 ha-513251 kubelet[805]: E1227 09:53:26.788090 805 reconstruct.go:188] "Failed to get Node status to reconstruct device paths" err="Get \"https://192.168.49.2:8443/api/v1/nodes/ha-513251\": dial tcp 192.168.49.2:8443: connect: connection refused"
Dec 27 09:53:26 ha-513251 kubelet[805]: E1227 09:53:26.888857 805 reconstruct.go:188] "Failed to get Node status to reconstruct device paths" err="Get \"https://192.168.49.2:8443/api/v1/nodes/ha-513251\": dial tcp 192.168.49.2:8443: connect: connection refused"
-- /stdout --
helpers_test.go:263: (dbg) Run: out/minikube-linux-arm64 status --format={{.APIServer}} -p ha-513251 -n ha-513251
helpers_test.go:263: (dbg) Non-zero exit: out/minikube-linux-arm64 status --format={{.APIServer}} -p ha-513251 -n ha-513251: exit status 2 (324.215185ms)
-- stdout --
Stopped
-- /stdout --
helpers_test.go:263: status error: exit status 2 (may be ok)
helpers_test.go:265: "ha-513251" apiserver is not running, skipping kubectl commands (state="Stopped")
--- FAIL: TestMultiControlPlane/serial/DegradedAfterClusterRestart (2.20s)