=== RUN TestMultiControlPlane/serial/DegradedAfterClusterRestart
ha_test.go:390: (dbg) Run: out/minikube-darwin-amd64 profile list --output json
ha_test.go:413: expected profile "ha-253000" in json of 'profile list' to have "Degraded" status but have "Stopped" status. got *"{\"invalid\":[],\"valid\":[{\"Name\":\"ha-253000\",\"Status\":\"Stopped\",\"Config\":{\"Name\":\"ha-253000\",\"KeepContext\":false,\"EmbedCerts\":false,\"MinikubeISO\":\"https://storage.googleapis.com/minikube-builds/iso/19616/minikube-v1.34.0-1726156389-19616-amd64.iso\",\"KicBaseImage\":\"gcr.io/k8s-minikube/kicbase-builds:v0.0.45-1726156396-19616@sha256:66b06a42534e914a5c8ad765d7508a93a34031939ec9a6b3a818ef0a444ff889\",\"Memory\":2200,\"CPUs\":2,\"DiskSize\":20000,\"Driver\":\"hyperkit\",\"HyperkitVpnKitSock\":\"\",\"HyperkitVSockPorts\":[],\"DockerEnv\":null,\"ContainerVolumeMounts\":null,\"InsecureRegistry\":null,\"RegistryMirror\":[],\"HostOnlyCIDR\":\"192.168.59.1/24\",\"HypervVirtualSwitch\":\"\",\"HypervUseExternalSwitch\":false,\"HypervExternalAdapter\":\"\",\"KVMNetwork\":\"default\",\"KVMQemuURI\":\"qemu:///system\",\"KVMGPU\":false,\"KVMHidden\":false,\"KVMNUMACoun
t\":1,\"APIServerPort\":8443,\"DockerOpt\":null,\"DisableDriverMounts\":false,\"NFSShare\":[],\"NFSSharesRoot\":\"/nfsshares\",\"UUID\":\"\",\"NoVTXCheck\":false,\"DNSProxy\":false,\"HostDNSResolver\":true,\"HostOnlyNicType\":\"virtio\",\"NatNicType\":\"virtio\",\"SSHIPAddress\":\"\",\"SSHUser\":\"root\",\"SSHKey\":\"\",\"SSHPort\":22,\"KubernetesConfig\":{\"KubernetesVersion\":\"v1.31.1\",\"ClusterName\":\"ha-253000\",\"Namespace\":\"default\",\"APIServerHAVIP\":\"192.169.0.254\",\"APIServerName\":\"minikubeCA\",\"APIServerNames\":null,\"APIServerIPs\":null,\"DNSDomain\":\"cluster.local\",\"ContainerRuntime\":\"docker\",\"CRISocket\":\"\",\"NetworkPlugin\":\"cni\",\"FeatureGates\":\"\",\"ServiceCIDR\":\"10.96.0.0/12\",\"ImageRepository\":\"\",\"LoadBalancerStartIP\":\"\",\"LoadBalancerEndIP\":\"\",\"CustomIngressCert\":\"\",\"RegistryAliases\":\"\",\"ExtraOptions\":null,\"ShouldLoadCachedImages\":true,\"EnableDefaultCNI\":false,\"CNI\":\"\"},\"Nodes\":[{\"Name\":\"\",\"IP\":\"192.169.0.117\",\"Port\":8443,\"
KubernetesVersion\":\"v1.31.1\",\"ContainerRuntime\":\"docker\",\"ControlPlane\":true,\"Worker\":true},{\"Name\":\"m02\",\"IP\":\"192.169.0.118\",\"Port\":8443,\"KubernetesVersion\":\"v1.31.1\",\"ContainerRuntime\":\"docker\",\"ControlPlane\":true,\"Worker\":true},{\"Name\":\"m04\",\"IP\":\"192.169.0.120\",\"Port\":0,\"KubernetesVersion\":\"v1.31.1\",\"ContainerRuntime\":\"docker\",\"ControlPlane\":false,\"Worker\":true}],\"Addons\":{\"ambassador\":false,\"auto-pause\":false,\"cloud-spanner\":false,\"csi-hostpath-driver\":false,\"dashboard\":false,\"default-storageclass\":false,\"efk\":false,\"freshpod\":false,\"gcp-auth\":false,\"gvisor\":false,\"headlamp\":false,\"helm-tiller\":false,\"inaccel\":false,\"ingress\":false,\"ingress-dns\":false,\"inspektor-gadget\":false,\"istio\":false,\"istio-provisioner\":false,\"kong\":false,\"kubeflow\":false,\"kubevirt\":false,\"logviewer\":false,\"metallb\":false,\"metrics-server\":false,\"nvidia-device-plugin\":false,\"nvidia-driver-installer\":false,\"nvidia-gpu-device
-plugin\":false,\"olm\":false,\"pod-security-policy\":false,\"portainer\":false,\"registry\":false,\"registry-aliases\":false,\"registry-creds\":false,\"storage-provisioner\":false,\"storage-provisioner-gluster\":false,\"storage-provisioner-rancher\":false,\"volcano\":false,\"volumesnapshots\":false,\"yakd\":false},\"CustomAddonImages\":null,\"CustomAddonRegistries\":null,\"VerifyComponents\":{\"apiserver\":true,\"apps_running\":true,\"default_sa\":true,\"extra\":true,\"kubelet\":true,\"node_ready\":true,\"system_pods\":true},\"StartHostTimeout\":360000000000,\"ScheduledStop\":null,\"ExposedPorts\":[],\"ListenAddress\":\"\",\"Network\":\"\",\"Subnet\":\"\",\"MultiNodeRequested\":true,\"ExtraDisks\":0,\"CertExpiration\":94608000000000000,\"Mount\":false,\"MountString\":\"/Users:/minikube-host\",\"Mount9PVersion\":\"9p2000.L\",\"MountGID\":\"docker\",\"MountIP\":\"\",\"MountMSize\":262144,\"MountOptions\":[],\"MountPort\":0,\"MountType\":\"9p\",\"MountUID\":\"docker\",\"BinaryMirror\":\"\",\"DisableOptimization
s\":false,\"DisableMetrics\":false,\"CustomQemuFirmwarePath\":\"\",\"SocketVMnetClientPath\":\"\",\"SocketVMnetPath\":\"\",\"StaticIP\":\"\",\"SSHAuthSock\":\"\",\"SSHAgentPID\":0,\"GPUs\":\"\",\"AutoPauseInterval\":60000000000},\"Active\":false,\"ActiveKubeContext\":true}]}"*. args: "out/minikube-darwin-amd64 profile list --output json"
helpers_test.go:222: -----------------------post-mortem--------------------------------
helpers_test.go:239: (dbg) Run: out/minikube-darwin-amd64 status --format={{.Host}} -p ha-253000 -n ha-253000
helpers_test.go:239: (dbg) Non-zero exit: out/minikube-darwin-amd64 status --format={{.Host}} -p ha-253000 -n ha-253000: exit status 2 (152.849199ms)
-- stdout --
Running
-- /stdout --
helpers_test.go:239: status error: exit status 2 (may be ok)
helpers_test.go:244: <<< TestMultiControlPlane/serial/DegradedAfterClusterRestart FAILED: start of post-mortem logs <<<
helpers_test.go:245: ======> post-mortem[TestMultiControlPlane/serial/DegradedAfterClusterRestart]: minikube logs <======
helpers_test.go:247: (dbg) Run: out/minikube-darwin-amd64 -p ha-253000 logs -n 25
helpers_test.go:247: (dbg) Done: out/minikube-darwin-amd64 -p ha-253000 logs -n 25: (2.1104221s)
helpers_test.go:252: TestMultiControlPlane/serial/DegradedAfterClusterRestart logs:
-- stdout --
==> Audit <==
|---------|-----------------------------------------------------------------------------------------------------------------------------|-----------|---------|---------|---------------------|---------------------|
| Command | Args | Profile | User | Version | Start Time | End Time |
|---------|-----------------------------------------------------------------------------------------------------------------------------|-----------|---------|---------|---------------------|---------------------|
| cp | ha-253000 cp ha-253000-m03:/home/docker/cp-test.txt | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:38 PDT | 12 Sep 24 16:38 PDT |
| | ha-253000-m04:/home/docker/cp-test_ha-253000-m03_ha-253000-m04.txt | | | | | |
| ssh | ha-253000 ssh -n | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:38 PDT | 12 Sep 24 16:38 PDT |
| | ha-253000-m03 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| ssh | ha-253000 ssh -n ha-253000-m04 sudo cat | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:38 PDT | 12 Sep 24 16:38 PDT |
| | /home/docker/cp-test_ha-253000-m03_ha-253000-m04.txt | | | | | |
| cp | ha-253000 cp testdata/cp-test.txt | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:38 PDT | 12 Sep 24 16:38 PDT |
| | ha-253000-m04:/home/docker/cp-test.txt | | | | | |
| ssh | ha-253000 ssh -n | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:38 PDT | 12 Sep 24 16:38 PDT |
| | ha-253000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| cp | ha-253000 cp ha-253000-m04:/home/docker/cp-test.txt | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:38 PDT | 12 Sep 24 16:38 PDT |
| | /var/folders/vq/yhv778t970xgml0dzm5fdwlr0000gp/T/TestMultiControlPlaneserialCopyFile443392125/001/cp-test_ha-253000-m04.txt | | | | | |
| ssh | ha-253000 ssh -n | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:38 PDT | 12 Sep 24 16:38 PDT |
| | ha-253000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| cp | ha-253000 cp ha-253000-m04:/home/docker/cp-test.txt | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:38 PDT | 12 Sep 24 16:38 PDT |
| | ha-253000:/home/docker/cp-test_ha-253000-m04_ha-253000.txt | | | | | |
| ssh | ha-253000 ssh -n | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:38 PDT | 12 Sep 24 16:38 PDT |
| | ha-253000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| ssh | ha-253000 ssh -n ha-253000 sudo cat | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:38 PDT | 12 Sep 24 16:38 PDT |
| | /home/docker/cp-test_ha-253000-m04_ha-253000.txt | | | | | |
| cp | ha-253000 cp ha-253000-m04:/home/docker/cp-test.txt | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:38 PDT | 12 Sep 24 16:38 PDT |
| | ha-253000-m02:/home/docker/cp-test_ha-253000-m04_ha-253000-m02.txt | | | | | |
| ssh | ha-253000 ssh -n | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:38 PDT | 12 Sep 24 16:38 PDT |
| | ha-253000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| ssh | ha-253000 ssh -n ha-253000-m02 sudo cat | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:38 PDT | 12 Sep 24 16:38 PDT |
| | /home/docker/cp-test_ha-253000-m04_ha-253000-m02.txt | | | | | |
| cp | ha-253000 cp ha-253000-m04:/home/docker/cp-test.txt | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:38 PDT | 12 Sep 24 16:38 PDT |
| | ha-253000-m03:/home/docker/cp-test_ha-253000-m04_ha-253000-m03.txt | | | | | |
| ssh | ha-253000 ssh -n | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:38 PDT | 12 Sep 24 16:38 PDT |
| | ha-253000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| ssh | ha-253000 ssh -n ha-253000-m03 sudo cat | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:38 PDT | 12 Sep 24 16:38 PDT |
| | /home/docker/cp-test_ha-253000-m04_ha-253000-m03.txt | | | | | |
| node | ha-253000 node stop m02 -v=7 | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:38 PDT | 12 Sep 24 16:38 PDT |
| | --alsologtostderr | | | | | |
| node | ha-253000 node start m02 -v=7 | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:38 PDT | 12 Sep 24 16:39 PDT |
| | --alsologtostderr | | | | | |
| node | list -p ha-253000 -v=7 | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:39 PDT | |
| | --alsologtostderr | | | | | |
| stop | -p ha-253000 -v=7 | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:39 PDT | 12 Sep 24 16:39 PDT |
| | --alsologtostderr | | | | | |
| start | -p ha-253000 --wait=true -v=7 | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:39 PDT | 12 Sep 24 16:42 PDT |
| | --alsologtostderr | | | | | |
| node | list -p ha-253000 | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:42 PDT | |
| node | ha-253000 node delete m03 -v=7 | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:42 PDT | 12 Sep 24 16:42 PDT |
| | --alsologtostderr | | | | | |
| stop | ha-253000 stop -v=7 | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:42 PDT | 12 Sep 24 16:42 PDT |
| | --alsologtostderr | | | | | |
| start | -p ha-253000 --wait=true | ha-253000 | jenkins | v1.34.0 | 12 Sep 24 16:42 PDT | |
| | -v=7 --alsologtostderr | | | | | |
| | --driver=hyperkit | | | | | |
|---------|-----------------------------------------------------------------------------------------------------------------------------|-----------|---------|---------|---------------------|---------------------|
==> Last Start <==
Log file created at: 2024/09/12 16:42:38
Running on machine: MacOS-Agent-2
Binary: Built with gc go1.22.5 for darwin/amd64
Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg
I0912 16:42:38.430140 38825 out.go:345] Setting OutFile to fd 1 ...
I0912 16:42:38.430320 38825 out.go:392] TERM=,COLORTERM=, which probably does not support color
I0912 16:42:38.430326 38825 out.go:358] Setting ErrFile to fd 2...
I0912 16:42:38.430330 38825 out.go:392] TERM=,COLORTERM=, which probably does not support color
I0912 16:42:38.430499 38825 root.go:338] Updating PATH: /Users/jenkins/minikube-integration/19616-35737/.minikube/bin
I0912 16:42:38.431868 38825 out.go:352] Setting JSON to false
I0912 16:42:38.454266 38825 start.go:129] hostinfo: {"hostname":"MacOS-Agent-2.local","uptime":27728,"bootTime":1726156830,"procs":477,"os":"darwin","platform":"darwin","platformFamily":"Standalone Workstation","platformVersion":"14.6.1","kernelVersion":"23.6.0","kernelArch":"x86_64","virtualizationSystem":"","virtualizationRole":"","hostId":"2965c349-98a5-5970-aaa9-9eedd3ae5959"}
W0912 16:42:38.454360 38825 start.go:137] gopshost.Virtualization returned error: not implemented yet
I0912 16:42:38.481285 38825 out.go:177] * [ha-253000] minikube v1.34.0 on Darwin 14.6.1
I0912 16:42:38.523148 38825 out.go:177] - MINIKUBE_LOCATION=19616
I0912 16:42:38.523195 38825 notify.go:220] Checking for updates...
I0912 16:42:38.565714 38825 out.go:177] - KUBECONFIG=/Users/jenkins/minikube-integration/19616-35737/kubeconfig
I0912 16:42:38.586945 38825 out.go:177] - MINIKUBE_BIN=out/minikube-darwin-amd64
I0912 16:42:38.607903 38825 out.go:177] - MINIKUBE_SUPPRESS_DOCKER_PERFORMANCE=true
I0912 16:42:38.629013 38825 out.go:177] - MINIKUBE_HOME=/Users/jenkins/minikube-integration/19616-35737/.minikube
I0912 16:42:38.649861 38825 out.go:177] - MINIKUBE_FORCE_SYSTEMD=
I0912 16:42:38.671524 38825 config.go:182] Loaded profile config "ha-253000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.1
I0912 16:42:38.672277 38825 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0912 16:42:38.672362 38825 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0912 16:42:38.682128 38825 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:64002
I0912 16:42:38.682494 38825 main.go:141] libmachine: () Calling .GetVersion
I0912 16:42:38.682914 38825 main.go:141] libmachine: Using API Version 1
I0912 16:42:38.682922 38825 main.go:141] libmachine: () Calling .SetConfigRaw
I0912 16:42:38.683129 38825 main.go:141] libmachine: () Calling .GetMachineName
I0912 16:42:38.683247 38825 main.go:141] libmachine: (ha-253000) Calling .DriverName
I0912 16:42:38.683447 38825 driver.go:394] Setting default libvirt URI to qemu:///system
I0912 16:42:38.683681 38825 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0912 16:42:38.683701 38825 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0912 16:42:38.692120 38825 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:64004
I0912 16:42:38.692513 38825 main.go:141] libmachine: () Calling .GetVersion
I0912 16:42:38.692837 38825 main.go:141] libmachine: Using API Version 1
I0912 16:42:38.692847 38825 main.go:141] libmachine: () Calling .SetConfigRaw
I0912 16:42:38.693129 38825 main.go:141] libmachine: () Calling .GetMachineName
I0912 16:42:38.693245 38825 main.go:141] libmachine: (ha-253000) Calling .DriverName
I0912 16:42:38.722026 38825 out.go:177] * Using the hyperkit driver based on existing profile
I0912 16:42:38.765647 38825 start.go:297] selected driver: hyperkit
I0912 16:42:38.765700 38825 start.go:901] validating driver "hyperkit" against &{Name:ha-253000 KeepContext:false EmbedCerts:false MinikubeISO:https://storage.googleapis.com/minikube-builds/iso/19616/minikube-v1.34.0-1726156389-19616-amd64.iso KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.45-1726156396-19616@sha256:66b06a42534e914a5c8ad765d7508a93a34031939ec9a6b3a818ef0a444ff889 Memory:2200 CPUs:2 DiskSize:20000 Driver:hyperkit HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{Kuber
netesVersion:v1.31.1 ClusterName:ha-253000 Namespace:default APIServerHAVIP:192.169.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.169.0.117 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m02 IP:192.169.0.118 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m04 IP:192.169.0.120 Port:0 KubernetesVersion:v1.31.1 ContainerRuntime: ControlPlane:false Worker:true}] Addons:map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false helm-tiller:false inaccel:false ingress:
false ingress-dns:false inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s Mount:false MountString:/Users:/minikube-host Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker Binar
yMirror: DisableOptimizations:false DisableMetrics:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s}
I0912 16:42:38.765914 38825 start.go:912] status for hyperkit: {Installed:true Healthy:true Running:true NeedsImprovement:false Error:<nil> Reason: Fix: Doc: Version:}
I0912 16:42:38.766111 38825 install.go:52] acquiring lock: {Name:mk4023283b30b374c3f04c8805d539e68824c0b8 Clock:{} Delay:500ms Timeout:10m0s Cancel:<nil>}
I0912 16:42:38.766319 38825 install.go:117] Validating docker-machine-driver-hyperkit, PATH=/Users/jenkins/minikube-integration/19616-35737/.minikube/bin:/Users/jenkins/workspace/out/:/usr/bin:/bin:/usr/sbin:/sbin:/Users/jenkins/google-cloud-sdk/bin:/usr/local/bin/:/usr/local/go/bin/:/Users/jenkins/go/bin
I0912 16:42:38.776394 38825 install.go:137] /Users/jenkins/workspace/out/docker-machine-driver-hyperkit version is 1.34.0
I0912 16:42:38.780270 38825 install.go:79] stdout: /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0912 16:42:38.780293 38825 install.go:81] /Users/jenkins/workspace/out/docker-machine-driver-hyperkit looks good
I0912 16:42:38.782996 38825 start_flags.go:947] Waiting for all components: map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true]
I0912 16:42:38.783035 38825 cni.go:84] Creating CNI manager for ""
I0912 16:42:38.783041 38825 cni.go:136] multinode detected (3 nodes found), recommending kindnet
I0912 16:42:38.783114 38825 start.go:340] cluster config:
{Name:ha-253000 KeepContext:false EmbedCerts:false MinikubeISO:https://storage.googleapis.com/minikube-builds/iso/19616/minikube-v1.34.0-1726156389-19616-amd64.iso KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.45-1726156396-19616@sha256:66b06a42534e914a5c8ad765d7508a93a34031939ec9a6b3a818ef0a444ff889 Memory:2200 CPUs:2 DiskSize:20000 Driver:hyperkit HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.31.1 ClusterName:ha-253000 Namespace:default APIServerHAVIP:192.16
9.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.169.0.117 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m02 IP:192.169.0.118 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m04 IP:192.169.0.120 Port:0 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:false Worker:true}] Addons:map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false helm-tiller:false inaccel:false ingress:false ingress-dns:false inspektor-gadget:false istio:false istio-provisioner
:false kong:false kubeflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s Mount:false MountString:/Users:/minikube-host Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:false DisableMetrics:false CustomQemuFirmwareP
ath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s}
I0912 16:42:38.783215 38825 iso.go:125] acquiring lock: {Name:mkb828e230b821a4c21cec3fdf1c004fbe312a1d Clock:{} Delay:500ms Timeout:10m0s Cancel:<nil>}
I0912 16:42:38.824584 38825 out.go:177] * Starting "ha-253000" primary control-plane node in "ha-253000" cluster
I0912 16:42:38.845946 38825 preload.go:131] Checking if preload exists for k8s version v1.31.1 and runtime docker
I0912 16:42:38.846016 38825 preload.go:146] Found local preload: /Users/jenkins/minikube-integration/19616-35737/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.31.1-docker-overlay2-amd64.tar.lz4
I0912 16:42:38.846053 38825 cache.go:56] Caching tarball of preloaded images
I0912 16:42:38.846265 38825 preload.go:172] Found /Users/jenkins/minikube-integration/19616-35737/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.31.1-docker-overlay2-amd64.tar.lz4 in cache, skipping download
I0912 16:42:38.846288 38825 cache.go:59] Finished verifying existence of preloaded tar for v1.31.1 on docker
I0912 16:42:38.846498 38825 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/config.json ...
I0912 16:42:38.847353 38825 start.go:360] acquireMachinesLock for ha-253000: {Name:mk1a336f8b7e1d023cbf697496a947d172cff4c4 Clock:{} Delay:500ms Timeout:13m0s Cancel:<nil>}
I0912 16:42:38.847473 38825 start.go:364] duration metric: took 96.788µs to acquireMachinesLock for "ha-253000"
I0912 16:42:38.847506 38825 start.go:96] Skipping create...Using existing machine configuration
I0912 16:42:38.847523 38825 fix.go:54] fixHost starting:
I0912 16:42:38.847940 38825 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0912 16:42:38.847969 38825 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0912 16:42:38.857172 38825 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:64006
I0912 16:42:38.857567 38825 main.go:141] libmachine: () Calling .GetVersion
I0912 16:42:38.857909 38825 main.go:141] libmachine: Using API Version 1
I0912 16:42:38.857924 38825 main.go:141] libmachine: () Calling .SetConfigRaw
I0912 16:42:38.858141 38825 main.go:141] libmachine: () Calling .GetMachineName
I0912 16:42:38.858254 38825 main.go:141] libmachine: (ha-253000) Calling .DriverName
I0912 16:42:38.858341 38825 main.go:141] libmachine: (ha-253000) Calling .GetState
I0912 16:42:38.858419 38825 main.go:141] libmachine: (ha-253000) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0912 16:42:38.858491 38825 main.go:141] libmachine: (ha-253000) DBG | hyperkit pid from json: 38724
I0912 16:42:38.859540 38825 fix.go:112] recreateIfNeeded on ha-253000: state=Stopped err=<nil>
I0912 16:42:38.859559 38825 main.go:141] libmachine: (ha-253000) Calling .DriverName
I0912 16:42:38.859566 38825 main.go:141] libmachine: (ha-253000) DBG | hyperkit pid 38724 missing from process table
W0912 16:42:38.859662 38825 fix.go:138] unexpected machine state, will restart: <nil>
I0912 16:42:38.901690 38825 out.go:177] * Restarting existing hyperkit VM for "ha-253000" ...
I0912 16:42:38.923802 38825 main.go:141] libmachine: (ha-253000) Calling .Start
I0912 16:42:38.924114 38825 main.go:141] libmachine: (ha-253000) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0912 16:42:38.924157 38825 main.go:141] libmachine: (ha-253000) minikube might have been shutdown in an unclean way, the hyperkit pid file still exists: /Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/hyperkit.pid
I0912 16:42:38.926084 38825 main.go:141] libmachine: (ha-253000) DBG | hyperkit pid 38724 missing from process table
I0912 16:42:38.926099 38825 main.go:141] libmachine: (ha-253000) DBG | pid 38724 is in state "Stopped"
I0912 16:42:38.926118 38825 main.go:141] libmachine: (ha-253000) DBG | Removing stale pid file /Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/hyperkit.pid...
I0912 16:42:38.926446 38825 main.go:141] libmachine: (ha-253000) DBG | Using UUID d8bd19b5-91c3-4315-a517-0d166605bdac
I0912 16:42:39.050649 38825 main.go:141] libmachine: (ha-253000) DBG | Generated MAC 82:27:44:ae:54:1
I0912 16:42:39.050677 38825 main.go:141] libmachine: (ha-253000) DBG | Starting with cmdline: loglevel=3 console=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-253000
I0912 16:42:39.050803 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:39 DEBUG: hyperkit: Start &hyperkit.HyperKit{HyperKit:"/usr/local/bin/hyperkit", Argv0:"", StateDir:"/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000", VPNKitSock:"", VPNKitUUID:"", VPNKitPreferredIPv4:"", UUID:"d8bd19b5-91c3-4315-a517-0d166605bdac", Disks:[]hyperkit.Disk{(*hyperkit.RawDisk)(0xc0003cb9b0)}, ISOImages:[]string{"/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/boot2docker.iso"}, VSock:false, VSockDir:"", VSockPorts:[]int(nil), VSockGuestCID:3, VMNet:true, Sockets9P:[]hyperkit.Socket9P(nil), Kernel:"/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/bzimage", Initrd:"/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/initrd", Bootrom:"", CPUs:2, Memory:2200, Console:1, Serials:[]hyperkit.Serial(nil), Pid:0, Arguments:[]string(nil), CmdLine:"", process:(*os.Process)(nil)}
I0912 16:42:39.050829 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:39 DEBUG: hyperkit: check &hyperkit.HyperKit{HyperKit:"/usr/local/bin/hyperkit", Argv0:"", StateDir:"/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000", VPNKitSock:"", VPNKitUUID:"", VPNKitPreferredIPv4:"", UUID:"d8bd19b5-91c3-4315-a517-0d166605bdac", Disks:[]hyperkit.Disk{(*hyperkit.RawDisk)(0xc0003cb9b0)}, ISOImages:[]string{"/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/boot2docker.iso"}, VSock:false, VSockDir:"", VSockPorts:[]int(nil), VSockGuestCID:3, VMNet:true, Sockets9P:[]hyperkit.Socket9P(nil), Kernel:"/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/bzimage", Initrd:"/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/initrd", Bootrom:"", CPUs:2, Memory:2200, Console:1, Serials:[]hyperkit.Serial(nil), Pid:0, Arguments:[]string(nil), CmdLine:"", process:(*os.Process)(nil)}
I0912 16:42:39.050873 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:39 DEBUG: hyperkit: Arguments: []string{"-A", "-u", "-F", "/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/hyperkit.pid", "-c", "2", "-m", "2200M", "-s", "0:0,hostbridge", "-s", "31,lpc", "-s", "1:0,virtio-net", "-U", "d8bd19b5-91c3-4315-a517-0d166605bdac", "-s", "2:0,virtio-blk,/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/ha-253000.rawdisk", "-s", "3,ahci-cd,/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/boot2docker.iso", "-s", "4,virtio-rnd", "-l", "com1,autopty=/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/tty,log=/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/console-ring", "-f", "kexec,/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/bzimage,/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/initrd,earlyp
rintk=serial loglevel=3 console=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-253000"}
I0912 16:42:39.050902 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:39 DEBUG: hyperkit: CmdLine: "/usr/local/bin/hyperkit -A -u -F /Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/hyperkit.pid -c 2 -m 2200M -s 0:0,hostbridge -s 31,lpc -s 1:0,virtio-net -U d8bd19b5-91c3-4315-a517-0d166605bdac -s 2:0,virtio-blk,/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/ha-253000.rawdisk -s 3,ahci-cd,/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/boot2docker.iso -s 4,virtio-rnd -l com1,autopty=/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/tty,log=/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/console-ring -f kexec,/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/bzimage,/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/initrd,earlyprintk=serial loglevel=3 console=ttyS0 console=tty0 noembed nom
odeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-253000"
I0912 16:42:39.050918 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:39 DEBUG: hyperkit: Redirecting stdout/stderr to logger
I0912 16:42:39.052559 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:39 DEBUG: hyperkit: Pid is 38839
I0912 16:42:39.053048 38825 main.go:141] libmachine: (ha-253000) DBG | Attempt 0
I0912 16:42:39.053063 38825 main.go:141] libmachine: (ha-253000) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0912 16:42:39.053126 38825 main.go:141] libmachine: (ha-253000) DBG | hyperkit pid from json: 38839
I0912 16:42:39.054650 38825 main.go:141] libmachine: (ha-253000) DBG | Searching for 82:27:44:ae:54:1 in /var/db/dhcpd_leases ...
I0912 16:42:39.054805 38825 main.go:141] libmachine: (ha-253000) DBG | Found 119 entries in /var/db/dhcpd_leases!
I0912 16:42:39.054828 38825 main.go:141] libmachine: (ha-253000) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.119 HWAddress:e6:f8:cc:1a:35:bd ID:1,e6:f8:cc:1a:35:bd Lease:0x66e37c53}
I0912 16:42:39.054869 38825 main.go:141] libmachine: (ha-253000) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.120 HWAddress:26:d:74:d3:a3:a1 ID:1,26:d:74:d3:a3:a1 Lease:0x66e4cdbe}
I0912 16:42:39.054901 38825 main.go:141] libmachine: (ha-253000) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.118 HWAddress:92:19:55:83:fe:25 ID:1,92:19:55:83:fe:25 Lease:0x66e4cd52}
I0912 16:42:39.054919 38825 main.go:141] libmachine: (ha-253000) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.117 HWAddress:82:27:44:ae:54:1 ID:1,82:27:44:ae:54:1 Lease:0x66e4cd3f}
I0912 16:42:39.054920 38825 main.go:141] libmachine: (ha-253000) Calling .GetConfigRaw
I0912 16:42:39.054925 38825 main.go:141] libmachine: (ha-253000) DBG | Found match: 82:27:44:ae:54:1
I0912 16:42:39.054933 38825 main.go:141] libmachine: (ha-253000) DBG | IP: 192.169.0.117
I0912 16:42:39.055570 38825 main.go:141] libmachine: (ha-253000) Calling .GetIP
I0912 16:42:39.055757 38825 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/config.json ...
I0912 16:42:39.056146 38825 machine.go:93] provisionDockerMachine start ...
I0912 16:42:39.056156 38825 main.go:141] libmachine: (ha-253000) Calling .DriverName
I0912 16:42:39.056296 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHHostname
I0912 16:42:39.056394 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHPort
I0912 16:42:39.056526 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHKeyPath
I0912 16:42:39.056636 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHKeyPath
I0912 16:42:39.056759 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHUsername
I0912 16:42:39.056898 38825 main.go:141] libmachine: Using SSH client type: native
I0912 16:42:39.057100 38825 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xefa9aa0] 0xefac800 <nil> [] 0s} 192.169.0.117 22 <nil> <nil>}
I0912 16:42:39.057108 38825 main.go:141] libmachine: About to run SSH command:
hostname
I0912 16:42:39.060350 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:39 INFO : hyperkit: stderr: Using fd 5 for I/O notifications
I0912 16:42:39.117799 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:39 INFO : hyperkit: stderr: /Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/boot2docker.iso: fcntl(F_PUNCHHOLE) Operation not permitted: block device will not support TRIM/DISCARD
I0912 16:42:39.118509 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:39 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 2 bit: 22 unspecified don't care: bit is 0
I0912 16:42:39.118533 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:39 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 12 unspecified don't care: bit is 0
I0912 16:42:39.118546 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:39 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 20 unspecified don't care: bit is 0
I0912 16:42:39.118555 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:39 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 3 bit: 13 unspecified don't care: bit is 0
I0912 16:42:39.507722 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:39 INFO : hyperkit: stderr: rdmsr to register 0x3a on vcpu 0
I0912 16:42:39.507740 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:39 INFO : hyperkit: stderr: rdmsr to register 0x140 on vcpu 0
I0912 16:42:39.622419 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:39 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 2 bit: 22 unspecified don't care: bit is 0
I0912 16:42:39.622440 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:39 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 12 unspecified don't care: bit is 0
I0912 16:42:39.622452 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:39 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 20 unspecified don't care: bit is 0
I0912 16:42:39.622471 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:39 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 3 bit: 13 unspecified don't care: bit is 0
I0912 16:42:39.623327 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:39 INFO : hyperkit: stderr: rdmsr to register 0x3a on vcpu 1
I0912 16:42:39.623338 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:39 INFO : hyperkit: stderr: rdmsr to register 0x140 on vcpu 1
I0912 16:42:45.228944 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:45 INFO : hyperkit: stderr: rdmsr to register 0x64d on vcpu 1
I0912 16:42:45.228975 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:45 INFO : hyperkit: stderr: rdmsr to register 0x64e on vcpu 1
I0912 16:42:45.228986 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:45 INFO : hyperkit: stderr: rdmsr to register 0x34 on vcpu 1
I0912 16:42:45.254405 38825 main.go:141] libmachine: (ha-253000) DBG | 2024/09/12 16:42:45 INFO : hyperkit: stderr: rdmsr to register 0xc0011029 on vcpu 1
I0912 16:42:49.192616 38825 main.go:141] libmachine: Error dialing TCP: dial tcp 192.169.0.117:22: connect: connection refused
I0912 16:42:52.258604 38825 main.go:141] libmachine: SSH cmd err, output: <nil>: minikube
I0912 16:42:52.258618 38825 main.go:141] libmachine: (ha-253000) Calling .GetMachineName
I0912 16:42:52.258757 38825 buildroot.go:166] provisioning hostname "ha-253000"
I0912 16:42:52.258767 38825 main.go:141] libmachine: (ha-253000) Calling .GetMachineName
I0912 16:42:52.258878 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHHostname
I0912 16:42:52.258988 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHPort
I0912 16:42:52.259118 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHKeyPath
I0912 16:42:52.259214 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHKeyPath
I0912 16:42:52.259296 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHUsername
I0912 16:42:52.259419 38825 main.go:141] libmachine: Using SSH client type: native
I0912 16:42:52.259573 38825 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xefa9aa0] 0xefac800 <nil> [] 0s} 192.169.0.117 22 <nil> <nil>}
I0912 16:42:52.259582 38825 main.go:141] libmachine: About to run SSH command:
sudo hostname ha-253000 && echo "ha-253000" | sudo tee /etc/hostname
I0912 16:42:52.331735 38825 main.go:141] libmachine: SSH cmd err, output: <nil>: ha-253000
I0912 16:42:52.331753 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHHostname
I0912 16:42:52.331886 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHPort
I0912 16:42:52.331994 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHKeyPath
I0912 16:42:52.332077 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHKeyPath
I0912 16:42:52.332187 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHUsername
I0912 16:42:52.332320 38825 main.go:141] libmachine: Using SSH client type: native
I0912 16:42:52.332461 38825 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xefa9aa0] 0xefac800 <nil> [] 0s} 192.169.0.117 22 <nil> <nil>}
I0912 16:42:52.332474 38825 main.go:141] libmachine: About to run SSH command:
if ! grep -xq '.*\sha-253000' /etc/hosts; then
if grep -xq '127.0.1.1\s.*' /etc/hosts; then
sudo sed -i 's/^127.0.1.1\s.*/127.0.1.1 ha-253000/g' /etc/hosts;
else
echo '127.0.1.1 ha-253000' | sudo tee -a /etc/hosts;
fi
fi
I0912 16:42:52.400568 38825 main.go:141] libmachine: SSH cmd err, output: <nil>:
I0912 16:42:52.400588 38825 buildroot.go:172] set auth options {CertDir:/Users/jenkins/minikube-integration/19616-35737/.minikube CaCertPath:/Users/jenkins/minikube-integration/19616-35737/.minikube/certs/ca.pem CaPrivateKeyPath:/Users/jenkins/minikube-integration/19616-35737/.minikube/certs/ca-key.pem CaCertRemotePath:/etc/docker/ca.pem ServerCertPath:/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/server.pem ServerKeyPath:/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/server-key.pem ClientKeyPath:/Users/jenkins/minikube-integration/19616-35737/.minikube/certs/key.pem ServerCertRemotePath:/etc/docker/server.pem ServerKeyRemotePath:/etc/docker/server-key.pem ClientCertPath:/Users/jenkins/minikube-integration/19616-35737/.minikube/certs/cert.pem ServerCertSANs:[] StorePath:/Users/jenkins/minikube-integration/19616-35737/.minikube}
I0912 16:42:52.400612 38825 buildroot.go:174] setting up certificates
I0912 16:42:52.400621 38825 provision.go:84] configureAuth start
I0912 16:42:52.400628 38825 main.go:141] libmachine: (ha-253000) Calling .GetMachineName
I0912 16:42:52.400766 38825 main.go:141] libmachine: (ha-253000) Calling .GetIP
I0912 16:42:52.400868 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHHostname
I0912 16:42:52.400969 38825 provision.go:143] copyHostCerts
I0912 16:42:52.400999 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/ca.pem -> /Users/jenkins/minikube-integration/19616-35737/.minikube/ca.pem
I0912 16:42:52.401072 38825 exec_runner.go:144] found /Users/jenkins/minikube-integration/19616-35737/.minikube/ca.pem, removing ...
I0912 16:42:52.401081 38825 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19616-35737/.minikube/ca.pem
I0912 16:42:52.401225 38825 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/ca.pem --> /Users/jenkins/minikube-integration/19616-35737/.minikube/ca.pem (1082 bytes)
I0912 16:42:52.401445 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/cert.pem -> /Users/jenkins/minikube-integration/19616-35737/.minikube/cert.pem
I0912 16:42:52.401485 38825 exec_runner.go:144] found /Users/jenkins/minikube-integration/19616-35737/.minikube/cert.pem, removing ...
I0912 16:42:52.401490 38825 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19616-35737/.minikube/cert.pem
I0912 16:42:52.401603 38825 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/cert.pem --> /Users/jenkins/minikube-integration/19616-35737/.minikube/cert.pem (1123 bytes)
I0912 16:42:52.401771 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/key.pem -> /Users/jenkins/minikube-integration/19616-35737/.minikube/key.pem
I0912 16:42:52.401816 38825 exec_runner.go:144] found /Users/jenkins/minikube-integration/19616-35737/.minikube/key.pem, removing ...
I0912 16:42:52.401822 38825 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19616-35737/.minikube/key.pem
I0912 16:42:52.401900 38825 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/key.pem --> /Users/jenkins/minikube-integration/19616-35737/.minikube/key.pem (1679 bytes)
I0912 16:42:52.402055 38825 provision.go:117] generating server cert: /Users/jenkins/minikube-integration/19616-35737/.minikube/machines/server.pem ca-key=/Users/jenkins/minikube-integration/19616-35737/.minikube/certs/ca.pem private-key=/Users/jenkins/minikube-integration/19616-35737/.minikube/certs/ca-key.pem org=jenkins.ha-253000 san=[127.0.0.1 192.169.0.117 ha-253000 localhost minikube]
I0912 16:42:52.474004 38825 provision.go:177] copyRemoteCerts
I0912 16:42:52.474054 38825 ssh_runner.go:195] Run: sudo mkdir -p /etc/docker /etc/docker /etc/docker
I0912 16:42:52.474068 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHHostname
I0912 16:42:52.474188 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHPort
I0912 16:42:52.474290 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHKeyPath
I0912 16:42:52.474382 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHUsername
I0912 16:42:52.474476 38825 sshutil.go:53] new ssh client: &{IP:192.169.0.117 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/id_rsa Username:docker}
I0912 16:42:52.512493 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/ca.pem -> /etc/docker/ca.pem
I0912 16:42:52.512560 38825 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/ca.pem --> /etc/docker/ca.pem (1082 bytes)
I0912 16:42:52.532249 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/machines/server.pem -> /etc/docker/server.pem
I0912 16:42:52.532313 38825 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19616-35737/.minikube/machines/server.pem --> /etc/docker/server.pem (1196 bytes)
I0912 16:42:52.552017 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/machines/server-key.pem -> /etc/docker/server-key.pem
I0912 16:42:52.552083 38825 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19616-35737/.minikube/machines/server-key.pem --> /etc/docker/server-key.pem (1679 bytes)
I0912 16:42:52.571233 38825 provision.go:87] duration metric: took 170.596675ms to configureAuth
I0912 16:42:52.571245 38825 buildroot.go:189] setting minikube options for container-runtime
I0912 16:42:52.571412 38825 config.go:182] Loaded profile config "ha-253000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.1
I0912 16:42:52.571425 38825 main.go:141] libmachine: (ha-253000) Calling .DriverName
I0912 16:42:52.571573 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHHostname
I0912 16:42:52.571654 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHPort
I0912 16:42:52.571749 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHKeyPath
I0912 16:42:52.571858 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHKeyPath
I0912 16:42:52.571941 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHUsername
I0912 16:42:52.572054 38825 main.go:141] libmachine: Using SSH client type: native
I0912 16:42:52.572180 38825 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xefa9aa0] 0xefac800 <nil> [] 0s} 192.169.0.117 22 <nil> <nil>}
I0912 16:42:52.572187 38825 main.go:141] libmachine: About to run SSH command:
df --output=fstype / | tail -n 1
I0912 16:42:52.631648 38825 main.go:141] libmachine: SSH cmd err, output: <nil>: tmpfs
I0912 16:42:52.631659 38825 buildroot.go:70] root file system type: tmpfs
I0912 16:42:52.631743 38825 provision.go:314] Updating docker unit: /lib/systemd/system/docker.service ...
I0912 16:42:52.631759 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHHostname
I0912 16:42:52.631890 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHPort
I0912 16:42:52.631984 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHKeyPath
I0912 16:42:52.632074 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHKeyPath
I0912 16:42:52.632161 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHUsername
I0912 16:42:52.632293 38825 main.go:141] libmachine: Using SSH client type: native
I0912 16:42:52.632431 38825 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xefa9aa0] 0xefac800 <nil> [] 0s} 192.169.0.117 22 <nil> <nil>}
I0912 16:42:52.632475 38825 main.go:141] libmachine: About to run SSH command:
sudo mkdir -p /lib/systemd/system && printf %s "[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target minikube-automount.service docker.socket
Requires= minikube-automount.service docker.socket
StartLimitBurst=3
StartLimitIntervalSec=60
[Service]
Type=notify
Restart=on-failure
# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
# Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=hyperkit --insecure-registry 10.96.0.0/12
ExecReload=/bin/kill -s HUP \$MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
[Install]
WantedBy=multi-user.target
" | sudo tee /lib/systemd/system/docker.service.new
I0912 16:42:52.705254 38825 main.go:141] libmachine: SSH cmd err, output: <nil>: [Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target minikube-automount.service docker.socket
Requires= minikube-automount.service docker.socket
StartLimitBurst=3
StartLimitIntervalSec=60
[Service]
Type=notify
Restart=on-failure
# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
# Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=hyperkit --insecure-registry 10.96.0.0/12
ExecReload=/bin/kill -s HUP $MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
[Install]
WantedBy=multi-user.target
I0912 16:42:52.705278 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHHostname
I0912 16:42:52.705417 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHPort
I0912 16:42:52.705502 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHKeyPath
I0912 16:42:52.705601 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHKeyPath
I0912 16:42:52.705715 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHUsername
I0912 16:42:52.705831 38825 main.go:141] libmachine: Using SSH client type: native
I0912 16:42:52.705987 38825 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xefa9aa0] 0xefac800 <nil> [] 0s} 192.169.0.117 22 <nil> <nil>}
I0912 16:42:52.705999 38825 main.go:141] libmachine: About to run SSH command:
sudo diff -u /lib/systemd/system/docker.service /lib/systemd/system/docker.service.new || { sudo mv /lib/systemd/system/docker.service.new /lib/systemd/system/docker.service; sudo systemctl -f daemon-reload && sudo systemctl -f enable docker && sudo systemctl -f restart docker; }
I0912 16:42:54.426361 38825 main.go:141] libmachine: SSH cmd err, output: <nil>: diff: can't stat '/lib/systemd/system/docker.service': No such file or directory
Created symlink /etc/systemd/system/multi-user.target.wants/docker.service → /usr/lib/systemd/system/docker.service.
I0912 16:42:54.426385 38825 machine.go:96] duration metric: took 15.3699374s to provisionDockerMachine
I0912 16:42:54.426393 38825 start.go:293] postStartSetup for "ha-253000" (driver="hyperkit")
I0912 16:42:54.426401 38825 start.go:322] creating required directories: [/etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs]
I0912 16:42:54.426418 38825 main.go:141] libmachine: (ha-253000) Calling .DriverName
I0912 16:42:54.426590 38825 ssh_runner.go:195] Run: sudo mkdir -p /etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs
I0912 16:42:54.426604 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHHostname
I0912 16:42:54.426705 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHPort
I0912 16:42:54.426792 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHKeyPath
I0912 16:42:54.426891 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHUsername
I0912 16:42:54.426996 38825 sshutil.go:53] new ssh client: &{IP:192.169.0.117 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/id_rsa Username:docker}
I0912 16:42:54.469077 38825 ssh_runner.go:195] Run: cat /etc/os-release
I0912 16:42:54.472609 38825 info.go:137] Remote host: Buildroot 2023.02.9
I0912 16:42:54.472622 38825 filesync.go:126] Scanning /Users/jenkins/minikube-integration/19616-35737/.minikube/addons for local assets ...
I0912 16:42:54.472720 38825 filesync.go:126] Scanning /Users/jenkins/minikube-integration/19616-35737/.minikube/files for local assets ...
I0912 16:42:54.472900 38825 filesync.go:149] local asset: /Users/jenkins/minikube-integration/19616-35737/.minikube/files/etc/ssl/certs/362852.pem -> 362852.pem in /etc/ssl/certs
I0912 16:42:54.472907 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/files/etc/ssl/certs/362852.pem -> /etc/ssl/certs/362852.pem
I0912 16:42:54.473112 38825 ssh_runner.go:195] Run: sudo mkdir -p /etc/ssl/certs
I0912 16:42:54.490033 38825 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19616-35737/.minikube/files/etc/ssl/certs/362852.pem --> /etc/ssl/certs/362852.pem (1708 bytes)
I0912 16:42:54.512576 38825 start.go:296] duration metric: took 86.162397ms for postStartSetup
I0912 16:42:54.512599 38825 main.go:141] libmachine: (ha-253000) Calling .DriverName
I0912 16:42:54.512766 38825 ssh_runner.go:195] Run: sudo ls --almost-all -1 /var/lib/minikube/backup
I0912 16:42:54.512778 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHHostname
I0912 16:42:54.512873 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHPort
I0912 16:42:54.512962 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHKeyPath
I0912 16:42:54.513054 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHUsername
I0912 16:42:54.513137 38825 sshutil.go:53] new ssh client: &{IP:192.169.0.117 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/id_rsa Username:docker}
I0912 16:42:54.550974 38825 machine.go:197] restoring vm config from /var/lib/minikube/backup: [etc]
I0912 16:42:54.551029 38825 ssh_runner.go:195] Run: sudo rsync --archive --update /var/lib/minikube/backup/etc /
I0912 16:42:54.582853 38825 fix.go:56] duration metric: took 15.73503382s for fixHost
I0912 16:42:54.582874 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHHostname
I0912 16:42:54.583005 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHPort
I0912 16:42:54.583103 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHKeyPath
I0912 16:42:54.583226 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHKeyPath
I0912 16:42:54.583320 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHUsername
I0912 16:42:54.583459 38825 main.go:141] libmachine: Using SSH client type: native
I0912 16:42:54.583610 38825 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xefa9aa0] 0xefac800 <nil> [] 0s} 192.169.0.117 22 <nil> <nil>}
I0912 16:42:54.583618 38825 main.go:141] libmachine: About to run SSH command:
date +%s.%N
I0912 16:42:54.644693 38825 main.go:141] libmachine: SSH cmd err, output: <nil>: 1726184574.681519438
I0912 16:42:54.644707 38825 fix.go:216] guest clock: 1726184574.681519438
I0912 16:42:54.644712 38825 fix.go:229] Guest: 2024-09-12 16:42:54.681519438 -0700 PDT Remote: 2024-09-12 16:42:54.582864 -0700 PDT m=+16.187853489 (delta=98.655438ms)
I0912 16:42:54.644730 38825 fix.go:200] guest clock delta is within tolerance: 98.655438ms
I0912 16:42:54.644733 38825 start.go:83] releasing machines lock for "ha-253000", held for 15.79694813s
I0912 16:42:54.644753 38825 main.go:141] libmachine: (ha-253000) Calling .DriverName
I0912 16:42:54.644885 38825 main.go:141] libmachine: (ha-253000) Calling .GetIP
I0912 16:42:54.644992 38825 main.go:141] libmachine: (ha-253000) Calling .DriverName
I0912 16:42:54.645278 38825 main.go:141] libmachine: (ha-253000) Calling .DriverName
I0912 16:42:54.645365 38825 main.go:141] libmachine: (ha-253000) Calling .DriverName
I0912 16:42:54.645439 38825 ssh_runner.go:195] Run: curl -sS -m 2 https://registry.k8s.io/
I0912 16:42:54.645470 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHHostname
I0912 16:42:54.645501 38825 ssh_runner.go:195] Run: cat /version.json
I0912 16:42:54.645511 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHHostname
I0912 16:42:54.645555 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHPort
I0912 16:42:54.645598 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHPort
I0912 16:42:54.645648 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHKeyPath
I0912 16:42:54.645674 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHKeyPath
I0912 16:42:54.645757 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHUsername
I0912 16:42:54.645769 38825 main.go:141] libmachine: (ha-253000) Calling .GetSSHUsername
I0912 16:42:54.645850 38825 sshutil.go:53] new ssh client: &{IP:192.169.0.117 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/id_rsa Username:docker}
I0912 16:42:54.645861 38825 sshutil.go:53] new ssh client: &{IP:192.169.0.117 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000/id_rsa Username:docker}
I0912 16:42:54.679845 38825 ssh_runner.go:195] Run: systemctl --version
I0912 16:42:54.730409 38825 ssh_runner.go:195] Run: sh -c "stat /etc/cni/net.d/*loopback.conf*"
W0912 16:42:54.735571 38825 cni.go:209] loopback cni configuration skipped: "/etc/cni/net.d/*loopback.conf*" not found
I0912 16:42:54.735616 38825 ssh_runner.go:195] Run: sudo find /etc/cni/net.d -maxdepth 1 -type f ( ( -name *bridge* -or -name *podman* ) -and -not -name *.mk_disabled ) -printf "%p, " -exec sh -c "sudo mv {} {}.mk_disabled" ;
I0912 16:42:54.747932 38825 cni.go:262] disabled [/etc/cni/net.d/87-podman-bridge.conflist] bridge cni config(s)
I0912 16:42:54.747947 38825 start.go:495] detecting cgroup driver to use...
I0912 16:42:54.748054 38825 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///run/containerd/containerd.sock
" | sudo tee /etc/crictl.yaml"
I0912 16:42:54.765836 38825 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)sandbox_image = .*$|\1sandbox_image = "registry.k8s.io/pause:3.10"|' /etc/containerd/config.toml"
I0912 16:42:54.774707 38825 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)restrict_oom_score_adj = .*$|\1restrict_oom_score_adj = false|' /etc/containerd/config.toml"
I0912 16:42:54.783358 38825 containerd.go:146] configuring containerd to use "cgroupfs" as cgroup driver...
I0912 16:42:54.783396 38825 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)SystemdCgroup = .*$|\1SystemdCgroup = false|g' /etc/containerd/config.toml"
I0912 16:42:54.792087 38825 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|"io.containerd.runtime.v1.linux"|"io.containerd.runc.v2"|g' /etc/containerd/config.toml"
I0912 16:42:54.800807 38825 ssh_runner.go:195] Run: sh -c "sudo sed -i '/systemd_cgroup/d' /etc/containerd/config.toml"
I0912 16:42:54.809667 38825 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|"io.containerd.runc.v1"|"io.containerd.runc.v2"|g' /etc/containerd/config.toml"
I0912 16:42:54.818168 38825 ssh_runner.go:195] Run: sh -c "sudo rm -rf /etc/cni/net.mk"
I0912 16:42:54.826930 38825 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)conf_dir = .*$|\1conf_dir = "/etc/cni/net.d"|g' /etc/containerd/config.toml"
I0912 16:42:54.835700 38825 ssh_runner.go:195] Run: sh -c "sudo sed -i '/^ *enable_unprivileged_ports = .*/d' /etc/containerd/config.toml"
I0912 16:42:54.844256 38825 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)\[plugins."io.containerd.grpc.v1.cri"\]|&\n\1 enable_unprivileged_ports = true|' /etc/containerd/config.toml"
I0912 16:42:54.853092 38825 ssh_runner.go:195] Run: sudo sysctl net.bridge.bridge-nf-call-iptables
I0912 16:42:54.861067 38825 ssh_runner.go:195] Run: sudo sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"
I0912 16:42:54.873463 38825 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0912 16:42:54.968510 38825 ssh_runner.go:195] Run: sudo systemctl restart containerd
I0912 16:42:54.987224 38825 start.go:495] detecting cgroup driver to use...
I0912 16:42:54.987300 38825 ssh_runner.go:195] Run: sudo systemctl cat docker.service
I0912 16:42:55.002855 38825 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I0912 16:42:55.015535 38825 ssh_runner.go:195] Run: sudo systemctl stop -f containerd
I0912 16:42:55.029272 38825 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I0912 16:42:55.040190 38825 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service crio
I0912 16:42:55.052091 38825 ssh_runner.go:195] Run: sudo systemctl stop -f crio
I0912 16:42:55.076807 38825 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service crio
I0912 16:42:55.087172 38825 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///var/run/cri-dockerd.sock
" | sudo tee /etc/crictl.yaml"
I0912 16:42:55.101838 38825 ssh_runner.go:195] Run: which cri-dockerd
I0912 16:42:55.104828 38825 ssh_runner.go:195] Run: sudo mkdir -p /etc/systemd/system/cri-docker.service.d
I0912 16:42:55.112047 38825 ssh_runner.go:362] scp memory --> /etc/systemd/system/cri-docker.service.d/10-cni.conf (190 bytes)
I0912 16:42:55.125332 38825 ssh_runner.go:195] Run: sudo systemctl unmask docker.service
I0912 16:42:55.218086 38825 ssh_runner.go:195] Run: sudo systemctl enable docker.socket
I0912 16:42:55.326978 38825 docker.go:574] configuring docker to use "cgroupfs" as cgroup driver...
I0912 16:42:55.327054 38825 ssh_runner.go:362] scp memory --> /etc/docker/daemon.json (130 bytes)
I0912 16:42:55.341857 38825 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0912 16:42:55.436240 38825 ssh_runner.go:195] Run: sudo systemctl restart docker
I0912 16:42:57.804642 38825 ssh_runner.go:235] Completed: sudo systemctl restart docker: (2.368337859s)
I0912 16:42:57.804710 38825 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service cri-docker.socket
I0912 16:42:57.815307 38825 ssh_runner.go:195] Run: sudo systemctl stop cri-docker.socket
I0912 16:42:57.828135 38825 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service cri-docker.service
I0912 16:42:57.838551 38825 ssh_runner.go:195] Run: sudo systemctl unmask cri-docker.socket
I0912 16:42:57.931465 38825 ssh_runner.go:195] Run: sudo systemctl enable cri-docker.socket
I0912 16:42:58.049070 38825 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0912 16:42:58.150280 38825 ssh_runner.go:195] Run: sudo systemctl restart cri-docker.socket
I0912 16:42:58.168993 38825 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service cri-docker.service
I0912 16:42:58.179634 38825 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0912 16:42:58.271207 38825 ssh_runner.go:195] Run: sudo systemctl restart cri-docker.service
I0912 16:42:58.335686 38825 start.go:542] Will wait 60s for socket path /var/run/cri-dockerd.sock
I0912 16:42:58.335761 38825 ssh_runner.go:195] Run: stat /var/run/cri-dockerd.sock
I0912 16:42:58.340279 38825 start.go:563] Will wait 60s for crictl version
I0912 16:42:58.340333 38825 ssh_runner.go:195] Run: which crictl
I0912 16:42:58.343500 38825 ssh_runner.go:195] Run: sudo /usr/bin/crictl version
I0912 16:42:58.370507 38825 start.go:579] Version: 0.1.0
RuntimeName: docker
RuntimeVersion: 27.2.1
RuntimeApiVersion: v1
I0912 16:42:58.370582 38825 ssh_runner.go:195] Run: docker version --format {{.Server.Version}}
I0912 16:42:58.389566 38825 ssh_runner.go:195] Run: docker version --format {{.Server.Version}}
I0912 16:42:58.448392 38825 out.go:235] * Preparing Kubernetes v1.31.1 on Docker 27.2.1 ...
I0912 16:42:58.448439 38825 main.go:141] libmachine: (ha-253000) Calling .GetIP
I0912 16:42:58.448802 38825 ssh_runner.go:195] Run: grep 192.169.0.1 host.minikube.internal$ /etc/hosts
I0912 16:42:58.452674 38825 ssh_runner.go:195] Run: /bin/bash -c "{ grep -v $'\thost.minikube.internal$' "/etc/hosts"; echo "192.169.0.1 host.minikube.internal"; } > /tmp/h.$$; sudo cp /tmp/h.$$ "/etc/hosts""
I0912 16:42:58.463577 38825 kubeadm.go:883] updating cluster {Name:ha-253000 KeepContext:false EmbedCerts:false MinikubeISO:https://storage.googleapis.com/minikube-builds/iso/19616/minikube-v1.34.0-1726156389-19616-amd64.iso KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.45-1726156396-19616@sha256:66b06a42534e914a5c8ad765d7508a93a34031939ec9a6b3a818ef0a444ff889 Memory:2200 CPUs:2 DiskSize:20000 Driver:hyperkit HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.31.
1 ClusterName:ha-253000 Namespace:default APIServerHAVIP:192.169.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.169.0.117 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m02 IP:192.169.0.118 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m04 IP:192.169.0.120 Port:0 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:false Worker:true}] Addons:map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false helm-tiller:false inaccel:false ingress:false ingress
-dns:false inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s Mount:false MountString:/Users:/minikube-host Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: Disa
bleOptimizations:false DisableMetrics:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s} ...
I0912 16:42:58.463671 38825 preload.go:131] Checking if preload exists for k8s version v1.31.1 and runtime docker
I0912 16:42:58.463721 38825 ssh_runner.go:195] Run: docker images --format {{.Repository}}:{{.Tag}}
I0912 16:42:58.477101 38825 docker.go:685] Got preloaded images: -- stdout --
registry.k8s.io/kube-apiserver:v1.31.1
registry.k8s.io/kube-scheduler:v1.31.1
registry.k8s.io/kube-controller-manager:v1.31.1
registry.k8s.io/kube-proxy:v1.31.1
kindest/kindnetd:v20240813-c6f155d6
registry.k8s.io/coredns/coredns:v1.11.3
registry.k8s.io/etcd:3.5.15-0
registry.k8s.io/pause:3.10
ghcr.io/kube-vip/kube-vip:v0.8.0
gcr.io/k8s-minikube/storage-provisioner:v5
gcr.io/k8s-minikube/busybox:1.28
-- /stdout --
I0912 16:42:58.477115 38825 docker.go:615] Images already preloaded, skipping extraction
I0912 16:42:58.477193 38825 ssh_runner.go:195] Run: docker images --format {{.Repository}}:{{.Tag}}
I0912 16:42:58.490964 38825 docker.go:685] Got preloaded images: -- stdout --
registry.k8s.io/kube-apiserver:v1.31.1
registry.k8s.io/kube-scheduler:v1.31.1
registry.k8s.io/kube-controller-manager:v1.31.1
registry.k8s.io/kube-proxy:v1.31.1
kindest/kindnetd:v20240813-c6f155d6
registry.k8s.io/coredns/coredns:v1.11.3
registry.k8s.io/etcd:3.5.15-0
registry.k8s.io/pause:3.10
ghcr.io/kube-vip/kube-vip:v0.8.0
gcr.io/k8s-minikube/storage-provisioner:v5
gcr.io/k8s-minikube/busybox:1.28
-- /stdout --
I0912 16:42:58.490984 38825 cache_images.go:84] Images are preloaded, skipping loading
I0912 16:42:58.490994 38825 kubeadm.go:934] updating node { 192.169.0.117 8443 v1.31.1 docker true true} ...
I0912 16:42:58.491069 38825 kubeadm.go:946] kubelet [Unit]
Wants=docker.socket
[Service]
ExecStart=
ExecStart=/var/lib/minikube/binaries/v1.31.1/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --config=/var/lib/kubelet/config.yaml --hostname-override=ha-253000 --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.169.0.117
[Install]
config:
{KubernetesVersion:v1.31.1 ClusterName:ha-253000 Namespace:default APIServerHAVIP:192.169.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:}
I0912 16:42:58.491146 38825 ssh_runner.go:195] Run: docker info --format {{.CgroupDriver}}
I0912 16:42:58.529026 38825 cni.go:84] Creating CNI manager for ""
I0912 16:42:58.529039 38825 cni.go:136] multinode detected (3 nodes found), recommending kindnet
I0912 16:42:58.529048 38825 kubeadm.go:84] Using pod CIDR: 10.244.0.0/16
I0912 16:42:58.529063 38825 kubeadm.go:181] kubeadm options: {CertDir:/var/lib/minikube/certs ServiceCIDR:10.96.0.0/12 PodSubnet:10.244.0.0/16 AdvertiseAddress:192.169.0.117 APIServerPort:8443 KubernetesVersion:v1.31.1 EtcdDataDir:/var/lib/minikube/etcd EtcdExtraArgs:map[] ClusterName:ha-253000 NodeName:ha-253000 DNSDomain:cluster.local CRISocket:/var/run/cri-dockerd.sock ImageRepository: ComponentOptions:[{Component:apiServer ExtraArgs:map[enable-admission-plugins:NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota] Pairs:map[certSANs:["127.0.0.1", "localhost", "192.169.0.117"]]} {Component:controllerManager ExtraArgs:map[allocate-node-cidrs:true leader-elect:false] Pairs:map[]} {Component:scheduler ExtraArgs:map[leader-elect:false] Pairs:map[]}] FeatureArgs:map[] NodeIP:192.169.0.117 CgroupDriver:cgroupfs ClientCAFile:/var/lib/minikube/certs/ca.crt StaticPodPath:/etc/kubernetes
/manifests ControlPlaneAddress:control-plane.minikube.internal KubeProxyOptions:map[] ResolvConfSearchRegression:false KubeletConfigOpts:map[containerRuntimeEndpoint:unix:///var/run/cri-dockerd.sock hairpinMode:hairpin-veth runtimeRequestTimeout:15m] PrependCriSocketUnix:true}
I0912 16:42:58.529151 38825 kubeadm.go:187] kubeadm config:
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 192.169.0.117
bindPort: 8443
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
ttl: 24h0m0s
usages:
- signing
- authentication
nodeRegistration:
criSocket: unix:///var/run/cri-dockerd.sock
name: "ha-253000"
kubeletExtraArgs:
node-ip: 192.169.0.117
taints: []
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
apiServer:
certSANs: ["127.0.0.1", "localhost", "192.169.0.117"]
extraArgs:
enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
controllerManager:
extraArgs:
allocate-node-cidrs: "true"
leader-elect: "false"
scheduler:
extraArgs:
leader-elect: "false"
certificatesDir: /var/lib/minikube/certs
clusterName: mk
controlPlaneEndpoint: control-plane.minikube.internal:8443
etcd:
local:
dataDir: /var/lib/minikube/etcd
extraArgs:
proxy-refresh-interval: "70000"
kubernetesVersion: v1.31.1
networking:
dnsDomain: cluster.local
podSubnet: "10.244.0.0/16"
serviceSubnet: 10.96.0.0/12
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
authentication:
x509:
clientCAFile: /var/lib/minikube/certs/ca.crt
cgroupDriver: cgroupfs
containerRuntimeEndpoint: unix:///var/run/cri-dockerd.sock
hairpinMode: hairpin-veth
runtimeRequestTimeout: 15m
clusterDomain: "cluster.local"
# disable disk resource management by default
imageGCHighThresholdPercent: 100
evictionHard:
nodefs.available: "0%"
nodefs.inodesFree: "0%"
imagefs.available: "0%"
failSwapOn: false
staticPodPath: /etc/kubernetes/manifests
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
clusterCIDR: "10.244.0.0/16"
metricsBindAddress: 0.0.0.0:10249
conntrack:
maxPerCore: 0
# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_established"
tcpEstablishedTimeout: 0s
# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_close"
tcpCloseWaitTimeout: 0s
I0912 16:42:58.529166 38825 kube-vip.go:115] generating kube-vip config ...
I0912 16:42:58.529226 38825 ssh_runner.go:195] Run: sudo sh -c "modprobe --all ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh nf_conntrack"
I0912 16:42:58.542282 38825 kube-vip.go:167] auto-enabling control-plane load-balancing in kube-vip
I0912 16:42:58.542349 38825 kube-vip.go:137] kube-vip config:
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
name: kube-vip
namespace: kube-system
spec:
containers:
- args:
- manager
env:
- name: vip_arp
value: "true"
- name: port
value: "8443"
- name: vip_nodename
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: vip_interface
value: eth0
- name: vip_cidr
value: "32"
- name: dns_mode
value: first
- name: cp_enable
value: "true"
- name: cp_namespace
value: kube-system
- name: vip_leaderelection
value: "true"
- name: vip_leasename
value: plndr-cp-lock
- name: vip_leaseduration
value: "5"
- name: vip_renewdeadline
value: "3"
- name: vip_retryperiod
value: "1"
- name: address
value: 192.169.0.254
- name: prometheus_server
value: :2112
- name : lb_enable
value: "true"
- name: lb_port
value: "8443"
image: ghcr.io/kube-vip/kube-vip:v0.8.0
imagePullPolicy: IfNotPresent
name: kube-vip
resources: {}
securityContext:
capabilities:
add:
- NET_ADMIN
- NET_RAW
volumeMounts:
- mountPath: /etc/kubernetes/admin.conf
name: kubeconfig
hostAliases:
- hostnames:
- kubernetes
ip: 127.0.0.1
hostNetwork: true
volumes:
- hostPath:
path: "/etc/kubernetes/admin.conf"
name: kubeconfig
status: {}
I0912 16:42:58.542402 38825 ssh_runner.go:195] Run: sudo ls /var/lib/minikube/binaries/v1.31.1
I0912 16:42:58.550850 38825 binaries.go:44] Found k8s binaries, skipping transfer
I0912 16:42:58.550895 38825 ssh_runner.go:195] Run: sudo mkdir -p /etc/systemd/system/kubelet.service.d /lib/systemd/system /var/tmp/minikube /etc/kubernetes/manifests
I0912 16:42:58.559004 38825 ssh_runner.go:362] scp memory --> /etc/systemd/system/kubelet.service.d/10-kubeadm.conf (309 bytes)
I0912 16:42:58.572515 38825 ssh_runner.go:362] scp memory --> /lib/systemd/system/kubelet.service (352 bytes)
I0912 16:42:58.585849 38825 ssh_runner.go:362] scp memory --> /var/tmp/minikube/kubeadm.yaml.new (2154 bytes)
I0912 16:42:58.599523 38825 ssh_runner.go:362] scp memory --> /etc/kubernetes/manifests/kube-vip.yaml (1440 bytes)
I0912 16:42:58.612915 38825 ssh_runner.go:195] Run: grep 192.169.0.254 control-plane.minikube.internal$ /etc/hosts
I0912 16:42:58.615674 38825 ssh_runner.go:195] Run: /bin/bash -c "{ grep -v $'\tcontrol-plane.minikube.internal$' "/etc/hosts"; echo "192.169.0.254 control-plane.minikube.internal"; } > /tmp/h.$$; sudo cp /tmp/h.$$ "/etc/hosts""
I0912 16:42:58.625619 38825 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0912 16:42:58.722704 38825 ssh_runner.go:195] Run: sudo systemctl start kubelet
I0912 16:42:58.737193 38825 certs.go:68] Setting up /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000 for IP: 192.169.0.117
I0912 16:42:58.737206 38825 certs.go:194] generating shared ca certs ...
I0912 16:42:58.737216 38825 certs.go:226] acquiring lock for ca certs: {Name:mka7ac38f1527bab8db77acd13744c4d87c27ef7 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0912 16:42:58.737390 38825 certs.go:235] skipping valid "minikubeCA" ca cert: /Users/jenkins/minikube-integration/19616-35737/.minikube/ca.key
I0912 16:42:58.737467 38825 certs.go:235] skipping valid "proxyClientCA" ca cert: /Users/jenkins/minikube-integration/19616-35737/.minikube/proxy-client-ca.key
I0912 16:42:58.737478 38825 certs.go:256] generating profile certs ...
I0912 16:42:58.737561 38825 certs.go:359] skipping valid signed profile cert regeneration for "minikube-user": /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/client.key
I0912 16:42:58.737583 38825 certs.go:363] generating signed profile cert for "minikube": /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/apiserver.key.492b3981
I0912 16:42:58.737601 38825 crypto.go:68] Generating cert /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/apiserver.crt.492b3981 with IP's: [10.96.0.1 127.0.0.1 10.0.0.1 192.169.0.117 192.169.0.118 192.169.0.254]
I0912 16:42:58.810055 38825 crypto.go:156] Writing cert to /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/apiserver.crt.492b3981 ...
I0912 16:42:58.810070 38825 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/apiserver.crt.492b3981: {Name:mk73a3232e8af8bb80f611e16c1832fa8c35d272 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0912 16:42:58.810605 38825 crypto.go:164] Writing key to /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/apiserver.key.492b3981 ...
I0912 16:42:58.810617 38825 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/apiserver.key.492b3981: {Name:mk53f23537ecb1425844b89fdc791cc7299f85c4 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0912 16:42:58.810872 38825 certs.go:381] copying /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/apiserver.crt.492b3981 -> /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/apiserver.crt
I0912 16:42:58.811120 38825 certs.go:385] copying /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/apiserver.key.492b3981 -> /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/apiserver.key
I0912 16:42:58.811366 38825 certs.go:359] skipping valid signed profile cert regeneration for "aggregator": /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/proxy-client.key
I0912 16:42:58.811376 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/ca.crt -> /var/lib/minikube/certs/ca.crt
I0912 16:42:58.811399 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/ca.key -> /var/lib/minikube/certs/ca.key
I0912 16:42:58.811425 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/proxy-client-ca.crt -> /var/lib/minikube/certs/proxy-client-ca.crt
I0912 16:42:58.811445 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/proxy-client-ca.key -> /var/lib/minikube/certs/proxy-client-ca.key
I0912 16:42:58.811464 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/apiserver.crt -> /var/lib/minikube/certs/apiserver.crt
I0912 16:42:58.811490 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/apiserver.key -> /var/lib/minikube/certs/apiserver.key
I0912 16:42:58.811508 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/proxy-client.crt -> /var/lib/minikube/certs/proxy-client.crt
I0912 16:42:58.811526 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/proxy-client.key -> /var/lib/minikube/certs/proxy-client.key
I0912 16:42:58.811623 38825 certs.go:484] found cert: /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/36285.pem (1338 bytes)
W0912 16:42:58.811676 38825 certs.go:480] ignoring /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/36285_empty.pem, impossibly tiny 0 bytes
I0912 16:42:58.811687 38825 certs.go:484] found cert: /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/ca-key.pem (1679 bytes)
I0912 16:42:58.811721 38825 certs.go:484] found cert: /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/ca.pem (1082 bytes)
I0912 16:42:58.811751 38825 certs.go:484] found cert: /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/cert.pem (1123 bytes)
I0912 16:42:58.811779 38825 certs.go:484] found cert: /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/key.pem (1679 bytes)
I0912 16:42:58.811845 38825 certs.go:484] found cert: /Users/jenkins/minikube-integration/19616-35737/.minikube/files/etc/ssl/certs/362852.pem (1708 bytes)
I0912 16:42:58.811883 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/files/etc/ssl/certs/362852.pem -> /usr/share/ca-certificates/362852.pem
I0912 16:42:58.811905 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/ca.crt -> /usr/share/ca-certificates/minikubeCA.pem
I0912 16:42:58.811923 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/36285.pem -> /usr/share/ca-certificates/36285.pem
I0912 16:42:58.812365 38825 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19616-35737/.minikube/ca.crt --> /var/lib/minikube/certs/ca.crt (1111 bytes)
I0912 16:42:58.835397 38825 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19616-35737/.minikube/ca.key --> /var/lib/minikube/certs/ca.key (1679 bytes)
I0912 16:42:58.860324 38825 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19616-35737/.minikube/proxy-client-ca.crt --> /var/lib/minikube/certs/proxy-client-ca.crt (1119 bytes)
I0912 16:42:58.890183 38825 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19616-35737/.minikube/proxy-client-ca.key --> /var/lib/minikube/certs/proxy-client-ca.key (1679 bytes)
I0912 16:42:58.922926 38825 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/apiserver.crt --> /var/lib/minikube/certs/apiserver.crt (1432 bytes)
I0912 16:42:58.965595 38825 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/apiserver.key --> /var/lib/minikube/certs/apiserver.key (1679 bytes)
I0912 16:42:59.004799 38825 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/proxy-client.crt --> /var/lib/minikube/certs/proxy-client.crt (1147 bytes)
I0912 16:42:59.055145 38825 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/proxy-client.key --> /var/lib/minikube/certs/proxy-client.key (1675 bytes)
I0912 16:42:59.130796 38825 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19616-35737/.minikube/files/etc/ssl/certs/362852.pem --> /usr/share/ca-certificates/362852.pem (1708 bytes)
I0912 16:42:59.166564 38825 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19616-35737/.minikube/ca.crt --> /usr/share/ca-certificates/minikubeCA.pem (1111 bytes)
I0912 16:42:59.200143 38825 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/36285.pem --> /usr/share/ca-certificates/36285.pem (1338 bytes)
I0912 16:42:59.224304 38825 ssh_runner.go:362] scp memory --> /var/lib/minikube/kubeconfig (738 bytes)
I0912 16:42:59.242124 38825 ssh_runner.go:195] Run: openssl version
I0912 16:42:59.251089 38825 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/36285.pem && ln -fs /usr/share/ca-certificates/36285.pem /etc/ssl/certs/36285.pem"
I0912 16:42:59.263207 38825 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/36285.pem
I0912 16:42:59.267299 38825 certs.go:528] hashing: -rw-r--r-- 1 root root 1338 Sep 12 23:29 /usr/share/ca-certificates/36285.pem
I0912 16:42:59.267343 38825 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/36285.pem
I0912 16:42:59.272690 38825 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/51391683.0 || ln -fs /etc/ssl/certs/36285.pem /etc/ssl/certs/51391683.0"
I0912 16:42:59.285188 38825 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/362852.pem && ln -fs /usr/share/ca-certificates/362852.pem /etc/ssl/certs/362852.pem"
I0912 16:42:59.294940 38825 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/362852.pem
I0912 16:42:59.298298 38825 certs.go:528] hashing: -rw-r--r-- 1 root root 1708 Sep 12 23:29 /usr/share/ca-certificates/362852.pem
I0912 16:42:59.298334 38825 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/362852.pem
I0912 16:42:59.302918 38825 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/3ec20f2e.0 || ln -fs /etc/ssl/certs/362852.pem /etc/ssl/certs/3ec20f2e.0"
I0912 16:42:59.311197 38825 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/minikubeCA.pem && ln -fs /usr/share/ca-certificates/minikubeCA.pem /etc/ssl/certs/minikubeCA.pem"
I0912 16:42:59.319416 38825 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/minikubeCA.pem
I0912 16:42:59.322866 38825 certs.go:528] hashing: -rw-r--r-- 1 root root 1111 Sep 12 23:12 /usr/share/ca-certificates/minikubeCA.pem
I0912 16:42:59.322900 38825 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/minikubeCA.pem
I0912 16:42:59.327166 38825 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/b5213941.0 || ln -fs /etc/ssl/certs/minikubeCA.pem /etc/ssl/certs/b5213941.0"
I0912 16:42:59.335480 38825 ssh_runner.go:195] Run: stat /var/lib/minikube/certs/apiserver-kubelet-client.crt
I0912 16:42:59.338943 38825 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/apiserver-etcd-client.crt -checkend 86400
I0912 16:42:59.343240 38825 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/apiserver-kubelet-client.crt -checkend 86400
I0912 16:42:59.347556 38825 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/etcd/server.crt -checkend 86400
I0912 16:42:59.351847 38825 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/etcd/healthcheck-client.crt -checkend 86400
I0912 16:42:59.356191 38825 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/etcd/peer.crt -checkend 86400
I0912 16:42:59.360385 38825 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/front-proxy-client.crt -checkend 86400
I0912 16:42:59.364865 38825 kubeadm.go:392] StartCluster: {Name:ha-253000 KeepContext:false EmbedCerts:false MinikubeISO:https://storage.googleapis.com/minikube-builds/iso/19616/minikube-v1.34.0-1726156389-19616-amd64.iso KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.45-1726156396-19616@sha256:66b06a42534e914a5c8ad765d7508a93a34031939ec9a6b3a818ef0a444ff889 Memory:2200 CPUs:2 DiskSize:20000 Driver:hyperkit HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.31.1 C
lusterName:ha-253000 Namespace:default APIServerHAVIP:192.169.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.169.0.117 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m02 IP:192.169.0.118 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m04 IP:192.169.0.120 Port:0 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:false Worker:true}] Addons:map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false helm-tiller:false inaccel:false ingress:false ingress-dn
s:false inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s Mount:false MountString:/Users:/minikube-host Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: Disable
Optimizations:false DisableMetrics:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s}
I0912 16:42:59.364979 38825 ssh_runner.go:195] Run: docker ps --filter status=paused --filter=name=k8s_.*_(kube-system)_ --format={{.ID}}
I0912 16:42:59.378134 38825 ssh_runner.go:195] Run: sudo ls /var/lib/kubelet/kubeadm-flags.env /var/lib/kubelet/config.yaml /var/lib/minikube/etcd
I0912 16:42:59.385935 38825 kubeadm.go:408] found existing configuration files, will attempt cluster restart
I0912 16:42:59.385949 38825 kubeadm.go:593] restartPrimaryControlPlane start ...
I0912 16:42:59.385987 38825 ssh_runner.go:195] Run: sudo test -d /data/minikube
I0912 16:42:59.393663 38825 kubeadm.go:130] /data/minikube skipping compat symlinks: sudo test -d /data/minikube: Process exited with status 1
stdout:
stderr:
I0912 16:42:59.393980 38825 kubeconfig.go:47] verify endpoint returned: get endpoint: "ha-253000" does not appear in /Users/jenkins/minikube-integration/19616-35737/kubeconfig
I0912 16:42:59.394061 38825 kubeconfig.go:62] /Users/jenkins/minikube-integration/19616-35737/kubeconfig needs updating (will repair): [kubeconfig missing "ha-253000" cluster setting kubeconfig missing "ha-253000" context setting]
I0912 16:42:59.394245 38825 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/19616-35737/kubeconfig: {Name:mk551f37071c12758c11027e3151edebeb8a0548 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0912 16:42:59.394911 38825 loader.go:395] Config loaded from file: /Users/jenkins/minikube-integration/19616-35737/kubeconfig
I0912 16:42:59.395114 38825 kapi.go:59] client config for ha-253000: &rest.Config{Host:"https://192.169.0.117:8443", APIPath:"", ContentConfig:rest.ContentConfig{AcceptContentTypes:"", ContentType:"", GroupVersion:(*schema.GroupVersion)(nil), NegotiatedSerializer:runtime.NegotiatedSerializer(nil)}, Username:"", Password:"", BearerToken:"", BearerTokenFile:"", Impersonate:rest.ImpersonationConfig{UserName:"", UID:"", Groups:[]string(nil), Extra:map[string][]string(nil)}, AuthProvider:<nil>, AuthConfigPersister:rest.AuthProviderConfigPersister(nil), ExecProvider:<nil>, TLSClientConfig:rest.sanitizedTLSClientConfig{Insecure:false, ServerName:"", CertFile:"/Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/client.crt", KeyFile:"/Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/client.key", CAFile:"/Users/jenkins/minikube-integration/19616-35737/.minikube/ca.crt", CertData:[]uint8(nil), KeyData:[]uint8(nil), CAData:[]uint8(nil), NextProtos:[]string(ni
l)}, UserAgent:"", DisableCompression:false, Transport:http.RoundTripper(nil), WrapTransport:(transport.WrapperFunc)(0x10674c40), QPS:0, Burst:0, RateLimiter:flowcontrol.RateLimiter(nil), WarningHandler:rest.WarningHandler(nil), Timeout:0, Dial:(func(context.Context, string, string) (net.Conn, error))(nil), Proxy:(func(*http.Request) (*url.URL, error))(nil)}
I0912 16:42:59.395438 38825 cert_rotation.go:140] Starting client certificate rotation controller
I0912 16:42:59.395632 38825 ssh_runner.go:195] Run: sudo diff -u /var/tmp/minikube/kubeadm.yaml /var/tmp/minikube/kubeadm.yaml.new
I0912 16:42:59.403337 38825 kubeadm.go:630] The running cluster does not require reconfiguration: 192.169.0.117
I0912 16:42:59.403348 38825 kubeadm.go:597] duration metric: took 17.39507ms to restartPrimaryControlPlane
I0912 16:42:59.403353 38825 kubeadm.go:394] duration metric: took 38.497957ms to StartCluster
I0912 16:42:59.403363 38825 settings.go:142] acquiring lock: {Name:mk627c7eb25773aca97a4529e1b40007adfe10d3 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0912 16:42:59.403442 38825 settings.go:150] Updating kubeconfig: /Users/jenkins/minikube-integration/19616-35737/kubeconfig
I0912 16:42:59.403824 38825 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/19616-35737/kubeconfig: {Name:mk551f37071c12758c11027e3151edebeb8a0548 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I0912 16:42:59.404063 38825 start.go:233] HA (multi-control plane) cluster: will skip waiting for primary control-plane node &{Name: IP:192.169.0.117 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true}
I0912 16:42:59.404078 38825 start.go:241] waiting for startup goroutines ...
I0912 16:42:59.404088 38825 addons.go:507] enable addons start: toEnable=map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false helm-tiller:false inaccel:false ingress:false ingress-dns:false inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false]
I0912 16:42:59.404202 38825 config.go:182] Loaded profile config "ha-253000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.1
I0912 16:42:59.449571 38825 out.go:177] * Enabled addons:
I0912 16:42:59.470516 38825 addons.go:510] duration metric: took 66.431118ms for enable addons: enabled=[]
I0912 16:42:59.470555 38825 start.go:246] waiting for cluster config update ...
I0912 16:42:59.470571 38825 start.go:255] writing updated cluster config ...
I0912 16:42:59.492863 38825 out.go:201]
I0912 16:42:59.513966 38825 config.go:182] Loaded profile config "ha-253000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.1
I0912 16:42:59.514101 38825 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/config.json ...
I0912 16:42:59.535678 38825 out.go:177] * Starting "ha-253000-m02" control-plane node in "ha-253000" cluster
I0912 16:42:59.577466 38825 preload.go:131] Checking if preload exists for k8s version v1.31.1 and runtime docker
I0912 16:42:59.577500 38825 cache.go:56] Caching tarball of preloaded images
I0912 16:42:59.577671 38825 preload.go:172] Found /Users/jenkins/minikube-integration/19616-35737/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.31.1-docker-overlay2-amd64.tar.lz4 in cache, skipping download
I0912 16:42:59.577686 38825 cache.go:59] Finished verifying existence of preloaded tar for v1.31.1 on docker
I0912 16:42:59.577840 38825 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/config.json ...
I0912 16:42:59.578781 38825 start.go:360] acquireMachinesLock for ha-253000-m02: {Name:mk1a336f8b7e1d023cbf697496a947d172cff4c4 Clock:{} Delay:500ms Timeout:13m0s Cancel:<nil>}
I0912 16:42:59.578972 38825 start.go:364] duration metric: took 165.217µs to acquireMachinesLock for "ha-253000-m02"
I0912 16:42:59.578994 38825 start.go:96] Skipping create...Using existing machine configuration
I0912 16:42:59.578999 38825 fix.go:54] fixHost starting: m02
I0912 16:42:59.579299 38825 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I0912 16:42:59.579331 38825 main.go:141] libmachine: Launching plugin server for driver hyperkit
I0912 16:42:59.588482 38825 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:64029
I0912 16:42:59.588984 38825 main.go:141] libmachine: () Calling .GetVersion
I0912 16:42:59.589397 38825 main.go:141] libmachine: Using API Version 1
I0912 16:42:59.589411 38825 main.go:141] libmachine: () Calling .SetConfigRaw
I0912 16:42:59.589711 38825 main.go:141] libmachine: () Calling .GetMachineName
I0912 16:42:59.589906 38825 main.go:141] libmachine: (ha-253000-m02) Calling .DriverName
I0912 16:42:59.590071 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetState
I0912 16:42:59.590171 38825 main.go:141] libmachine: (ha-253000-m02) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0912 16:42:59.590260 38825 main.go:141] libmachine: (ha-253000-m02) DBG | hyperkit pid from json: 38731
I0912 16:42:59.591269 38825 main.go:141] libmachine: (ha-253000-m02) DBG | hyperkit pid 38731 missing from process table
I0912 16:42:59.591288 38825 fix.go:112] recreateIfNeeded on ha-253000-m02: state=Stopped err=<nil>
I0912 16:42:59.591299 38825 main.go:141] libmachine: (ha-253000-m02) Calling .DriverName
W0912 16:42:59.591385 38825 fix.go:138] unexpected machine state, will restart: <nil>
I0912 16:42:59.633702 38825 out.go:177] * Restarting existing hyperkit VM for "ha-253000-m02" ...
I0912 16:42:59.654446 38825 main.go:141] libmachine: (ha-253000-m02) Calling .Start
I0912 16:42:59.654723 38825 main.go:141] libmachine: (ha-253000-m02) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0912 16:42:59.654769 38825 main.go:141] libmachine: (ha-253000-m02) minikube might have been shutdown in an unclean way, the hyperkit pid file still exists: /Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/hyperkit.pid
I0912 16:42:59.656677 38825 main.go:141] libmachine: (ha-253000-m02) DBG | hyperkit pid 38731 missing from process table
I0912 16:42:59.656690 38825 main.go:141] libmachine: (ha-253000-m02) DBG | pid 38731 is in state "Stopped"
I0912 16:42:59.656710 38825 main.go:141] libmachine: (ha-253000-m02) DBG | Removing stale pid file /Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/hyperkit.pid...
I0912 16:42:59.656957 38825 main.go:141] libmachine: (ha-253000-m02) DBG | Using UUID d29f3897-07f8-45ed-8ed9-5e8de05b00c8
I0912 16:42:59.686455 38825 main.go:141] libmachine: (ha-253000-m02) DBG | Generated MAC 92:19:55:83:fe:25
I0912 16:42:59.686477 38825 main.go:141] libmachine: (ha-253000-m02) DBG | Starting with cmdline: loglevel=3 console=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-253000
I0912 16:42:59.686637 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:42:59 DEBUG: hyperkit: Start &hyperkit.HyperKit{HyperKit:"/usr/local/bin/hyperkit", Argv0:"", StateDir:"/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02", VPNKitSock:"", VPNKitUUID:"", VPNKitPreferredIPv4:"", UUID:"d29f3897-07f8-45ed-8ed9-5e8de05b00c8", Disks:[]hyperkit.Disk{(*hyperkit.RawDisk)(0xc0003b79e0)}, ISOImages:[]string{"/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/boot2docker.iso"}, VSock:false, VSockDir:"", VSockPorts:[]int(nil), VSockGuestCID:3, VMNet:true, Sockets9P:[]hyperkit.Socket9P(nil), Kernel:"/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/bzimage", Initrd:"/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/initrd", Bootrom:"", CPUs:2, Memory:2200, Console:1, Serials:[]hyperkit.Serial(nil), Pid:0, Arguments:[]string(nil), CmdLine:"", process:(*os.Process)(nil)}
I0912 16:42:59.686694 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:42:59 DEBUG: hyperkit: check &hyperkit.HyperKit{HyperKit:"/usr/local/bin/hyperkit", Argv0:"", StateDir:"/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02", VPNKitSock:"", VPNKitUUID:"", VPNKitPreferredIPv4:"", UUID:"d29f3897-07f8-45ed-8ed9-5e8de05b00c8", Disks:[]hyperkit.Disk{(*hyperkit.RawDisk)(0xc0003b79e0)}, ISOImages:[]string{"/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/boot2docker.iso"}, VSock:false, VSockDir:"", VSockPorts:[]int(nil), VSockGuestCID:3, VMNet:true, Sockets9P:[]hyperkit.Socket9P(nil), Kernel:"/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/bzimage", Initrd:"/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/initrd", Bootrom:"", CPUs:2, Memory:2200, Console:1, Serials:[]hyperkit.Serial(nil), Pid:0, Arguments:[]string(nil), CmdLine:"", process:(*os.Process)(nil)}
I0912 16:42:59.686744 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:42:59 DEBUG: hyperkit: Arguments: []string{"-A", "-u", "-F", "/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/hyperkit.pid", "-c", "2", "-m", "2200M", "-s", "0:0,hostbridge", "-s", "31,lpc", "-s", "1:0,virtio-net", "-U", "d29f3897-07f8-45ed-8ed9-5e8de05b00c8", "-s", "2:0,virtio-blk,/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/ha-253000-m02.rawdisk", "-s", "3,ahci-cd,/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/boot2docker.iso", "-s", "4,virtio-rnd", "-l", "com1,autopty=/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/tty,log=/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/console-ring", "-f", "kexec,/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/bzimage,/Users/jenkins/minikube-integration/19616-35737/.minikube/
machines/ha-253000-m02/initrd,earlyprintk=serial loglevel=3 console=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-253000"}
I0912 16:42:59.686791 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:42:59 DEBUG: hyperkit: CmdLine: "/usr/local/bin/hyperkit -A -u -F /Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/hyperkit.pid -c 2 -m 2200M -s 0:0,hostbridge -s 31,lpc -s 1:0,virtio-net -U d29f3897-07f8-45ed-8ed9-5e8de05b00c8 -s 2:0,virtio-blk,/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/ha-253000-m02.rawdisk -s 3,ahci-cd,/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/boot2docker.iso -s 4,virtio-rnd -l com1,autopty=/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/tty,log=/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/console-ring -f kexec,/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/bzimage,/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/initrd,earlyprintk=serial loglevel=3 co
nsole=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-253000"
I0912 16:42:59.686802 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:42:59 DEBUG: hyperkit: Redirecting stdout/stderr to logger
I0912 16:42:59.688152 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:42:59 DEBUG: hyperkit: Pid is 38846
I0912 16:42:59.688548 38825 main.go:141] libmachine: (ha-253000-m02) DBG | Attempt 0
I0912 16:42:59.688564 38825 main.go:141] libmachine: (ha-253000-m02) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I0912 16:42:59.688702 38825 main.go:141] libmachine: (ha-253000-m02) DBG | hyperkit pid from json: 38846
I0912 16:42:59.690342 38825 main.go:141] libmachine: (ha-253000-m02) DBG | Searching for 92:19:55:83:fe:25 in /var/db/dhcpd_leases ...
I0912 16:42:59.690547 38825 main.go:141] libmachine: (ha-253000-m02) DBG | Found 119 entries in /var/db/dhcpd_leases!
I0912 16:42:59.690565 38825 main.go:141] libmachine: (ha-253000-m02) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.117 HWAddress:82:27:44:ae:54:1 ID:1,82:27:44:ae:54:1 Lease:0x66e4cdf8}
I0912 16:42:59.690592 38825 main.go:141] libmachine: (ha-253000-m02) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.119 HWAddress:e6:f8:cc:1a:35:bd ID:1,e6:f8:cc:1a:35:bd Lease:0x66e37c53}
I0912 16:42:59.690603 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetConfigRaw
I0912 16:42:59.690615 38825 main.go:141] libmachine: (ha-253000-m02) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.120 HWAddress:26:d:74:d3:a3:a1 ID:1,26:d:74:d3:a3:a1 Lease:0x66e4cdbe}
I0912 16:42:59.690664 38825 main.go:141] libmachine: (ha-253000-m02) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.118 HWAddress:92:19:55:83:fe:25 ID:1,92:19:55:83:fe:25 Lease:0x66e4cd52}
I0912 16:42:59.690685 38825 main.go:141] libmachine: (ha-253000-m02) DBG | Found match: 92:19:55:83:fe:25
I0912 16:42:59.690696 38825 main.go:141] libmachine: (ha-253000-m02) DBG | IP: 192.169.0.118
I0912 16:42:59.691253 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetIP
I0912 16:42:59.691472 38825 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19616-35737/.minikube/profiles/ha-253000/config.json ...
I0912 16:42:59.691875 38825 machine.go:93] provisionDockerMachine start ...
I0912 16:42:59.691885 38825 main.go:141] libmachine: (ha-253000-m02) Calling .DriverName
I0912 16:42:59.692024 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHHostname
I0912 16:42:59.692162 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHPort
I0912 16:42:59.692269 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHKeyPath
I0912 16:42:59.692373 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHKeyPath
I0912 16:42:59.692465 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHUsername
I0912 16:42:59.692577 38825 main.go:141] libmachine: Using SSH client type: native
I0912 16:42:59.692729 38825 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xefa9aa0] 0xefac800 <nil> [] 0s} 192.169.0.118 22 <nil> <nil>}
I0912 16:42:59.692738 38825 main.go:141] libmachine: About to run SSH command:
hostname
I0912 16:42:59.696335 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:42:59 INFO : hyperkit: stderr: Using fd 5 for I/O notifications
I0912 16:42:59.705018 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:42:59 INFO : hyperkit: stderr: /Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/boot2docker.iso: fcntl(F_PUNCHHOLE) Operation not permitted: block device will not support TRIM/DISCARD
I0912 16:42:59.706122 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:42:59 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 2 bit: 22 unspecified don't care: bit is 0
I0912 16:42:59.706147 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:42:59 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 12 unspecified don't care: bit is 0
I0912 16:42:59.706170 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:42:59 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 20 unspecified don't care: bit is 0
I0912 16:42:59.706190 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:42:59 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 3 bit: 13 unspecified don't care: bit is 0
I0912 16:43:00.097601 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:43:00 INFO : hyperkit: stderr: rdmsr to register 0x3a on vcpu 0
I0912 16:43:00.097615 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:43:00 INFO : hyperkit: stderr: rdmsr to register 0x140 on vcpu 0
I0912 16:43:00.212595 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:43:00 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 2 bit: 22 unspecified don't care: bit is 0
I0912 16:43:00.212613 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:43:00 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 12 unspecified don't care: bit is 0
I0912 16:43:00.212620 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:43:00 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 20 unspecified don't care: bit is 0
I0912 16:43:00.212626 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:43:00 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 3 bit: 13 unspecified don't care: bit is 0
I0912 16:43:00.213483 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:43:00 INFO : hyperkit: stderr: rdmsr to register 0x3a on vcpu 1
I0912 16:43:00.213495 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:43:00 INFO : hyperkit: stderr: rdmsr to register 0x140 on vcpu 1
I0912 16:43:05.836726 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:43:05 INFO : hyperkit: stderr: rdmsr to register 0x64d on vcpu 1
I0912 16:43:05.836802 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:43:05 INFO : hyperkit: stderr: rdmsr to register 0x64e on vcpu 1
I0912 16:43:05.836812 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:43:05 INFO : hyperkit: stderr: rdmsr to register 0x34 on vcpu 1
I0912 16:43:05.861273 38825 main.go:141] libmachine: (ha-253000-m02) DBG | 2024/09/12 16:43:05 INFO : hyperkit: stderr: rdmsr to register 0xc0011029 on vcpu 1
I0912 16:43:18.763912 38825 main.go:141] libmachine: SSH cmd err, output: <nil>: minikube
I0912 16:43:18.763925 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetMachineName
I0912 16:43:18.764098 38825 buildroot.go:166] provisioning hostname "ha-253000-m02"
I0912 16:43:18.764109 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetMachineName
I0912 16:43:18.764189 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHHostname
I0912 16:43:18.764265 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHPort
I0912 16:43:18.764359 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHKeyPath
I0912 16:43:18.764459 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHKeyPath
I0912 16:43:18.764551 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHUsername
I0912 16:43:18.764681 38825 main.go:141] libmachine: Using SSH client type: native
I0912 16:43:18.764843 38825 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xefa9aa0] 0xefac800 <nil> [] 0s} 192.169.0.118 22 <nil> <nil>}
I0912 16:43:18.764852 38825 main.go:141] libmachine: About to run SSH command:
sudo hostname ha-253000-m02 && echo "ha-253000-m02" | sudo tee /etc/hostname
I0912 16:43:18.839647 38825 main.go:141] libmachine: SSH cmd err, output: <nil>: ha-253000-m02
I0912 16:43:18.839665 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHHostname
I0912 16:43:18.839810 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHPort
I0912 16:43:18.839904 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHKeyPath
I0912 16:43:18.839994 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHKeyPath
I0912 16:43:18.840106 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHUsername
I0912 16:43:18.840246 38825 main.go:141] libmachine: Using SSH client type: native
I0912 16:43:18.840400 38825 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xefa9aa0] 0xefac800 <nil> [] 0s} 192.169.0.118 22 <nil> <nil>}
I0912 16:43:18.840411 38825 main.go:141] libmachine: About to run SSH command:
if ! grep -xq '.*\sha-253000-m02' /etc/hosts; then
if grep -xq '127.0.1.1\s.*' /etc/hosts; then
sudo sed -i 's/^127.0.1.1\s.*/127.0.1.1 ha-253000-m02/g' /etc/hosts;
else
echo '127.0.1.1 ha-253000-m02' | sudo tee -a /etc/hosts;
fi
fi
I0912 16:43:18.910184 38825 main.go:141] libmachine: SSH cmd err, output: <nil>:
I0912 16:43:18.910201 38825 buildroot.go:172] set auth options {CertDir:/Users/jenkins/minikube-integration/19616-35737/.minikube CaCertPath:/Users/jenkins/minikube-integration/19616-35737/.minikube/certs/ca.pem CaPrivateKeyPath:/Users/jenkins/minikube-integration/19616-35737/.minikube/certs/ca-key.pem CaCertRemotePath:/etc/docker/ca.pem ServerCertPath:/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/server.pem ServerKeyPath:/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/server-key.pem ClientKeyPath:/Users/jenkins/minikube-integration/19616-35737/.minikube/certs/key.pem ServerCertRemotePath:/etc/docker/server.pem ServerKeyRemotePath:/etc/docker/server-key.pem ClientCertPath:/Users/jenkins/minikube-integration/19616-35737/.minikube/certs/cert.pem ServerCertSANs:[] StorePath:/Users/jenkins/minikube-integration/19616-35737/.minikube}
I0912 16:43:18.910210 38825 buildroot.go:174] setting up certificates
I0912 16:43:18.910215 38825 provision.go:84] configureAuth start
I0912 16:43:18.910222 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetMachineName
I0912 16:43:18.910351 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetIP
I0912 16:43:18.910440 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHHostname
I0912 16:43:18.910514 38825 provision.go:143] copyHostCerts
I0912 16:43:18.910543 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/ca.pem -> /Users/jenkins/minikube-integration/19616-35737/.minikube/ca.pem
I0912 16:43:18.910616 38825 exec_runner.go:144] found /Users/jenkins/minikube-integration/19616-35737/.minikube/ca.pem, removing ...
I0912 16:43:18.910622 38825 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19616-35737/.minikube/ca.pem
I0912 16:43:18.910778 38825 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/ca.pem --> /Users/jenkins/minikube-integration/19616-35737/.minikube/ca.pem (1082 bytes)
I0912 16:43:18.910977 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/cert.pem -> /Users/jenkins/minikube-integration/19616-35737/.minikube/cert.pem
I0912 16:43:18.911020 38825 exec_runner.go:144] found /Users/jenkins/minikube-integration/19616-35737/.minikube/cert.pem, removing ...
I0912 16:43:18.911024 38825 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19616-35737/.minikube/cert.pem
I0912 16:43:18.911387 38825 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/cert.pem --> /Users/jenkins/minikube-integration/19616-35737/.minikube/cert.pem (1123 bytes)
I0912 16:43:18.911538 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/key.pem -> /Users/jenkins/minikube-integration/19616-35737/.minikube/key.pem
I0912 16:43:18.911575 38825 exec_runner.go:144] found /Users/jenkins/minikube-integration/19616-35737/.minikube/key.pem, removing ...
I0912 16:43:18.911583 38825 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19616-35737/.minikube/key.pem
I0912 16:43:18.911656 38825 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/key.pem --> /Users/jenkins/minikube-integration/19616-35737/.minikube/key.pem (1679 bytes)
I0912 16:43:18.911814 38825 provision.go:117] generating server cert: /Users/jenkins/minikube-integration/19616-35737/.minikube/machines/server.pem ca-key=/Users/jenkins/minikube-integration/19616-35737/.minikube/certs/ca.pem private-key=/Users/jenkins/minikube-integration/19616-35737/.minikube/certs/ca-key.pem org=jenkins.ha-253000-m02 san=[127.0.0.1 192.169.0.118 ha-253000-m02 localhost minikube]
I0912 16:43:18.981905 38825 provision.go:177] copyRemoteCerts
I0912 16:43:18.981957 38825 ssh_runner.go:195] Run: sudo mkdir -p /etc/docker /etc/docker /etc/docker
I0912 16:43:18.981973 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHHostname
I0912 16:43:18.982137 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHPort
I0912 16:43:18.982237 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHKeyPath
I0912 16:43:18.982310 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHUsername
I0912 16:43:18.982393 38825 sshutil.go:53] new ssh client: &{IP:192.169.0.118 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/id_rsa Username:docker}
I0912 16:43:19.020374 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/ca.pem -> /etc/docker/ca.pem
I0912 16:43:19.020459 38825 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19616-35737/.minikube/certs/ca.pem --> /etc/docker/ca.pem (1082 bytes)
I0912 16:43:19.040149 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/machines/server.pem -> /etc/docker/server.pem
I0912 16:43:19.040216 38825 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19616-35737/.minikube/machines/server.pem --> /etc/docker/server.pem (1208 bytes)
I0912 16:43:19.059817 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/machines/server-key.pem -> /etc/docker/server-key.pem
I0912 16:43:19.059896 38825 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19616-35737/.minikube/machines/server-key.pem --> /etc/docker/server-key.pem (1679 bytes)
I0912 16:43:19.079630 38825 provision.go:87] duration metric: took 169.40432ms to configureAuth
I0912 16:43:19.079642 38825 buildroot.go:189] setting minikube options for container-runtime
I0912 16:43:19.079810 38825 config.go:182] Loaded profile config "ha-253000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.1
I0912 16:43:19.079824 38825 main.go:141] libmachine: (ha-253000-m02) Calling .DriverName
I0912 16:43:19.079996 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHHostname
I0912 16:43:19.080090 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHPort
I0912 16:43:19.080181 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHKeyPath
I0912 16:43:19.080268 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHKeyPath
I0912 16:43:19.080356 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHUsername
I0912 16:43:19.080465 38825 main.go:141] libmachine: Using SSH client type: native
I0912 16:43:19.080586 38825 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xefa9aa0] 0xefac800 <nil> [] 0s} 192.169.0.118 22 <nil> <nil>}
I0912 16:43:19.080593 38825 main.go:141] libmachine: About to run SSH command:
df --output=fstype / | tail -n 1
I0912 16:43:19.145523 38825 main.go:141] libmachine: SSH cmd err, output: <nil>: tmpfs
I0912 16:43:19.145537 38825 buildroot.go:70] root file system type: tmpfs
I0912 16:43:19.145622 38825 provision.go:314] Updating docker unit: /lib/systemd/system/docker.service ...
I0912 16:43:19.145635 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHHostname
I0912 16:43:19.145765 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHPort
I0912 16:43:19.145847 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHKeyPath
I0912 16:43:19.145938 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHKeyPath
I0912 16:43:19.146035 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHUsername
I0912 16:43:19.146175 38825 main.go:141] libmachine: Using SSH client type: native
I0912 16:43:19.146311 38825 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xefa9aa0] 0xefac800 <nil> [] 0s} 192.169.0.118 22 <nil> <nil>}
I0912 16:43:19.146357 38825 main.go:141] libmachine: About to run SSH command:
sudo mkdir -p /lib/systemd/system && printf %s "[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target minikube-automount.service docker.socket
Requires= minikube-automount.service docker.socket
StartLimitBurst=3
StartLimitIntervalSec=60
[Service]
Type=notify
Restart=on-failure
Environment="NO_PROXY=192.169.0.117"
# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
# Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=hyperkit --insecure-registry 10.96.0.0/12
ExecReload=/bin/kill -s HUP \$MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
[Install]
WantedBy=multi-user.target
" | sudo tee /lib/systemd/system/docker.service.new
I0912 16:43:19.220152 38825 main.go:141] libmachine: SSH cmd err, output: <nil>: [Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target minikube-automount.service docker.socket
Requires= minikube-automount.service docker.socket
StartLimitBurst=3
StartLimitIntervalSec=60
[Service]
Type=notify
Restart=on-failure
Environment=NO_PROXY=192.169.0.117
# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
# Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=hyperkit --insecure-registry 10.96.0.0/12
ExecReload=/bin/kill -s HUP $MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
[Install]
WantedBy=multi-user.target
I0912 16:43:19.220174 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHHostname
I0912 16:43:19.220308 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHPort
I0912 16:43:19.220424 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHKeyPath
I0912 16:43:19.220525 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHKeyPath
I0912 16:43:19.220610 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHUsername
I0912 16:43:19.220747 38825 main.go:141] libmachine: Using SSH client type: native
I0912 16:43:19.220888 38825 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xefa9aa0] 0xefac800 <nil> [] 0s} 192.169.0.118 22 <nil> <nil>}
I0912 16:43:19.220900 38825 main.go:141] libmachine: About to run SSH command:
sudo diff -u /lib/systemd/system/docker.service /lib/systemd/system/docker.service.new || { sudo mv /lib/systemd/system/docker.service.new /lib/systemd/system/docker.service; sudo systemctl -f daemon-reload && sudo systemctl -f enable docker && sudo systemctl -f restart docker; }
I0912 16:43:20.920661 38825 main.go:141] libmachine: SSH cmd err, output: <nil>: diff: can't stat '/lib/systemd/system/docker.service': No such file or directory
Created symlink /etc/systemd/system/multi-user.target.wants/docker.service → /usr/lib/systemd/system/docker.service.
I0912 16:43:20.920679 38825 machine.go:96] duration metric: took 21.228391095s to provisionDockerMachine
I0912 16:43:20.920687 38825 start.go:293] postStartSetup for "ha-253000-m02" (driver="hyperkit")
I0912 16:43:20.920695 38825 start.go:322] creating required directories: [/etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs]
I0912 16:43:20.920706 38825 main.go:141] libmachine: (ha-253000-m02) Calling .DriverName
I0912 16:43:20.920886 38825 ssh_runner.go:195] Run: sudo mkdir -p /etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs
I0912 16:43:20.920907 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHHostname
I0912 16:43:20.921014 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHPort
I0912 16:43:20.921113 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHKeyPath
I0912 16:43:20.921188 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHUsername
I0912 16:43:20.921270 38825 sshutil.go:53] new ssh client: &{IP:192.169.0.118 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/id_rsa Username:docker}
I0912 16:43:20.961031 38825 ssh_runner.go:195] Run: cat /etc/os-release
I0912 16:43:20.964192 38825 info.go:137] Remote host: Buildroot 2023.02.9
I0912 16:43:20.964203 38825 filesync.go:126] Scanning /Users/jenkins/minikube-integration/19616-35737/.minikube/addons for local assets ...
I0912 16:43:20.964310 38825 filesync.go:126] Scanning /Users/jenkins/minikube-integration/19616-35737/.minikube/files for local assets ...
I0912 16:43:20.964487 38825 filesync.go:149] local asset: /Users/jenkins/minikube-integration/19616-35737/.minikube/files/etc/ssl/certs/362852.pem -> 362852.pem in /etc/ssl/certs
I0912 16:43:20.964493 38825 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19616-35737/.minikube/files/etc/ssl/certs/362852.pem -> /etc/ssl/certs/362852.pem
I0912 16:43:20.964690 38825 ssh_runner.go:195] Run: sudo mkdir -p /etc/ssl/certs
I0912 16:43:20.971749 38825 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19616-35737/.minikube/files/etc/ssl/certs/362852.pem --> /etc/ssl/certs/362852.pem (1708 bytes)
I0912 16:43:20.991458 38825 start.go:296] duration metric: took 70.761558ms for postStartSetup
I0912 16:43:20.991477 38825 main.go:141] libmachine: (ha-253000-m02) Calling .DriverName
I0912 16:43:20.991660 38825 ssh_runner.go:195] Run: sudo ls --almost-all -1 /var/lib/minikube/backup
I0912 16:43:20.991671 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHHostname
I0912 16:43:20.991767 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHPort
I0912 16:43:20.991846 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHKeyPath
I0912 16:43:20.991948 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHUsername
I0912 16:43:20.992026 38825 sshutil.go:53] new ssh client: &{IP:192.169.0.118 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/id_rsa Username:docker}
I0912 16:43:21.030957 38825 machine.go:197] restoring vm config from /var/lib/minikube/backup: [etc]
I0912 16:43:21.031029 38825 ssh_runner.go:195] Run: sudo rsync --archive --update /var/lib/minikube/backup/etc /
I0912 16:43:21.063980 38825 fix.go:56] duration metric: took 21.484565549s for fixHost
I0912 16:43:21.064007 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHHostname
I0912 16:43:21.064140 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHPort
I0912 16:43:21.064232 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHKeyPath
I0912 16:43:21.064349 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHKeyPath
I0912 16:43:21.064440 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHUsername
I0912 16:43:21.064580 38825 main.go:141] libmachine: Using SSH client type: native
I0912 16:43:21.064738 38825 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0xefa9aa0] 0xefac800 <nil> [] 0s} 192.169.0.118 22 <nil> <nil>}
I0912 16:43:21.064771 38825 main.go:141] libmachine: About to run SSH command:
date +%s.%N
I0912 16:43:21.129872 38825 main.go:141] libmachine: SSH cmd err, output: <nil>: 1726184601.132822786
I0912 16:43:21.129886 38825 fix.go:216] guest clock: 1726184601.132822786
I0912 16:43:21.129892 38825 fix.go:229] Guest: 2024-09-12 16:43:21.132822786 -0700 PDT Remote: 2024-09-12 16:43:21.063996 -0700 PDT m=+42.668480997 (delta=68.826786ms)
I0912 16:43:21.129902 38825 fix.go:200] guest clock delta is within tolerance: 68.826786ms
I0912 16:43:21.129906 38825 start.go:83] releasing machines lock for "ha-253000-m02", held for 21.550512411s
I0912 16:43:21.129922 38825 main.go:141] libmachine: (ha-253000-m02) Calling .DriverName
I0912 16:43:21.130046 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetIP
I0912 16:43:21.151551 38825 out.go:177] * Found network options:
I0912 16:43:21.172543 38825 out.go:177] - NO_PROXY=192.169.0.117
W0912 16:43:21.194443 38825 proxy.go:119] fail to check proxy env: Error ip not in block
I0912 16:43:21.194482 38825 main.go:141] libmachine: (ha-253000-m02) Calling .DriverName
I0912 16:43:21.195316 38825 main.go:141] libmachine: (ha-253000-m02) Calling .DriverName
I0912 16:43:21.195617 38825 main.go:141] libmachine: (ha-253000-m02) Calling .DriverName
I0912 16:43:21.195745 38825 ssh_runner.go:195] Run: curl -sS -m 2 https://registry.k8s.io/
I0912 16:43:21.195787 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHHostname
W0912 16:43:21.195893 38825 proxy.go:119] fail to check proxy env: Error ip not in block
I0912 16:43:21.195994 38825 ssh_runner.go:195] Run: sh -c "stat /etc/cni/net.d/*loopback.conf*"
I0912 16:43:21.196014 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHHostname
I0912 16:43:21.196078 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHPort
I0912 16:43:21.196305 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHPort
I0912 16:43:21.196335 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHKeyPath
I0912 16:43:21.196530 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHKeyPath
I0912 16:43:21.196575 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHUsername
I0912 16:43:21.196712 38825 main.go:141] libmachine: (ha-253000-m02) Calling .GetSSHUsername
I0912 16:43:21.196728 38825 sshutil.go:53] new ssh client: &{IP:192.169.0.118 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/id_rsa Username:docker}
I0912 16:43:21.196823 38825 sshutil.go:53] new ssh client: &{IP:192.169.0.118 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19616-35737/.minikube/machines/ha-253000-m02/id_rsa Username:docker}
W0912 16:43:21.232728 38825 cni.go:209] loopback cni configuration skipped: "/etc/cni/net.d/*loopback.conf*" not found
I0912 16:43:21.232787 38825 ssh_runner.go:195] Run: sudo find /etc/cni/net.d -maxdepth 1 -type f ( ( -name *bridge* -or -name *podman* ) -and -not -name *.mk_disabled ) -printf "%p, " -exec sh -c "sudo mv {} {}.mk_disabled" ;
I0912 16:43:21.287739 38825 cni.go:262] disabled [/etc/cni/net.d/87-podman-bridge.conflist] bridge cni config(s)
I0912 16:43:21.287762 38825 start.go:495] detecting cgroup driver to use...
I0912 16:43:21.287869 38825 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///run/containerd/containerd.sock
" | sudo tee /etc/crictl.yaml"
I0912 16:43:21.304013 38825 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)sandbox_image = .*$|\1sandbox_image = "registry.k8s.io/pause:3.10"|' /etc/containerd/config.toml"
I0912 16:43:21.313177 38825 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)restrict_oom_score_adj = .*$|\1restrict_oom_score_adj = false|' /etc/containerd/config.toml"
I0912 16:43:21.322064 38825 containerd.go:146] configuring containerd to use "cgroupfs" as cgroup driver...
I0912 16:43:21.322105 38825 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)SystemdCgroup = .*$|\1SystemdCgroup = false|g' /etc/containerd/config.toml"
I0912 16:43:21.331145 38825 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|"io.containerd.runtime.v1.linux"|"io.containerd.runc.v2"|g' /etc/containerd/config.toml"
I0912 16:43:21.340151 38825 ssh_runner.go:195] Run: sh -c "sudo sed -i '/systemd_cgroup/d' /etc/containerd/config.toml"
I0912 16:43:21.349347 38825 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|"io.containerd.runc.v1"|"io.containerd.runc.v2"|g' /etc/containerd/config.toml"
I0912 16:43:21.358534 38825 ssh_runner.go:195] Run: sh -c "sudo rm -rf /etc/cni/net.mk"
I0912 16:43:21.367648 38825 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)conf_dir = .*$|\1conf_dir = "/etc/cni/net.d"|g' /etc/containerd/config.toml"
I0912 16:43:21.376646 38825 ssh_runner.go:195] Run: sh -c "sudo sed -i '/^ *enable_unprivileged_ports = .*/d' /etc/containerd/config.toml"
I0912 16:43:21.385751 38825 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)\[plugins."io.containerd.grpc.v1.cri"\]|&\n\1 enable_unprivileged_ports = true|' /etc/containerd/config.toml"
I0912 16:43:21.395230 38825 ssh_runner.go:195] Run: sudo sysctl net.bridge.bridge-nf-call-iptables
I0912 16:43:21.403556 38825 ssh_runner.go:195] Run: sudo sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"
I0912 16:43:21.411842 38825 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0912 16:43:21.505139 38825 ssh_runner.go:195] Run: sudo systemctl restart containerd
I0912 16:43:21.524199 38825 start.go:495] detecting cgroup driver to use...
I0912 16:43:21.524264 38825 ssh_runner.go:195] Run: sudo systemctl cat docker.service
I0912 16:43:21.545632 38825 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I0912 16:43:21.558055 38825 ssh_runner.go:195] Run: sudo systemctl stop -f containerd
I0912 16:43:21.579009 38825 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I0912 16:43:21.591013 38825 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service crio
I0912 16:43:21.602335 38825 ssh_runner.go:195] Run: sudo systemctl stop -f crio
I0912 16:43:21.625328 38825 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service crio
I0912 16:43:21.636978 38825 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///var/run/cri-dockerd.sock
" | sudo tee /etc/crictl.yaml"
I0912 16:43:21.651787 38825 ssh_runner.go:195] Run: which cri-dockerd
I0912 16:43:21.654849 38825 ssh_runner.go:195] Run: sudo mkdir -p /etc/systemd/system/cri-docker.service.d
I0912 16:43:21.663201 38825 ssh_runner.go:362] scp memory --> /etc/systemd/system/cri-docker.service.d/10-cni.conf (190 bytes)
I0912 16:43:21.676638 38825 ssh_runner.go:195] Run: sudo systemctl unmask docker.service
I0912 16:43:21.767154 38825 ssh_runner.go:195] Run: sudo systemctl enable docker.socket
I0912 16:43:21.868966 38825 docker.go:574] configuring docker to use "cgroupfs" as cgroup driver...
I0912 16:43:21.868994 38825 ssh_runner.go:362] scp memory --> /etc/docker/daemon.json (130 bytes)
I0912 16:43:21.885365 38825 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I0912 16:43:21.986969 38825 ssh_runner.go:195] Run: sudo systemctl restart docker
I0912 16:44:22.972052 38825 ssh_runner.go:235] Completed: sudo systemctl restart docker: (1m0.985153798s)
I0912 16:44:22.972115 38825 ssh_runner.go:195] Run: sudo journalctl --no-pager -u docker
I0912 16:44:23.006512 38825 out.go:201]
W0912 16:44:23.027373 38825 out.go:270] X Exiting due to RUNTIME_ENABLE: Failed to enable container runtime: sudo systemctl restart docker: Process exited with status 1
stdout:
stderr:
Job for docker.service failed because the control process exited with error code.
See "systemctl status docker.service" and "journalctl -xeu docker.service" for details.
sudo journalctl --no-pager -u docker:
-- stdout --
Sep 12 23:43:19 ha-253000-m02 systemd[1]: Starting Docker Application Container Engine...
Sep 12 23:43:19 ha-253000-m02 dockerd[501]: time="2024-09-12T23:43:19.542477514Z" level=info msg="Starting up"
Sep 12 23:43:19 ha-253000-m02 dockerd[501]: time="2024-09-12T23:43:19.543273892Z" level=info msg="containerd not running, starting managed containerd"
Sep 12 23:43:19 ha-253000-m02 dockerd[501]: time="2024-09-12T23:43:19.544311343Z" level=info msg="started new containerd process" address=/var/run/docker/containerd/containerd.sock module=libcontainerd pid=508
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.560651579Z" level=info msg="starting containerd" revision=472731909fa34bd7bc9c087e4c27943f9835f111 version=v1.7.21
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.576874209Z" level=info msg="loading plugin \"io.containerd.event.v1.exchange\"..." type=io.containerd.event.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.576896535Z" level=info msg="loading plugin \"io.containerd.internal.v1.opt\"..." type=io.containerd.internal.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.576934374Z" level=info msg="loading plugin \"io.containerd.warning.v1.deprecations\"..." type=io.containerd.warning.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.576944185Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.blockfile\"..." type=io.containerd.snapshotter.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.577062257Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.blockfile\"..." error="no scratch file generator: skip plugin" type=io.containerd.snapshotter.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.577096865Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.btrfs\"..." type=io.containerd.snapshotter.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.577244648Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.btrfs\"..." error="path /var/lib/docker/containerd/daemon/io.containerd.snapshotter.v1.btrfs (ext4) must be a btrfs filesystem to be used with the btrfs snapshotter: skip plugin" type=io.containerd.snapshotter.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.577278882Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.devmapper\"..." type=io.containerd.snapshotter.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.577290899Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.devmapper\"..." error="devmapper not configured: skip plugin" type=io.containerd.snapshotter.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.577298862Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.native\"..." type=io.containerd.snapshotter.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.577479772Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.overlayfs\"..." type=io.containerd.snapshotter.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.577693060Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.aufs\"..." type=io.containerd.snapshotter.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.579277133Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.aufs\"..." error="aufs is not supported (modprobe aufs failed: exit status 1 \"modprobe: FATAL: Module aufs not found in directory /lib/modules/5.10.207\\n\"): skip plugin" type=io.containerd.snapshotter.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.579315062Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.zfs\"..." type=io.containerd.snapshotter.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.579473927Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.zfs\"..." error="path /var/lib/docker/containerd/daemon/io.containerd.snapshotter.v1.zfs must be a zfs filesystem to be used with the zfs snapshotter: skip plugin" type=io.containerd.snapshotter.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.579507321Z" level=info msg="loading plugin \"io.containerd.content.v1.content\"..." type=io.containerd.content.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.579642536Z" level=info msg="loading plugin \"io.containerd.metadata.v1.bolt\"..." type=io.containerd.metadata.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.579718072Z" level=info msg="metadata content store policy set" policy=shared
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583431554Z" level=info msg="loading plugin \"io.containerd.gc.v1.scheduler\"..." type=io.containerd.gc.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583479063Z" level=info msg="loading plugin \"io.containerd.differ.v1.walking\"..." type=io.containerd.differ.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583492745Z" level=info msg="loading plugin \"io.containerd.lease.v1.manager\"..." type=io.containerd.lease.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583503139Z" level=info msg="loading plugin \"io.containerd.streaming.v1.manager\"..." type=io.containerd.streaming.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583513468Z" level=info msg="loading plugin \"io.containerd.runtime.v1.linux\"..." type=io.containerd.runtime.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583561475Z" level=info msg="loading plugin \"io.containerd.monitor.v1.cgroups\"..." type=io.containerd.monitor.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583721342Z" level=info msg="loading plugin \"io.containerd.runtime.v2.task\"..." type=io.containerd.runtime.v2
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583790856Z" level=info msg="loading plugin \"io.containerd.runtime.v2.shim\"..." type=io.containerd.runtime.v2
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583825643Z" level=info msg="loading plugin \"io.containerd.sandbox.store.v1.local\"..." type=io.containerd.sandbox.store.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583836979Z" level=info msg="loading plugin \"io.containerd.sandbox.controller.v1.local\"..." type=io.containerd.sandbox.controller.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583846091Z" level=info msg="loading plugin \"io.containerd.service.v1.containers-service\"..." type=io.containerd.service.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583854705Z" level=info msg="loading plugin \"io.containerd.service.v1.content-service\"..." type=io.containerd.service.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583862338Z" level=info msg="loading plugin \"io.containerd.service.v1.diff-service\"..." type=io.containerd.service.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583871034Z" level=info msg="loading plugin \"io.containerd.service.v1.images-service\"..." type=io.containerd.service.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583879853Z" level=info msg="loading plugin \"io.containerd.service.v1.introspection-service\"..." type=io.containerd.service.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583887896Z" level=info msg="loading plugin \"io.containerd.service.v1.namespaces-service\"..." type=io.containerd.service.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583895577Z" level=info msg="loading plugin \"io.containerd.service.v1.snapshots-service\"..." type=io.containerd.service.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583902924Z" level=info msg="loading plugin \"io.containerd.service.v1.tasks-service\"..." type=io.containerd.service.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583916339Z" level=info msg="loading plugin \"io.containerd.grpc.v1.containers\"..." type=io.containerd.grpc.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583925297Z" level=info msg="loading plugin \"io.containerd.grpc.v1.content\"..." type=io.containerd.grpc.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583937222Z" level=info msg="loading plugin \"io.containerd.grpc.v1.diff\"..." type=io.containerd.grpc.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583951613Z" level=info msg="loading plugin \"io.containerd.grpc.v1.events\"..." type=io.containerd.grpc.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583962252Z" level=info msg="loading plugin \"io.containerd.grpc.v1.images\"..." type=io.containerd.grpc.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583970511Z" level=info msg="loading plugin \"io.containerd.grpc.v1.introspection\"..." type=io.containerd.grpc.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583978488Z" level=info msg="loading plugin \"io.containerd.grpc.v1.leases\"..." type=io.containerd.grpc.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583987448Z" level=info msg="loading plugin \"io.containerd.grpc.v1.namespaces\"..." type=io.containerd.grpc.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.583996029Z" level=info msg="loading plugin \"io.containerd.grpc.v1.sandbox-controllers\"..." type=io.containerd.grpc.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.584005228Z" level=info msg="loading plugin \"io.containerd.grpc.v1.sandboxes\"..." type=io.containerd.grpc.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.584012496Z" level=info msg="loading plugin \"io.containerd.grpc.v1.snapshots\"..." type=io.containerd.grpc.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.584019863Z" level=info msg="loading plugin \"io.containerd.grpc.v1.streaming\"..." type=io.containerd.grpc.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.584027388Z" level=info msg="loading plugin \"io.containerd.grpc.v1.tasks\"..." type=io.containerd.grpc.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.584037727Z" level=info msg="loading plugin \"io.containerd.transfer.v1.local\"..." type=io.containerd.transfer.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.584051226Z" level=info msg="loading plugin \"io.containerd.grpc.v1.transfer\"..." type=io.containerd.grpc.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.584059787Z" level=info msg="loading plugin \"io.containerd.grpc.v1.version\"..." type=io.containerd.grpc.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.584066858Z" level=info msg="loading plugin \"io.containerd.internal.v1.restart\"..." type=io.containerd.internal.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.584115945Z" level=info msg="loading plugin \"io.containerd.tracing.processor.v1.otlp\"..." type=io.containerd.tracing.processor.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.584137546Z" level=info msg="skip loading plugin \"io.containerd.tracing.processor.v1.otlp\"..." error="skip plugin: tracing endpoint not configured" type=io.containerd.tracing.processor.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.584146693Z" level=info msg="loading plugin \"io.containerd.internal.v1.tracing\"..." type=io.containerd.internal.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.584154778Z" level=info msg="skip loading plugin \"io.containerd.internal.v1.tracing\"..." error="skip plugin: tracing endpoint not configured" type=io.containerd.internal.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.584161587Z" level=info msg="loading plugin \"io.containerd.grpc.v1.healthcheck\"..." type=io.containerd.grpc.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.584169376Z" level=info msg="loading plugin \"io.containerd.nri.v1.nri\"..." type=io.containerd.nri.v1
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.584175876Z" level=info msg="NRI interface is disabled by configuration."
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.584336179Z" level=info msg=serving... address=/var/run/docker/containerd/containerd-debug.sock
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.584390949Z" level=info msg=serving... address=/var/run/docker/containerd/containerd.sock.ttrpc
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.584419385Z" level=info msg=serving... address=/var/run/docker/containerd/containerd.sock
Sep 12 23:43:19 ha-253000-m02 dockerd[508]: time="2024-09-12T23:43:19.584699970Z" level=info msg="containerd successfully booted in 0.024947s"
Sep 12 23:43:20 ha-253000-m02 dockerd[501]: time="2024-09-12T23:43:20.571233646Z" level=info msg="[graphdriver] trying configured driver: overlay2"
Sep 12 23:43:20 ha-253000-m02 dockerd[501]: time="2024-09-12T23:43:20.601540469Z" level=info msg="Loading containers: start."
Sep 12 23:43:20 ha-253000-m02 dockerd[501]: time="2024-09-12T23:43:20.783385765Z" level=warning msg="ip6tables is enabled, but cannot set up ip6tables chains" error="failed to create NAT chain DOCKER: iptables failed: ip6tables --wait -t nat -N DOCKER: ip6tables v1.8.9 (legacy): can't initialize ip6tables table `nat': Table does not exist (do you need to insmod?)\nPerhaps ip6tables or your kernel needs to be upgraded.\n (exit status 3)"
Sep 12 23:43:20 ha-253000-m02 dockerd[501]: time="2024-09-12T23:43:20.842757080Z" level=info msg="Default bridge (docker0) is assigned with an IP address 172.17.0.0/16. Daemon option --bip can be used to set a preferred IP address"
Sep 12 23:43:20 ha-253000-m02 dockerd[501]: time="2024-09-12T23:43:20.886491137Z" level=info msg="Loading containers: done."
Sep 12 23:43:20 ha-253000-m02 dockerd[501]: time="2024-09-12T23:43:20.898173270Z" level=info msg="Docker daemon" commit=8b539b8 containerd-snapshotter=false storage-driver=overlay2 version=27.2.1
Sep 12 23:43:20 ha-253000-m02 dockerd[501]: time="2024-09-12T23:43:20.898363295Z" level=info msg="Daemon has completed initialization"
Sep 12 23:43:20 ha-253000-m02 dockerd[501]: time="2024-09-12T23:43:20.921778310Z" level=info msg="API listen on /var/run/docker.sock"
Sep 12 23:43:20 ha-253000-m02 dockerd[501]: time="2024-09-12T23:43:20.921932935Z" level=info msg="API listen on [::]:2376"
Sep 12 23:43:20 ha-253000-m02 systemd[1]: Started Docker Application Container Engine.
Sep 12 23:43:22 ha-253000-m02 dockerd[501]: time="2024-09-12T23:43:22.003249641Z" level=info msg="Processing signal 'terminated'"
Sep 12 23:43:22 ha-253000-m02 dockerd[501]: time="2024-09-12T23:43:22.004113525Z" level=info msg="stopping event stream following graceful shutdown" error="<nil>" module=libcontainerd namespace=moby
Sep 12 23:43:22 ha-253000-m02 dockerd[501]: time="2024-09-12T23:43:22.004302507Z" level=info msg="Daemon shutdown complete"
Sep 12 23:43:22 ha-253000-m02 dockerd[501]: time="2024-09-12T23:43:22.004335750Z" level=info msg="stopping healthcheck following graceful shutdown" module=libcontainerd
Sep 12 23:43:22 ha-253000-m02 dockerd[501]: time="2024-09-12T23:43:22.004344772Z" level=info msg="stopping event stream following graceful shutdown" error="context canceled" module=libcontainerd namespace=plugins.moby
Sep 12 23:43:22 ha-253000-m02 systemd[1]: Stopping Docker Application Container Engine...
Sep 12 23:43:23 ha-253000-m02 systemd[1]: docker.service: Deactivated successfully.
Sep 12 23:43:23 ha-253000-m02 systemd[1]: Stopped Docker Application Container Engine.
Sep 12 23:43:23 ha-253000-m02 systemd[1]: Starting Docker Application Container Engine...
Sep 12 23:43:23 ha-253000-m02 dockerd[1081]: time="2024-09-12T23:43:23.038564212Z" level=info msg="Starting up"
Sep 12 23:44:23 ha-253000-m02 dockerd[1081]: failed to start daemon: failed to dial "/run/containerd/containerd.sock": failed to dial "/run/containerd/containerd.sock": context deadline exceeded
Sep 12 23:44:23 ha-253000-m02 systemd[1]: docker.service: Main process exited, code=exited, status=1/FAILURE
Sep 12 23:44:23 ha-253000-m02 systemd[1]: docker.service: Failed with result 'exit-code'.
Sep 12 23:44:23 ha-253000-m02 systemd[1]: Failed to start Docker Application Container Engine.
-- /stdout --
W0912 16:44:23.027475 38825 out.go:270] *
W0912 16:44:23.028715 38825 out.go:293] ╭─────────────────────────────────────────────────────────────────────────────────────────────╮
│ │
│ * If the above advice does not help, please let us know: │
│ https://github.com/kubernetes/minikube/issues/new/choose │
│ │
│ * Please run `minikube logs --file=logs.txt` and attach logs.txt to the GitHub issue. │
│ │
╰─────────────────────────────────────────────────────────────────────────────────────────────╯
I0912 16:44:23.090264 38825 out.go:201]
==> Docker <==
Sep 12 23:43:05 ha-253000 dockerd[1171]: time="2024-09-12T23:43:05.896030763Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.pause\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Sep 12 23:43:27 ha-253000 dockerd[1164]: time="2024-09-12T23:43:27.094791431Z" level=info msg="ignoring event" container=81000b6c37c93daac51379117b0e199f60aad41ecc9049ff3b4782607ba433e7 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 12 23:43:27 ha-253000 dockerd[1171]: time="2024-09-12T23:43:27.095463388Z" level=info msg="shim disconnected" id=81000b6c37c93daac51379117b0e199f60aad41ecc9049ff3b4782607ba433e7 namespace=moby
Sep 12 23:43:27 ha-253000 dockerd[1171]: time="2024-09-12T23:43:27.095519095Z" level=warning msg="cleaning up after shim disconnected" id=81000b6c37c93daac51379117b0e199f60aad41ecc9049ff3b4782607ba433e7 namespace=moby
Sep 12 23:43:27 ha-253000 dockerd[1171]: time="2024-09-12T23:43:27.095527791Z" level=info msg="cleaning up dead shim" namespace=moby
Sep 12 23:43:28 ha-253000 dockerd[1171]: time="2024-09-12T23:43:28.106766439Z" level=info msg="shim disconnected" id=bda6c96eaf4f67ea998c930b015c4060e579532ea2f977d165549eee5625f550 namespace=moby
Sep 12 23:43:28 ha-253000 dockerd[1171]: time="2024-09-12T23:43:28.106817450Z" level=warning msg="cleaning up after shim disconnected" id=bda6c96eaf4f67ea998c930b015c4060e579532ea2f977d165549eee5625f550 namespace=moby
Sep 12 23:43:28 ha-253000 dockerd[1171]: time="2024-09-12T23:43:28.106826128Z" level=info msg="cleaning up dead shim" namespace=moby
Sep 12 23:43:28 ha-253000 dockerd[1164]: time="2024-09-12T23:43:28.107015149Z" level=info msg="ignoring event" container=bda6c96eaf4f67ea998c930b015c4060e579532ea2f977d165549eee5625f550 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 12 23:43:38 ha-253000 dockerd[1171]: time="2024-09-12T23:43:38.426955933Z" level=info msg="loading plugin \"io.containerd.event.v1.publisher\"..." runtime=io.containerd.runc.v2 type=io.containerd.event.v1
Sep 12 23:43:38 ha-253000 dockerd[1171]: time="2024-09-12T23:43:38.427031665Z" level=info msg="loading plugin \"io.containerd.internal.v1.shutdown\"..." runtime=io.containerd.runc.v2 type=io.containerd.internal.v1
Sep 12 23:43:38 ha-253000 dockerd[1171]: time="2024-09-12T23:43:38.427062216Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.task\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Sep 12 23:43:38 ha-253000 dockerd[1171]: time="2024-09-12T23:43:38.427200623Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.pause\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Sep 12 23:43:43 ha-253000 dockerd[1171]: time="2024-09-12T23:43:43.045273171Z" level=info msg="loading plugin \"io.containerd.event.v1.publisher\"..." runtime=io.containerd.runc.v2 type=io.containerd.event.v1
Sep 12 23:43:43 ha-253000 dockerd[1171]: time="2024-09-12T23:43:43.045308356Z" level=info msg="loading plugin \"io.containerd.internal.v1.shutdown\"..." runtime=io.containerd.runc.v2 type=io.containerd.internal.v1
Sep 12 23:43:43 ha-253000 dockerd[1171]: time="2024-09-12T23:43:43.045316120Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.task\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Sep 12 23:43:43 ha-253000 dockerd[1171]: time="2024-09-12T23:43:43.045424517Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.pause\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Sep 12 23:44:03 ha-253000 dockerd[1171]: time="2024-09-12T23:44:03.301092361Z" level=info msg="shim disconnected" id=e8d3a41e6ab25f94223bbcf0f0115c2507995c287ca94e9afc14daee53a73845 namespace=moby
Sep 12 23:44:03 ha-253000 dockerd[1171]: time="2024-09-12T23:44:03.301158948Z" level=warning msg="cleaning up after shim disconnected" id=e8d3a41e6ab25f94223bbcf0f0115c2507995c287ca94e9afc14daee53a73845 namespace=moby
Sep 12 23:44:03 ha-253000 dockerd[1171]: time="2024-09-12T23:44:03.301168176Z" level=info msg="cleaning up dead shim" namespace=moby
Sep 12 23:44:03 ha-253000 dockerd[1164]: time="2024-09-12T23:44:03.301308328Z" level=info msg="ignoring event" container=e8d3a41e6ab25f94223bbcf0f0115c2507995c287ca94e9afc14daee53a73845 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 12 23:44:04 ha-253000 dockerd[1164]: time="2024-09-12T23:44:04.320168581Z" level=info msg="ignoring event" container=9c94eaa958da2700183df9261cda195a92962437936490991ed8ee5658a183aa module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Sep 12 23:44:04 ha-253000 dockerd[1171]: time="2024-09-12T23:44:04.320793337Z" level=info msg="shim disconnected" id=9c94eaa958da2700183df9261cda195a92962437936490991ed8ee5658a183aa namespace=moby
Sep 12 23:44:04 ha-253000 dockerd[1171]: time="2024-09-12T23:44:04.320842570Z" level=warning msg="cleaning up after shim disconnected" id=9c94eaa958da2700183df9261cda195a92962437936490991ed8ee5658a183aa namespace=moby
Sep 12 23:44:04 ha-253000 dockerd[1171]: time="2024-09-12T23:44:04.320851102Z" level=info msg="cleaning up dead shim" namespace=moby
==> container status <==
CONTAINER IMAGE CREATED STATE NAME ATTEMPT POD ID POD
e8d3a41e6ab25 6bab7719df100 43 seconds ago Exited kube-apiserver 3 20554f2d55246 kube-apiserver-ha-253000
9c94eaa958da2 175ffd71cce3d 48 seconds ago Exited kube-controller-manager 3 557d2b5cc337f kube-controller-manager-ha-253000
44f539c735403 9aa1fad941575 About a minute ago Running kube-scheduler 2 40ebaa4939766 kube-scheduler-ha-253000
a7340ceca4379 2e96e5913fc06 About a minute ago Running etcd 2 d76ee6275aa2c etcd-ha-253000
22168afd0987c 38af8ddebf499 About a minute ago Running kube-vip 1 2f15fc5ec421a kube-vip-ha-253000
7fb1ca2669339 6e38f40d628db 2 minutes ago Exited storage-provisioner 2 4b88d73dc5d58 storage-provisioner
4e94248e9da38 8c811b4aec35f 3 minutes ago Exited busybox 1 243c293630977 busybox-7dff88458-rfm8g
0fa64922a61ea 12968670680f4 3 minutes ago Exited kindnet-cni 1 ad5c73f973380 kindnet-czfs6
260df2eb6f90e c69fa2e9cbf5f 3 minutes ago Exited coredns 1 0ad1481b73727 coredns-7c65d6cfc9-pjlxt
7f8b7aa83d2b5 60c005f310ff3 3 minutes ago Exited kube-proxy 1 1a6a2cf60d0bf kube-proxy-97mxr
012df70ef569a c69fa2e9cbf5f 3 minutes ago Exited coredns 1 4227528eb6ad8 coredns-7c65d6cfc9-ldhxz
1661d4640c85c 38af8ddebf499 4 minutes ago Exited kube-vip 0 0d8e0cb428b56 kube-vip-ha-253000
a8f623b2ba8cb 9aa1fad941575 4 minutes ago Exited kube-scheduler 1 61109627f7b2e kube-scheduler-ha-253000
11193e6af9751 2e96e5913fc06 4 minutes ago Exited etcd 1 ebaabf68be3dd etcd-ha-253000
==> coredns [012df70ef569] <==
.:53
[INFO] plugin/reload: Running configuration SHA512 = 257e111468ef6f1e36f10df061303186c353cd0e51aed8f50f4e4fd21cec02687aef97084fe1f82262f5cee88179d311670a6ae21ae185759728216fc264125f
CoreDNS-1.11.3
linux/amd64, go1.21.11, a6338e9
[INFO] 127.0.0.1:49928 - 53804 "HINFO IN 5704259437763674289.202657279360214362. udp 56 false 512" NXDOMAIN qr,rd,ra 131 0.051145177s
[INFO] plugin/ready: Still waiting on: "kubernetes"
[INFO] plugin/ready: Still waiting on: "kubernetes"
[INFO] plugin/ready: Still waiting on: "kubernetes"
[INFO] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229: failed to list *v1.Namespace: Get "https://10.96.0.1:443/api/v1/namespaces?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: Trace[614376894]: "Reflector ListAndWatch" name:pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229 (12-Sep-2024 23:40:44.755) (total time: 30003ms):
Trace[614376894]: ---"Objects listed" error:Get "https://10.96.0.1:443/api/v1/namespaces?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout 30003ms (23:41:14.759)
Trace[614376894]: [30.003908911s] [30.003908911s] END
[ERROR] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229: Failed to watch *v1.Namespace: failed to list *v1.Namespace: Get "https://10.96.0.1:443/api/v1/namespaces?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229: failed to list *v1.Service: Get "https://10.96.0.1:443/api/v1/services?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: Trace[1855073097]: "Reflector ListAndWatch" name:pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229 (12-Sep-2024 23:40:44.757) (total time: 30002ms):
Trace[1855073097]: ---"Objects listed" error:Get "https://10.96.0.1:443/api/v1/services?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout 30002ms (23:41:14.760)
Trace[1855073097]: [30.002544244s] [30.002544244s] END
[ERROR] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229: Failed to watch *v1.Service: failed to list *v1.Service: Get "https://10.96.0.1:443/api/v1/services?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229: failed to list *v1.EndpointSlice: Get "https://10.96.0.1:443/apis/discovery.k8s.io/v1/endpointslices?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: Trace[1332869769]: "Reflector ListAndWatch" name:pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229 (12-Sep-2024 23:40:44.758) (total time: 30003ms):
Trace[1332869769]: ---"Objects listed" error:Get "https://10.96.0.1:443/apis/discovery.k8s.io/v1/endpointslices?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout 30002ms (23:41:14.761)
Trace[1332869769]: [30.003009859s] [30.003009859s] END
[ERROR] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229: Failed to watch *v1.EndpointSlice: failed to list *v1.EndpointSlice: Get "https://10.96.0.1:443/apis/discovery.k8s.io/v1/endpointslices?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] SIGTERM: Shutting down servers then terminating
[INFO] plugin/health: Going into lameduck mode for 5s
==> coredns [260df2eb6f90] <==
.:53
[INFO] plugin/reload: Running configuration SHA512 = 257e111468ef6f1e36f10df061303186c353cd0e51aed8f50f4e4fd21cec02687aef97084fe1f82262f5cee88179d311670a6ae21ae185759728216fc264125f
CoreDNS-1.11.3
linux/amd64, go1.21.11, a6338e9
[INFO] 127.0.0.1:41827 - 12879 "HINFO IN 8438143683253022015.5008403823119686861. udp 57 false 512" NXDOMAIN qr,rd,ra 132 0.011499751s
[INFO] plugin/ready: Still waiting on: "kubernetes"
[INFO] plugin/ready: Still waiting on: "kubernetes"
[INFO] plugin/ready: Still waiting on: "kubernetes"
[INFO] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229: failed to list *v1.Namespace: Get "https://10.96.0.1:443/api/v1/namespaces?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: Trace[87447775]: "Reflector ListAndWatch" name:pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229 (12-Sep-2024 23:40:44.852) (total time: 30002ms):
Trace[87447775]: ---"Objects listed" error:Get "https://10.96.0.1:443/api/v1/namespaces?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout 30001ms (23:41:14.854)
Trace[87447775]: [30.002327785s] [30.002327785s] END
[ERROR] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229: Failed to watch *v1.Namespace: failed to list *v1.Namespace: Get "https://10.96.0.1:443/api/v1/namespaces?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229: failed to list *v1.Service: Get "https://10.96.0.1:443/api/v1/services?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: Trace[1816436743]: "Reflector ListAndWatch" name:pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229 (12-Sep-2024 23:40:44.855) (total time: 30001ms):
Trace[1816436743]: ---"Objects listed" error:Get "https://10.96.0.1:443/api/v1/services?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout 30001ms (23:41:14.856)
Trace[1816436743]: [30.00111807s] [30.00111807s] END
[ERROR] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229: Failed to watch *v1.Service: failed to list *v1.Service: Get "https://10.96.0.1:443/api/v1/services?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229: failed to list *v1.EndpointSlice: Get "https://10.96.0.1:443/apis/discovery.k8s.io/v1/endpointslices?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] plugin/kubernetes: Trace[363139867]: "Reflector ListAndWatch" name:pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229 (12-Sep-2024 23:40:44.856) (total time: 30000ms):
Trace[363139867]: ---"Objects listed" error:Get "https://10.96.0.1:443/apis/discovery.k8s.io/v1/endpointslices?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout 30000ms (23:41:14.856)
Trace[363139867]: [30.000428757s] [30.000428757s] END
[ERROR] plugin/kubernetes: pkg/mod/k8s.io/client-go@v0.29.3/tools/cache/reflector.go:229: Failed to watch *v1.EndpointSlice: failed to list *v1.EndpointSlice: Get "https://10.96.0.1:443/apis/discovery.k8s.io/v1/endpointslices?limit=500&resourceVersion=0": dial tcp 10.96.0.1:443: i/o timeout
[INFO] SIGTERM: Shutting down servers then terminating
[INFO] plugin/health: Going into lameduck mode for 5s
==> describe nodes <==
command /bin/bash -c "sudo /var/lib/minikube/binaries/v1.31.1/kubectl describe nodes --kubeconfig=/var/lib/minikube/kubeconfig" failed with error: /bin/bash -c "sudo /var/lib/minikube/binaries/v1.31.1/kubectl describe nodes --kubeconfig=/var/lib/minikube/kubeconfig": Process exited with status 1
stdout:
stderr:
E0912 23:44:27.035325 2886 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
E0912 23:44:27.036749 2886 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
E0912 23:44:27.038024 2886 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
E0912 23:44:27.039531 2886 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
E0912 23:44:27.040775 2886 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
The connection to the server localhost:8443 was refused - did you specify the right host or port?
==> dmesg <==
[ +0.000000] Any video related functionality will be severely degraded, and you may not even be able to suspend the system properly
[ +0.000001] Unless you actually understand what nomodeset does, you should reboot without enabling it
[ +0.035998] ACPI BIOS Warning (bug): Incorrect checksum in table [DSDT] - 0xBE, should be 0x1B (20200925/tbprint-173)
[ +0.006975] RETBleed: WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!
[ +5.710969] ACPI Error: Could not enable RealTimeClock event (20200925/evxfevnt-182)
[ +0.000002] ACPI Warning: Could not enable fixed event - RealTimeClock (4) (20200925/evxface-618)
[ +0.006858] platform regulatory.0: Direct firmware load for regulatory.db failed with error -2
[ +2.795373] systemd-fstab-generator[127]: Ignoring "noauto" option for root device
[ +2.268187] NFSD: Using /var/lib/nfs/v4recovery as the NFSv4 state recovery directory
[ +0.000007] NFSD: unable to find recovery directory /var/lib/nfs/v4recovery
[ +0.000001] NFSD: Unable to initialize client recovery tracking! (-2)
[ +2.481103] systemd-fstab-generator[467]: Ignoring "noauto" option for root device
[ +0.100652] systemd-fstab-generator[479]: Ignoring "noauto" option for root device
[ +1.321112] kauditd_printk_skb: 42 callbacks suppressed
[ +0.678362] systemd-fstab-generator[1092]: Ignoring "noauto" option for root device
[ +0.257060] systemd-fstab-generator[1130]: Ignoring "noauto" option for root device
[ +0.104508] systemd-fstab-generator[1142]: Ignoring "noauto" option for root device
[ +0.112721] systemd-fstab-generator[1156]: Ignoring "noauto" option for root device
[ +2.497877] systemd-fstab-generator[1375]: Ignoring "noauto" option for root device
[ +0.099535] systemd-fstab-generator[1387]: Ignoring "noauto" option for root device
[ +0.120654] systemd-fstab-generator[1399]: Ignoring "noauto" option for root device
[ +0.118728] systemd-fstab-generator[1415]: Ignoring "noauto" option for root device
[ +0.449023] systemd-fstab-generator[1576]: Ignoring "noauto" option for root device
[Sep12 23:43] kauditd_printk_skb: 271 callbacks suppressed
[ +21.577611] kauditd_printk_skb: 40 callbacks suppressed
==> etcd [11193e6af975] <==
2024/09/12 23:42:30 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing"
{"level":"warn","ts":"2024-09-12T23:42:30.484626Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"279.501279ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/configmaps/\" range_end:\"/registry/configmaps0\" count_only:true ","response":"","error":"context canceled"}
{"level":"info","ts":"2024-09-12T23:42:30.484638Z","caller":"traceutil/trace.go:171","msg":"trace[1005771260] range","detail":"{range_begin:/registry/configmaps/; range_end:/registry/configmaps0; }","duration":"279.515897ms","start":"2024-09-12T23:42:30.205120Z","end":"2024-09-12T23:42:30.484636Z","steps":["trace[1005771260] 'agreement among raft nodes before linearized reading' (duration: 279.501051ms)"],"step_count":1}
2024/09/12 23:42:30 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing"
{"level":"warn","ts":"2024-09-12T23:42:30.484687Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"2.629807305s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/\" range_end:\"/registry/events0\" count_only:true ","response":"","error":"context canceled"}
{"level":"info","ts":"2024-09-12T23:42:30.484698Z","caller":"traceutil/trace.go:171","msg":"trace[1452044378] range","detail":"{range_begin:/registry/events/; range_end:/registry/events0; }","duration":"2.629820155s","start":"2024-09-12T23:42:27.854875Z","end":"2024-09-12T23:42:30.484695Z","steps":["trace[1452044378] 'agreement among raft nodes before linearized reading' (duration: 2.629807533s)"],"step_count":1}
{"level":"warn","ts":"2024-09-12T23:42:30.484708Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2024-09-12T23:42:27.854858Z","time spent":"2.629846604s","remote":"127.0.0.1:33582","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":0,"response size":0,"request content":"key:\"/registry/events/\" range_end:\"/registry/events0\" count_only:true "}
2024/09/12 23:42:30 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing"
{"level":"warn","ts":"2024-09-12T23:42:30.484782Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"316.402921ms","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/priorityclasses/\" range_end:\"/registry/priorityclasses0\" count_only:true ","response":"","error":"context canceled"}
{"level":"info","ts":"2024-09-12T23:42:30.484794Z","caller":"traceutil/trace.go:171","msg":"trace[1043205888] range","detail":"{range_begin:/registry/priorityclasses/; range_end:/registry/priorityclasses0; }","duration":"316.415381ms","start":"2024-09-12T23:42:30.168375Z","end":"2024-09-12T23:42:30.484790Z","steps":["trace[1043205888] 'agreement among raft nodes before linearized reading' (duration: 316.402713ms)"],"step_count":1}
{"level":"warn","ts":"2024-09-12T23:42:30.484804Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2024-09-12T23:42:30.168368Z","time spent":"316.43339ms","remote":"127.0.0.1:33406","response type":"/etcdserverpb.KV/Range","request count":0,"request size":58,"response count":0,"response size":0,"request content":"key:\"/registry/priorityclasses/\" range_end:\"/registry/priorityclasses0\" count_only:true "}
2024/09/12 23:42:30 WARNING: [core] [Server #8] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing"
{"level":"warn","ts":"2024-09-12T23:42:30.515302Z","caller":"embed/serve.go:212","msg":"stopping secure grpc server due to error","error":"accept tcp 192.169.0.117:2379: use of closed network connection"}
{"level":"warn","ts":"2024-09-12T23:42:30.515362Z","caller":"embed/serve.go:214","msg":"stopped secure grpc server due to error","error":"accept tcp 192.169.0.117:2379: use of closed network connection"}
{"level":"info","ts":"2024-09-12T23:42:30.515417Z","caller":"etcdserver/server.go:1512","msg":"skipped leadership transfer; local server is not leader","local-member-id":"cea40a72315446c","current-leader-member-id":"0"}
{"level":"info","ts":"2024-09-12T23:42:30.515502Z","caller":"rafthttp/peer.go:330","msg":"stopping remote peer","remote-peer-id":"7320e887fd922001"}
{"level":"info","ts":"2024-09-12T23:42:30.515512Z","caller":"rafthttp/stream.go:294","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","remote-peer-id":"7320e887fd922001"}
{"level":"info","ts":"2024-09-12T23:42:30.515525Z","caller":"rafthttp/stream.go:294","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream Message","remote-peer-id":"7320e887fd922001"}
{"level":"info","ts":"2024-09-12T23:42:30.515573Z","caller":"rafthttp/pipeline.go:85","msg":"stopped HTTP pipelining with remote peer","local-member-id":"cea40a72315446c","remote-peer-id":"7320e887fd922001"}
{"level":"info","ts":"2024-09-12T23:42:30.515595Z","caller":"rafthttp/stream.go:442","msg":"stopped stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"cea40a72315446c","remote-peer-id":"7320e887fd922001"}
{"level":"info","ts":"2024-09-12T23:42:30.515619Z","caller":"rafthttp/stream.go:442","msg":"stopped stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"cea40a72315446c","remote-peer-id":"7320e887fd922001"}
{"level":"info","ts":"2024-09-12T23:42:30.515627Z","caller":"rafthttp/peer.go:335","msg":"stopped remote peer","remote-peer-id":"7320e887fd922001"}
{"level":"info","ts":"2024-09-12T23:42:30.520917Z","caller":"embed/etcd.go:581","msg":"stopping serving peer traffic","address":"192.169.0.117:2380"}
{"level":"info","ts":"2024-09-12T23:42:30.521012Z","caller":"embed/etcd.go:586","msg":"stopped serving peer traffic","address":"192.169.0.117:2380"}
{"level":"info","ts":"2024-09-12T23:42:30.521029Z","caller":"embed/etcd.go:379","msg":"closed etcd server","name":"ha-253000","data-dir":"/var/lib/minikube/etcd","advertise-peer-urls":["https://192.169.0.117:2380"],"advertise-client-urls":["https://192.169.0.117:2379"]}
==> etcd [a7340ceca437] <==
{"level":"warn","ts":"2024-09-12T23:44:21.752182Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":4930476120341209873,"retry-timeout":"500ms"}
{"level":"warn","ts":"2024-09-12T23:44:22.253396Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":4930476120341209873,"retry-timeout":"500ms"}
{"level":"warn","ts":"2024-09-12T23:44:22.758024Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":4930476120341209873,"retry-timeout":"500ms"}
{"level":"info","ts":"2024-09-12T23:44:23.012911Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"cea40a72315446c is starting a new election at term 3"}
{"level":"info","ts":"2024-09-12T23:44:23.013028Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"cea40a72315446c became pre-candidate at term 3"}
{"level":"info","ts":"2024-09-12T23:44:23.013046Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"cea40a72315446c received MsgPreVoteResp from cea40a72315446c at term 3"}
{"level":"info","ts":"2024-09-12T23:44:23.013062Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"cea40a72315446c [logterm: 3, index: 2778] sent MsgPreVote request to 7320e887fd922001 at term 3"}
{"level":"warn","ts":"2024-09-12T23:44:23.255839Z","caller":"etcdserver/server.go:2139","msg":"failed to publish local member to cluster through raft","local-member-id":"cea40a72315446c","local-member-attributes":"{Name:ha-253000 ClientURLs:[https://192.169.0.117:2379]}","request-path":"/0/members/cea40a72315446c/attributes","publish-timeout":"7s","error":"etcdserver: request timed out"}
{"level":"warn","ts":"2024-09-12T23:44:23.258729Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":4930476120341209873,"retry-timeout":"500ms"}
{"level":"warn","ts":"2024-09-12T23:44:23.758902Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":4930476120341209873,"retry-timeout":"500ms"}
{"level":"warn","ts":"2024-09-12T23:44:24.265509Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":4930476120341209873,"retry-timeout":"500ms"}
{"level":"info","ts":"2024-09-12T23:44:24.612911Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"cea40a72315446c is starting a new election at term 3"}
{"level":"info","ts":"2024-09-12T23:44:24.612948Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"cea40a72315446c became pre-candidate at term 3"}
{"level":"info","ts":"2024-09-12T23:44:24.612960Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"cea40a72315446c received MsgPreVoteResp from cea40a72315446c at term 3"}
{"level":"info","ts":"2024-09-12T23:44:24.612973Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"cea40a72315446c [logterm: 3, index: 2778] sent MsgPreVote request to 7320e887fd922001 at term 3"}
{"level":"warn","ts":"2024-09-12T23:44:24.737316Z","caller":"etcdserver/v3_server.go:932","msg":"timed out waiting for read index response (local node might have slow network)","timeout":"7s"}
{"level":"warn","ts":"2024-09-12T23:44:24.737658Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"7.006544092s","expected-duration":"100ms","prefix":"read-only range ","request":"limit:1 keys_only:true ","response":"","error":"etcdserver: request timed out"}
{"level":"info","ts":"2024-09-12T23:44:24.737747Z","caller":"traceutil/trace.go:171","msg":"trace[1023661117] range","detail":"{range_begin:; range_end:; }","duration":"7.006644404s","start":"2024-09-12T23:44:17.731094Z","end":"2024-09-12T23:44:24.737739Z","steps":["trace[1023661117] 'agreement among raft nodes before linearized reading' (duration: 7.006541714s)"],"step_count":1}
{"level":"error","ts":"2024-09-12T23:44:24.737772Z","caller":"etcdhttp/health.go:367","msg":"Health check error","path":"/readyz","reason":"[+]data_corruption ok\n[+]serializable_read ok\n[-]linearizable_read failed: etcdserver: request timed out\n","status-code":503,"stacktrace":"go.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp.(*CheckRegistry).installRootHttpEndpoint.newHealthHandler.func2\n\tgo.etcd.io/etcd/server/v3/etcdserver/api/etcdhttp/health.go:367\nnet/http.HandlerFunc.ServeHTTP\n\tnet/http/server.go:2141\nnet/http.(*ServeMux).ServeHTTP\n\tnet/http/server.go:2519\nnet/http.serverHandler.ServeHTTP\n\tnet/http/server.go:2943\nnet/http.(*conn).serve\n\tnet/http/server.go:2014"}
{"level":"info","ts":"2024-09-12T23:44:26.211834Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"cea40a72315446c is starting a new election at term 3"}
{"level":"info","ts":"2024-09-12T23:44:26.211900Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"cea40a72315446c became pre-candidate at term 3"}
{"level":"info","ts":"2024-09-12T23:44:26.211914Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"cea40a72315446c received MsgPreVoteResp from cea40a72315446c at term 3"}
{"level":"info","ts":"2024-09-12T23:44:26.211926Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"cea40a72315446c [logterm: 3, index: 2778] sent MsgPreVote request to 7320e887fd922001 at term 3"}
{"level":"warn","ts":"2024-09-12T23:44:26.256114Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_RAFT_MESSAGE","remote-peer-id":"7320e887fd922001","rtt":"0s","error":"dial tcp 192.169.0.118:2380: connect: connection refused"}
{"level":"warn","ts":"2024-09-12T23:44:26.256264Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_SNAPSHOT","remote-peer-id":"7320e887fd922001","rtt":"0s","error":"dial tcp 192.169.0.118:2380: connect: connection refused"}
==> kernel <==
23:44:27 up 1 min, 0 users, load average: 0.24, 0.11, 0.04
Linux ha-253000 5.10.207 #1 SMP Thu Sep 12 19:03:33 UTC 2024 x86_64 GNU/Linux
PRETTY_NAME="Buildroot 2023.02.9"
==> kindnet [0fa64922a61e] <==
I0912 23:41:56.064709 1 main.go:322] Node ha-253000-m02 has CIDR [10.244.1.0/24]
I0912 23:41:56.064793 1 main.go:295] Handling node with IPs: map[192.169.0.119:{}]
I0912 23:41:56.064863 1 main.go:322] Node ha-253000-m03 has CIDR [10.244.2.0/24]
I0912 23:41:56.064925 1 main.go:295] Handling node with IPs: map[192.169.0.120:{}]
I0912 23:41:56.065063 1 main.go:322] Node ha-253000-m04 has CIDR [10.244.3.0/24]
I0912 23:42:06.064466 1 main.go:295] Handling node with IPs: map[192.169.0.117:{}]
I0912 23:42:06.064489 1 main.go:299] handling current node
I0912 23:42:06.064500 1 main.go:295] Handling node with IPs: map[192.169.0.118:{}]
I0912 23:42:06.064504 1 main.go:322] Node ha-253000-m02 has CIDR [10.244.1.0/24]
I0912 23:42:06.064613 1 main.go:295] Handling node with IPs: map[192.169.0.119:{}]
I0912 23:42:06.064652 1 main.go:322] Node ha-253000-m03 has CIDR [10.244.2.0/24]
I0912 23:42:06.064706 1 main.go:295] Handling node with IPs: map[192.169.0.120:{}]
I0912 23:42:06.064744 1 main.go:322] Node ha-253000-m04 has CIDR [10.244.3.0/24]
I0912 23:42:16.065220 1 main.go:295] Handling node with IPs: map[192.169.0.117:{}]
I0912 23:42:16.065468 1 main.go:299] handling current node
I0912 23:42:16.065523 1 main.go:295] Handling node with IPs: map[192.169.0.118:{}]
I0912 23:42:16.065900 1 main.go:322] Node ha-253000-m02 has CIDR [10.244.1.0/24]
I0912 23:42:16.066273 1 main.go:295] Handling node with IPs: map[192.169.0.120:{}]
I0912 23:42:16.066482 1 main.go:322] Node ha-253000-m04 has CIDR [10.244.3.0/24]
I0912 23:42:26.073208 1 main.go:295] Handling node with IPs: map[192.169.0.120:{}]
I0912 23:42:26.073304 1 main.go:322] Node ha-253000-m04 has CIDR [10.244.3.0/24]
I0912 23:42:26.073569 1 main.go:295] Handling node with IPs: map[192.169.0.117:{}]
I0912 23:42:26.073623 1 main.go:299] handling current node
I0912 23:42:26.073646 1 main.go:295] Handling node with IPs: map[192.169.0.118:{}]
I0912 23:42:26.074016 1 main.go:322] Node ha-253000-m02 has CIDR [10.244.1.0/24]
==> kube-apiserver [e8d3a41e6ab2] <==
I0912 23:43:43.143519 1 options.go:228] external host was not specified, using 192.169.0.117
I0912 23:43:43.145026 1 server.go:142] Version: v1.31.1
I0912 23:43:43.145065 1 server.go:144] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I0912 23:43:43.276830 1 shared_informer.go:313] Waiting for caches to sync for node_authorizer
I0912 23:43:43.281423 1 shared_informer.go:313] Waiting for caches to sync for *generic.policySource[*k8s.io/api/admissionregistration/v1.ValidatingAdmissionPolicy,*k8s.io/api/admissionregistration/v1.ValidatingAdmissionPolicyBinding,k8s.io/apiserver/pkg/admission/plugin/policy/validating.Validator]
I0912 23:43:43.283992 1 plugins.go:157] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
I0912 23:43:43.284003 1 plugins.go:160] Loaded 13 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,PodSecurity,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,ClusterTrustBundleAttest,CertificateSubjectRestriction,ValidatingAdmissionPolicy,ValidatingAdmissionWebhook,ResourceQuota.
I0912 23:43:43.284201 1 instance.go:232] Using reconciler: lease
W0912 23:44:03.279631 1 logging.go:55] [core] [Channel #2 SubChannel #4]grpc: addrConn.createTransport failed to connect to {Addr: "127.0.0.1:2379", ServerName: "127.0.0.1:2379", }. Err: connection error: desc = "transport: authentication handshake failed: context deadline exceeded"
W0912 23:44:03.279699 1 logging.go:55] [core] [Channel #1 SubChannel #3]grpc: addrConn.createTransport failed to connect to {Addr: "127.0.0.1:2379", ServerName: "127.0.0.1:2379", }. Err: connection error: desc = "transport: authentication handshake failed: context deadline exceeded"
F0912 23:44:03.284897 1 instance.go:225] Error creating leases: error creating storage factory: context deadline exceeded
==> kube-controller-manager [9c94eaa958da] <==
I0912 23:43:38.840154 1 serving.go:386] Generated self-signed cert in-memory
I0912 23:43:39.317735 1 controllermanager.go:197] "Starting" version="v1.31.1"
I0912 23:43:39.317803 1 controllermanager.go:199] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I0912 23:43:39.319233 1 secure_serving.go:213] Serving securely on 127.0.0.1:10257
I0912 23:43:39.319943 1 dynamic_cafile_content.go:160] "Starting controller" name="request-header::/var/lib/minikube/certs/front-proxy-ca.crt"
I0912 23:43:39.320012 1 tlsconfig.go:243] "Starting DynamicServingCertificateController"
I0912 23:43:39.320077 1 dynamic_cafile_content.go:160] "Starting controller" name="client-ca-bundle::/var/lib/minikube/certs/ca.crt"
E0912 23:44:04.302770 1 controllermanager.go:242] "Error building controller context" err="failed to wait for apiserver being healthy: timed out waiting for the condition: failed to get apiserver /healthz status: Get \"https://192.169.0.117:8443/healthz\": dial tcp 192.169.0.117:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.117:43952->192.169.0.117:8443: read: connection reset by peer"
==> kube-proxy [7f8b7aa83d2b] <==
add table ip kube-proxy
^^^^^^^^^^^^^^^^^^^^^^^^
>
E0912 23:40:45.174456 1 proxier.go:734] "Error cleaning up nftables rules" err=<
could not run nftables command: /dev/stdin:1:1-25: Error: Could not process rule: Operation not supported
add table ip6 kube-proxy
^^^^^^^^^^^^^^^^^^^^^^^^^
>
I0912 23:40:45.194725 1 server.go:677] "Successfully retrieved node IP(s)" IPs=["192.169.0.117"]
E0912 23:40:45.194941 1 server.go:234] "Kube-proxy configuration may be incomplete or incorrect" err="nodePortAddresses is unset; NodePort connections will be accepted on all local IPs. Consider using `--nodeport-addresses primary`"
I0912 23:40:45.234850 1 server_linux.go:146] "No iptables support for family" ipFamily="IPv6"
I0912 23:40:45.234876 1 server.go:245] "kube-proxy running in single-stack mode" ipFamily="IPv4"
I0912 23:40:45.234894 1 server_linux.go:169] "Using iptables Proxier"
I0912 23:40:45.241426 1 proxier.go:255] "Setting route_localnet=1 to allow node-ports on localhost; to change this either disable iptables.localhostNodePorts (--iptables-localhost-nodeports) or set nodePortAddresses (--nodeport-addresses) to filter loopback addresses" ipFamily="IPv4"
I0912 23:40:45.242975 1 server.go:483] "Version info" version="v1.31.1"
I0912 23:40:45.243175 1 server.go:485] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I0912 23:40:45.247944 1 config.go:199] "Starting service config controller"
I0912 23:40:45.248599 1 config.go:105] "Starting endpoint slice config controller"
I0912 23:40:45.248753 1 shared_informer.go:313] Waiting for caches to sync for service config
I0912 23:40:45.248824 1 shared_informer.go:313] Waiting for caches to sync for endpoint slice config
I0912 23:40:45.249269 1 config.go:328] "Starting node config controller"
I0912 23:40:45.249424 1 shared_informer.go:313] Waiting for caches to sync for node config
I0912 23:40:45.349730 1 shared_informer.go:320] Caches are synced for node config
I0912 23:40:45.349730 1 shared_informer.go:320] Caches are synced for endpoint slice config
I0912 23:40:45.349742 1 shared_informer.go:320] Caches are synced for service config
==> kube-scheduler [44f539c73540] <==
E0912 23:44:12.406413 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.StorageClass: failed to list *v1.StorageClass: Get \"https://192.169.0.117:8443/apis/storage.k8s.io/v1/storageclasses?limit=500&resourceVersion=0\": dial tcp 192.169.0.117:8443: connect: connection refused" logger="UnhandledError"
W0912 23:44:13.294889 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.Pod: Get "https://192.169.0.117:8443/api/v1/pods?fieldSelector=status.phase%21%3DSucceeded%2Cstatus.phase%21%3DFailed&limit=500&resourceVersion=0": dial tcp 192.169.0.117:8443: connect: connection refused
E0912 23:44:13.294966 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Pod: failed to list *v1.Pod: Get \"https://192.169.0.117:8443/api/v1/pods?fieldSelector=status.phase%21%3DSucceeded%2Cstatus.phase%21%3DFailed&limit=500&resourceVersion=0\": dial tcp 192.169.0.117:8443: connect: connection refused" logger="UnhandledError"
W0912 23:44:13.368834 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.Node: Get "https://192.169.0.117:8443/api/v1/nodes?limit=500&resourceVersion=0": dial tcp 192.169.0.117:8443: connect: connection refused
E0912 23:44:13.368889 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Node: failed to list *v1.Node: Get \"https://192.169.0.117:8443/api/v1/nodes?limit=500&resourceVersion=0\": dial tcp 192.169.0.117:8443: connect: connection refused" logger="UnhandledError"
W0912 23:44:13.548869 1 reflector.go:561] runtime/asm_amd64.s:1695: failed to list *v1.ConfigMap: Get "https://192.169.0.117:8443/api/v1/namespaces/kube-system/configmaps?fieldSelector=metadata.name%3Dextension-apiserver-authentication&limit=500&resourceVersion=0": dial tcp 192.169.0.117:8443: connect: connection refused
E0912 23:44:13.548963 1 reflector.go:158] "Unhandled Error" err="runtime/asm_amd64.s:1695: Failed to watch *v1.ConfigMap: failed to list *v1.ConfigMap: Get \"https://192.169.0.117:8443/api/v1/namespaces/kube-system/configmaps?fieldSelector=metadata.name%3Dextension-apiserver-authentication&limit=500&resourceVersion=0\": dial tcp 192.169.0.117:8443: connect: connection refused" logger="UnhandledError"
W0912 23:44:15.339175 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.Namespace: Get "https://192.169.0.117:8443/api/v1/namespaces?limit=500&resourceVersion=0": dial tcp 192.169.0.117:8443: connect: connection refused
E0912 23:44:15.339276 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Namespace: failed to list *v1.Namespace: Get \"https://192.169.0.117:8443/api/v1/namespaces?limit=500&resourceVersion=0\": dial tcp 192.169.0.117:8443: connect: connection refused" logger="UnhandledError"
W0912 23:44:15.357298 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.PodDisruptionBudget: Get "https://192.169.0.117:8443/apis/policy/v1/poddisruptionbudgets?limit=500&resourceVersion=0": dial tcp 192.169.0.117:8443: connect: connection refused
E0912 23:44:15.357395 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.PodDisruptionBudget: failed to list *v1.PodDisruptionBudget: Get \"https://192.169.0.117:8443/apis/policy/v1/poddisruptionbudgets?limit=500&resourceVersion=0\": dial tcp 192.169.0.117:8443: connect: connection refused" logger="UnhandledError"
W0912 23:44:15.521160 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.CSINode: Get "https://192.169.0.117:8443/apis/storage.k8s.io/v1/csinodes?limit=500&resourceVersion=0": dial tcp 192.169.0.117:8443: connect: connection refused
E0912 23:44:15.521263 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.CSINode: failed to list *v1.CSINode: Get \"https://192.169.0.117:8443/apis/storage.k8s.io/v1/csinodes?limit=500&resourceVersion=0\": dial tcp 192.169.0.117:8443: connect: connection refused" logger="UnhandledError"
W0912 23:44:16.098736 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.CSIStorageCapacity: Get "https://192.169.0.117:8443/apis/storage.k8s.io/v1/csistoragecapacities?limit=500&resourceVersion=0": dial tcp 192.169.0.117:8443: connect: connection refused
E0912 23:44:16.098840 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.CSIStorageCapacity: failed to list *v1.CSIStorageCapacity: Get \"https://192.169.0.117:8443/apis/storage.k8s.io/v1/csistoragecapacities?limit=500&resourceVersion=0\": dial tcp 192.169.0.117:8443: connect: connection refused" logger="UnhandledError"
W0912 23:44:16.198934 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.CSIDriver: Get "https://192.169.0.117:8443/apis/storage.k8s.io/v1/csidrivers?limit=500&resourceVersion=0": dial tcp 192.169.0.117:8443: connect: connection refused
E0912 23:44:16.199042 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.CSIDriver: failed to list *v1.CSIDriver: Get \"https://192.169.0.117:8443/apis/storage.k8s.io/v1/csidrivers?limit=500&resourceVersion=0\": dial tcp 192.169.0.117:8443: connect: connection refused" logger="UnhandledError"
W0912 23:44:16.671003 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.ReplicaSet: Get "https://192.169.0.117:8443/apis/apps/v1/replicasets?limit=500&resourceVersion=0": dial tcp 192.169.0.117:8443: connect: connection refused
E0912 23:44:16.671172 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.ReplicaSet: failed to list *v1.ReplicaSet: Get \"https://192.169.0.117:8443/apis/apps/v1/replicasets?limit=500&resourceVersion=0\": dial tcp 192.169.0.117:8443: connect: connection refused" logger="UnhandledError"
W0912 23:44:18.112896 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.ReplicationController: Get "https://192.169.0.117:8443/api/v1/replicationcontrollers?limit=500&resourceVersion=0": dial tcp 192.169.0.117:8443: connect: connection refused
E0912 23:44:18.113217 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.ReplicationController: failed to list *v1.ReplicationController: Get \"https://192.169.0.117:8443/api/v1/replicationcontrollers?limit=500&resourceVersion=0\": dial tcp 192.169.0.117:8443: connect: connection refused" logger="UnhandledError"
W0912 23:44:20.436247 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.PersistentVolumeClaim: Get "https://192.169.0.117:8443/api/v1/persistentvolumeclaims?limit=500&resourceVersion=0": dial tcp 192.169.0.117:8443: connect: connection refused
E0912 23:44:20.436673 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.PersistentVolumeClaim: failed to list *v1.PersistentVolumeClaim: Get \"https://192.169.0.117:8443/api/v1/persistentvolumeclaims?limit=500&resourceVersion=0\": dial tcp 192.169.0.117:8443: connect: connection refused" logger="UnhandledError"
W0912 23:44:21.769199 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.PersistentVolume: Get "https://192.169.0.117:8443/api/v1/persistentvolumes?limit=500&resourceVersion=0": dial tcp 192.169.0.117:8443: connect: connection refused
E0912 23:44:21.769277 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.PersistentVolume: failed to list *v1.PersistentVolume: Get \"https://192.169.0.117:8443/api/v1/persistentvolumes?limit=500&resourceVersion=0\": dial tcp 192.169.0.117:8443: connect: connection refused" logger="UnhandledError"
==> kube-scheduler [a8f623b2ba8c] <==
I0912 23:40:00.517752 1 serving.go:386] Generated self-signed cert in-memory
W0912 23:40:11.074658 1 authentication.go:370] Error looking up in-cluster authentication configuration: Get "https://192.169.0.117:8443/api/v1/namespaces/kube-system/configmaps/extension-apiserver-authentication": net/http: TLS handshake timeout
W0912 23:40:11.074810 1 authentication.go:371] Continuing without authentication configuration. This may treat all requests as anonymous.
W0912 23:40:11.074883 1 authentication.go:372] To require authentication configuration lookup to succeed, set --authentication-tolerate-lookup-failure=false
I0912 23:40:19.380158 1 server.go:167] "Starting Kubernetes Scheduler" version="v1.31.1"
I0912 23:40:19.380203 1 server.go:169] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I0912 23:40:19.390746 1 secure_serving.go:213] Serving securely on 127.0.0.1:10259
I0912 23:40:19.393474 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file"
I0912 23:40:19.393523 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
I0912 23:40:19.393684 1 tlsconfig.go:243] "Starting DynamicServingCertificateController"
I0912 23:40:19.493874 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
E0912 23:42:06.354102 1 framework.go:1305] "Plugin Failed" err="Operation cannot be fulfilled on pods/binding \"busybox-7dff88458-9fd8r\": pod busybox-7dff88458-9fd8r is already assigned to node \"ha-253000-m04\"" plugin="DefaultBinder" pod="default/busybox-7dff88458-9fd8r" node="ha-253000-m04"
E0912 23:42:06.354215 1 schedule_one.go:348] "scheduler cache ForgetPod failed" err="pod ce95fde0-fbb1-456d-8bcd-3a3f8b1a5a3f(default/busybox-7dff88458-9fd8r) wasn't assumed so cannot be forgotten" pod="default/busybox-7dff88458-9fd8r"
E0912 23:42:06.354527 1 schedule_one.go:1057] "Error scheduling pod; retrying" err="running Bind plugin \"DefaultBinder\": Operation cannot be fulfilled on pods/binding \"busybox-7dff88458-9fd8r\": pod busybox-7dff88458-9fd8r is already assigned to node \"ha-253000-m04\"" pod="default/busybox-7dff88458-9fd8r"
I0912 23:42:06.356912 1 schedule_one.go:1070] "Pod has been assigned to node. Abort adding it back to queue." pod="default/busybox-7dff88458-9fd8r" node="ha-253000-m04"
E0912 23:42:30.424278 1 run.go:72] "command failed" err="finished without leader elect"
==> kubelet <==
Sep 12 23:44:08 ha-253000 kubelet[1583]: E0912 23:44:08.186329 1583 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-apiserver\" with CrashLoopBackOff: \"back-off 20s restarting failed container=kube-apiserver pod=kube-apiserver-ha-253000_kube-system(e59ab57c02b15bcf5baf4e47d80d82ca)\"" pod="kube-system/kube-apiserver-ha-253000" podUID="e59ab57c02b15bcf5baf4e47d80d82ca"
Sep 12 23:44:08 ha-253000 kubelet[1583]: I0912 23:44:08.373210 1583 scope.go:117] "RemoveContainer" containerID="9c94eaa958da2700183df9261cda195a92962437936490991ed8ee5658a183aa"
Sep 12 23:44:08 ha-253000 kubelet[1583]: E0912 23:44:08.373610 1583 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-controller-manager\" with CrashLoopBackOff: \"back-off 20s restarting failed container=kube-controller-manager pod=kube-controller-manager-ha-253000_kube-system(f4da0f8debed1bc7ea1235223ab90b7b)\"" pod="kube-system/kube-controller-manager-ha-253000" podUID="f4da0f8debed1bc7ea1235223ab90b7b"
Sep 12 23:44:08 ha-253000 kubelet[1583]: I0912 23:44:08.884918 1583 scope.go:117] "RemoveContainer" containerID="e8d3a41e6ab25f94223bbcf0f0115c2507995c287ca94e9afc14daee53a73845"
Sep 12 23:44:08 ha-253000 kubelet[1583]: E0912 23:44:08.885042 1583 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-apiserver\" with CrashLoopBackOff: \"back-off 20s restarting failed container=kube-apiserver pod=kube-apiserver-ha-253000_kube-system(e59ab57c02b15bcf5baf4e47d80d82ca)\"" pod="kube-system/kube-apiserver-ha-253000" podUID="e59ab57c02b15bcf5baf4e47d80d82ca"
Sep 12 23:44:09 ha-253000 kubelet[1583]: E0912 23:44:09.026590 1583 eviction_manager.go:285] "Eviction manager: failed to get summary stats" err="failed to get node info: node \"ha-253000\" not found"
Sep 12 23:44:09 ha-253000 kubelet[1583]: E0912 23:44:09.589976 1583 event.go:368] "Unable to write event (may retry after sleeping)" err="Post \"https://control-plane.minikube.internal:8443/api/v1/namespaces/default/events\": dial tcp 192.169.0.254:8443: connect: no route to host" event="&Event{ObjectMeta:{ha-253000.17f4a3ba93ec3dc3 default 0 0001-01-01 00:00:00 +0000 UTC <nil> <nil> map[] map[] [] [] []},InvolvedObject:ObjectReference{Kind:Node,Namespace:,Name:ha-253000,UID:ha-253000,APIVersion:,ResourceVersion:,FieldPath:,},Reason:Starting,Message:Starting kubelet.,Source:EventSource{Component:kubelet,Host:ha-253000,},FirstTimestamp:2024-09-12 23:42:58.930720195 +0000 UTC m=+0.140656114,LastTimestamp:2024-09-12 23:42:58.930720195 +0000 UTC m=+0.140656114,Count:1,Type:Normal,EventTime:0001-01-01 00:00:00 +0000 UTC,Series:nil,Action:,Related:nil,ReportingController:kubelet,ReportingInstance:ha-253000,}"
Sep 12 23:44:12 ha-253000 kubelet[1583]: W0912 23:44:12.659056 1583 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.RuntimeClass: Get "https://control-plane.minikube.internal:8443/apis/node.k8s.io/v1/runtimeclasses?limit=500&resourceVersion=0": dial tcp 192.169.0.254:8443: connect: no route to host
Sep 12 23:44:12 ha-253000 kubelet[1583]: E0912 23:44:12.659770 1583 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.RuntimeClass: failed to list *v1.RuntimeClass: Get \"https://control-plane.minikube.internal:8443/apis/node.k8s.io/v1/runtimeclasses?limit=500&resourceVersion=0\": dial tcp 192.169.0.254:8443: connect: no route to host" logger="UnhandledError"
Sep 12 23:44:13 ha-253000 kubelet[1583]: I0912 23:44:13.516605 1583 kubelet_node_status.go:72] "Attempting to register node" node="ha-253000"
Sep 12 23:44:15 ha-253000 kubelet[1583]: E0912 23:44:15.737515 1583 kubelet_node_status.go:95] "Unable to register node with API server" err="Post \"https://control-plane.minikube.internal:8443/api/v1/nodes\": dial tcp 192.169.0.254:8443: connect: no route to host" node="ha-253000"
Sep 12 23:44:15 ha-253000 kubelet[1583]: E0912 23:44:15.737566 1583 controller.go:145] "Failed to ensure lease exists, will retry" err="Get \"https://control-plane.minikube.internal:8443/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/ha-253000?timeout=10s\": dial tcp 192.169.0.254:8443: connect: no route to host" interval="7s"
Sep 12 23:44:18 ha-253000 kubelet[1583]: W0912 23:44:18.802868 1583 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.Service: Get "https://control-plane.minikube.internal:8443/api/v1/services?fieldSelector=spec.clusterIP%21%3DNone&limit=500&resourceVersion=0": dial tcp 192.169.0.254:8443: connect: no route to host
Sep 12 23:44:18 ha-253000 kubelet[1583]: E0912 23:44:18.802995 1583 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Service: failed to list *v1.Service: Get \"https://control-plane.minikube.internal:8443/api/v1/services?fieldSelector=spec.clusterIP%21%3DNone&limit=500&resourceVersion=0\": dial tcp 192.169.0.254:8443: connect: no route to host" logger="UnhandledError"
Sep 12 23:44:19 ha-253000 kubelet[1583]: E0912 23:44:19.026895 1583 eviction_manager.go:285] "Eviction manager: failed to get summary stats" err="failed to get node info: node \"ha-253000\" not found"
Sep 12 23:44:20 ha-253000 kubelet[1583]: I0912 23:44:20.006296 1583 scope.go:117] "RemoveContainer" containerID="9c94eaa958da2700183df9261cda195a92962437936490991ed8ee5658a183aa"
Sep 12 23:44:20 ha-253000 kubelet[1583]: E0912 23:44:20.006774 1583 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-controller-manager\" with CrashLoopBackOff: \"back-off 20s restarting failed container=kube-controller-manager pod=kube-controller-manager-ha-253000_kube-system(f4da0f8debed1bc7ea1235223ab90b7b)\"" pod="kube-system/kube-controller-manager-ha-253000" podUID="f4da0f8debed1bc7ea1235223ab90b7b"
Sep 12 23:44:21 ha-253000 kubelet[1583]: I0912 23:44:21.006065 1583 scope.go:117] "RemoveContainer" containerID="e8d3a41e6ab25f94223bbcf0f0115c2507995c287ca94e9afc14daee53a73845"
Sep 12 23:44:21 ha-253000 kubelet[1583]: E0912 23:44:21.006201 1583 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-apiserver\" with CrashLoopBackOff: \"back-off 20s restarting failed container=kube-apiserver pod=kube-apiserver-ha-253000_kube-system(e59ab57c02b15bcf5baf4e47d80d82ca)\"" pod="kube-system/kube-apiserver-ha-253000" podUID="e59ab57c02b15bcf5baf4e47d80d82ca"
Sep 12 23:44:21 ha-253000 kubelet[1583]: E0912 23:44:21.877318 1583 event.go:368] "Unable to write event (may retry after sleeping)" err="Post \"https://control-plane.minikube.internal:8443/api/v1/namespaces/default/events\": dial tcp 192.169.0.254:8443: connect: no route to host" event="&Event{ObjectMeta:{ha-253000.17f4a3ba93ec3dc3 default 0 0001-01-01 00:00:00 +0000 UTC <nil> <nil> map[] map[] [] [] []},InvolvedObject:ObjectReference{Kind:Node,Namespace:,Name:ha-253000,UID:ha-253000,APIVersion:,ResourceVersion:,FieldPath:,},Reason:Starting,Message:Starting kubelet.,Source:EventSource{Component:kubelet,Host:ha-253000,},FirstTimestamp:2024-09-12 23:42:58.930720195 +0000 UTC m=+0.140656114,LastTimestamp:2024-09-12 23:42:58.930720195 +0000 UTC m=+0.140656114,Count:1,Type:Normal,EventTime:0001-01-01 00:00:00 +0000 UTC,Series:nil,Action:,Related:nil,ReportingController:kubelet,ReportingInstance:ha-253000,}"
Sep 12 23:44:22 ha-253000 kubelet[1583]: I0912 23:44:22.738977 1583 kubelet_node_status.go:72] "Attempting to register node" node="ha-253000"
Sep 12 23:44:24 ha-253000 kubelet[1583]: E0912 23:44:24.948597 1583 kubelet_node_status.go:95] "Unable to register node with API server" err="Post \"https://control-plane.minikube.internal:8443/api/v1/nodes\": dial tcp 192.169.0.254:8443: connect: no route to host" node="ha-253000"
Sep 12 23:44:24 ha-253000 kubelet[1583]: E0912 23:44:24.948614 1583 controller.go:145] "Failed to ensure lease exists, will retry" err="Get \"https://control-plane.minikube.internal:8443/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/ha-253000?timeout=10s\": dial tcp 192.169.0.254:8443: connect: no route to host" interval="7s"
Sep 12 23:44:28 ha-253000 kubelet[1583]: W0912 23:44:28.019244 1583 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.CSIDriver: Get "https://control-plane.minikube.internal:8443/apis/storage.k8s.io/v1/csidrivers?limit=500&resourceVersion=0": dial tcp 192.169.0.254:8443: connect: no route to host
Sep 12 23:44:28 ha-253000 kubelet[1583]: E0912 23:44:28.019344 1583 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.CSIDriver: failed to list *v1.CSIDriver: Get \"https://control-plane.minikube.internal:8443/apis/storage.k8s.io/v1/csidrivers?limit=500&resourceVersion=0\": dial tcp 192.169.0.254:8443: connect: no route to host" logger="UnhandledError"
-- /stdout --
helpers_test.go:254: (dbg) Run: out/minikube-darwin-amd64 status --format={{.APIServer}} -p ha-253000 -n ha-253000
helpers_test.go:254: (dbg) Non-zero exit: out/minikube-darwin-amd64 status --format={{.APIServer}} -p ha-253000 -n ha-253000: exit status 2 (153.861664ms)
-- stdout --
Stopped
-- /stdout --
helpers_test.go:254: status error: exit status 2 (may be ok)
helpers_test.go:256: "ha-253000" apiserver is not running, skipping kubectl commands (state="Stopped")
--- FAIL: TestMultiControlPlane/serial/DegradedAfterClusterRestart (2.70s)