=== RUN TestMultiControlPlane/serial/DegradedAfterClusterRestart
ha_test.go:392: (dbg) Run: out/minikube-darwin-amd64 profile list --output json
ha_test.go:415: expected profile "ha-626000" in json of 'profile list' to have "Degraded" status but have "Starting" status. got *"{\"invalid\":[],\"valid\":[{\"Name\":\"ha-626000\",\"Status\":\"Starting\",\"Config\":{\"Name\":\"ha-626000\",\"KeepContext\":false,\"EmbedCerts\":false,\"MinikubeISO\":\"https://storage.googleapis.com/minikube-builds/iso/19774/minikube-v1.34.0-1728382514-19774-amd64.iso\",\"KicBaseImage\":\"gcr.io/k8s-minikube/kicbase-builds:v0.0.45-1728382586-19774@sha256:5d8c4f6d838467365e214e2194dd0153a763e3f78723b5f2a8e06ef7b47409ec\",\"Memory\":2200,\"CPUs\":2,\"DiskSize\":20000,\"Driver\":\"hyperkit\",\"HyperkitVpnKitSock\":\"\",\"HyperkitVSockPorts\":[],\"DockerEnv\":null,\"ContainerVolumeMounts\":null,\"InsecureRegistry\":null,\"RegistryMirror\":[],\"HostOnlyCIDR\":\"192.168.59.1/24\",\"HypervVirtualSwitch\":\"\",\"HypervUseExternalSwitch\":false,\"HypervExternalAdapter\":\"\",\"KVMNetwork\":\"default\",\"KVMQemuURI\":\"qemu:///system\",\"KVMGPU\":false,\"KVMHidden\":false,\"KVMNUMACo
unt\":1,\"APIServerPort\":8443,\"DockerOpt\":null,\"DisableDriverMounts\":false,\"NFSShare\":[],\"NFSSharesRoot\":\"/nfsshares\",\"UUID\":\"\",\"NoVTXCheck\":false,\"DNSProxy\":false,\"HostDNSResolver\":true,\"HostOnlyNicType\":\"virtio\",\"NatNicType\":\"virtio\",\"SSHIPAddress\":\"\",\"SSHUser\":\"root\",\"SSHKey\":\"\",\"SSHPort\":22,\"KubernetesConfig\":{\"KubernetesVersion\":\"v1.31.1\",\"ClusterName\":\"ha-626000\",\"Namespace\":\"default\",\"APIServerHAVIP\":\"192.169.0.254\",\"APIServerName\":\"minikubeCA\",\"APIServerNames\":null,\"APIServerIPs\":null,\"DNSDomain\":\"cluster.local\",\"ContainerRuntime\":\"docker\",\"CRISocket\":\"\",\"NetworkPlugin\":\"cni\",\"FeatureGates\":\"\",\"ServiceCIDR\":\"10.96.0.0/12\",\"ImageRepository\":\"\",\"LoadBalancerStartIP\":\"\",\"LoadBalancerEndIP\":\"\",\"CustomIngressCert\":\"\",\"RegistryAliases\":\"\",\"ExtraOptions\":null,\"ShouldLoadCachedImages\":true,\"EnableDefaultCNI\":false,\"CNI\":\"\"},\"Nodes\":[{\"Name\":\"\",\"IP\":\"192.169.0.41\",\"Port\":8443,\
"KubernetesVersion\":\"v1.31.1\",\"ContainerRuntime\":\"docker\",\"ControlPlane\":true,\"Worker\":true},{\"Name\":\"m02\",\"IP\":\"192.169.0.42\",\"Port\":8443,\"KubernetesVersion\":\"v1.31.1\",\"ContainerRuntime\":\"docker\",\"ControlPlane\":true,\"Worker\":true},{\"Name\":\"m04\",\"IP\":\"192.169.0.44\",\"Port\":0,\"KubernetesVersion\":\"v1.31.1\",\"ContainerRuntime\":\"docker\",\"ControlPlane\":false,\"Worker\":true}],\"Addons\":{\"ambassador\":false,\"auto-pause\":false,\"cloud-spanner\":false,\"csi-hostpath-driver\":false,\"dashboard\":false,\"default-storageclass\":false,\"efk\":false,\"freshpod\":false,\"gcp-auth\":false,\"gvisor\":false,\"headlamp\":false,\"inaccel\":false,\"ingress\":false,\"ingress-dns\":false,\"inspektor-gadget\":false,\"istio\":false,\"istio-provisioner\":false,\"kong\":false,\"kubeflow\":false,\"kubevirt\":false,\"logviewer\":false,\"metallb\":false,\"metrics-server\":false,\"nvidia-device-plugin\":false,\"nvidia-driver-installer\":false,\"nvidia-gpu-device-plugin\":false,\"olm\"
:false,\"pod-security-policy\":false,\"portainer\":false,\"registry\":false,\"registry-aliases\":false,\"registry-creds\":false,\"storage-provisioner\":false,\"storage-provisioner-gluster\":false,\"storage-provisioner-rancher\":false,\"volcano\":false,\"volumesnapshots\":false,\"yakd\":false},\"CustomAddonImages\":null,\"CustomAddonRegistries\":null,\"VerifyComponents\":{\"apiserver\":true,\"apps_running\":true,\"default_sa\":true,\"extra\":true,\"kubelet\":true,\"node_ready\":true,\"system_pods\":true},\"StartHostTimeout\":360000000000,\"ScheduledStop\":null,\"ExposedPorts\":[],\"ListenAddress\":\"\",\"Network\":\"\",\"Subnet\":\"\",\"MultiNodeRequested\":true,\"ExtraDisks\":0,\"CertExpiration\":94608000000000000,\"Mount\":false,\"MountString\":\"/Users:/minikube-host\",\"Mount9PVersion\":\"9p2000.L\",\"MountGID\":\"docker\",\"MountIP\":\"\",\"MountMSize\":262144,\"MountOptions\":[],\"MountPort\":0,\"MountType\":\"9p\",\"MountUID\":\"docker\",\"BinaryMirror\":\"\",\"DisableOptimizations\":false,\"DisableMetr
ics\":false,\"CustomQemuFirmwarePath\":\"\",\"SocketVMnetClientPath\":\"\",\"SocketVMnetPath\":\"\",\"StaticIP\":\"\",\"SSHAuthSock\":\"\",\"SSHAgentPID\":0,\"GPUs\":\"\",\"AutoPauseInterval\":60000000000},\"Active\":false,\"ActiveKubeContext\":true}]}"*. args: "out/minikube-darwin-amd64 profile list --output json"
helpers_test.go:222: -----------------------post-mortem--------------------------------
helpers_test.go:239: (dbg) Run: out/minikube-darwin-amd64 status --format={{.Host}} -p ha-626000 -n ha-626000
helpers_test.go:239: (dbg) Non-zero exit: out/minikube-darwin-amd64 status --format={{.Host}} -p ha-626000 -n ha-626000: exit status 2 (176.615039ms)
-- stdout --
Running
-- /stdout --
helpers_test.go:239: status error: exit status 2 (may be ok)
helpers_test.go:244: <<< TestMultiControlPlane/serial/DegradedAfterClusterRestart FAILED: start of post-mortem logs <<<
helpers_test.go:245: ======> post-mortem[TestMultiControlPlane/serial/DegradedAfterClusterRestart]: minikube logs <======
helpers_test.go:247: (dbg) Run: out/minikube-darwin-amd64 -p ha-626000 logs -n 25
helpers_test.go:247: (dbg) Done: out/minikube-darwin-amd64 -p ha-626000 logs -n 25: (2.207244473s)
helpers_test.go:252: TestMultiControlPlane/serial/DegradedAfterClusterRestart logs:
-- stdout --
==> Audit <==
|---------|------------------------------------------------------------------------------------------------------------------------------|-----------|---------|---------|---------------------|---------------------|
| Command | Args | Profile | User | Version | Start Time | End Time |
|---------|------------------------------------------------------------------------------------------------------------------------------|-----------|---------|---------|---------------------|---------------------|
| cp | ha-626000 cp ha-626000-m03:/home/docker/cp-test.txt | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:12 PDT | 10 Oct 24 13:12 PDT |
| | ha-626000-m04:/home/docker/cp-test_ha-626000-m03_ha-626000-m04.txt | | | | | |
| ssh | ha-626000 ssh -n | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:12 PDT | 10 Oct 24 13:12 PDT |
| | ha-626000-m03 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| ssh | ha-626000 ssh -n ha-626000-m04 sudo cat | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:12 PDT | 10 Oct 24 13:12 PDT |
| | /home/docker/cp-test_ha-626000-m03_ha-626000-m04.txt | | | | | |
| cp | ha-626000 cp testdata/cp-test.txt | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:12 PDT | 10 Oct 24 13:12 PDT |
| | ha-626000-m04:/home/docker/cp-test.txt | | | | | |
| ssh | ha-626000 ssh -n | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:12 PDT | 10 Oct 24 13:12 PDT |
| | ha-626000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| cp | ha-626000 cp ha-626000-m04:/home/docker/cp-test.txt | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:12 PDT | 10 Oct 24 13:12 PDT |
| | /var/folders/0y/_8hvl7v13q38_kkh25vpxkz00000gp/T/TestMultiControlPlaneserialCopyFile2720842674/001/cp-test_ha-626000-m04.txt | | | | | |
| ssh | ha-626000 ssh -n | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:12 PDT | 10 Oct 24 13:12 PDT |
| | ha-626000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| cp | ha-626000 cp ha-626000-m04:/home/docker/cp-test.txt | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:12 PDT | 10 Oct 24 13:12 PDT |
| | ha-626000:/home/docker/cp-test_ha-626000-m04_ha-626000.txt | | | | | |
| ssh | ha-626000 ssh -n | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:12 PDT | 10 Oct 24 13:12 PDT |
| | ha-626000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| ssh | ha-626000 ssh -n ha-626000 sudo cat | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:12 PDT | 10 Oct 24 13:12 PDT |
| | /home/docker/cp-test_ha-626000-m04_ha-626000.txt | | | | | |
| cp | ha-626000 cp ha-626000-m04:/home/docker/cp-test.txt | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:12 PDT | 10 Oct 24 13:12 PDT |
| | ha-626000-m02:/home/docker/cp-test_ha-626000-m04_ha-626000-m02.txt | | | | | |
| ssh | ha-626000 ssh -n | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:12 PDT | 10 Oct 24 13:12 PDT |
| | ha-626000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| ssh | ha-626000 ssh -n ha-626000-m02 sudo cat | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:12 PDT | 10 Oct 24 13:12 PDT |
| | /home/docker/cp-test_ha-626000-m04_ha-626000-m02.txt | | | | | |
| cp | ha-626000 cp ha-626000-m04:/home/docker/cp-test.txt | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:12 PDT | 10 Oct 24 13:12 PDT |
| | ha-626000-m03:/home/docker/cp-test_ha-626000-m04_ha-626000-m03.txt | | | | | |
| ssh | ha-626000 ssh -n | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:12 PDT | 10 Oct 24 13:12 PDT |
| | ha-626000-m04 sudo cat | | | | | |
| | /home/docker/cp-test.txt | | | | | |
| ssh | ha-626000 ssh -n ha-626000-m03 sudo cat | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:12 PDT | 10 Oct 24 13:12 PDT |
| | /home/docker/cp-test_ha-626000-m04_ha-626000-m03.txt | | | | | |
| node | ha-626000 node stop m02 -v=7 | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:12 PDT | 10 Oct 24 13:12 PDT |
| | --alsologtostderr | | | | | |
| node | ha-626000 node start m02 -v=7 | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:12 PDT | 10 Oct 24 13:12 PDT |
| | --alsologtostderr | | | | | |
| node | list -p ha-626000 -v=7 | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:12 PDT | |
| | --alsologtostderr | | | | | |
| stop | -p ha-626000 -v=7 | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:12 PDT | 10 Oct 24 13:13 PDT |
| | --alsologtostderr | | | | | |
| start | -p ha-626000 --wait=true -v=7 | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:13 PDT | 10 Oct 24 13:15 PDT |
| | --alsologtostderr | | | | | |
| node | list -p ha-626000 | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:15 PDT | |
| node | ha-626000 node delete m03 -v=7 | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:15 PDT | 10 Oct 24 13:15 PDT |
| | --alsologtostderr | | | | | |
| stop | ha-626000 stop -v=7 | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:15 PDT | 10 Oct 24 13:16 PDT |
| | --alsologtostderr | | | | | |
| start | -p ha-626000 --wait=true | ha-626000 | jenkins | v1.34.0 | 10 Oct 24 13:16 PDT | |
| | -v=7 --alsologtostderr | | | | | |
| | --driver=hyperkit | | | | | |
|---------|------------------------------------------------------------------------------------------------------------------------------|-----------|---------|---------|---------------------|---------------------|
==> Last Start <==
Log file created at: 2024/10/10 13:16:24
Running on machine: MacOS-Agent-4
Binary: Built with gc go1.23.1 for darwin/amd64
Log line format: [IWEF]mmdd hh:mm:ss.uuuuuu threadid file:line] msg
I1010 13:16:24.889019 29815 out.go:345] Setting OutFile to fd 1 ...
I1010 13:16:24.889799 29815 out.go:392] TERM=,COLORTERM=, which probably does not support color
I1010 13:16:24.889808 29815 out.go:358] Setting ErrFile to fd 2...
I1010 13:16:24.889813 29815 out.go:392] TERM=,COLORTERM=, which probably does not support color
I1010 13:16:24.890334 29815 root.go:338] Updating PATH: /Users/jenkins/minikube-integration/19787-26748/.minikube/bin
I1010 13:16:24.891908 29815 out.go:352] Setting JSON to false
I1010 13:16:24.921696 29815 start.go:129] hostinfo: {"hostname":"MacOS-Agent-4.local","uptime":13552,"bootTime":1728577832,"procs":502,"os":"darwin","platform":"darwin","platformFamily":"Standalone Workstation","platformVersion":"15.0.1","kernelVersion":"24.0.0","kernelArch":"x86_64","virtualizationSystem":"","virtualizationRole":"","hostId":"f2f27e25-cfda-5ffd-9706-e98286194e62"}
W1010 13:16:24.921864 29815 start.go:137] gopshost.Virtualization returned error: not implemented yet
I1010 13:16:24.943506 29815 out.go:177] * [ha-626000] minikube v1.34.0 on Darwin 15.0.1
I1010 13:16:24.985213 29815 out.go:177] - MINIKUBE_LOCATION=19787
I1010 13:16:24.985239 29815 notify.go:220] Checking for updates...
I1010 13:16:25.028346 29815 out.go:177] - KUBECONFIG=/Users/jenkins/minikube-integration/19787-26748/kubeconfig
I1010 13:16:25.049375 29815 out.go:177] - MINIKUBE_BIN=out/minikube-darwin-amd64
I1010 13:16:25.070570 29815 out.go:177] - MINIKUBE_SUPPRESS_DOCKER_PERFORMANCE=true
I1010 13:16:25.092648 29815 out.go:177] - MINIKUBE_HOME=/Users/jenkins/minikube-integration/19787-26748/.minikube
I1010 13:16:25.114409 29815 out.go:177] - MINIKUBE_FORCE_SYSTEMD=
I1010 13:16:25.140941 29815 config.go:182] Loaded profile config "ha-626000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.1
I1010 13:16:25.141439 29815 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I1010 13:16:25.141490 29815 main.go:141] libmachine: Launching plugin server for driver hyperkit
I1010 13:16:25.153326 29815 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:51024
I1010 13:16:25.153661 29815 main.go:141] libmachine: () Calling .GetVersion
I1010 13:16:25.154064 29815 main.go:141] libmachine: Using API Version 1
I1010 13:16:25.154087 29815 main.go:141] libmachine: () Calling .SetConfigRaw
I1010 13:16:25.154304 29815 main.go:141] libmachine: () Calling .GetMachineName
I1010 13:16:25.154432 29815 main.go:141] libmachine: (ha-626000) Calling .DriverName
I1010 13:16:25.154685 29815 driver.go:394] Setting default libvirt URI to qemu:///system
I1010 13:16:25.154970 29815 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I1010 13:16:25.155009 29815 main.go:141] libmachine: Launching plugin server for driver hyperkit
I1010 13:16:25.166018 29815 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:51026
I1010 13:16:25.166359 29815 main.go:141] libmachine: () Calling .GetVersion
I1010 13:16:25.166687 29815 main.go:141] libmachine: Using API Version 1
I1010 13:16:25.166698 29815 main.go:141] libmachine: () Calling .SetConfigRaw
I1010 13:16:25.166944 29815 main.go:141] libmachine: () Calling .GetMachineName
I1010 13:16:25.167068 29815 main.go:141] libmachine: (ha-626000) Calling .DriverName
I1010 13:16:25.198342 29815 out.go:177] * Using the hyperkit driver based on existing profile
I1010 13:16:25.240085 29815 start.go:297] selected driver: hyperkit
I1010 13:16:25.240103 29815 start.go:901] validating driver "hyperkit" against &{Name:ha-626000 KeepContext:false EmbedCerts:false MinikubeISO:https://storage.googleapis.com/minikube-builds/iso/19774/minikube-v1.34.0-1728382514-19774-amd64.iso KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.45-1728382586-19774@sha256:5d8c4f6d838467365e214e2194dd0153a763e3f78723b5f2a8e06ef7b47409ec Memory:2200 CPUs:2 DiskSize:20000 Driver:hyperkit HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{Kuber
netesVersion:v1.31.1 ClusterName:ha-626000 Namespace:default APIServerHAVIP:192.169.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.169.0.41 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m02 IP:192.169.0.42 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m04 IP:192.169.0.44 Port:0 KubernetesVersion:v1.31.1 ContainerRuntime: ControlPlane:false Worker:true}] Addons:map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false inaccel:false ingress:false ingress-dns:fal
se inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s Mount:false MountString:/Users:/minikube-host Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptim
izations:false DisableMetrics:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s}
I1010 13:16:25.240240 29815 start.go:912] status for hyperkit: {Installed:true Healthy:true Running:true NeedsImprovement:false Error:<nil> Reason: Fix: Doc: Version:}
I1010 13:16:25.240347 29815 install.go:52] acquiring lock: {Name:mk4023283b30b374c3f04c8805d539e68824c0b8 Clock:{} Delay:500ms Timeout:10m0s Cancel:<nil>}
I1010 13:16:25.240478 29815 install.go:117] Validating docker-machine-driver-hyperkit, PATH=/Users/jenkins/minikube-integration/19787-26748/.minikube/bin:/Users/jenkins/workspace/out/:/usr/bin:/bin:/usr/sbin:/sbin:/Users/jenkins/google-cloud-sdk/bin:/usr/local/bin/:/usr/local/go/bin/:/Users/jenkins/go/bin
I1010 13:16:25.251408 29815 install.go:137] /Users/jenkins/workspace/out/docker-machine-driver-hyperkit version is 1.34.0
I1010 13:16:25.259037 29815 install.go:79] stdout: /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I1010 13:16:25.259069 29815 install.go:81] /Users/jenkins/workspace/out/docker-machine-driver-hyperkit looks good
I1010 13:16:25.265950 29815 start_flags.go:947] Waiting for all components: map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true]
I1010 13:16:25.265995 29815 cni.go:84] Creating CNI manager for ""
I1010 13:16:25.266033 29815 cni.go:136] multinode detected (3 nodes found), recommending kindnet
I1010 13:16:25.266111 29815 start.go:340] cluster config:
{Name:ha-626000 KeepContext:false EmbedCerts:false MinikubeISO:https://storage.googleapis.com/minikube-builds/iso/19774/minikube-v1.34.0-1728382514-19774-amd64.iso KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.45-1728382586-19774@sha256:5d8c4f6d838467365e214e2194dd0153a763e3f78723b5f2a8e06ef7b47409ec Memory:2200 CPUs:2 DiskSize:20000 Driver:hyperkit HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.31.1 ClusterName:ha-626000 Namespace:default APIServerHAVIP:192.16
9.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.169.0.41 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m02 IP:192.169.0.42 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m04 IP:192.169.0.44 Port:0 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:false Worker:true}] Addons:map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false inaccel:false ingress:false ingress-dns:false inspektor-gadget:false istio:false istio-provisioner:false kong:false kub
eflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s Mount:false MountString:/Users:/minikube-host Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:false DisableMetrics:false CustomQemuFirmwarePath: SocketVMnetClien
tPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s}
I1010 13:16:25.266245 29815 iso.go:125] acquiring lock: {Name:mk7d3cfd77611518fbebdb5d4a9ba2d8234d7ce3 Clock:{} Delay:500ms Timeout:10m0s Cancel:<nil>}
I1010 13:16:25.288849 29815 out.go:177] * Starting "ha-626000" primary control-plane node in "ha-626000" cluster
I1010 13:16:25.332121 29815 preload.go:131] Checking if preload exists for k8s version v1.31.1 and runtime docker
I1010 13:16:25.332192 29815 preload.go:146] Found local preload: /Users/jenkins/minikube-integration/19787-26748/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.31.1-docker-overlay2-amd64.tar.lz4
I1010 13:16:25.332219 29815 cache.go:56] Caching tarball of preloaded images
I1010 13:16:25.332445 29815 preload.go:172] Found /Users/jenkins/minikube-integration/19787-26748/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.31.1-docker-overlay2-amd64.tar.lz4 in cache, skipping download
I1010 13:16:25.332464 29815 cache.go:59] Finished verifying existence of preloaded tar for v1.31.1 on docker
I1010 13:16:25.332656 29815 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/config.json ...
I1010 13:16:25.333550 29815 start.go:360] acquireMachinesLock for ha-626000: {Name:mk9fc820136d807cdfeb8abc8a165ddd26f4799c Clock:{} Delay:500ms Timeout:13m0s Cancel:<nil>}
I1010 13:16:25.333694 29815 start.go:364] duration metric: took 120.536µs to acquireMachinesLock for "ha-626000"
I1010 13:16:25.333731 29815 start.go:96] Skipping create...Using existing machine configuration
I1010 13:16:25.333747 29815 fix.go:54] fixHost starting:
I1010 13:16:25.334185 29815 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I1010 13:16:25.334216 29815 main.go:141] libmachine: Launching plugin server for driver hyperkit
I1010 13:16:25.345836 29815 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:51028
I1010 13:16:25.346164 29815 main.go:141] libmachine: () Calling .GetVersion
I1010 13:16:25.346534 29815 main.go:141] libmachine: Using API Version 1
I1010 13:16:25.346548 29815 main.go:141] libmachine: () Calling .SetConfigRaw
I1010 13:16:25.346782 29815 main.go:141] libmachine: () Calling .GetMachineName
I1010 13:16:25.346900 29815 main.go:141] libmachine: (ha-626000) Calling .DriverName
I1010 13:16:25.346998 29815 main.go:141] libmachine: (ha-626000) Calling .GetState
I1010 13:16:25.347071 29815 main.go:141] libmachine: (ha-626000) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I1010 13:16:25.347166 29815 main.go:141] libmachine: (ha-626000) DBG | hyperkit pid from json: 29705
I1010 13:16:25.348240 29815 main.go:141] libmachine: (ha-626000) DBG | hyperkit pid 29705 missing from process table
I1010 13:16:25.348273 29815 fix.go:112] recreateIfNeeded on ha-626000: state=Stopped err=<nil>
I1010 13:16:25.348290 29815 main.go:141] libmachine: (ha-626000) Calling .DriverName
W1010 13:16:25.348380 29815 fix.go:138] unexpected machine state, will restart: <nil>
I1010 13:16:25.389971 29815 out.go:177] * Restarting existing hyperkit VM for "ha-626000" ...
I1010 13:16:25.411047 29815 main.go:141] libmachine: (ha-626000) Calling .Start
I1010 13:16:25.411303 29815 main.go:141] libmachine: (ha-626000) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I1010 13:16:25.411354 29815 main.go:141] libmachine: (ha-626000) minikube might have been shutdown in an unclean way, the hyperkit pid file still exists: /Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/hyperkit.pid
I1010 13:16:25.413717 29815 main.go:141] libmachine: (ha-626000) DBG | hyperkit pid 29705 missing from process table
I1010 13:16:25.413731 29815 main.go:141] libmachine: (ha-626000) DBG | pid 29705 is in state "Stopped"
I1010 13:16:25.413747 29815 main.go:141] libmachine: (ha-626000) DBG | Removing stale pid file /Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/hyperkit.pid...
I1010 13:16:25.414037 29815 main.go:141] libmachine: (ha-626000) DBG | Using UUID f878efc1-1e1d-4c9e-901c-8a76fff7c00e
I1010 13:16:25.526236 29815 main.go:141] libmachine: (ha-626000) DBG | Generated MAC a6:9:ee:ef:40:24
I1010 13:16:25.526268 29815 main.go:141] libmachine: (ha-626000) DBG | Starting with cmdline: loglevel=3 console=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-626000
I1010 13:16:25.526425 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:25 DEBUG: hyperkit: Start &hyperkit.HyperKit{HyperKit:"/usr/local/bin/hyperkit", Argv0:"", StateDir:"/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000", VPNKitSock:"", VPNKitUUID:"", VPNKitPreferredIPv4:"", UUID:"f878efc1-1e1d-4c9e-901c-8a76fff7c00e", Disks:[]hyperkit.Disk{(*hyperkit.RawDisk)(0xc0004003c0)}, ISOImages:[]string{"/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/boot2docker.iso"}, VSock:false, VSockDir:"", VSockPorts:[]int(nil), VSockGuestCID:3, VMNet:true, Sockets9P:[]hyperkit.Socket9P(nil), Kernel:"/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/bzimage", Initrd:"/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/initrd", Bootrom:"", CPUs:2, Memory:2200, Console:1, Serials:[]hyperkit.Serial(nil), Pid:0, Arguments:[]string(nil), CmdLine:"", process:(*os.Process)(nil)}
I1010 13:16:25.526456 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:25 DEBUG: hyperkit: check &hyperkit.HyperKit{HyperKit:"/usr/local/bin/hyperkit", Argv0:"", StateDir:"/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000", VPNKitSock:"", VPNKitUUID:"", VPNKitPreferredIPv4:"", UUID:"f878efc1-1e1d-4c9e-901c-8a76fff7c00e", Disks:[]hyperkit.Disk{(*hyperkit.RawDisk)(0xc0004003c0)}, ISOImages:[]string{"/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/boot2docker.iso"}, VSock:false, VSockDir:"", VSockPorts:[]int(nil), VSockGuestCID:3, VMNet:true, Sockets9P:[]hyperkit.Socket9P(nil), Kernel:"/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/bzimage", Initrd:"/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/initrd", Bootrom:"", CPUs:2, Memory:2200, Console:1, Serials:[]hyperkit.Serial(nil), Pid:0, Arguments:[]string(nil), CmdLine:"", process:(*os.Process)(nil)}
I1010 13:16:25.526509 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:25 DEBUG: hyperkit: Arguments: []string{"-A", "-u", "-F", "/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/hyperkit.pid", "-c", "2", "-m", "2200M", "-s", "0:0,hostbridge", "-s", "31,lpc", "-s", "1:0,virtio-net", "-U", "f878efc1-1e1d-4c9e-901c-8a76fff7c00e", "-s", "2:0,virtio-blk,/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/ha-626000.rawdisk", "-s", "3,ahci-cd,/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/boot2docker.iso", "-s", "4,virtio-rnd", "-l", "com1,autopty=/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/tty,log=/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/console-ring", "-f", "kexec,/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/bzimage,/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/initrd,earlyp
rintk=serial loglevel=3 console=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-626000"}
I1010 13:16:25.526558 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:25 DEBUG: hyperkit: CmdLine: "/usr/local/bin/hyperkit -A -u -F /Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/hyperkit.pid -c 2 -m 2200M -s 0:0,hostbridge -s 31,lpc -s 1:0,virtio-net -U f878efc1-1e1d-4c9e-901c-8a76fff7c00e -s 2:0,virtio-blk,/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/ha-626000.rawdisk -s 3,ahci-cd,/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/boot2docker.iso -s 4,virtio-rnd -l com1,autopty=/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/tty,log=/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/console-ring -f kexec,/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/bzimage,/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/initrd,earlyprintk=serial loglevel=3 console=ttyS0 console=tty0 noembed nom
odeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-626000"
I1010 13:16:25.526575 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:25 DEBUG: hyperkit: Redirecting stdout/stderr to logger
I1010 13:16:25.528225 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:25 DEBUG: hyperkit: Pid is 29829
I1010 13:16:25.528627 29815 main.go:141] libmachine: (ha-626000) DBG | Attempt 0
I1010 13:16:25.528647 29815 main.go:141] libmachine: (ha-626000) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I1010 13:16:25.528701 29815 main.go:141] libmachine: (ha-626000) DBG | hyperkit pid from json: 29829
I1010 13:16:25.530722 29815 main.go:141] libmachine: (ha-626000) DBG | Searching for a6:9:ee:ef:40:24 in /var/db/dhcpd_leases ...
I1010 13:16:25.530887 29815 main.go:141] libmachine: (ha-626000) DBG | Found 43 entries in /var/db/dhcpd_leases!
I1010 13:16:25.530905 29815 main.go:141] libmachine: (ha-626000) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.43 HWAddress:8e:ee:cb:41:b6:d7 ID:1,8e:ee:cb:41:b6:d7 Lease:0x670835fd}
I1010 13:16:25.530941 29815 main.go:141] libmachine: (ha-626000) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.44 HWAddress:ba:c3:6f:f1:51:bf ID:1,ba:c3:6f:f1:51:bf Lease:0x670843f8}
I1010 13:16:25.530957 29815 main.go:141] libmachine: (ha-626000) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.42 HWAddress:d6:2c:95:0:73:df ID:1,d6:2c:95:0:73:df Lease:0x6708438f}
I1010 13:16:25.530967 29815 main.go:141] libmachine: (ha-626000) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.41 HWAddress:a6:9:ee:ef:40:24 ID:1,a6:9:ee:ef:40:24 Lease:0x6708437d}
I1010 13:16:25.530977 29815 main.go:141] libmachine: (ha-626000) DBG | Found match: a6:9:ee:ef:40:24
I1010 13:16:25.530984 29815 main.go:141] libmachine: (ha-626000) DBG | IP: 192.169.0.41
I1010 13:16:25.531038 29815 main.go:141] libmachine: (ha-626000) Calling .GetConfigRaw
I1010 13:16:25.531726 29815 main.go:141] libmachine: (ha-626000) Calling .GetIP
I1010 13:16:25.531903 29815 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/config.json ...
I1010 13:16:25.532356 29815 machine.go:93] provisionDockerMachine start ...
I1010 13:16:25.532368 29815 main.go:141] libmachine: (ha-626000) Calling .DriverName
I1010 13:16:25.532505 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHHostname
I1010 13:16:25.532651 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHPort
I1010 13:16:25.532772 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHKeyPath
I1010 13:16:25.532895 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHKeyPath
I1010 13:16:25.533037 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHUsername
I1010 13:16:25.533490 29815 main.go:141] libmachine: Using SSH client type: native
I1010 13:16:25.533704 29815 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x2a33520] 0x2a36200 <nil> [] 0s} 192.169.0.41 22 <nil> <nil>}
I1010 13:16:25.533719 29815 main.go:141] libmachine: About to run SSH command:
hostname
I1010 13:16:25.539220 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:25 INFO : hyperkit: stderr: Using fd 5 for I/O notifications
I1010 13:16:25.595853 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:25 INFO : hyperkit: stderr: /Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/boot2docker.iso: fcntl(F_PUNCHHOLE) Operation not permitted: block device will not support TRIM/DISCARD
I1010 13:16:25.597129 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:25 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 2 bit: 22 unspecified don't care: bit is 0
I1010 13:16:25.597152 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:25 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 12 unspecified don't care: bit is 0
I1010 13:16:25.597160 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:25 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 20 unspecified don't care: bit is 0
I1010 13:16:25.597167 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:25 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 3 bit: 13 unspecified don't care: bit is 0
I1010 13:16:25.994985 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:25 INFO : hyperkit: stderr: rdmsr to register 0x3a on vcpu 0
I1010 13:16:25.995001 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:25 INFO : hyperkit: stderr: rdmsr to register 0x140 on vcpu 0
I1010 13:16:26.110267 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:26 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 2 bit: 22 unspecified don't care: bit is 0
I1010 13:16:26.110290 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:26 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 12 unspecified don't care: bit is 0
I1010 13:16:26.110302 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:26 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 20 unspecified don't care: bit is 0
I1010 13:16:26.110312 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:26 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 3 bit: 13 unspecified don't care: bit is 0
I1010 13:16:26.111166 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:26 INFO : hyperkit: stderr: rdmsr to register 0x3a on vcpu 1
I1010 13:16:26.111176 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:26 INFO : hyperkit: stderr: rdmsr to register 0x140 on vcpu 1
I1010 13:16:31.818821 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:31 INFO : hyperkit: stderr: rdmsr to register 0x64d on vcpu 1
I1010 13:16:31.818870 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:31 INFO : hyperkit: stderr: rdmsr to register 0x64e on vcpu 1
I1010 13:16:31.818883 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:31 INFO : hyperkit: stderr: rdmsr to register 0x34 on vcpu 1
I1010 13:16:31.843257 29815 main.go:141] libmachine: (ha-626000) DBG | 2024/10/10 13:16:31 INFO : hyperkit: stderr: rdmsr to register 0xc0011029 on vcpu 1
I1010 13:16:36.602640 29815 main.go:141] libmachine: SSH cmd err, output: <nil>: minikube
I1010 13:16:36.602656 29815 main.go:141] libmachine: (ha-626000) Calling .GetMachineName
I1010 13:16:36.602795 29815 buildroot.go:166] provisioning hostname "ha-626000"
I1010 13:16:36.602806 29815 main.go:141] libmachine: (ha-626000) Calling .GetMachineName
I1010 13:16:36.602899 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHHostname
I1010 13:16:36.602992 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHPort
I1010 13:16:36.603090 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHKeyPath
I1010 13:16:36.603179 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHKeyPath
I1010 13:16:36.603270 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHUsername
I1010 13:16:36.603414 29815 main.go:141] libmachine: Using SSH client type: native
I1010 13:16:36.603555 29815 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x2a33520] 0x2a36200 <nil> [] 0s} 192.169.0.41 22 <nil> <nil>}
I1010 13:16:36.603564 29815 main.go:141] libmachine: About to run SSH command:
sudo hostname ha-626000 && echo "ha-626000" | sudo tee /etc/hostname
I1010 13:16:36.673811 29815 main.go:141] libmachine: SSH cmd err, output: <nil>: ha-626000
I1010 13:16:36.673833 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHHostname
I1010 13:16:36.673977 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHPort
I1010 13:16:36.674103 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHKeyPath
I1010 13:16:36.674206 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHKeyPath
I1010 13:16:36.674311 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHUsername
I1010 13:16:36.674503 29815 main.go:141] libmachine: Using SSH client type: native
I1010 13:16:36.674661 29815 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x2a33520] 0x2a36200 <nil> [] 0s} 192.169.0.41 22 <nil> <nil>}
I1010 13:16:36.674673 29815 main.go:141] libmachine: About to run SSH command:
if ! grep -xq '.*\sha-626000' /etc/hosts; then
if grep -xq '127.0.1.1\s.*' /etc/hosts; then
sudo sed -i 's/^127.0.1.1\s.*/127.0.1.1 ha-626000/g' /etc/hosts;
else
echo '127.0.1.1 ha-626000' | sudo tee -a /etc/hosts;
fi
fi
I1010 13:16:36.742516 29815 main.go:141] libmachine: SSH cmd err, output: <nil>:
I1010 13:16:36.742539 29815 buildroot.go:172] set auth options {CertDir:/Users/jenkins/minikube-integration/19787-26748/.minikube CaCertPath:/Users/jenkins/minikube-integration/19787-26748/.minikube/certs/ca.pem CaPrivateKeyPath:/Users/jenkins/minikube-integration/19787-26748/.minikube/certs/ca-key.pem CaCertRemotePath:/etc/docker/ca.pem ServerCertPath:/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/server.pem ServerKeyPath:/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/server-key.pem ClientKeyPath:/Users/jenkins/minikube-integration/19787-26748/.minikube/certs/key.pem ServerCertRemotePath:/etc/docker/server.pem ServerKeyRemotePath:/etc/docker/server-key.pem ClientCertPath:/Users/jenkins/minikube-integration/19787-26748/.minikube/certs/cert.pem ServerCertSANs:[] StorePath:/Users/jenkins/minikube-integration/19787-26748/.minikube}
I1010 13:16:36.742557 29815 buildroot.go:174] setting up certificates
I1010 13:16:36.742563 29815 provision.go:84] configureAuth start
I1010 13:16:36.742570 29815 main.go:141] libmachine: (ha-626000) Calling .GetMachineName
I1010 13:16:36.742706 29815 main.go:141] libmachine: (ha-626000) Calling .GetIP
I1010 13:16:36.742815 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHHostname
I1010 13:16:36.742915 29815 provision.go:143] copyHostCerts
I1010 13:16:36.742946 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/ca.pem -> /Users/jenkins/minikube-integration/19787-26748/.minikube/ca.pem
I1010 13:16:36.743020 29815 exec_runner.go:144] found /Users/jenkins/minikube-integration/19787-26748/.minikube/ca.pem, removing ...
I1010 13:16:36.743027 29815 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19787-26748/.minikube/ca.pem
I1010 13:16:36.743163 29815 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/ca.pem --> /Users/jenkins/minikube-integration/19787-26748/.minikube/ca.pem (1082 bytes)
I1010 13:16:36.743381 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/cert.pem -> /Users/jenkins/minikube-integration/19787-26748/.minikube/cert.pem
I1010 13:16:36.743436 29815 exec_runner.go:144] found /Users/jenkins/minikube-integration/19787-26748/.minikube/cert.pem, removing ...
I1010 13:16:36.743441 29815 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19787-26748/.minikube/cert.pem
I1010 13:16:36.743530 29815 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/cert.pem --> /Users/jenkins/minikube-integration/19787-26748/.minikube/cert.pem (1123 bytes)
I1010 13:16:36.743696 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/key.pem -> /Users/jenkins/minikube-integration/19787-26748/.minikube/key.pem
I1010 13:16:36.743742 29815 exec_runner.go:144] found /Users/jenkins/minikube-integration/19787-26748/.minikube/key.pem, removing ...
I1010 13:16:36.743747 29815 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19787-26748/.minikube/key.pem
I1010 13:16:36.743831 29815 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/key.pem --> /Users/jenkins/minikube-integration/19787-26748/.minikube/key.pem (1679 bytes)
I1010 13:16:36.743991 29815 provision.go:117] generating server cert: /Users/jenkins/minikube-integration/19787-26748/.minikube/machines/server.pem ca-key=/Users/jenkins/minikube-integration/19787-26748/.minikube/certs/ca.pem private-key=/Users/jenkins/minikube-integration/19787-26748/.minikube/certs/ca-key.pem org=jenkins.ha-626000 san=[127.0.0.1 192.169.0.41 ha-626000 localhost minikube]
I1010 13:16:36.775824 29815 provision.go:177] copyRemoteCerts
I1010 13:16:36.775937 29815 ssh_runner.go:195] Run: sudo mkdir -p /etc/docker /etc/docker /etc/docker
I1010 13:16:36.775959 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHHostname
I1010 13:16:36.776125 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHPort
I1010 13:16:36.776250 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHKeyPath
I1010 13:16:36.776352 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHUsername
I1010 13:16:36.776455 29815 sshutil.go:53] new ssh client: &{IP:192.169.0.41 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/id_rsa Username:docker}
I1010 13:16:36.813364 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/machines/server-key.pem -> /etc/docker/server-key.pem
I1010 13:16:36.813445 29815 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19787-26748/.minikube/machines/server-key.pem --> /etc/docker/server-key.pem (1675 bytes)
I1010 13:16:36.833112 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/ca.pem -> /etc/docker/ca.pem
I1010 13:16:36.833171 29815 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/ca.pem --> /etc/docker/ca.pem (1082 bytes)
I1010 13:16:36.852801 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/machines/server.pem -> /etc/docker/server.pem
I1010 13:16:36.852858 29815 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19787-26748/.minikube/machines/server.pem --> /etc/docker/server.pem (1196 bytes)
I1010 13:16:36.876613 29815 provision.go:87] duration metric: took 134.031367ms to configureAuth
I1010 13:16:36.876627 29815 buildroot.go:189] setting minikube options for container-runtime
I1010 13:16:36.876789 29815 config.go:182] Loaded profile config "ha-626000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.1
I1010 13:16:36.876802 29815 main.go:141] libmachine: (ha-626000) Calling .DriverName
I1010 13:16:36.876932 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHHostname
I1010 13:16:36.877022 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHPort
I1010 13:16:36.877118 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHKeyPath
I1010 13:16:36.877202 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHKeyPath
I1010 13:16:36.877295 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHUsername
I1010 13:16:36.877434 29815 main.go:141] libmachine: Using SSH client type: native
I1010 13:16:36.877563 29815 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x2a33520] 0x2a36200 <nil> [] 0s} 192.169.0.41 22 <nil> <nil>}
I1010 13:16:36.877571 29815 main.go:141] libmachine: About to run SSH command:
df --output=fstype / | tail -n 1
I1010 13:16:36.939780 29815 main.go:141] libmachine: SSH cmd err, output: <nil>: tmpfs
I1010 13:16:36.939793 29815 buildroot.go:70] root file system type: tmpfs
I1010 13:16:36.939867 29815 provision.go:314] Updating docker unit: /lib/systemd/system/docker.service ...
I1010 13:16:36.939879 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHHostname
I1010 13:16:36.940023 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHPort
I1010 13:16:36.940142 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHKeyPath
I1010 13:16:36.940251 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHKeyPath
I1010 13:16:36.940346 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHUsername
I1010 13:16:36.940502 29815 main.go:141] libmachine: Using SSH client type: native
I1010 13:16:36.940664 29815 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x2a33520] 0x2a36200 <nil> [] 0s} 192.169.0.41 22 <nil> <nil>}
I1010 13:16:36.940710 29815 main.go:141] libmachine: About to run SSH command:
sudo mkdir -p /lib/systemd/system && printf %s "[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target minikube-automount.service docker.socket
Requires= minikube-automount.service docker.socket
StartLimitBurst=3
StartLimitIntervalSec=60
[Service]
Type=notify
Restart=on-failure
# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
# Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=hyperkit --insecure-registry 10.96.0.0/12
ExecReload=/bin/kill -s HUP \$MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
[Install]
WantedBy=multi-user.target
" | sudo tee /lib/systemd/system/docker.service.new
I1010 13:16:37.012360 29815 main.go:141] libmachine: SSH cmd err, output: <nil>: [Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target minikube-automount.service docker.socket
Requires= minikube-automount.service docker.socket
StartLimitBurst=3
StartLimitIntervalSec=60
[Service]
Type=notify
Restart=on-failure
# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
# Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=hyperkit --insecure-registry 10.96.0.0/12
ExecReload=/bin/kill -s HUP $MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
[Install]
WantedBy=multi-user.target
I1010 13:16:37.012381 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHHostname
I1010 13:16:37.012533 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHPort
I1010 13:16:37.012624 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHKeyPath
I1010 13:16:37.012733 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHKeyPath
I1010 13:16:37.012824 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHUsername
I1010 13:16:37.012983 29815 main.go:141] libmachine: Using SSH client type: native
I1010 13:16:37.013127 29815 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x2a33520] 0x2a36200 <nil> [] 0s} 192.169.0.41 22 <nil> <nil>}
I1010 13:16:37.013139 29815 main.go:141] libmachine: About to run SSH command:
sudo diff -u /lib/systemd/system/docker.service /lib/systemd/system/docker.service.new || { sudo mv /lib/systemd/system/docker.service.new /lib/systemd/system/docker.service; sudo systemctl -f daemon-reload && sudo systemctl -f enable docker && sudo systemctl -f restart docker; }
I1010 13:16:38.782813 29815 main.go:141] libmachine: SSH cmd err, output: <nil>: diff: can't stat '/lib/systemd/system/docker.service': No such file or directory
Created symlink /etc/systemd/system/multi-user.target.wants/docker.service → /usr/lib/systemd/system/docker.service.
I1010 13:16:38.782828 29815 machine.go:96] duration metric: took 13.25013022s to provisionDockerMachine
I1010 13:16:38.782843 29815 start.go:293] postStartSetup for "ha-626000" (driver="hyperkit")
I1010 13:16:38.782850 29815 start.go:322] creating required directories: [/etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs]
I1010 13:16:38.782861 29815 main.go:141] libmachine: (ha-626000) Calling .DriverName
I1010 13:16:38.783072 29815 ssh_runner.go:195] Run: sudo mkdir -p /etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs
I1010 13:16:38.783092 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHHostname
I1010 13:16:38.783195 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHPort
I1010 13:16:38.783304 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHKeyPath
I1010 13:16:38.783398 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHUsername
I1010 13:16:38.783511 29815 sshutil.go:53] new ssh client: &{IP:192.169.0.41 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/id_rsa Username:docker}
I1010 13:16:38.825535 29815 ssh_runner.go:195] Run: cat /etc/os-release
I1010 13:16:38.829333 29815 info.go:137] Remote host: Buildroot 2023.02.9
I1010 13:16:38.829348 29815 filesync.go:126] Scanning /Users/jenkins/minikube-integration/19787-26748/.minikube/addons for local assets ...
I1010 13:16:38.829450 29815 filesync.go:126] Scanning /Users/jenkins/minikube-integration/19787-26748/.minikube/files for local assets ...
I1010 13:16:38.829666 29815 filesync.go:149] local asset: /Users/jenkins/minikube-integration/19787-26748/.minikube/files/etc/ssl/certs/273152.pem -> 273152.pem in /etc/ssl/certs
I1010 13:16:38.829677 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/files/etc/ssl/certs/273152.pem -> /etc/ssl/certs/273152.pem
I1010 13:16:38.829934 29815 ssh_runner.go:195] Run: sudo mkdir -p /etc/ssl/certs
I1010 13:16:38.840033 29815 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19787-26748/.minikube/files/etc/ssl/certs/273152.pem --> /etc/ssl/certs/273152.pem (1708 bytes)
I1010 13:16:38.869793 29815 start.go:296] duration metric: took 86.939361ms for postStartSetup
I1010 13:16:38.869816 29815 main.go:141] libmachine: (ha-626000) Calling .DriverName
I1010 13:16:38.870011 29815 ssh_runner.go:195] Run: sudo ls --almost-all -1 /var/lib/minikube/backup
I1010 13:16:38.870024 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHHostname
I1010 13:16:38.870139 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHPort
I1010 13:16:38.870249 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHKeyPath
I1010 13:16:38.870337 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHUsername
I1010 13:16:38.870423 29815 sshutil.go:53] new ssh client: &{IP:192.169.0.41 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/id_rsa Username:docker}
I1010 13:16:38.907852 29815 machine.go:197] restoring vm config from /var/lib/minikube/backup: [etc]
I1010 13:16:38.907924 29815 ssh_runner.go:195] Run: sudo rsync --archive --update /var/lib/minikube/backup/etc /
I1010 13:16:38.962760 29815 fix.go:56] duration metric: took 13.628674129s for fixHost
I1010 13:16:38.962784 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHHostname
I1010 13:16:38.962928 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHPort
I1010 13:16:38.963021 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHKeyPath
I1010 13:16:38.963112 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHKeyPath
I1010 13:16:38.963192 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHUsername
I1010 13:16:38.963341 29815 main.go:141] libmachine: Using SSH client type: native
I1010 13:16:38.963488 29815 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x2a33520] 0x2a36200 <nil> [] 0s} 192.169.0.41 22 <nil> <nil>}
I1010 13:16:38.963495 29815 main.go:141] libmachine: About to run SSH command:
date +%s.%N
I1010 13:16:39.024332 29815 main.go:141] libmachine: SSH cmd err, output: <nil>: 1728591399.080601266
I1010 13:16:39.024344 29815 fix.go:216] guest clock: 1728591399.080601266
I1010 13:16:39.024350 29815 fix.go:229] Guest: 2024-10-10 13:16:39.080601266 -0700 PDT Remote: 2024-10-10 13:16:38.962773 -0700 PDT m=+14.114943313 (delta=117.828266ms)
I1010 13:16:39.024366 29815 fix.go:200] guest clock delta is within tolerance: 117.828266ms
I1010 13:16:39.024373 29815 start.go:83] releasing machines lock for "ha-626000", held for 13.690323392s
I1010 13:16:39.024392 29815 main.go:141] libmachine: (ha-626000) Calling .DriverName
I1010 13:16:39.024526 29815 main.go:141] libmachine: (ha-626000) Calling .GetIP
I1010 13:16:39.024660 29815 main.go:141] libmachine: (ha-626000) Calling .DriverName
I1010 13:16:39.024997 29815 main.go:141] libmachine: (ha-626000) Calling .DriverName
I1010 13:16:39.025095 29815 main.go:141] libmachine: (ha-626000) Calling .DriverName
I1010 13:16:39.025262 29815 ssh_runner.go:195] Run: cat /version.json
I1010 13:16:39.025274 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHHostname
I1010 13:16:39.025373 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHPort
I1010 13:16:39.025461 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHKeyPath
I1010 13:16:39.025535 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHUsername
I1010 13:16:39.025602 29815 sshutil.go:53] new ssh client: &{IP:192.169.0.41 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/id_rsa Username:docker}
I1010 13:16:39.025976 29815 ssh_runner.go:195] Run: curl -sS -m 2 https://registry.k8s.io/
I1010 13:16:39.026010 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHHostname
I1010 13:16:39.026092 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHPort
I1010 13:16:39.026175 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHKeyPath
I1010 13:16:39.026259 29815 main.go:141] libmachine: (ha-626000) Calling .GetSSHUsername
I1010 13:16:39.026333 29815 sshutil.go:53] new ssh client: &{IP:192.169.0.41 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000/id_rsa Username:docker}
I1010 13:16:39.058879 29815 ssh_runner.go:195] Run: systemctl --version
I1010 13:16:39.107668 29815 ssh_runner.go:195] Run: sh -c "stat /etc/cni/net.d/*loopback.conf*"
W1010 13:16:39.112647 29815 cni.go:209] loopback cni configuration skipped: "/etc/cni/net.d/*loopback.conf*" not found
I1010 13:16:39.112703 29815 ssh_runner.go:195] Run: sudo find /etc/cni/net.d -maxdepth 1 -type f ( ( -name *bridge* -or -name *podman* ) -and -not -name *.mk_disabled ) -printf "%p, " -exec sh -c "sudo mv {} {}.mk_disabled" ;
I1010 13:16:39.124846 29815 cni.go:262] disabled [/etc/cni/net.d/87-podman-bridge.conflist] bridge cni config(s)
I1010 13:16:39.124863 29815 start.go:495] detecting cgroup driver to use...
I1010 13:16:39.124983 29815 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///run/containerd/containerd.sock
" | sudo tee /etc/crictl.yaml"
I1010 13:16:39.140613 29815 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)sandbox_image = .*$|\1sandbox_image = "registry.k8s.io/pause:3.10"|' /etc/containerd/config.toml"
I1010 13:16:39.149878 29815 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)restrict_oom_score_adj = .*$|\1restrict_oom_score_adj = false|' /etc/containerd/config.toml"
I1010 13:16:39.158703 29815 containerd.go:146] configuring containerd to use "cgroupfs" as cgroup driver...
I1010 13:16:39.158757 29815 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)SystemdCgroup = .*$|\1SystemdCgroup = false|g' /etc/containerd/config.toml"
I1010 13:16:39.167540 29815 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|"io.containerd.runtime.v1.linux"|"io.containerd.runc.v2"|g' /etc/containerd/config.toml"
I1010 13:16:39.176366 29815 ssh_runner.go:195] Run: sh -c "sudo sed -i '/systemd_cgroup/d' /etc/containerd/config.toml"
I1010 13:16:39.185279 29815 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|"io.containerd.runc.v1"|"io.containerd.runc.v2"|g' /etc/containerd/config.toml"
I1010 13:16:39.194398 29815 ssh_runner.go:195] Run: sh -c "sudo rm -rf /etc/cni/net.mk"
I1010 13:16:39.203294 29815 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)conf_dir = .*$|\1conf_dir = "/etc/cni/net.d"|g' /etc/containerd/config.toml"
I1010 13:16:39.211896 29815 ssh_runner.go:195] Run: sh -c "sudo sed -i '/^ *enable_unprivileged_ports = .*/d' /etc/containerd/config.toml"
I1010 13:16:39.220590 29815 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)\[plugins."io.containerd.grpc.v1.cri"\]|&\n\1 enable_unprivileged_ports = true|' /etc/containerd/config.toml"
I1010 13:16:39.229328 29815 ssh_runner.go:195] Run: sudo sysctl net.bridge.bridge-nf-call-iptables
I1010 13:16:39.237317 29815 crio.go:166] couldn't verify netfilter by "sudo sysctl net.bridge.bridge-nf-call-iptables" which might be okay. error: sudo sysctl net.bridge.bridge-nf-call-iptables: Process exited with status 255
stdout:
stderr:
sysctl: cannot stat /proc/sys/net/bridge/bridge-nf-call-iptables: No such file or directory
I1010 13:16:39.237370 29815 ssh_runner.go:195] Run: sudo modprobe br_netfilter
I1010 13:16:39.246226 29815 ssh_runner.go:195] Run: sudo sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"
I1010 13:16:39.254333 29815 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1010 13:16:39.354925 29815 ssh_runner.go:195] Run: sudo systemctl restart containerd
I1010 13:16:39.376994 29815 start.go:495] detecting cgroup driver to use...
I1010 13:16:39.377087 29815 ssh_runner.go:195] Run: sudo systemctl cat docker.service
I1010 13:16:39.393031 29815 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I1010 13:16:39.403783 29815 ssh_runner.go:195] Run: sudo systemctl stop -f containerd
I1010 13:16:39.416300 29815 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I1010 13:16:39.427647 29815 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service crio
I1010 13:16:39.438977 29815 ssh_runner.go:195] Run: sudo systemctl stop -f crio
I1010 13:16:39.459178 29815 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service crio
I1010 13:16:39.470637 29815 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///var/run/cri-dockerd.sock
" | sudo tee /etc/crictl.yaml"
I1010 13:16:39.486272 29815 ssh_runner.go:195] Run: which cri-dockerd
I1010 13:16:39.489288 29815 ssh_runner.go:195] Run: sudo mkdir -p /etc/systemd/system/cri-docker.service.d
I1010 13:16:39.497409 29815 ssh_runner.go:362] scp memory --> /etc/systemd/system/cri-docker.service.d/10-cni.conf (190 bytes)
I1010 13:16:39.510710 29815 ssh_runner.go:195] Run: sudo systemctl unmask docker.service
I1010 13:16:39.613389 29815 ssh_runner.go:195] Run: sudo systemctl enable docker.socket
I1010 13:16:39.715737 29815 docker.go:574] configuring docker to use "cgroupfs" as cgroup driver...
I1010 13:16:39.715829 29815 ssh_runner.go:362] scp memory --> /etc/docker/daemon.json (130 bytes)
I1010 13:16:39.730277 29815 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1010 13:16:39.823052 29815 ssh_runner.go:195] Run: sudo systemctl restart docker
I1010 13:16:42.216703 29815 ssh_runner.go:235] Completed: sudo systemctl restart docker: (2.39356892s)
I1010 13:16:42.216792 29815 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service cri-docker.socket
I1010 13:16:42.227955 29815 ssh_runner.go:195] Run: sudo systemctl stop cri-docker.socket
I1010 13:16:42.241637 29815 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service cri-docker.service
I1010 13:16:42.252765 29815 ssh_runner.go:195] Run: sudo systemctl unmask cri-docker.socket
I1010 13:16:42.357835 29815 ssh_runner.go:195] Run: sudo systemctl enable cri-docker.socket
I1010 13:16:42.472664 29815 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1010 13:16:42.582945 29815 ssh_runner.go:195] Run: sudo systemctl restart cri-docker.socket
I1010 13:16:42.595883 29815 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service cri-docker.service
I1010 13:16:42.606258 29815 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1010 13:16:42.701742 29815 ssh_runner.go:195] Run: sudo systemctl restart cri-docker.service
I1010 13:16:42.767935 29815 start.go:542] Will wait 60s for socket path /var/run/cri-dockerd.sock
I1010 13:16:42.769169 29815 ssh_runner.go:195] Run: stat /var/run/cri-dockerd.sock
I1010 13:16:42.773584 29815 start.go:563] Will wait 60s for crictl version
I1010 13:16:42.773640 29815 ssh_runner.go:195] Run: which crictl
I1010 13:16:42.776972 29815 ssh_runner.go:195] Run: sudo /usr/bin/crictl version
I1010 13:16:42.803951 29815 start.go:579] Version: 0.1.0
RuntimeName: docker
RuntimeVersion: 27.3.1
RuntimeApiVersion: v1
I1010 13:16:42.804042 29815 ssh_runner.go:195] Run: docker version --format {{.Server.Version}}
I1010 13:16:42.821027 29815 ssh_runner.go:195] Run: docker version --format {{.Server.Version}}
I1010 13:16:42.858514 29815 out.go:235] * Preparing Kubernetes v1.31.1 on Docker 27.3.1 ...
I1010 13:16:42.858563 29815 main.go:141] libmachine: (ha-626000) Calling .GetIP
I1010 13:16:42.859504 29815 ssh_runner.go:195] Run: grep 192.169.0.1 host.minikube.internal$ /etc/hosts
I1010 13:16:42.864004 29815 ssh_runner.go:195] Run: /bin/bash -c "{ grep -v $'\thost.minikube.internal$' "/etc/hosts"; echo "192.169.0.1 host.minikube.internal"; } > /tmp/h.$$; sudo cp /tmp/h.$$ "/etc/hosts""
I1010 13:16:42.873893 29815 kubeadm.go:883] updating cluster {Name:ha-626000 KeepContext:false EmbedCerts:false MinikubeISO:https://storage.googleapis.com/minikube-builds/iso/19774/minikube-v1.34.0-1728382514-19774-amd64.iso KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.45-1728382586-19774@sha256:5d8c4f6d838467365e214e2194dd0153a763e3f78723b5f2a8e06ef7b47409ec Memory:2200 CPUs:2 DiskSize:20000 Driver:hyperkit HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.31.
1 ClusterName:ha-626000 Namespace:default APIServerHAVIP:192.169.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.169.0.41 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m02 IP:192.169.0.42 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m04 IP:192.169.0.44 Port:0 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:false Worker:true}] Addons:map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false inaccel:false ingress:false ingress-dns:false inspektor-
gadget:false istio:false istio-provisioner:false kong:false kubeflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s Mount:false MountString:/Users:/minikube-host Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:fals
e DisableMetrics:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s} ...
I1010 13:16:42.873986 29815 preload.go:131] Checking if preload exists for k8s version v1.31.1 and runtime docker
I1010 13:16:42.874064 29815 ssh_runner.go:195] Run: docker images --format {{.Repository}}:{{.Tag}}
I1010 13:16:42.887582 29815 docker.go:685] Got preloaded images: -- stdout --
registry.k8s.io/kube-apiserver:v1.31.1
registry.k8s.io/kube-controller-manager:v1.31.1
registry.k8s.io/kube-scheduler:v1.31.1
registry.k8s.io/kube-proxy:v1.31.1
ghcr.io/kube-vip/kube-vip:v0.8.3
kindest/kindnetd:v20240813-c6f155d6
registry.k8s.io/coredns/coredns:v1.11.3
registry.k8s.io/etcd:3.5.15-0
registry.k8s.io/pause:3.10
gcr.io/k8s-minikube/storage-provisioner:v5
gcr.io/k8s-minikube/busybox:1.28
-- /stdout --
I1010 13:16:42.887600 29815 docker.go:615] Images already preloaded, skipping extraction
I1010 13:16:42.887681 29815 ssh_runner.go:195] Run: docker images --format {{.Repository}}:{{.Tag}}
I1010 13:16:42.900599 29815 docker.go:685] Got preloaded images: -- stdout --
registry.k8s.io/kube-apiserver:v1.31.1
registry.k8s.io/kube-controller-manager:v1.31.1
registry.k8s.io/kube-scheduler:v1.31.1
registry.k8s.io/kube-proxy:v1.31.1
ghcr.io/kube-vip/kube-vip:v0.8.3
kindest/kindnetd:v20240813-c6f155d6
registry.k8s.io/coredns/coredns:v1.11.3
registry.k8s.io/etcd:3.5.15-0
registry.k8s.io/pause:3.10
gcr.io/k8s-minikube/storage-provisioner:v5
gcr.io/k8s-minikube/busybox:1.28
-- /stdout --
I1010 13:16:42.900620 29815 cache_images.go:84] Images are preloaded, skipping loading
I1010 13:16:42.900630 29815 kubeadm.go:934] updating node { 192.169.0.41 8443 v1.31.1 docker true true} ...
I1010 13:16:42.900715 29815 kubeadm.go:946] kubelet [Unit]
Wants=docker.socket
[Service]
ExecStart=
ExecStart=/var/lib/minikube/binaries/v1.31.1/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --config=/var/lib/kubelet/config.yaml --hostname-override=ha-626000 --kubeconfig=/etc/kubernetes/kubelet.conf --node-ip=192.169.0.41
[Install]
config:
{KubernetesVersion:v1.31.1 ClusterName:ha-626000 Namespace:default APIServerHAVIP:192.169.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:}
I1010 13:16:42.900796 29815 ssh_runner.go:195] Run: docker info --format {{.CgroupDriver}}
I1010 13:16:42.934276 29815 cni.go:84] Creating CNI manager for ""
I1010 13:16:42.934289 29815 cni.go:136] multinode detected (3 nodes found), recommending kindnet
I1010 13:16:42.934299 29815 kubeadm.go:84] Using pod CIDR: 10.244.0.0/16
I1010 13:16:42.934312 29815 kubeadm.go:181] kubeadm options: {CertDir:/var/lib/minikube/certs ServiceCIDR:10.96.0.0/12 PodSubnet:10.244.0.0/16 AdvertiseAddress:192.169.0.41 APIServerPort:8443 KubernetesVersion:v1.31.1 EtcdDataDir:/var/lib/minikube/etcd EtcdExtraArgs:map[] ClusterName:ha-626000 NodeName:ha-626000 DNSDomain:cluster.local CRISocket:/var/run/cri-dockerd.sock ImageRepository: ComponentOptions:[{Component:apiServer ExtraArgs:map[enable-admission-plugins:NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota] Pairs:map[certSANs:["127.0.0.1", "localhost", "192.169.0.41"]]} {Component:controllerManager ExtraArgs:map[allocate-node-cidrs:true leader-elect:false] Pairs:map[]} {Component:scheduler ExtraArgs:map[leader-elect:false] Pairs:map[]}] FeatureArgs:map[] NodeIP:192.169.0.41 CgroupDriver:cgroupfs ClientCAFile:/var/lib/minikube/certs/ca.crt StaticPodPath:/etc/kubernetes/ma
nifests ControlPlaneAddress:control-plane.minikube.internal KubeProxyOptions:map[] ResolvConfSearchRegression:false KubeletConfigOpts:map[containerRuntimeEndpoint:unix:///var/run/cri-dockerd.sock hairpinMode:hairpin-veth runtimeRequestTimeout:15m] PrependCriSocketUnix:true}
I1010 13:16:42.934392 29815 kubeadm.go:187] kubeadm config:
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 192.169.0.41
bindPort: 8443
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
ttl: 24h0m0s
usages:
- signing
- authentication
nodeRegistration:
criSocket: unix:///var/run/cri-dockerd.sock
name: "ha-626000"
kubeletExtraArgs:
node-ip: 192.169.0.41
taints: []
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
apiServer:
certSANs: ["127.0.0.1", "localhost", "192.169.0.41"]
extraArgs:
enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
controllerManager:
extraArgs:
allocate-node-cidrs: "true"
leader-elect: "false"
scheduler:
extraArgs:
leader-elect: "false"
certificatesDir: /var/lib/minikube/certs
clusterName: mk
controlPlaneEndpoint: control-plane.minikube.internal:8443
etcd:
local:
dataDir: /var/lib/minikube/etcd
extraArgs:
proxy-refresh-interval: "70000"
kubernetesVersion: v1.31.1
networking:
dnsDomain: cluster.local
podSubnet: "10.244.0.0/16"
serviceSubnet: 10.96.0.0/12
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
authentication:
x509:
clientCAFile: /var/lib/minikube/certs/ca.crt
cgroupDriver: cgroupfs
containerRuntimeEndpoint: unix:///var/run/cri-dockerd.sock
hairpinMode: hairpin-veth
runtimeRequestTimeout: 15m
clusterDomain: "cluster.local"
# disable disk resource management by default
imageGCHighThresholdPercent: 100
evictionHard:
nodefs.available: "0%"
nodefs.inodesFree: "0%"
imagefs.available: "0%"
failSwapOn: false
staticPodPath: /etc/kubernetes/manifests
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
clusterCIDR: "10.244.0.0/16"
metricsBindAddress: 0.0.0.0:10249
conntrack:
maxPerCore: 0
# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_established"
tcpEstablishedTimeout: 0s
# Skip setting "net.netfilter.nf_conntrack_tcp_timeout_close"
tcpCloseWaitTimeout: 0s
I1010 13:16:42.934402 29815 kube-vip.go:115] generating kube-vip config ...
I1010 13:16:42.934468 29815 ssh_runner.go:195] Run: sudo sh -c "modprobe --all ip_vs ip_vs_rr ip_vs_wrr ip_vs_sh nf_conntrack"
I1010 13:16:42.947080 29815 kube-vip.go:167] auto-enabling control-plane load-balancing in kube-vip
I1010 13:16:42.947141 29815 kube-vip.go:137] kube-vip config:
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
name: kube-vip
namespace: kube-system
spec:
containers:
- args:
- manager
env:
- name: vip_arp
value: "true"
- name: port
value: "8443"
- name: vip_nodename
valueFrom:
fieldRef:
fieldPath: spec.nodeName
- name: vip_interface
value: eth0
- name: vip_cidr
value: "32"
- name: dns_mode
value: first
- name: cp_enable
value: "true"
- name: cp_namespace
value: kube-system
- name: vip_leaderelection
value: "true"
- name: vip_leasename
value: plndr-cp-lock
- name: vip_leaseduration
value: "5"
- name: vip_renewdeadline
value: "3"
- name: vip_retryperiod
value: "1"
- name: address
value: 192.169.0.254
- name: prometheus_server
value: :2112
- name : lb_enable
value: "true"
- name: lb_port
value: "8443"
image: ghcr.io/kube-vip/kube-vip:v0.8.3
imagePullPolicy: IfNotPresent
name: kube-vip
resources: {}
securityContext:
capabilities:
add:
- NET_ADMIN
- NET_RAW
volumeMounts:
- mountPath: /etc/kubernetes/admin.conf
name: kubeconfig
hostAliases:
- hostnames:
- kubernetes
ip: 127.0.0.1
hostNetwork: true
volumes:
- hostPath:
path: "/etc/kubernetes/admin.conf"
name: kubeconfig
status: {}
I1010 13:16:42.947203 29815 ssh_runner.go:195] Run: sudo ls /var/lib/minikube/binaries/v1.31.1
I1010 13:16:42.954760 29815 binaries.go:44] Found k8s binaries, skipping transfer
I1010 13:16:42.954818 29815 ssh_runner.go:195] Run: sudo mkdir -p /etc/systemd/system/kubelet.service.d /lib/systemd/system /var/tmp/minikube /etc/kubernetes/manifests
I1010 13:16:42.962348 29815 ssh_runner.go:362] scp memory --> /etc/systemd/system/kubelet.service.d/10-kubeadm.conf (308 bytes)
I1010 13:16:42.975847 29815 ssh_runner.go:362] scp memory --> /lib/systemd/system/kubelet.service (352 bytes)
I1010 13:16:42.989282 29815 ssh_runner.go:362] scp memory --> /var/tmp/minikube/kubeadm.yaml.new (2151 bytes)
I1010 13:16:43.003125 29815 ssh_runner.go:362] scp memory --> /etc/kubernetes/manifests/kube-vip.yaml (1440 bytes)
I1010 13:16:43.016588 29815 ssh_runner.go:195] Run: grep 192.169.0.254 control-plane.minikube.internal$ /etc/hosts
I1010 13:16:43.019584 29815 ssh_runner.go:195] Run: /bin/bash -c "{ grep -v $'\tcontrol-plane.minikube.internal$' "/etc/hosts"; echo "192.169.0.254 control-plane.minikube.internal"; } > /tmp/h.$$; sudo cp /tmp/h.$$ "/etc/hosts""
I1010 13:16:43.028987 29815 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1010 13:16:43.138574 29815 ssh_runner.go:195] Run: sudo systemctl start kubelet
I1010 13:16:43.150562 29815 certs.go:68] Setting up /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000 for IP: 192.169.0.41
I1010 13:16:43.150577 29815 certs.go:194] generating shared ca certs ...
I1010 13:16:43.150587 29815 certs.go:226] acquiring lock for ca certs: {Name:mk4d04e58ed88d08cffb9df8d85d783349bbd157 Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1010 13:16:43.150815 29815 certs.go:235] skipping valid "minikubeCA" ca cert: /Users/jenkins/minikube-integration/19787-26748/.minikube/ca.key
I1010 13:16:43.150911 29815 certs.go:235] skipping valid "proxyClientCA" ca cert: /Users/jenkins/minikube-integration/19787-26748/.minikube/proxy-client-ca.key
I1010 13:16:43.150922 29815 certs.go:256] generating profile certs ...
I1010 13:16:43.151045 29815 certs.go:359] skipping valid signed profile cert regeneration for "minikube-user": /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/client.key
I1010 13:16:43.151066 29815 certs.go:363] generating signed profile cert for "minikube": /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/apiserver.key.5ae831cf
I1010 13:16:43.151081 29815 crypto.go:68] Generating cert /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/apiserver.crt.5ae831cf with IP's: [10.96.0.1 127.0.0.1 10.0.0.1 192.169.0.41 192.169.0.42 192.169.0.254]
I1010 13:16:43.341768 29815 crypto.go:156] Writing cert to /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/apiserver.crt.5ae831cf ...
I1010 13:16:43.341786 29815 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/apiserver.crt.5ae831cf: {Name:mkcb647eb6800b3ff2ddd4db6e4d8712b550af8c Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1010 13:16:43.343217 29815 crypto.go:164] Writing key to /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/apiserver.key.5ae831cf ...
I1010 13:16:43.343227 29815 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/apiserver.key.5ae831cf: {Name:mkbecf0e31ef0011312c87070a863f40e430629e Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1010 13:16:43.343865 29815 certs.go:381] copying /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/apiserver.crt.5ae831cf -> /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/apiserver.crt
I1010 13:16:43.344243 29815 certs.go:385] copying /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/apiserver.key.5ae831cf -> /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/apiserver.key
I1010 13:16:43.344531 29815 certs.go:359] skipping valid signed profile cert regeneration for "aggregator": /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/proxy-client.key
I1010 13:16:43.344540 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/ca.crt -> /var/lib/minikube/certs/ca.crt
I1010 13:16:43.344564 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/ca.key -> /var/lib/minikube/certs/ca.key
I1010 13:16:43.344584 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/proxy-client-ca.crt -> /var/lib/minikube/certs/proxy-client-ca.crt
I1010 13:16:43.344606 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/proxy-client-ca.key -> /var/lib/minikube/certs/proxy-client-ca.key
I1010 13:16:43.344625 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/apiserver.crt -> /var/lib/minikube/certs/apiserver.crt
I1010 13:16:43.344643 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/apiserver.key -> /var/lib/minikube/certs/apiserver.key
I1010 13:16:43.344661 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/proxy-client.crt -> /var/lib/minikube/certs/proxy-client.crt
I1010 13:16:43.344679 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/proxy-client.key -> /var/lib/minikube/certs/proxy-client.key
I1010 13:16:43.344781 29815 certs.go:484] found cert: /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/27315.pem (1338 bytes)
W1010 13:16:43.344838 29815 certs.go:480] ignoring /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/27315_empty.pem, impossibly tiny 0 bytes
I1010 13:16:43.344846 29815 certs.go:484] found cert: /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/ca-key.pem (1675 bytes)
I1010 13:16:43.344885 29815 certs.go:484] found cert: /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/ca.pem (1082 bytes)
I1010 13:16:43.344918 29815 certs.go:484] found cert: /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/cert.pem (1123 bytes)
I1010 13:16:43.344947 29815 certs.go:484] found cert: /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/key.pem (1679 bytes)
I1010 13:16:43.345013 29815 certs.go:484] found cert: /Users/jenkins/minikube-integration/19787-26748/.minikube/files/etc/ssl/certs/273152.pem (1708 bytes)
I1010 13:16:43.345053 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/27315.pem -> /usr/share/ca-certificates/27315.pem
I1010 13:16:43.345081 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/files/etc/ssl/certs/273152.pem -> /usr/share/ca-certificates/273152.pem
I1010 13:16:43.345101 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/ca.crt -> /usr/share/ca-certificates/minikubeCA.pem
I1010 13:16:43.345596 29815 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19787-26748/.minikube/ca.crt --> /var/lib/minikube/certs/ca.crt (1111 bytes)
I1010 13:16:43.366045 29815 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19787-26748/.minikube/ca.key --> /var/lib/minikube/certs/ca.key (1679 bytes)
I1010 13:16:43.390573 29815 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19787-26748/.minikube/proxy-client-ca.crt --> /var/lib/minikube/certs/proxy-client-ca.crt (1119 bytes)
I1010 13:16:43.416054 29815 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19787-26748/.minikube/proxy-client-ca.key --> /var/lib/minikube/certs/proxy-client-ca.key (1679 bytes)
I1010 13:16:43.448807 29815 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/apiserver.crt --> /var/lib/minikube/certs/apiserver.crt (1432 bytes)
I1010 13:16:43.472086 29815 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/apiserver.key --> /var/lib/minikube/certs/apiserver.key (1679 bytes)
I1010 13:16:43.521197 29815 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/proxy-client.crt --> /var/lib/minikube/certs/proxy-client.crt (1147 bytes)
I1010 13:16:43.556068 29815 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/proxy-client.key --> /var/lib/minikube/certs/proxy-client.key (1679 bytes)
I1010 13:16:43.610483 29815 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/27315.pem --> /usr/share/ca-certificates/27315.pem (1338 bytes)
I1010 13:16:43.662512 29815 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19787-26748/.minikube/files/etc/ssl/certs/273152.pem --> /usr/share/ca-certificates/273152.pem (1708 bytes)
I1010 13:16:43.708102 29815 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19787-26748/.minikube/ca.crt --> /usr/share/ca-certificates/minikubeCA.pem (1111 bytes)
I1010 13:16:43.735654 29815 ssh_runner.go:362] scp memory --> /var/lib/minikube/kubeconfig (738 bytes)
I1010 13:16:43.755965 29815 ssh_runner.go:195] Run: openssl version
I1010 13:16:43.762495 29815 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/27315.pem && ln -fs /usr/share/ca-certificates/27315.pem /etc/ssl/certs/27315.pem"
I1010 13:16:43.775826 29815 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/27315.pem
I1010 13:16:43.783979 29815 certs.go:528] hashing: -rw-r--r-- 1 root root 1338 Oct 10 20:01 /usr/share/ca-certificates/27315.pem
I1010 13:16:43.784042 29815 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/27315.pem
I1010 13:16:43.790429 29815 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/51391683.0 || ln -fs /etc/ssl/certs/27315.pem /etc/ssl/certs/51391683.0"
I1010 13:16:43.799053 29815 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/273152.pem && ln -fs /usr/share/ca-certificates/273152.pem /etc/ssl/certs/273152.pem"
I1010 13:16:43.807503 29815 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/273152.pem
I1010 13:16:43.811033 29815 certs.go:528] hashing: -rw-r--r-- 1 root root 1708 Oct 10 20:01 /usr/share/ca-certificates/273152.pem
I1010 13:16:43.811080 29815 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/273152.pem
I1010 13:16:43.815391 29815 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/3ec20f2e.0 || ln -fs /etc/ssl/certs/273152.pem /etc/ssl/certs/3ec20f2e.0"
I1010 13:16:43.823848 29815 ssh_runner.go:195] Run: sudo /bin/bash -c "test -s /usr/share/ca-certificates/minikubeCA.pem && ln -fs /usr/share/ca-certificates/minikubeCA.pem /etc/ssl/certs/minikubeCA.pem"
I1010 13:16:43.832298 29815 ssh_runner.go:195] Run: ls -la /usr/share/ca-certificates/minikubeCA.pem
I1010 13:16:43.836018 29815 certs.go:528] hashing: -rw-r--r-- 1 root root 1111 Oct 10 19:51 /usr/share/ca-certificates/minikubeCA.pem
I1010 13:16:43.836071 29815 ssh_runner.go:195] Run: openssl x509 -hash -noout -in /usr/share/ca-certificates/minikubeCA.pem
I1010 13:16:43.840406 29815 ssh_runner.go:195] Run: sudo /bin/bash -c "test -L /etc/ssl/certs/b5213941.0 || ln -fs /etc/ssl/certs/minikubeCA.pem /etc/ssl/certs/b5213941.0"
I1010 13:16:43.848852 29815 ssh_runner.go:195] Run: stat /var/lib/minikube/certs/apiserver-kubelet-client.crt
I1010 13:16:43.852445 29815 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/apiserver-etcd-client.crt -checkend 86400
I1010 13:16:43.856829 29815 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/apiserver-kubelet-client.crt -checkend 86400
I1010 13:16:43.861744 29815 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/etcd/server.crt -checkend 86400
I1010 13:16:43.866124 29815 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/etcd/healthcheck-client.crt -checkend 86400
I1010 13:16:43.870508 29815 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/etcd/peer.crt -checkend 86400
I1010 13:16:43.874749 29815 ssh_runner.go:195] Run: openssl x509 -noout -in /var/lib/minikube/certs/front-proxy-client.crt -checkend 86400
I1010 13:16:43.879370 29815 kubeadm.go:392] StartCluster: {Name:ha-626000 KeepContext:false EmbedCerts:false MinikubeISO:https://storage.googleapis.com/minikube-builds/iso/19774/minikube-v1.34.0-1728382514-19774-amd64.iso KicBaseImage:gcr.io/k8s-minikube/kicbase-builds:v0.0.45-1728382586-19774@sha256:5d8c4f6d838467365e214e2194dd0153a763e3f78723b5f2a8e06ef7b47409ec Memory:2200 CPUs:2 DiskSize:20000 Driver:hyperkit HyperkitVpnKitSock: HyperkitVSockPorts:[] DockerEnv:[] ContainerVolumeMounts:[] InsecureRegistry:[] RegistryMirror:[] HostOnlyCIDR:192.168.59.1/24 HypervVirtualSwitch: HypervUseExternalSwitch:false HypervExternalAdapter: KVMNetwork:default KVMQemuURI:qemu:///system KVMGPU:false KVMHidden:false KVMNUMACount:1 APIServerPort:8443 DockerOpt:[] DisableDriverMounts:false NFSShare:[] NFSSharesRoot:/nfsshares UUID: NoVTXCheck:false DNSProxy:false HostDNSResolver:true HostOnlyNicType:virtio NatNicType:virtio SSHIPAddress: SSHUser:root SSHKey: SSHPort:22 KubernetesConfig:{KubernetesVersion:v1.31.1 C
lusterName:ha-626000 Namespace:default APIServerHAVIP:192.169.0.254 APIServerName:minikubeCA APIServerNames:[] APIServerIPs:[] DNSDomain:cluster.local ContainerRuntime:docker CRISocket: NetworkPlugin:cni FeatureGates: ServiceCIDR:10.96.0.0/12 ImageRepository: LoadBalancerStartIP: LoadBalancerEndIP: CustomIngressCert: RegistryAliases: ExtraOptions:[] ShouldLoadCachedImages:true EnableDefaultCNI:false CNI:} Nodes:[{Name: IP:192.169.0.41 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m02 IP:192.169.0.42 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true} {Name:m04 IP:192.169.0.44 Port:0 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:false Worker:true}] Addons:map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false inaccel:false ingress:false ingress-dns:false inspektor-gad
get:false istio:false istio-provisioner:false kong:false kubeflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false] CustomAddonImages:map[] CustomAddonRegistries:map[] VerifyComponents:map[apiserver:true apps_running:true default_sa:true extra:true kubelet:true node_ready:true system_pods:true] StartHostTimeout:6m0s ScheduledStop:<nil> ExposedPorts:[] ListenAddress: Network: Subnet: MultiNodeRequested:true ExtraDisks:0 CertExpiration:26280h0m0s Mount:false MountString:/Users:/minikube-host Mount9PVersion:9p2000.L MountGID:docker MountIP: MountMSize:262144 MountOptions:[] MountPort:0 MountType:9p MountUID:docker BinaryMirror: DisableOptimizations:false D
isableMetrics:false CustomQemuFirmwarePath: SocketVMnetClientPath: SocketVMnetPath: StaticIP: SSHAuthSock: SSHAgentPID:0 GPUs: AutoPauseInterval:1m0s}
I1010 13:16:43.879491 29815 ssh_runner.go:195] Run: docker ps --filter status=paused --filter=name=k8s_.*_(kube-system)_ --format={{.ID}}
I1010 13:16:43.890801 29815 ssh_runner.go:195] Run: sudo ls /var/lib/kubelet/kubeadm-flags.env /var/lib/kubelet/config.yaml /var/lib/minikube/etcd
I1010 13:16:43.898587 29815 kubeadm.go:408] found existing configuration files, will attempt cluster restart
I1010 13:16:43.898599 29815 kubeadm.go:593] restartPrimaryControlPlane start ...
I1010 13:16:43.898660 29815 ssh_runner.go:195] Run: sudo test -d /data/minikube
I1010 13:16:43.906295 29815 kubeadm.go:130] /data/minikube skipping compat symlinks: sudo test -d /data/minikube: Process exited with status 1
stdout:
stderr:
I1010 13:16:43.906614 29815 kubeconfig.go:47] verify endpoint returned: get endpoint: "ha-626000" does not appear in /Users/jenkins/minikube-integration/19787-26748/kubeconfig
I1010 13:16:43.906702 29815 kubeconfig.go:62] /Users/jenkins/minikube-integration/19787-26748/kubeconfig needs updating (will repair): [kubeconfig missing "ha-626000" cluster setting kubeconfig missing "ha-626000" context setting]
I1010 13:16:43.906894 29815 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/19787-26748/kubeconfig: {Name:mk3e06d3193ef6e4a3a37d1bc0eb9b008118d82b Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1010 13:16:43.907305 29815 loader.go:395] Config loaded from file: /Users/jenkins/minikube-integration/19787-26748/kubeconfig
I1010 13:16:43.907528 29815 kapi.go:59] client config for ha-626000: &rest.Config{Host:"https://192.169.0.41:8443", APIPath:"", ContentConfig:rest.ContentConfig{AcceptContentTypes:"", ContentType:"", GroupVersion:(*schema.GroupVersion)(nil), NegotiatedSerializer:runtime.NegotiatedSerializer(nil)}, Username:"", Password:"", BearerToken:"", BearerTokenFile:"", Impersonate:rest.ImpersonationConfig{UserName:"", UID:"", Groups:[]string(nil), Extra:map[string][]string(nil)}, AuthProvider:<nil>, AuthConfigPersister:rest.AuthProviderConfigPersister(nil), ExecProvider:<nil>, TLSClientConfig:rest.sanitizedTLSClientConfig{Insecure:false, ServerName:"", CertFile:"/Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/client.crt", KeyFile:"/Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/client.key", CAFile:"/Users/jenkins/minikube-integration/19787-26748/.minikube/ca.crt", CertData:[]uint8(nil), KeyData:[]uint8(nil), CAData:[]uint8(nil), NextProtos:[]string(nil
)}, UserAgent:"", DisableCompression:false, Transport:http.RoundTripper(nil), WrapTransport:(transport.WrapperFunc)(0x45d3140), QPS:0, Burst:0, RateLimiter:flowcontrol.RateLimiter(nil), WarningHandler:rest.WarningHandler(nil), Timeout:0, Dial:(func(context.Context, string, string) (net.Conn, error))(nil), Proxy:(func(*http.Request) (*url.URL, error))(nil)}
I1010 13:16:43.907845 29815 cert_rotation.go:140] Starting client certificate rotation controller
I1010 13:16:43.908047 29815 ssh_runner.go:195] Run: sudo diff -u /var/tmp/minikube/kubeadm.yaml /var/tmp/minikube/kubeadm.yaml.new
I1010 13:16:43.915706 29815 kubeadm.go:630] The running cluster does not require reconfiguration: 192.169.0.41
I1010 13:16:43.915718 29815 kubeadm.go:597] duration metric: took 17.114685ms to restartPrimaryControlPlane
I1010 13:16:43.915723 29815 kubeadm.go:394] duration metric: took 36.357373ms to StartCluster
I1010 13:16:43.915733 29815 settings.go:142] acquiring lock: {Name:mk7fe819cc8f0c2ad533f729afc7f23dbff23d5d Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1010 13:16:43.915824 29815 settings.go:150] Updating kubeconfig: /Users/jenkins/minikube-integration/19787-26748/kubeconfig
I1010 13:16:43.916213 29815 lock.go:35] WriteFile acquiring /Users/jenkins/minikube-integration/19787-26748/kubeconfig: {Name:mk3e06d3193ef6e4a3a37d1bc0eb9b008118d82b Clock:{} Delay:500ms Timeout:1m0s Cancel:<nil>}
I1010 13:16:43.916420 29815 start.go:233] HA (multi-control plane) cluster: will skip waiting for primary control-plane node &{Name: IP:192.169.0.41 Port:8443 KubernetesVersion:v1.31.1 ContainerRuntime:docker ControlPlane:true Worker:true}
I1010 13:16:43.916432 29815 start.go:241] waiting for startup goroutines ...
I1010 13:16:43.916444 29815 addons.go:507] enable addons start: toEnable=map[ambassador:false auto-pause:false cloud-spanner:false csi-hostpath-driver:false dashboard:false default-storageclass:false efk:false freshpod:false gcp-auth:false gvisor:false headlamp:false inaccel:false ingress:false ingress-dns:false inspektor-gadget:false istio:false istio-provisioner:false kong:false kubeflow:false kubevirt:false logviewer:false metallb:false metrics-server:false nvidia-device-plugin:false nvidia-driver-installer:false nvidia-gpu-device-plugin:false olm:false pod-security-policy:false portainer:false registry:false registry-aliases:false registry-creds:false storage-provisioner:false storage-provisioner-gluster:false storage-provisioner-rancher:false volcano:false volumesnapshots:false yakd:false]
I1010 13:16:43.916548 29815 config.go:182] Loaded profile config "ha-626000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.1
I1010 13:16:43.958932 29815 out.go:177] * Enabled addons:
I1010 13:16:43.979898 29815 addons.go:510] duration metric: took 63.402493ms for enable addons: enabled=[]
I1010 13:16:43.979940 29815 start.go:246] waiting for cluster config update ...
I1010 13:16:43.979952 29815 start.go:255] writing updated cluster config ...
I1010 13:16:44.001823 29815 out.go:201]
I1010 13:16:44.023160 29815 config.go:182] Loaded profile config "ha-626000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.1
I1010 13:16:44.023319 29815 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/config.json ...
I1010 13:16:44.045912 29815 out.go:177] * Starting "ha-626000-m02" control-plane node in "ha-626000" cluster
I1010 13:16:44.087861 29815 preload.go:131] Checking if preload exists for k8s version v1.31.1 and runtime docker
I1010 13:16:44.087895 29815 cache.go:56] Caching tarball of preloaded images
I1010 13:16:44.088074 29815 preload.go:172] Found /Users/jenkins/minikube-integration/19787-26748/.minikube/cache/preloaded-tarball/preloaded-images-k8s-v18-v1.31.1-docker-overlay2-amd64.tar.lz4 in cache, skipping download
I1010 13:16:44.088092 29815 cache.go:59] Finished verifying existence of preloaded tar for v1.31.1 on docker
I1010 13:16:44.088231 29815 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/config.json ...
I1010 13:16:44.089190 29815 start.go:360] acquireMachinesLock for ha-626000-m02: {Name:mk9fc820136d807cdfeb8abc8a165ddd26f4799c Clock:{} Delay:500ms Timeout:13m0s Cancel:<nil>}
I1010 13:16:44.089302 29815 start.go:364] duration metric: took 87.69µs to acquireMachinesLock for "ha-626000-m02"
I1010 13:16:44.089328 29815 start.go:96] Skipping create...Using existing machine configuration
I1010 13:16:44.089349 29815 fix.go:54] fixHost starting: m02
I1010 13:16:44.089784 29815 main.go:141] libmachine: Found binary path at /Users/jenkins/workspace/out/docker-machine-driver-hyperkit
I1010 13:16:44.089819 29815 main.go:141] libmachine: Launching plugin server for driver hyperkit
I1010 13:16:44.101685 29815 main.go:141] libmachine: Plugin server listening at address 127.0.0.1:51050
I1010 13:16:44.102041 29815 main.go:141] libmachine: () Calling .GetVersion
I1010 13:16:44.102368 29815 main.go:141] libmachine: Using API Version 1
I1010 13:16:44.102380 29815 main.go:141] libmachine: () Calling .SetConfigRaw
I1010 13:16:44.102622 29815 main.go:141] libmachine: () Calling .GetMachineName
I1010 13:16:44.102745 29815 main.go:141] libmachine: (ha-626000-m02) Calling .DriverName
I1010 13:16:44.102841 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetState
I1010 13:16:44.102926 29815 main.go:141] libmachine: (ha-626000-m02) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I1010 13:16:44.103035 29815 main.go:141] libmachine: (ha-626000-m02) DBG | hyperkit pid from json: 29715
I1010 13:16:44.104078 29815 main.go:141] libmachine: (ha-626000-m02) DBG | hyperkit pid 29715 missing from process table
I1010 13:16:44.104132 29815 fix.go:112] recreateIfNeeded on ha-626000-m02: state=Stopped err=<nil>
I1010 13:16:44.104153 29815 main.go:141] libmachine: (ha-626000-m02) Calling .DriverName
W1010 13:16:44.104238 29815 fix.go:138] unexpected machine state, will restart: <nil>
I1010 13:16:44.125793 29815 out.go:177] * Restarting existing hyperkit VM for "ha-626000-m02" ...
I1010 13:16:44.167872 29815 main.go:141] libmachine: (ha-626000-m02) Calling .Start
I1010 13:16:44.168287 29815 main.go:141] libmachine: (ha-626000-m02) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I1010 13:16:44.168309 29815 main.go:141] libmachine: (ha-626000-m02) minikube might have been shutdown in an unclean way, the hyperkit pid file still exists: /Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/hyperkit.pid
I1010 13:16:44.170219 29815 main.go:141] libmachine: (ha-626000-m02) DBG | hyperkit pid 29715 missing from process table
I1010 13:16:44.170232 29815 main.go:141] libmachine: (ha-626000-m02) DBG | pid 29715 is in state "Stopped"
I1010 13:16:44.170254 29815 main.go:141] libmachine: (ha-626000-m02) DBG | Removing stale pid file /Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/hyperkit.pid...
I1010 13:16:44.170466 29815 main.go:141] libmachine: (ha-626000-m02) DBG | Using UUID 8f610dd8-f5bc-4544-a344-edb546cfa578
I1010 13:16:44.195787 29815 main.go:141] libmachine: (ha-626000-m02) DBG | Generated MAC d6:2c:95:0:73:df
I1010 13:16:44.195807 29815 main.go:141] libmachine: (ha-626000-m02) DBG | Starting with cmdline: loglevel=3 console=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-626000
I1010 13:16:44.195971 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:44 DEBUG: hyperkit: Start &hyperkit.HyperKit{HyperKit:"/usr/local/bin/hyperkit", Argv0:"", StateDir:"/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02", VPNKitSock:"", VPNKitUUID:"", VPNKitPreferredIPv4:"", UUID:"8f610dd8-f5bc-4544-a344-edb546cfa578", Disks:[]hyperkit.Disk{(*hyperkit.RawDisk)(0xc0003ca2d0)}, ISOImages:[]string{"/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/boot2docker.iso"}, VSock:false, VSockDir:"", VSockPorts:[]int(nil), VSockGuestCID:3, VMNet:true, Sockets9P:[]hyperkit.Socket9P(nil), Kernel:"/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/bzimage", Initrd:"/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/initrd", Bootrom:"", CPUs:2, Memory:2200, Console:1, Serials:[]hyperkit.Serial(nil), Pid:0, Arguments:[]string(nil), CmdLine:"", process:(*os.Process)(nil)}
I1010 13:16:44.195999 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:44 DEBUG: hyperkit: check &hyperkit.HyperKit{HyperKit:"/usr/local/bin/hyperkit", Argv0:"", StateDir:"/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02", VPNKitSock:"", VPNKitUUID:"", VPNKitPreferredIPv4:"", UUID:"8f610dd8-f5bc-4544-a344-edb546cfa578", Disks:[]hyperkit.Disk{(*hyperkit.RawDisk)(0xc0003ca2d0)}, ISOImages:[]string{"/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/boot2docker.iso"}, VSock:false, VSockDir:"", VSockPorts:[]int(nil), VSockGuestCID:3, VMNet:true, Sockets9P:[]hyperkit.Socket9P(nil), Kernel:"/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/bzimage", Initrd:"/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/initrd", Bootrom:"", CPUs:2, Memory:2200, Console:1, Serials:[]hyperkit.Serial(nil), Pid:0, Arguments:[]string(nil), CmdLine:"", process:(*os.Process)(nil)}
I1010 13:16:44.196094 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:44 DEBUG: hyperkit: Arguments: []string{"-A", "-u", "-F", "/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/hyperkit.pid", "-c", "2", "-m", "2200M", "-s", "0:0,hostbridge", "-s", "31,lpc", "-s", "1:0,virtio-net", "-U", "8f610dd8-f5bc-4544-a344-edb546cfa578", "-s", "2:0,virtio-blk,/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/ha-626000-m02.rawdisk", "-s", "3,ahci-cd,/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/boot2docker.iso", "-s", "4,virtio-rnd", "-l", "com1,autopty=/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/tty,log=/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/console-ring", "-f", "kexec,/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/bzimage,/Users/jenkins/minikube-integration/19787-26748/.minikube/
machines/ha-626000-m02/initrd,earlyprintk=serial loglevel=3 console=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-626000"}
I1010 13:16:44.196136 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:44 DEBUG: hyperkit: CmdLine: "/usr/local/bin/hyperkit -A -u -F /Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/hyperkit.pid -c 2 -m 2200M -s 0:0,hostbridge -s 31,lpc -s 1:0,virtio-net -U 8f610dd8-f5bc-4544-a344-edb546cfa578 -s 2:0,virtio-blk,/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/ha-626000-m02.rawdisk -s 3,ahci-cd,/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/boot2docker.iso -s 4,virtio-rnd -l com1,autopty=/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/tty,log=/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/console-ring -f kexec,/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/bzimage,/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/initrd,earlyprintk=serial loglevel=3 co
nsole=ttyS0 console=tty0 noembed nomodeset norestore waitusb=10 systemd.legacy_systemd_cgroup_controller=yes random.trust_cpu=on hw_rng_model=virtio base host=ha-626000"
I1010 13:16:44.196186 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:44 DEBUG: hyperkit: Redirecting stdout/stderr to logger
I1010 13:16:44.197513 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:44 DEBUG: hyperkit: Pid is 29839
I1010 13:16:44.199039 29815 main.go:141] libmachine: (ha-626000-m02) DBG | Attempt 0
I1010 13:16:44.199051 29815 main.go:141] libmachine: (ha-626000-m02) DBG | exe=/Users/jenkins/workspace/out/docker-machine-driver-hyperkit uid=0
I1010 13:16:44.199170 29815 main.go:141] libmachine: (ha-626000-m02) DBG | hyperkit pid from json: 29839
I1010 13:16:44.200208 29815 main.go:141] libmachine: (ha-626000-m02) DBG | Searching for d6:2c:95:0:73:df in /var/db/dhcpd_leases ...
I1010 13:16:44.200307 29815 main.go:141] libmachine: (ha-626000-m02) DBG | Found 43 entries in /var/db/dhcpd_leases!
I1010 13:16:44.200322 29815 main.go:141] libmachine: (ha-626000-m02) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.41 HWAddress:a6:9:ee:ef:40:24 ID:1,a6:9:ee:ef:40:24 Lease:0x67084432}
I1010 13:16:44.200358 29815 main.go:141] libmachine: (ha-626000-m02) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.43 HWAddress:8e:ee:cb:41:b6:d7 ID:1,8e:ee:cb:41:b6:d7 Lease:0x670835fd}
I1010 13:16:44.200379 29815 main.go:141] libmachine: (ha-626000-m02) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.44 HWAddress:ba:c3:6f:f1:51:bf ID:1,ba:c3:6f:f1:51:bf Lease:0x670843f8}
I1010 13:16:44.200393 29815 main.go:141] libmachine: (ha-626000-m02) DBG | dhcp entry: {Name:minikube IPAddress:192.169.0.42 HWAddress:d6:2c:95:0:73:df ID:1,d6:2c:95:0:73:df Lease:0x6708438f}
I1010 13:16:44.200405 29815 main.go:141] libmachine: (ha-626000-m02) DBG | Found match: d6:2c:95:0:73:df
I1010 13:16:44.200416 29815 main.go:141] libmachine: (ha-626000-m02) DBG | IP: 192.169.0.42
I1010 13:16:44.200424 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetConfigRaw
I1010 13:16:44.201100 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetIP
I1010 13:16:44.201294 29815 profile.go:143] Saving config to /Users/jenkins/minikube-integration/19787-26748/.minikube/profiles/ha-626000/config.json ...
I1010 13:16:44.201789 29815 machine.go:93] provisionDockerMachine start ...
I1010 13:16:44.201800 29815 main.go:141] libmachine: (ha-626000-m02) Calling .DriverName
I1010 13:16:44.201945 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHHostname
I1010 13:16:44.202049 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHPort
I1010 13:16:44.202154 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHKeyPath
I1010 13:16:44.202248 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHKeyPath
I1010 13:16:44.202379 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHUsername
I1010 13:16:44.202528 29815 main.go:141] libmachine: Using SSH client type: native
I1010 13:16:44.202660 29815 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x2a33520] 0x2a36200 <nil> [] 0s} 192.169.0.42 22 <nil> <nil>}
I1010 13:16:44.202667 29815 main.go:141] libmachine: About to run SSH command:
hostname
I1010 13:16:44.208456 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:44 INFO : hyperkit: stderr: Using fd 5 for I/O notifications
I1010 13:16:44.217123 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:44 INFO : hyperkit: stderr: /Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/boot2docker.iso: fcntl(F_PUNCHHOLE) Operation not permitted: block device will not support TRIM/DISCARD
I1010 13:16:44.218126 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:44 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 2 bit: 22 unspecified don't care: bit is 0
I1010 13:16:44.218143 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:44 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 12 unspecified don't care: bit is 0
I1010 13:16:44.218152 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:44 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 20 unspecified don't care: bit is 0
I1010 13:16:44.218160 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:44 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 3 bit: 13 unspecified don't care: bit is 0
I1010 13:16:44.614079 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:44 INFO : hyperkit: stderr: rdmsr to register 0x3a on vcpu 0
I1010 13:16:44.614092 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:44 INFO : hyperkit: stderr: rdmsr to register 0x140 on vcpu 0
I1010 13:16:44.729949 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:44 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 2 bit: 22 unspecified don't care: bit is 0
I1010 13:16:44.729969 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:44 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 12 unspecified don't care: bit is 0
I1010 13:16:44.730012 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:44 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 4 bit: 20 unspecified don't care: bit is 0
I1010 13:16:44.730029 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:44 INFO : hyperkit: stderr: vmx_set_ctlreg: cap_field: 3 bit: 13 unspecified don't care: bit is 0
I1010 13:16:44.730844 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:44 INFO : hyperkit: stderr: rdmsr to register 0x3a on vcpu 1
I1010 13:16:44.730859 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:44 INFO : hyperkit: stderr: rdmsr to register 0x140 on vcpu 1
I1010 13:16:50.432860 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:50 INFO : hyperkit: stderr: rdmsr to register 0x64d on vcpu 1
I1010 13:16:50.432950 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:50 INFO : hyperkit: stderr: rdmsr to register 0x64e on vcpu 1
I1010 13:16:50.432959 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:50 INFO : hyperkit: stderr: rdmsr to register 0x34 on vcpu 1
I1010 13:16:50.457274 29815 main.go:141] libmachine: (ha-626000-m02) DBG | 2024/10/10 13:16:50 INFO : hyperkit: stderr: rdmsr to register 0xc0011029 on vcpu 1
I1010 13:16:55.271946 29815 main.go:141] libmachine: SSH cmd err, output: <nil>: minikube
I1010 13:16:55.271964 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetMachineName
I1010 13:16:55.272120 29815 buildroot.go:166] provisioning hostname "ha-626000-m02"
I1010 13:16:55.272130 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetMachineName
I1010 13:16:55.272234 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHHostname
I1010 13:16:55.272326 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHPort
I1010 13:16:55.272428 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHKeyPath
I1010 13:16:55.272542 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHKeyPath
I1010 13:16:55.272650 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHUsername
I1010 13:16:55.272834 29815 main.go:141] libmachine: Using SSH client type: native
I1010 13:16:55.272979 29815 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x2a33520] 0x2a36200 <nil> [] 0s} 192.169.0.42 22 <nil> <nil>}
I1010 13:16:55.272991 29815 main.go:141] libmachine: About to run SSH command:
sudo hostname ha-626000-m02 && echo "ha-626000-m02" | sudo tee /etc/hostname
I1010 13:16:55.341969 29815 main.go:141] libmachine: SSH cmd err, output: <nil>: ha-626000-m02
I1010 13:16:55.341987 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHHostname
I1010 13:16:55.342134 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHPort
I1010 13:16:55.342233 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHKeyPath
I1010 13:16:55.342320 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHKeyPath
I1010 13:16:55.342398 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHUsername
I1010 13:16:55.342550 29815 main.go:141] libmachine: Using SSH client type: native
I1010 13:16:55.342708 29815 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x2a33520] 0x2a36200 <nil> [] 0s} 192.169.0.42 22 <nil> <nil>}
I1010 13:16:55.342720 29815 main.go:141] libmachine: About to run SSH command:
if ! grep -xq '.*\sha-626000-m02' /etc/hosts; then
if grep -xq '127.0.1.1\s.*' /etc/hosts; then
sudo sed -i 's/^127.0.1.1\s.*/127.0.1.1 ha-626000-m02/g' /etc/hosts;
else
echo '127.0.1.1 ha-626000-m02' | sudo tee -a /etc/hosts;
fi
fi
I1010 13:16:55.408163 29815 main.go:141] libmachine: SSH cmd err, output: <nil>:
I1010 13:16:55.408188 29815 buildroot.go:172] set auth options {CertDir:/Users/jenkins/minikube-integration/19787-26748/.minikube CaCertPath:/Users/jenkins/minikube-integration/19787-26748/.minikube/certs/ca.pem CaPrivateKeyPath:/Users/jenkins/minikube-integration/19787-26748/.minikube/certs/ca-key.pem CaCertRemotePath:/etc/docker/ca.pem ServerCertPath:/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/server.pem ServerKeyPath:/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/server-key.pem ClientKeyPath:/Users/jenkins/minikube-integration/19787-26748/.minikube/certs/key.pem ServerCertRemotePath:/etc/docker/server.pem ServerKeyRemotePath:/etc/docker/server-key.pem ClientCertPath:/Users/jenkins/minikube-integration/19787-26748/.minikube/certs/cert.pem ServerCertSANs:[] StorePath:/Users/jenkins/minikube-integration/19787-26748/.minikube}
I1010 13:16:55.408198 29815 buildroot.go:174] setting up certificates
I1010 13:16:55.408205 29815 provision.go:84] configureAuth start
I1010 13:16:55.408211 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetMachineName
I1010 13:16:55.408351 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetIP
I1010 13:16:55.408438 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHHostname
I1010 13:16:55.408529 29815 provision.go:143] copyHostCerts
I1010 13:16:55.408555 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/ca.pem -> /Users/jenkins/minikube-integration/19787-26748/.minikube/ca.pem
I1010 13:16:55.408609 29815 exec_runner.go:144] found /Users/jenkins/minikube-integration/19787-26748/.minikube/ca.pem, removing ...
I1010 13:16:55.408615 29815 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19787-26748/.minikube/ca.pem
I1010 13:16:55.408742 29815 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/ca.pem --> /Users/jenkins/minikube-integration/19787-26748/.minikube/ca.pem (1082 bytes)
I1010 13:16:55.408949 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/cert.pem -> /Users/jenkins/minikube-integration/19787-26748/.minikube/cert.pem
I1010 13:16:55.408979 29815 exec_runner.go:144] found /Users/jenkins/minikube-integration/19787-26748/.minikube/cert.pem, removing ...
I1010 13:16:55.408984 29815 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19787-26748/.minikube/cert.pem
I1010 13:16:55.409051 29815 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/cert.pem --> /Users/jenkins/minikube-integration/19787-26748/.minikube/cert.pem (1123 bytes)
I1010 13:16:55.409211 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/key.pem -> /Users/jenkins/minikube-integration/19787-26748/.minikube/key.pem
I1010 13:16:55.409239 29815 exec_runner.go:144] found /Users/jenkins/minikube-integration/19787-26748/.minikube/key.pem, removing ...
I1010 13:16:55.409244 29815 exec_runner.go:203] rm: /Users/jenkins/minikube-integration/19787-26748/.minikube/key.pem
I1010 13:16:55.409308 29815 exec_runner.go:151] cp: /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/key.pem --> /Users/jenkins/minikube-integration/19787-26748/.minikube/key.pem (1679 bytes)
I1010 13:16:55.409457 29815 provision.go:117] generating server cert: /Users/jenkins/minikube-integration/19787-26748/.minikube/machines/server.pem ca-key=/Users/jenkins/minikube-integration/19787-26748/.minikube/certs/ca.pem private-key=/Users/jenkins/minikube-integration/19787-26748/.minikube/certs/ca-key.pem org=jenkins.ha-626000-m02 san=[127.0.0.1 192.169.0.42 ha-626000-m02 localhost minikube]
I1010 13:16:55.498128 29815 provision.go:177] copyRemoteCerts
I1010 13:16:55.498196 29815 ssh_runner.go:195] Run: sudo mkdir -p /etc/docker /etc/docker /etc/docker
I1010 13:16:55.498213 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHHostname
I1010 13:16:55.498366 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHPort
I1010 13:16:55.498479 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHKeyPath
I1010 13:16:55.498567 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHUsername
I1010 13:16:55.498652 29815 sshutil.go:53] new ssh client: &{IP:192.169.0.42 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/id_rsa Username:docker}
I1010 13:16:55.535419 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/machines/server.pem -> /etc/docker/server.pem
I1010 13:16:55.535493 29815 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19787-26748/.minikube/machines/server.pem --> /etc/docker/server.pem (1208 bytes)
I1010 13:16:55.555384 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/machines/server-key.pem -> /etc/docker/server-key.pem
I1010 13:16:55.555450 29815 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19787-26748/.minikube/machines/server-key.pem --> /etc/docker/server-key.pem (1675 bytes)
I1010 13:16:55.575172 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/ca.pem -> /etc/docker/ca.pem
I1010 13:16:55.575236 29815 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19787-26748/.minikube/certs/ca.pem --> /etc/docker/ca.pem (1082 bytes)
I1010 13:16:55.594699 29815 provision.go:87] duration metric: took 186.481581ms to configureAuth
I1010 13:16:55.594712 29815 buildroot.go:189] setting minikube options for container-runtime
I1010 13:16:55.594870 29815 config.go:182] Loaded profile config "ha-626000": Driver=hyperkit, ContainerRuntime=docker, KubernetesVersion=v1.31.1
I1010 13:16:55.594883 29815 main.go:141] libmachine: (ha-626000-m02) Calling .DriverName
I1010 13:16:55.595019 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHHostname
I1010 13:16:55.595122 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHPort
I1010 13:16:55.595215 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHKeyPath
I1010 13:16:55.595289 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHKeyPath
I1010 13:16:55.595355 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHUsername
I1010 13:16:55.595476 29815 main.go:141] libmachine: Using SSH client type: native
I1010 13:16:55.595590 29815 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x2a33520] 0x2a36200 <nil> [] 0s} 192.169.0.42 22 <nil> <nil>}
I1010 13:16:55.595597 29815 main.go:141] libmachine: About to run SSH command:
df --output=fstype / | tail -n 1
I1010 13:16:55.658982 29815 main.go:141] libmachine: SSH cmd err, output: <nil>: tmpfs
I1010 13:16:55.658994 29815 buildroot.go:70] root file system type: tmpfs
I1010 13:16:55.659075 29815 provision.go:314] Updating docker unit: /lib/systemd/system/docker.service ...
I1010 13:16:55.659086 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHHostname
I1010 13:16:55.659228 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHPort
I1010 13:16:55.659314 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHKeyPath
I1010 13:16:55.659435 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHKeyPath
I1010 13:16:55.659523 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHUsername
I1010 13:16:55.659666 29815 main.go:141] libmachine: Using SSH client type: native
I1010 13:16:55.659800 29815 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x2a33520] 0x2a36200 <nil> [] 0s} 192.169.0.42 22 <nil> <nil>}
I1010 13:16:55.659848 29815 main.go:141] libmachine: About to run SSH command:
sudo mkdir -p /lib/systemd/system && printf %s "[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target minikube-automount.service docker.socket
Requires= minikube-automount.service docker.socket
StartLimitBurst=3
StartLimitIntervalSec=60
[Service]
Type=notify
Restart=on-failure
Environment="NO_PROXY=192.169.0.41"
# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
# Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=hyperkit --insecure-registry 10.96.0.0/12
ExecReload=/bin/kill -s HUP \$MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
[Install]
WantedBy=multi-user.target
" | sudo tee /lib/systemd/system/docker.service.new
I1010 13:16:55.730730 29815 main.go:141] libmachine: SSH cmd err, output: <nil>: [Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network.target minikube-automount.service docker.socket
Requires= minikube-automount.service docker.socket
StartLimitBurst=3
StartLimitIntervalSec=60
[Service]
Type=notify
Restart=on-failure
Environment=NO_PROXY=192.169.0.41
# This file is a systemd drop-in unit that inherits from the base dockerd configuration.
# The base configuration already specifies an 'ExecStart=...' command. The first directive
# here is to clear out that command inherited from the base configuration. Without this,
# the command from the base configuration and the command specified here are treated as
# a sequence of commands, which is not the desired behavior, nor is it valid -- systemd
# will catch this invalid input and refuse to start the service with an error like:
# Service has more than one ExecStart= setting, which is only allowed for Type=oneshot services.
# NOTE: default-ulimit=nofile is set to an arbitrary number for consistency with other
# container runtimes. If left unlimited, it may result in OOM issues with MySQL.
ExecStart=
ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --default-ulimit=nofile=1048576:1048576 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=hyperkit --insecure-registry 10.96.0.0/12
ExecReload=/bin/kill -s HUP $MAINPID
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
# Uncomment TasksMax if your systemd version supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
TimeoutStartSec=0
# set delegate yes so that systemd does not reset the cgroups of docker containers
Delegate=yes
# kill only the docker process, not all processes in the cgroup
KillMode=process
[Install]
WantedBy=multi-user.target
I1010 13:16:55.730747 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHHostname
I1010 13:16:55.730887 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHPort
I1010 13:16:55.730992 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHKeyPath
I1010 13:16:55.731085 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHKeyPath
I1010 13:16:55.731164 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHUsername
I1010 13:16:55.731306 29815 main.go:141] libmachine: Using SSH client type: native
I1010 13:16:55.731448 29815 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x2a33520] 0x2a36200 <nil> [] 0s} 192.169.0.42 22 <nil> <nil>}
I1010 13:16:55.731463 29815 main.go:141] libmachine: About to run SSH command:
sudo diff -u /lib/systemd/system/docker.service /lib/systemd/system/docker.service.new || { sudo mv /lib/systemd/system/docker.service.new /lib/systemd/system/docker.service; sudo systemctl -f daemon-reload && sudo systemctl -f enable docker && sudo systemctl -f restart docker; }
I1010 13:16:57.425348 29815 main.go:141] libmachine: SSH cmd err, output: <nil>: diff: can't stat '/lib/systemd/system/docker.service': No such file or directory
Created symlink /etc/systemd/system/multi-user.target.wants/docker.service → /usr/lib/systemd/system/docker.service.
I1010 13:16:57.425363 29815 machine.go:96] duration metric: took 13.22323319s to provisionDockerMachine
I1010 13:16:57.425370 29815 start.go:293] postStartSetup for "ha-626000-m02" (driver="hyperkit")
I1010 13:16:57.425378 29815 start.go:322] creating required directories: [/etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs]
I1010 13:16:57.425394 29815 main.go:141] libmachine: (ha-626000-m02) Calling .DriverName
I1010 13:16:57.425601 29815 ssh_runner.go:195] Run: sudo mkdir -p /etc/kubernetes/addons /etc/kubernetes/manifests /var/tmp/minikube /var/lib/minikube /var/lib/minikube/certs /var/lib/minikube/images /var/lib/minikube/binaries /tmp/gvisor /usr/share/ca-certificates /etc/ssl/certs
I1010 13:16:57.425613 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHHostname
I1010 13:16:57.425745 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHPort
I1010 13:16:57.425840 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHKeyPath
I1010 13:16:57.425925 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHUsername
I1010 13:16:57.426012 29815 sshutil.go:53] new ssh client: &{IP:192.169.0.42 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/id_rsa Username:docker}
I1010 13:16:57.467111 29815 ssh_runner.go:195] Run: cat /etc/os-release
I1010 13:16:57.471722 29815 info.go:137] Remote host: Buildroot 2023.02.9
I1010 13:16:57.471735 29815 filesync.go:126] Scanning /Users/jenkins/minikube-integration/19787-26748/.minikube/addons for local assets ...
I1010 13:16:57.471823 29815 filesync.go:126] Scanning /Users/jenkins/minikube-integration/19787-26748/.minikube/files for local assets ...
I1010 13:16:57.471957 29815 filesync.go:149] local asset: /Users/jenkins/minikube-integration/19787-26748/.minikube/files/etc/ssl/certs/273152.pem -> 273152.pem in /etc/ssl/certs
I1010 13:16:57.471963 29815 vm_assets.go:164] NewFileAsset: /Users/jenkins/minikube-integration/19787-26748/.minikube/files/etc/ssl/certs/273152.pem -> /etc/ssl/certs/273152.pem
I1010 13:16:57.472127 29815 ssh_runner.go:195] Run: sudo mkdir -p /etc/ssl/certs
I1010 13:16:57.481143 29815 ssh_runner.go:362] scp /Users/jenkins/minikube-integration/19787-26748/.minikube/files/etc/ssl/certs/273152.pem --> /etc/ssl/certs/273152.pem (1708 bytes)
I1010 13:16:57.510595 29815 start.go:296] duration metric: took 85.214219ms for postStartSetup
I1010 13:16:57.510619 29815 main.go:141] libmachine: (ha-626000-m02) Calling .DriverName
I1010 13:16:57.510828 29815 ssh_runner.go:195] Run: sudo ls --almost-all -1 /var/lib/minikube/backup
I1010 13:16:57.510841 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHHostname
I1010 13:16:57.510934 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHPort
I1010 13:16:57.511035 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHKeyPath
I1010 13:16:57.511123 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHUsername
I1010 13:16:57.511201 29815 sshutil.go:53] new ssh client: &{IP:192.169.0.42 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/id_rsa Username:docker}
I1010 13:16:57.548286 29815 machine.go:197] restoring vm config from /var/lib/minikube/backup: [etc]
I1010 13:16:57.548362 29815 ssh_runner.go:195] Run: sudo rsync --archive --update /var/lib/minikube/backup/etc /
I1010 13:16:57.581410 29815 fix.go:56] duration metric: took 13.491717864s for fixHost
I1010 13:16:57.581436 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHHostname
I1010 13:16:57.581572 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHPort
I1010 13:16:57.581679 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHKeyPath
I1010 13:16:57.581768 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHKeyPath
I1010 13:16:57.581850 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHUsername
I1010 13:16:57.581995 29815 main.go:141] libmachine: Using SSH client type: native
I1010 13:16:57.582138 29815 main.go:141] libmachine: &{{{<nil> 0 [] [] []} docker [0x2a33520] 0x2a36200 <nil> [] 0s} 192.169.0.42 22 <nil> <nil>}
I1010 13:16:57.582145 29815 main.go:141] libmachine: About to run SSH command:
date +%s.%N
I1010 13:16:57.644592 29815 main.go:141] libmachine: SSH cmd err, output: <nil>: 1728591417.704983125
I1010 13:16:57.644609 29815 fix.go:216] guest clock: 1728591417.704983125
I1010 13:16:57.644615 29815 fix.go:229] Guest: 2024-10-10 13:16:57.704983125 -0700 PDT Remote: 2024-10-10 13:16:57.581426 -0700 PDT m=+32.733128083 (delta=123.557125ms)
I1010 13:16:57.644626 29815 fix.go:200] guest clock delta is within tolerance: 123.557125ms
I1010 13:16:57.644630 29815 start.go:83] releasing machines lock for "ha-626000-m02", held for 13.554976043s
I1010 13:16:57.644646 29815 main.go:141] libmachine: (ha-626000-m02) Calling .DriverName
I1010 13:16:57.644778 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetIP
I1010 13:16:57.666322 29815 out.go:177] * Found network options:
I1010 13:16:57.686268 29815 out.go:177] - NO_PROXY=192.169.0.41
W1010 13:16:57.708158 29815 proxy.go:119] fail to check proxy env: Error ip not in block
I1010 13:16:57.708197 29815 main.go:141] libmachine: (ha-626000-m02) Calling .DriverName
I1010 13:16:57.709017 29815 main.go:141] libmachine: (ha-626000-m02) Calling .DriverName
I1010 13:16:57.709282 29815 main.go:141] libmachine: (ha-626000-m02) Calling .DriverName
I1010 13:16:57.709427 29815 ssh_runner.go:195] Run: curl -sS -m 2 https://registry.k8s.io/
I1010 13:16:57.709465 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHHostname
W1010 13:16:57.709488 29815 proxy.go:119] fail to check proxy env: Error ip not in block
I1010 13:16:57.709618 29815 ssh_runner.go:195] Run: sh -c "stat /etc/cni/net.d/*loopback.conf*"
I1010 13:16:57.709638 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHHostname
I1010 13:16:57.709693 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHPort
I1010 13:16:57.709889 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHKeyPath
I1010 13:16:57.709921 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHPort
I1010 13:16:57.710109 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHUsername
I1010 13:16:57.710143 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHKeyPath
I1010 13:16:57.710330 29815 sshutil.go:53] new ssh client: &{IP:192.169.0.42 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/id_rsa Username:docker}
I1010 13:16:57.710363 29815 main.go:141] libmachine: (ha-626000-m02) Calling .GetSSHUsername
I1010 13:16:57.710488 29815 sshutil.go:53] new ssh client: &{IP:192.169.0.42 Port:22 SSHKeyPath:/Users/jenkins/minikube-integration/19787-26748/.minikube/machines/ha-626000-m02/id_rsa Username:docker}
W1010 13:16:57.746768 29815 cni.go:209] loopback cni configuration skipped: "/etc/cni/net.d/*loopback.conf*" not found
I1010 13:16:57.746845 29815 ssh_runner.go:195] Run: sudo find /etc/cni/net.d -maxdepth 1 -type f ( ( -name *bridge* -or -name *podman* ) -and -not -name *.mk_disabled ) -printf "%p, " -exec sh -c "sudo mv {} {}.mk_disabled" ;
I1010 13:16:57.793732 29815 cni.go:262] disabled [/etc/cni/net.d/87-podman-bridge.conflist] bridge cni config(s)
I1010 13:16:57.793747 29815 start.go:495] detecting cgroup driver to use...
I1010 13:16:57.793818 29815 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///run/containerd/containerd.sock
" | sudo tee /etc/crictl.yaml"
I1010 13:16:57.809334 29815 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)sandbox_image = .*$|\1sandbox_image = "registry.k8s.io/pause:3.10"|' /etc/containerd/config.toml"
I1010 13:16:57.817604 29815 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)restrict_oom_score_adj = .*$|\1restrict_oom_score_adj = false|' /etc/containerd/config.toml"
I1010 13:16:57.825718 29815 containerd.go:146] configuring containerd to use "cgroupfs" as cgroup driver...
I1010 13:16:57.825773 29815 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)SystemdCgroup = .*$|\1SystemdCgroup = false|g' /etc/containerd/config.toml"
I1010 13:16:57.833783 29815 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|"io.containerd.runtime.v1.linux"|"io.containerd.runc.v2"|g' /etc/containerd/config.toml"
I1010 13:16:57.841873 29815 ssh_runner.go:195] Run: sh -c "sudo sed -i '/systemd_cgroup/d' /etc/containerd/config.toml"
I1010 13:16:57.850156 29815 ssh_runner.go:195] Run: sh -c "sudo sed -i 's|"io.containerd.runc.v1"|"io.containerd.runc.v2"|g' /etc/containerd/config.toml"
I1010 13:16:57.858484 29815 ssh_runner.go:195] Run: sh -c "sudo rm -rf /etc/cni/net.mk"
I1010 13:16:57.867015 29815 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)conf_dir = .*$|\1conf_dir = "/etc/cni/net.d"|g' /etc/containerd/config.toml"
I1010 13:16:57.875402 29815 ssh_runner.go:195] Run: sh -c "sudo sed -i '/^ *enable_unprivileged_ports = .*/d' /etc/containerd/config.toml"
I1010 13:16:57.883575 29815 ssh_runner.go:195] Run: sh -c "sudo sed -i -r 's|^( *)\[plugins."io.containerd.grpc.v1.cri"\]|&\n\1 enable_unprivileged_ports = true|' /etc/containerd/config.toml"
I1010 13:16:57.891912 29815 ssh_runner.go:195] Run: sudo sysctl net.bridge.bridge-nf-call-iptables
I1010 13:16:57.899420 29815 crio.go:166] couldn't verify netfilter by "sudo sysctl net.bridge.bridge-nf-call-iptables" which might be okay. error: sudo sysctl net.bridge.bridge-nf-call-iptables: Process exited with status 255
stdout:
stderr:
sysctl: cannot stat /proc/sys/net/bridge/bridge-nf-call-iptables: No such file or directory
I1010 13:16:57.899472 29815 ssh_runner.go:195] Run: sudo modprobe br_netfilter
I1010 13:16:57.908126 29815 ssh_runner.go:195] Run: sudo sh -c "echo 1 > /proc/sys/net/ipv4/ip_forward"
I1010 13:16:57.915520 29815 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1010 13:16:58.015759 29815 ssh_runner.go:195] Run: sudo systemctl restart containerd
I1010 13:16:58.034890 29815 start.go:495] detecting cgroup driver to use...
I1010 13:16:58.034983 29815 ssh_runner.go:195] Run: sudo systemctl cat docker.service
I1010 13:16:58.051295 29815 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I1010 13:16:58.066419 29815 ssh_runner.go:195] Run: sudo systemctl stop -f containerd
I1010 13:16:58.083439 29815 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service containerd
I1010 13:16:58.094858 29815 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service crio
I1010 13:16:58.105773 29815 ssh_runner.go:195] Run: sudo systemctl stop -f crio
I1010 13:16:58.127768 29815 ssh_runner.go:195] Run: sudo systemctl is-active --quiet service crio
I1010 13:16:58.138494 29815 ssh_runner.go:195] Run: /bin/bash -c "sudo mkdir -p /etc && printf %s "runtime-endpoint: unix:///var/run/cri-dockerd.sock
" | sudo tee /etc/crictl.yaml"
I1010 13:16:58.153537 29815 ssh_runner.go:195] Run: which cri-dockerd
I1010 13:16:58.156334 29815 ssh_runner.go:195] Run: sudo mkdir -p /etc/systemd/system/cri-docker.service.d
I1010 13:16:58.163267 29815 ssh_runner.go:362] scp memory --> /etc/systemd/system/cri-docker.service.d/10-cni.conf (190 bytes)
I1010 13:16:58.176702 29815 ssh_runner.go:195] Run: sudo systemctl unmask docker.service
I1010 13:16:58.272635 29815 ssh_runner.go:195] Run: sudo systemctl enable docker.socket
I1010 13:16:58.377383 29815 docker.go:574] configuring docker to use "cgroupfs" as cgroup driver...
I1010 13:16:58.377410 29815 ssh_runner.go:362] scp memory --> /etc/docker/daemon.json (130 bytes)
I1010 13:16:58.391908 29815 ssh_runner.go:195] Run: sudo systemctl daemon-reload
I1010 13:16:58.500320 29815 ssh_runner.go:195] Run: sudo systemctl restart docker
I1010 13:17:59.526939 29815 ssh_runner.go:235] Completed: sudo systemctl restart docker: (1m1.025068228s)
I1010 13:17:59.527020 29815 ssh_runner.go:195] Run: sudo journalctl --no-pager -u docker
I1010 13:17:59.564268 29815 out.go:201]
W1010 13:17:59.585359 29815 out.go:270] X Exiting due to RUNTIME_ENABLE: Failed to enable container runtime: sudo systemctl restart docker: Process exited with status 1
stdout:
stderr:
Job for docker.service failed because the control process exited with error code.
See "systemctl status docker.service" and "journalctl -xeu docker.service" for details.
sudo journalctl --no-pager -u docker:
-- stdout --
Oct 10 20:16:56 ha-626000-m02 systemd[1]: Starting Docker Application Container Engine...
Oct 10 20:16:56 ha-626000-m02 dockerd[483]: time="2024-10-10T20:16:56.095421343Z" level=info msg="Starting up"
Oct 10 20:16:56 ha-626000-m02 dockerd[483]: time="2024-10-10T20:16:56.095872070Z" level=info msg="containerd not running, starting managed containerd"
Oct 10 20:16:56 ha-626000-m02 dockerd[483]: time="2024-10-10T20:16:56.096491528Z" level=info msg="started new containerd process" address=/var/run/docker/containerd/containerd.sock module=libcontainerd pid=489
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.111979883Z" level=info msg="starting containerd" revision=7f7fdf5fed64eb6a7caf99b3e12efcf9d60e311c version=v1.7.22
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.127031401Z" level=info msg="loading plugin \"io.containerd.event.v1.exchange\"..." type=io.containerd.event.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.127077109Z" level=info msg="loading plugin \"io.containerd.internal.v1.opt\"..." type=io.containerd.internal.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.127143316Z" level=info msg="loading plugin \"io.containerd.warning.v1.deprecations\"..." type=io.containerd.warning.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.127197830Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.blockfile\"..." type=io.containerd.snapshotter.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.127320620Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.blockfile\"..." error="no scratch file generator: skip plugin" type=io.containerd.snapshotter.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.127332692Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.btrfs\"..." type=io.containerd.snapshotter.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.127439239Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.btrfs\"..." error="path /var/lib/docker/containerd/daemon/io.containerd.snapshotter.v1.btrfs (ext4) must be a btrfs filesystem to be used with the btrfs snapshotter: skip plugin" type=io.containerd.snapshotter.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.127473520Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.devmapper\"..." type=io.containerd.snapshotter.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.127485281Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.devmapper\"..." error="devmapper not configured: skip plugin" type=io.containerd.snapshotter.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.127493492Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.native\"..." type=io.containerd.snapshotter.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.127572077Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.overlayfs\"..." type=io.containerd.snapshotter.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.127725747Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.aufs\"..." type=io.containerd.snapshotter.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.129281793Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.aufs\"..." error="aufs is not supported (modprobe aufs failed: exit status 1 \"modprobe: FATAL: Module aufs not found in directory /lib/modules/5.10.207\\n\"): skip plugin" type=io.containerd.snapshotter.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.129366255Z" level=info msg="loading plugin \"io.containerd.snapshotter.v1.zfs\"..." type=io.containerd.snapshotter.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.129462911Z" level=info msg="skip loading plugin \"io.containerd.snapshotter.v1.zfs\"..." error="path /var/lib/docker/containerd/daemon/io.containerd.snapshotter.v1.zfs must be a zfs filesystem to be used with the zfs snapshotter: skip plugin" type=io.containerd.snapshotter.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.129496425Z" level=info msg="loading plugin \"io.containerd.content.v1.content\"..." type=io.containerd.content.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.129578606Z" level=info msg="loading plugin \"io.containerd.metadata.v1.bolt\"..." type=io.containerd.metadata.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.129622849Z" level=info msg="metadata content store policy set" policy=shared
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.131670697Z" level=info msg="loading plugin \"io.containerd.gc.v1.scheduler\"..." type=io.containerd.gc.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.131720823Z" level=info msg="loading plugin \"io.containerd.differ.v1.walking\"..." type=io.containerd.differ.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.131743782Z" level=info msg="loading plugin \"io.containerd.lease.v1.manager\"..." type=io.containerd.lease.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.131761792Z" level=info msg="loading plugin \"io.containerd.streaming.v1.manager\"..." type=io.containerd.streaming.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.131782024Z" level=info msg="loading plugin \"io.containerd.runtime.v1.linux\"..." type=io.containerd.runtime.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.131828377Z" level=info msg="loading plugin \"io.containerd.monitor.v1.cgroups\"..." type=io.containerd.monitor.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132034739Z" level=info msg="loading plugin \"io.containerd.runtime.v2.task\"..." type=io.containerd.runtime.v2
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132105597Z" level=info msg="loading plugin \"io.containerd.runtime.v2.shim\"..." type=io.containerd.runtime.v2
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132140461Z" level=info msg="loading plugin \"io.containerd.sandbox.store.v1.local\"..." type=io.containerd.sandbox.store.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132151782Z" level=info msg="loading plugin \"io.containerd.sandbox.controller.v1.local\"..." type=io.containerd.sandbox.controller.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132161449Z" level=info msg="loading plugin \"io.containerd.service.v1.containers-service\"..." type=io.containerd.service.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132169811Z" level=info msg="loading plugin \"io.containerd.service.v1.content-service\"..." type=io.containerd.service.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132177453Z" level=info msg="loading plugin \"io.containerd.service.v1.diff-service\"..." type=io.containerd.service.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132189274Z" level=info msg="loading plugin \"io.containerd.service.v1.images-service\"..." type=io.containerd.service.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132204527Z" level=info msg="loading plugin \"io.containerd.service.v1.introspection-service\"..." type=io.containerd.service.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132213976Z" level=info msg="loading plugin \"io.containerd.service.v1.namespaces-service\"..." type=io.containerd.service.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132221961Z" level=info msg="loading plugin \"io.containerd.service.v1.snapshots-service\"..." type=io.containerd.service.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132229313Z" level=info msg="loading plugin \"io.containerd.service.v1.tasks-service\"..." type=io.containerd.service.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132242414Z" level=info msg="loading plugin \"io.containerd.grpc.v1.containers\"..." type=io.containerd.grpc.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132251159Z" level=info msg="loading plugin \"io.containerd.grpc.v1.content\"..." type=io.containerd.grpc.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132259491Z" level=info msg="loading plugin \"io.containerd.grpc.v1.diff\"..." type=io.containerd.grpc.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132267949Z" level=info msg="loading plugin \"io.containerd.grpc.v1.events\"..." type=io.containerd.grpc.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132275752Z" level=info msg="loading plugin \"io.containerd.grpc.v1.images\"..." type=io.containerd.grpc.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132283992Z" level=info msg="loading plugin \"io.containerd.grpc.v1.introspection\"..." type=io.containerd.grpc.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132325057Z" level=info msg="loading plugin \"io.containerd.grpc.v1.leases\"..." type=io.containerd.grpc.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132344390Z" level=info msg="loading plugin \"io.containerd.grpc.v1.namespaces\"..." type=io.containerd.grpc.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132356479Z" level=info msg="loading plugin \"io.containerd.grpc.v1.sandbox-controllers\"..." type=io.containerd.grpc.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132366018Z" level=info msg="loading plugin \"io.containerd.grpc.v1.sandboxes\"..." type=io.containerd.grpc.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132373320Z" level=info msg="loading plugin \"io.containerd.grpc.v1.snapshots\"..." type=io.containerd.grpc.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132382742Z" level=info msg="loading plugin \"io.containerd.grpc.v1.streaming\"..." type=io.containerd.grpc.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132390887Z" level=info msg="loading plugin \"io.containerd.grpc.v1.tasks\"..." type=io.containerd.grpc.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132400288Z" level=info msg="loading plugin \"io.containerd.transfer.v1.local\"..." type=io.containerd.transfer.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132413378Z" level=info msg="loading plugin \"io.containerd.grpc.v1.transfer\"..." type=io.containerd.grpc.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132421644Z" level=info msg="loading plugin \"io.containerd.grpc.v1.version\"..." type=io.containerd.grpc.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132429500Z" level=info msg="loading plugin \"io.containerd.internal.v1.restart\"..." type=io.containerd.internal.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132475778Z" level=info msg="loading plugin \"io.containerd.tracing.processor.v1.otlp\"..." type=io.containerd.tracing.processor.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132488943Z" level=info msg="skip loading plugin \"io.containerd.tracing.processor.v1.otlp\"..." error="skip plugin: tracing endpoint not configured" type=io.containerd.tracing.processor.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132496503Z" level=info msg="loading plugin \"io.containerd.internal.v1.tracing\"..." type=io.containerd.internal.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132503826Z" level=info msg="skip loading plugin \"io.containerd.internal.v1.tracing\"..." error="skip plugin: tracing endpoint not configured" type=io.containerd.internal.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132511504Z" level=info msg="loading plugin \"io.containerd.grpc.v1.healthcheck\"..." type=io.containerd.grpc.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132519401Z" level=info msg="loading plugin \"io.containerd.nri.v1.nri\"..." type=io.containerd.nri.v1
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132525897Z" level=info msg="NRI interface is disabled by configuration."
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132652026Z" level=info msg=serving... address=/var/run/docker/containerd/containerd-debug.sock
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132703318Z" level=info msg=serving... address=/var/run/docker/containerd/containerd.sock.ttrpc
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132728821Z" level=info msg=serving... address=/var/run/docker/containerd/containerd.sock
Oct 10 20:16:56 ha-626000-m02 dockerd[489]: time="2024-10-10T20:16:56.132773065Z" level=info msg="containerd successfully booted in 0.021642s"
Oct 10 20:16:57 ha-626000-m02 dockerd[483]: time="2024-10-10T20:16:57.114420162Z" level=info msg="[graphdriver] trying configured driver: overlay2"
Oct 10 20:16:57 ha-626000-m02 dockerd[483]: time="2024-10-10T20:16:57.159763133Z" level=info msg="Loading containers: start."
Oct 10 20:16:57 ha-626000-m02 dockerd[483]: time="2024-10-10T20:16:57.346634551Z" level=warning msg="ip6tables is enabled, but cannot set up ip6tables chains" error="failed to create NAT chain DOCKER: iptables failed: ip6tables --wait -t nat -N DOCKER: ip6tables v1.8.9 (legacy): can't initialize ip6tables table `nat': Table does not exist (do you need to insmod?)\nPerhaps ip6tables or your kernel needs to be upgraded.\n (exit status 3)"
Oct 10 20:16:57 ha-626000-m02 dockerd[483]: time="2024-10-10T20:16:57.411672568Z" level=info msg="Default bridge (docker0) is assigned with an IP address 172.17.0.0/16. Daemon option --bip can be used to set a preferred IP address"
Oct 10 20:16:57 ha-626000-m02 dockerd[483]: time="2024-10-10T20:16:57.455323344Z" level=warning msg="error locating sandbox id a71db8c63849ab76e46083059136c4b0c2200d8f5a7b4382eb2927a8f8d6264f: sandbox a71db8c63849ab76e46083059136c4b0c2200d8f5a7b4382eb2927a8f8d6264f not found"
Oct 10 20:16:57 ha-626000-m02 dockerd[483]: time="2024-10-10T20:16:57.455575567Z" level=info msg="Loading containers: done."
Oct 10 20:16:57 ha-626000-m02 dockerd[483]: time="2024-10-10T20:16:57.463861524Z" level=warning msg="WARNING: bridge-nf-call-iptables is disabled"
Oct 10 20:16:57 ha-626000-m02 dockerd[483]: time="2024-10-10T20:16:57.463894672Z" level=warning msg="WARNING: bridge-nf-call-ip6tables is disabled"
Oct 10 20:16:57 ha-626000-m02 dockerd[483]: time="2024-10-10T20:16:57.463913319Z" level=info msg="Docker daemon" commit=41ca978 containerd-snapshotter=false storage-driver=overlay2 version=27.3.1
Oct 10 20:16:57 ha-626000-m02 dockerd[483]: time="2024-10-10T20:16:57.464045946Z" level=info msg="Daemon has completed initialization"
Oct 10 20:16:57 ha-626000-m02 dockerd[483]: time="2024-10-10T20:16:57.483219555Z" level=info msg="API listen on /var/run/docker.sock"
Oct 10 20:16:57 ha-626000-m02 dockerd[483]: time="2024-10-10T20:16:57.483349899Z" level=info msg="API listen on [::]:2376"
Oct 10 20:16:57 ha-626000-m02 systemd[1]: Started Docker Application Container Engine.
Oct 10 20:16:58 ha-626000-m02 dockerd[483]: time="2024-10-10T20:16:58.573176461Z" level=info msg="Processing signal 'terminated'"
Oct 10 20:16:58 ha-626000-m02 dockerd[483]: time="2024-10-10T20:16:58.574027128Z" level=info msg="stopping event stream following graceful shutdown" error="<nil>" module=libcontainerd namespace=moby
Oct 10 20:16:58 ha-626000-m02 dockerd[483]: time="2024-10-10T20:16:58.574539755Z" level=info msg="Daemon shutdown complete"
Oct 10 20:16:58 ha-626000-m02 dockerd[483]: time="2024-10-10T20:16:58.574658950Z" level=info msg="stopping healthcheck following graceful shutdown" module=libcontainerd
Oct 10 20:16:58 ha-626000-m02 dockerd[483]: time="2024-10-10T20:16:58.574828586Z" level=info msg="stopping event stream following graceful shutdown" error="context canceled" module=libcontainerd namespace=plugins.moby
Oct 10 20:16:58 ha-626000-m02 systemd[1]: Stopping Docker Application Container Engine...
Oct 10 20:16:59 ha-626000-m02 systemd[1]: docker.service: Deactivated successfully.
Oct 10 20:16:59 ha-626000-m02 systemd[1]: Stopped Docker Application Container Engine.
Oct 10 20:16:59 ha-626000-m02 systemd[1]: Starting Docker Application Container Engine...
Oct 10 20:16:59 ha-626000-m02 dockerd[1157]: time="2024-10-10T20:16:59.611347297Z" level=info msg="Starting up"
Oct 10 20:17:59 ha-626000-m02 dockerd[1157]: failed to start daemon: failed to dial "/run/containerd/containerd.sock": failed to dial "/run/containerd/containerd.sock": context deadline exceeded
Oct 10 20:17:59 ha-626000-m02 systemd[1]: docker.service: Main process exited, code=exited, status=1/FAILURE
Oct 10 20:17:59 ha-626000-m02 systemd[1]: docker.service: Failed with result 'exit-code'.
Oct 10 20:17:59 ha-626000-m02 systemd[1]: Failed to start Docker Application Container Engine.
-- /stdout --
W1010 13:17:59.585474 29815 out.go:270] *
W1010 13:17:59.586720 29815 out.go:293] ╭─────────────────────────────────────────────────────────────────────────────────────────────╮
│ │
│ * If the above advice does not help, please let us know: │
│ https://github.com/kubernetes/minikube/issues/new/choose │
│ │
│ * Please run `minikube logs --file=logs.txt` and attach logs.txt to the GitHub issue. │
│ │
╰─────────────────────────────────────────────────────────────────────────────────────────────╯
I1010 13:17:59.649261 29815 out.go:201]
==> Docker <==
Oct 10 20:16:50 ha-626000 dockerd[1178]: time="2024-10-10T20:16:50.294975308Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.pause\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Oct 10 20:17:10 ha-626000 dockerd[1171]: time="2024-10-10T20:17:10.983701176Z" level=info msg="ignoring event" container=6f1d7d582b23ead17384e918f7a33c671ead28b4478fc65ec18f4c47b34cb812 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Oct 10 20:17:10 ha-626000 dockerd[1178]: time="2024-10-10T20:17:10.984275842Z" level=info msg="shim disconnected" id=6f1d7d582b23ead17384e918f7a33c671ead28b4478fc65ec18f4c47b34cb812 namespace=moby
Oct 10 20:17:10 ha-626000 dockerd[1178]: time="2024-10-10T20:17:10.984344850Z" level=warning msg="cleaning up after shim disconnected" id=6f1d7d582b23ead17384e918f7a33c671ead28b4478fc65ec18f4c47b34cb812 namespace=moby
Oct 10 20:17:10 ha-626000 dockerd[1178]: time="2024-10-10T20:17:10.984353541Z" level=info msg="cleaning up dead shim" namespace=moby
Oct 10 20:17:11 ha-626000 dockerd[1178]: time="2024-10-10T20:17:11.993020147Z" level=info msg="shim disconnected" id=2170d784f89990cead44a18e60b634457082e2522cf71171f420e3836cd39685 namespace=moby
Oct 10 20:17:11 ha-626000 dockerd[1178]: time="2024-10-10T20:17:11.993132404Z" level=warning msg="cleaning up after shim disconnected" id=2170d784f89990cead44a18e60b634457082e2522cf71171f420e3836cd39685 namespace=moby
Oct 10 20:17:11 ha-626000 dockerd[1178]: time="2024-10-10T20:17:11.993159339Z" level=info msg="cleaning up dead shim" namespace=moby
Oct 10 20:17:11 ha-626000 dockerd[1171]: time="2024-10-10T20:17:11.993548693Z" level=info msg="ignoring event" container=2170d784f89990cead44a18e60b634457082e2522cf71171f420e3836cd39685 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Oct 10 20:17:28 ha-626000 dockerd[1178]: time="2024-10-10T20:17:28.579714881Z" level=info msg="loading plugin \"io.containerd.event.v1.publisher\"..." runtime=io.containerd.runc.v2 type=io.containerd.event.v1
Oct 10 20:17:28 ha-626000 dockerd[1178]: time="2024-10-10T20:17:28.579791023Z" level=info msg="loading plugin \"io.containerd.internal.v1.shutdown\"..." runtime=io.containerd.runc.v2 type=io.containerd.internal.v1
Oct 10 20:17:28 ha-626000 dockerd[1178]: time="2024-10-10T20:17:28.579846211Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.task\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Oct 10 20:17:28 ha-626000 dockerd[1178]: time="2024-10-10T20:17:28.579961526Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.pause\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Oct 10 20:17:35 ha-626000 dockerd[1178]: time="2024-10-10T20:17:35.584618557Z" level=info msg="loading plugin \"io.containerd.event.v1.publisher\"..." runtime=io.containerd.runc.v2 type=io.containerd.event.v1
Oct 10 20:17:35 ha-626000 dockerd[1178]: time="2024-10-10T20:17:35.584695017Z" level=info msg="loading plugin \"io.containerd.internal.v1.shutdown\"..." runtime=io.containerd.runc.v2 type=io.containerd.internal.v1
Oct 10 20:17:35 ha-626000 dockerd[1178]: time="2024-10-10T20:17:35.584707878Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.task\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Oct 10 20:17:35 ha-626000 dockerd[1178]: time="2024-10-10T20:17:35.584810767Z" level=info msg="loading plugin \"io.containerd.ttrpc.v1.pause\"..." runtime=io.containerd.runc.v2 type=io.containerd.ttrpc.v1
Oct 10 20:17:56 ha-626000 dockerd[1171]: time="2024-10-10T20:17:56.033442824Z" level=info msg="ignoring event" container=8a44e652b1b7d9348d611ffd8b57b790bc0a44e22f3a1fec0ff10f02e62641bd module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Oct 10 20:17:56 ha-626000 dockerd[1178]: time="2024-10-10T20:17:56.033370016Z" level=info msg="shim disconnected" id=8a44e652b1b7d9348d611ffd8b57b790bc0a44e22f3a1fec0ff10f02e62641bd namespace=moby
Oct 10 20:17:56 ha-626000 dockerd[1178]: time="2024-10-10T20:17:56.033497532Z" level=warning msg="cleaning up after shim disconnected" id=8a44e652b1b7d9348d611ffd8b57b790bc0a44e22f3a1fec0ff10f02e62641bd namespace=moby
Oct 10 20:17:56 ha-626000 dockerd[1178]: time="2024-10-10T20:17:56.033506428Z" level=info msg="cleaning up dead shim" namespace=moby
Oct 10 20:17:57 ha-626000 dockerd[1171]: time="2024-10-10T20:17:57.045225267Z" level=info msg="ignoring event" container=3afa164adce3e656bdd4ed1f7b7b72d03786a891465ca12364466813a3fc9b37 module=libcontainerd namespace=moby topic=/tasks/delete type="*events.TaskDelete"
Oct 10 20:17:57 ha-626000 dockerd[1178]: time="2024-10-10T20:17:57.045232805Z" level=info msg="shim disconnected" id=3afa164adce3e656bdd4ed1f7b7b72d03786a891465ca12364466813a3fc9b37 namespace=moby
Oct 10 20:17:57 ha-626000 dockerd[1178]: time="2024-10-10T20:17:57.045796535Z" level=warning msg="cleaning up after shim disconnected" id=3afa164adce3e656bdd4ed1f7b7b72d03786a891465ca12364466813a3fc9b37 namespace=moby
Oct 10 20:17:57 ha-626000 dockerd[1178]: time="2024-10-10T20:17:57.045829771Z" level=info msg="cleaning up dead shim" namespace=moby
==> container status <==
CONTAINER IMAGE CREATED STATE NAME ATTEMPT POD ID POD
8a44e652b1b7d 6bab7719df100 28 seconds ago Exited kube-apiserver 3 3ea5c098ef852 kube-apiserver-ha-626000
3afa164adce3e 175ffd71cce3d 35 seconds ago Exited kube-controller-manager 4 d53271b993283 kube-controller-manager-ha-626000
7f09d36ffe6fa 2e96e5913fc06 About a minute ago Running etcd 2 5b0cd556276a8 etcd-ha-626000
550da91d9eb94 9aa1fad941575 About a minute ago Running kube-scheduler 2 93b17088f23b4 kube-scheduler-ha-626000
60b49c7c17f69 18b729c2288dc About a minute ago Running kube-vip 1 f6e3e7cd8338f kube-vip-ha-626000
27350291354d8 c69fa2e9cbf5f 2 minutes ago Exited coredns 1 bfd48cf87677c coredns-7c65d6cfc9-pq2g5
4c69b43a782cd 8c811b4aec35f 3 minutes ago Exited busybox 1 016cbc70be9cd busybox-7dff88458-xgc5g
c357241ea58d6 6e38f40d628db 3 minutes ago Exited storage-provisioner 1 e84f9e9fda933 storage-provisioner
a5a2cb1326b18 c69fa2e9cbf5f 3 minutes ago Exited coredns 1 6696a2f9e13f5 coredns-7c65d6cfc9-7ll56
61a3e6937d22c 12968670680f4 3 minutes ago Exited kindnet-cni 1 4782d629ccfaa kindnet-jrj94
596f082e571ac 60c005f310ff3 3 minutes ago Exited kube-proxy 1 a8948192377da kube-proxy-mc4wh
4689120a3801b 18b729c2288dc 4 minutes ago Exited kube-vip 0 faea2f96a2089 kube-vip-ha-626000
78f0f5ac9315a 9aa1fad941575 4 minutes ago Exited kube-scheduler 1 8971faa78644c kube-scheduler-ha-626000
b736986d17456 2e96e5913fc06 4 minutes ago Exited etcd 1 84391c82a303b etcd-ha-626000
==> coredns [27350291354d] <==
.:53
[INFO] plugin/reload: Running configuration SHA512 = 257e111468ef6f1e36f10df061303186c353cd0e51aed8f50f4e4fd21cec02687aef97084fe1f82262f5cee88179d311670a6ae21ae185759728216fc264125f
CoreDNS-1.11.3
linux/amd64, go1.21.11, a6338e9
[INFO] 127.0.0.1:47656 - 14135 "HINFO IN 8135881964715630157.6396861469927917988. udp 57 false 512" NXDOMAIN qr,rd,ra 132 0.009979586s
[INFO] SIGTERM: Shutting down servers then terminating
[INFO] plugin/health: Going into lameduck mode for 5s
==> coredns [a5a2cb1326b1] <==
.:53
[INFO] plugin/reload: Running configuration SHA512 = 257e111468ef6f1e36f10df061303186c353cd0e51aed8f50f4e4fd21cec02687aef97084fe1f82262f5cee88179d311670a6ae21ae185759728216fc264125f
CoreDNS-1.11.3
linux/amd64, go1.21.11, a6338e9
[INFO] 127.0.0.1:35391 - 47738 "HINFO IN 6733301182857557160.2123788221555309508. udp 57 false 512" NXDOMAIN qr,rd,ra 132 2.00402397s
[INFO] 127.0.0.1:39768 - 64521 "HINFO IN 6733301182857557160.2123788221555309508. udp 57 false 512" NXDOMAIN qr,aa,rd,ra 132 0.000166745s
[INFO] SIGTERM: Shutting down servers then terminating
[INFO] plugin/health: Going into lameduck mode for 5s
==> describe nodes <==
command /bin/bash -c "sudo /var/lib/minikube/binaries/v1.31.1/kubectl describe nodes --kubeconfig=/var/lib/minikube/kubeconfig" failed with error: /bin/bash -c "sudo /var/lib/minikube/binaries/v1.31.1/kubectl describe nodes --kubeconfig=/var/lib/minikube/kubeconfig": Process exited with status 1
stdout:
stderr:
E1010 20:18:03.811391 2862 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
E1010 20:18:03.812863 2862 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
E1010 20:18:03.814302 2862 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
E1010 20:18:03.815988 2862 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
E1010 20:18:03.817961 2862 memcache.go:265] "Unhandled Error" err="couldn't get current server API group list: Get \"https://localhost:8443/api?timeout=32s\": dial tcp 127.0.0.1:8443: connect: connection refused"
The connection to the server localhost:8443 was refused - did you specify the right host or port?
==> dmesg <==
[ +0.000000] Any video related functionality will be severely degraded, and you may not even be able to suspend the system properly
[ +0.000001] Unless you actually understand what nomodeset does, you should reboot without enabling it
[ +0.036536] ACPI BIOS Warning (bug): Incorrect checksum in table [DSDT] - 0xBE, should be 0x1B (20200925/tbprint-173)
[ +0.007971] RETBleed: WARNING: Spectre v2 mitigation leaves CPU vulnerable to RETBleed attacks, data leaks possible!
[ +5.811935] ACPI Error: Could not enable RealTimeClock event (20200925/evxfevnt-182)
[ +0.000002] ACPI Warning: Could not enable fixed event - RealTimeClock (4) (20200925/evxface-618)
[ +0.007087] platform regulatory.0: Direct firmware load for regulatory.db failed with error -2
[ +2.772106] systemd-fstab-generator[127]: Ignoring "noauto" option for root device
[ +2.243757] NFSD: Using /var/lib/nfs/v4recovery as the NFSv4 state recovery directory
[ +0.000003] NFSD: unable to find recovery directory /var/lib/nfs/v4recovery
[ +0.000002] NFSD: Unable to initialize client recovery tracking! (-2)
[ +0.251640] systemd-fstab-generator[465]: Ignoring "noauto" option for root device
[ +0.102249] systemd-fstab-generator[477]: Ignoring "noauto" option for root device
[ +2.075176] systemd-fstab-generator[1100]: Ignoring "noauto" option for root device
[ +0.258167] systemd-fstab-generator[1137]: Ignoring "noauto" option for root device
[ +0.104293] systemd-fstab-generator[1149]: Ignoring "noauto" option for root device
[ +0.108460] systemd-fstab-generator[1163]: Ignoring "noauto" option for root device
[ +0.049332] kauditd_printk_skb: 145 callbacks suppressed
[ +2.478387] systemd-fstab-generator[1380]: Ignoring "noauto" option for root device
[ +0.123360] systemd-fstab-generator[1392]: Ignoring "noauto" option for root device
[ +0.110539] systemd-fstab-generator[1404]: Ignoring "noauto" option for root device
[ +0.119482] systemd-fstab-generator[1419]: Ignoring "noauto" option for root device
[ +0.426400] systemd-fstab-generator[1581]: Ignoring "noauto" option for root device
[ +6.693600] kauditd_printk_skb: 168 callbacks suppressed
[Oct10 20:17] kauditd_printk_skb: 40 callbacks suppressed
==> etcd [7f09d36ffe6f] <==
{"level":"warn","ts":"2024-10-10T20:17:59.468008Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1222606868278815760,"retry-timeout":"500ms"}
{"level":"warn","ts":"2024-10-10T20:17:59.968533Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1222606868278815760,"retry-timeout":"500ms"}
{"level":"warn","ts":"2024-10-10T20:18:00.469632Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1222606868278815760,"retry-timeout":"500ms"}
{"level":"info","ts":"2024-10-10T20:18:00.533398Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"a8706e527d7f10f7 is starting a new election at term 3"}
{"level":"info","ts":"2024-10-10T20:18:00.533502Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"a8706e527d7f10f7 became pre-candidate at term 3"}
{"level":"info","ts":"2024-10-10T20:18:00.533522Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"a8706e527d7f10f7 received MsgPreVoteResp from a8706e527d7f10f7 at term 3"}
{"level":"info","ts":"2024-10-10T20:18:00.533540Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"a8706e527d7f10f7 [logterm: 3, index: 2839] sent MsgPreVote request to b051c8b0a160f6d8 at term 3"}
{"level":"warn","ts":"2024-10-10T20:18:00.779558Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_RAFT_MESSAGE","remote-peer-id":"b051c8b0a160f6d8","rtt":"0s","error":"dial tcp 192.169.0.42:2380: connect: connection refused"}
{"level":"warn","ts":"2024-10-10T20:18:00.780696Z","caller":"rafthttp/probing_status.go:68","msg":"prober detected unhealthy status","round-tripper-name":"ROUND_TRIPPER_SNAPSHOT","remote-peer-id":"b051c8b0a160f6d8","rtt":"0s","error":"dial tcp 192.169.0.42:2380: connect: connection refused"}
{"level":"warn","ts":"2024-10-10T20:18:00.883716Z","caller":"etcdserver/server.go:2139","msg":"failed to publish local member to cluster through raft","local-member-id":"a8706e527d7f10f7","local-member-attributes":"{Name:ha-626000 ClientURLs:[https://192.169.0.41:2379]}","request-path":"/0/members/a8706e527d7f10f7/attributes","publish-timeout":"7s","error":"etcdserver: request timed out"}
{"level":"warn","ts":"2024-10-10T20:18:00.970634Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1222606868278815760,"retry-timeout":"500ms"}
{"level":"warn","ts":"2024-10-10T20:18:01.470680Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1222606868278815760,"retry-timeout":"500ms"}
{"level":"warn","ts":"2024-10-10T20:18:01.970823Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1222606868278815760,"retry-timeout":"500ms"}
{"level":"info","ts":"2024-10-10T20:18:02.233439Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"a8706e527d7f10f7 is starting a new election at term 3"}
{"level":"info","ts":"2024-10-10T20:18:02.233488Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"a8706e527d7f10f7 became pre-candidate at term 3"}
{"level":"info","ts":"2024-10-10T20:18:02.233516Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"a8706e527d7f10f7 received MsgPreVoteResp from a8706e527d7f10f7 at term 3"}
{"level":"info","ts":"2024-10-10T20:18:02.233527Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"a8706e527d7f10f7 [logterm: 3, index: 2839] sent MsgPreVote request to b051c8b0a160f6d8 at term 3"}
{"level":"warn","ts":"2024-10-10T20:18:02.471809Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1222606868278815760,"retry-timeout":"500ms"}
{"level":"warn","ts":"2024-10-10T20:18:02.971959Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1222606868278815760,"retry-timeout":"500ms"}
{"level":"warn","ts":"2024-10-10T20:18:03.473096Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1222606868278815760,"retry-timeout":"500ms"}
{"level":"info","ts":"2024-10-10T20:18:03.933805Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"a8706e527d7f10f7 is starting a new election at term 3"}
{"level":"info","ts":"2024-10-10T20:18:03.933851Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"a8706e527d7f10f7 became pre-candidate at term 3"}
{"level":"info","ts":"2024-10-10T20:18:03.933866Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"a8706e527d7f10f7 received MsgPreVoteResp from a8706e527d7f10f7 at term 3"}
{"level":"info","ts":"2024-10-10T20:18:03.933951Z","logger":"raft","caller":"etcdserver/zap_raft.go:77","msg":"a8706e527d7f10f7 [logterm: 3, index: 2839] sent MsgPreVote request to b051c8b0a160f6d8 at term 3"}
{"level":"warn","ts":"2024-10-10T20:18:03.974434Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":1222606868278815760,"retry-timeout":"500ms"}
==> etcd [b736986d1745] <==
{"level":"warn","ts":"2024-10-10T20:16:16.999682Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"6.64172095s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/masterleases/192.169.0.41\" ","response":"","error":"context canceled"}
{"level":"info","ts":"2024-10-10T20:16:16.999699Z","caller":"traceutil/trace.go:171","msg":"trace[2020268020] range","detail":"{range_begin:/registry/masterleases/192.169.0.41; range_end:; }","duration":"6.641741225s","start":"2024-10-10T20:16:10.357953Z","end":"2024-10-10T20:16:16.999694Z","steps":["trace[2020268020] 'agreement among raft nodes before linearized reading' (duration: 6.641720495s)"],"step_count":1}
{"level":"warn","ts":"2024-10-10T20:16:16.999713Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2024-10-10T20:16:10.357930Z","time spent":"6.641780391s","remote":"127.0.0.1:58438","response type":"/etcdserverpb.KV/Range","request count":0,"request size":37,"response count":0,"response size":0,"request content":"key:\"/registry/masterleases/192.169.0.41\" "}
2024/10/10 20:16:16 WARNING: [core] [Server #7] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing"
{"level":"warn","ts":"2024-10-10T20:16:16.999772Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"8.163992644s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/events/\" range_end:\"/registry/events0\" count_only:true ","response":"","error":"context canceled"}
{"level":"info","ts":"2024-10-10T20:16:16.999784Z","caller":"traceutil/trace.go:171","msg":"trace[284656230] range","detail":"{range_begin:/registry/events/; range_end:/registry/events0; }","duration":"8.164005636s","start":"2024-10-10T20:16:08.835774Z","end":"2024-10-10T20:16:16.999780Z","steps":["trace[284656230] 'agreement among raft nodes before linearized reading' (duration: 8.163992685s)"],"step_count":1}
{"level":"warn","ts":"2024-10-10T20:16:16.999795Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2024-10-10T20:16:08.835750Z","time spent":"8.164041488s","remote":"127.0.0.1:58482","response type":"/etcdserverpb.KV/Range","request count":0,"request size":40,"response count":0,"response size":0,"request content":"key:\"/registry/events/\" range_end:\"/registry/events0\" count_only:true "}
2024/10/10 20:16:16 WARNING: [core] [Server #7] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing"
{"level":"warn","ts":"2024-10-10T20:16:16.999844Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"6.863849071s","expected-duration":"100ms","prefix":"read-only range ","request":"key:\"/registry/minions/\" range_end:\"/registry/minions0\" count_only:true ","response":"","error":"context canceled"}
{"level":"info","ts":"2024-10-10T20:16:16.999872Z","caller":"traceutil/trace.go:171","msg":"trace[105546159] range","detail":"{range_begin:/registry/minions/; range_end:/registry/minions0; }","duration":"6.863877322s","start":"2024-10-10T20:16:10.135991Z","end":"2024-10-10T20:16:16.999868Z","steps":["trace[105546159] 'agreement among raft nodes before linearized reading' (duration: 6.863849051s)"],"step_count":1}
{"level":"warn","ts":"2024-10-10T20:16:16.999886Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2024-10-10T20:16:10.135969Z","time spent":"6.863912877s","remote":"127.0.0.1:58578","response type":"/etcdserverpb.KV/Range","request count":0,"request size":42,"response count":0,"response size":0,"request content":"key:\"/registry/minions/\" range_end:\"/registry/minions0\" count_only:true "}
2024/10/10 20:16:16 WARNING: [core] [Server #7] grpc: Server.processUnaryRPC failed to write status: connection error: desc = "transport is closing"
{"level":"warn","ts":"2024-10-10T20:16:17.047809Z","caller":"embed/serve.go:212","msg":"stopping secure grpc server due to error","error":"accept tcp 192.169.0.41:2379: use of closed network connection"}
{"level":"warn","ts":"2024-10-10T20:16:17.047856Z","caller":"embed/serve.go:214","msg":"stopped secure grpc server due to error","error":"accept tcp 192.169.0.41:2379: use of closed network connection"}
{"level":"info","ts":"2024-10-10T20:16:17.047928Z","caller":"etcdserver/server.go:1512","msg":"skipped leadership transfer; local server is not leader","local-member-id":"a8706e527d7f10f7","current-leader-member-id":"0"}
{"level":"info","ts":"2024-10-10T20:16:17.048079Z","caller":"rafthttp/peer.go:330","msg":"stopping remote peer","remote-peer-id":"b051c8b0a160f6d8"}
{"level":"info","ts":"2024-10-10T20:16:17.048110Z","caller":"rafthttp/stream.go:294","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream MsgApp v2","remote-peer-id":"b051c8b0a160f6d8"}
{"level":"info","ts":"2024-10-10T20:16:17.048127Z","caller":"rafthttp/stream.go:294","msg":"stopped TCP streaming connection with remote peer","stream-writer-type":"stream Message","remote-peer-id":"b051c8b0a160f6d8"}
{"level":"info","ts":"2024-10-10T20:16:17.048223Z","caller":"rafthttp/pipeline.go:85","msg":"stopped HTTP pipelining with remote peer","local-member-id":"a8706e527d7f10f7","remote-peer-id":"b051c8b0a160f6d8"}
{"level":"info","ts":"2024-10-10T20:16:17.048345Z","caller":"rafthttp/stream.go:442","msg":"stopped stream reader with remote peer","stream-reader-type":"stream MsgApp v2","local-member-id":"a8706e527d7f10f7","remote-peer-id":"b051c8b0a160f6d8"}
{"level":"info","ts":"2024-10-10T20:16:17.048392Z","caller":"rafthttp/stream.go:442","msg":"stopped stream reader with remote peer","stream-reader-type":"stream Message","local-member-id":"a8706e527d7f10f7","remote-peer-id":"b051c8b0a160f6d8"}
{"level":"info","ts":"2024-10-10T20:16:17.048421Z","caller":"rafthttp/peer.go:335","msg":"stopped remote peer","remote-peer-id":"b051c8b0a160f6d8"}
{"level":"info","ts":"2024-10-10T20:16:17.051104Z","caller":"embed/etcd.go:581","msg":"stopping serving peer traffic","address":"192.169.0.41:2380"}
{"level":"info","ts":"2024-10-10T20:16:17.051147Z","caller":"embed/etcd.go:586","msg":"stopped serving peer traffic","address":"192.169.0.41:2380"}
{"level":"info","ts":"2024-10-10T20:16:17.051155Z","caller":"embed/etcd.go:379","msg":"closed etcd server","name":"ha-626000","data-dir":"/var/lib/minikube/etcd","advertise-peer-urls":["https://192.169.0.41:2380"],"advertise-client-urls":["https://192.169.0.41:2379"]}
==> kernel <==
20:18:04 up 1 min, 0 users, load average: 0.18, 0.10, 0.04
Linux ha-626000 5.10.207 #1 SMP Tue Oct 8 15:16:25 UTC 2024 x86_64 GNU/Linux
PRETTY_NAME="Buildroot 2023.02.9"
==> kindnet [61a3e6937d22] <==
I1010 20:15:38.825073 1 main.go:322] Node ha-626000-m02 has CIDR [10.244.1.0/24]
I1010 20:15:38.825197 1 main.go:295] Handling node with IPs: map[192.169.0.43:{}]
I1010 20:15:38.825243 1 main.go:322] Node ha-626000-m03 has CIDR [10.244.2.0/24]
I1010 20:15:38.825419 1 main.go:295] Handling node with IPs: map[192.169.0.44:{}]
I1010 20:15:38.825464 1 main.go:322] Node ha-626000-m04 has CIDR [10.244.3.0/24]
I1010 20:15:48.825389 1 main.go:295] Handling node with IPs: map[192.169.0.41:{}]
I1010 20:15:48.825441 1 main.go:299] handling current node
I1010 20:15:48.825455 1 main.go:295] Handling node with IPs: map[192.169.0.42:{}]
I1010 20:15:48.825462 1 main.go:322] Node ha-626000-m02 has CIDR [10.244.1.0/24]
I1010 20:15:48.825757 1 main.go:295] Handling node with IPs: map[192.169.0.43:{}]
I1010 20:15:48.825850 1 main.go:322] Node ha-626000-m03 has CIDR [10.244.2.0/24]
I1010 20:15:48.826019 1 main.go:295] Handling node with IPs: map[192.169.0.44:{}]
I1010 20:15:48.826057 1 main.go:322] Node ha-626000-m04 has CIDR [10.244.3.0/24]
I1010 20:15:58.824593 1 main.go:295] Handling node with IPs: map[192.169.0.44:{}]
I1010 20:15:58.824779 1 main.go:322] Node ha-626000-m04 has CIDR [10.244.3.0/24]
I1010 20:15:58.824960 1 main.go:295] Handling node with IPs: map[192.169.0.41:{}]
I1010 20:15:58.825038 1 main.go:299] handling current node
I1010 20:15:58.825059 1 main.go:295] Handling node with IPs: map[192.169.0.42:{}]
I1010 20:15:58.825134 1 main.go:322] Node ha-626000-m02 has CIDR [10.244.1.0/24]
I1010 20:16:08.832784 1 main.go:295] Handling node with IPs: map[192.169.0.41:{}]
I1010 20:16:08.832807 1 main.go:299] handling current node
I1010 20:16:08.832816 1 main.go:295] Handling node with IPs: map[192.169.0.42:{}]
I1010 20:16:08.832820 1 main.go:322] Node ha-626000-m02 has CIDR [10.244.1.0/24]
I1010 20:16:08.833071 1 main.go:295] Handling node with IPs: map[192.169.0.44:{}]
I1010 20:16:08.833101 1 main.go:322] Node ha-626000-m04 has CIDR [10.244.3.0/24]
==> kube-apiserver [8a44e652b1b7] <==
I1010 20:17:35.673130 1 options.go:228] external host was not specified, using 192.169.0.41
I1010 20:17:35.677018 1 server.go:142] Version: v1.31.1
I1010 20:17:35.677064 1 server.go:144] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I1010 20:17:36.007593 1 shared_informer.go:313] Waiting for caches to sync for node_authorizer
I1010 20:17:36.011281 1 shared_informer.go:313] Waiting for caches to sync for *generic.policySource[*k8s.io/api/admissionregistration/v1.ValidatingAdmissionPolicy,*k8s.io/api/admissionregistration/v1.ValidatingAdmissionPolicyBinding,k8s.io/apiserver/pkg/admission/plugin/policy/validating.Validator]
I1010 20:17:36.013907 1 plugins.go:157] Loaded 12 mutating admission controller(s) successfully in the following order: NamespaceLifecycle,LimitRanger,ServiceAccount,NodeRestriction,TaintNodesByCondition,Priority,DefaultTolerationSeconds,DefaultStorageClass,StorageObjectInUseProtection,RuntimeClass,DefaultIngressClass,MutatingAdmissionWebhook.
I1010 20:17:36.014052 1 plugins.go:160] Loaded 13 validating admission controller(s) successfully in the following order: LimitRanger,ServiceAccount,PodSecurity,Priority,PersistentVolumeClaimResize,RuntimeClass,CertificateApproval,CertificateSigning,ClusterTrustBundleAttest,CertificateSubjectRestriction,ValidatingAdmissionPolicy,ValidatingAdmissionWebhook,ResourceQuota.
I1010 20:17:36.014330 1 instance.go:232] Using reconciler: lease
W1010 20:17:56.007089 1 logging.go:55] [core] [Channel #1 SubChannel #3]grpc: addrConn.createTransport failed to connect to {Addr: "127.0.0.1:2379", ServerName: "127.0.0.1:2379", }. Err: connection error: desc = "transport: authentication handshake failed: context deadline exceeded"
W1010 20:17:56.007558 1 logging.go:55] [core] [Channel #2 SubChannel #4]grpc: addrConn.createTransport failed to connect to {Addr: "127.0.0.1:2379", ServerName: "127.0.0.1:2379", }. Err: connection error: desc = "transport: authentication handshake failed: context deadline exceeded"
F1010 20:17:56.016219 1 instance.go:225] Error creating leases: error creating storage factory: context deadline exceeded
==> kube-controller-manager [3afa164adce3] <==
I1010 20:17:29.069271 1 serving.go:386] Generated self-signed cert in-memory
I1010 20:17:29.509879 1 controllermanager.go:197] "Starting" version="v1.31.1"
I1010 20:17:29.510016 1 controllermanager.go:199] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I1010 20:17:29.511410 1 dynamic_cafile_content.go:160] "Starting controller" name="request-header::/var/lib/minikube/certs/front-proxy-ca.crt"
I1010 20:17:29.511516 1 secure_serving.go:213] Serving securely on 127.0.0.1:10257
I1010 20:17:29.511671 1 dynamic_cafile_content.go:160] "Starting controller" name="client-ca-bundle::/var/lib/minikube/certs/ca.crt"
I1010 20:17:29.511878 1 tlsconfig.go:243] "Starting DynamicServingCertificateController"
E1010 20:17:57.029236 1 controllermanager.go:242] "Error building controller context" err="failed to wait for apiserver being healthy: timed out waiting for the condition: failed to get apiserver /healthz status: Get \"https://192.169.0.41:8443/healthz\": dial tcp 192.169.0.41:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.41:36936->192.169.0.41:8443: read: connection reset by peer"
==> kube-proxy [596f082e571a] <==
add table ip kube-proxy
^^^^^^^^^^^^^^^^^^^^^^^^
>
E1010 20:14:35.634306 1 proxier.go:734] "Error cleaning up nftables rules" err=<
could not run nftables command: /dev/stdin:1:1-25: Error: Could not process rule: Operation not supported
add table ip6 kube-proxy
^^^^^^^^^^^^^^^^^^^^^^^^^
>
I1010 20:14:35.648652 1 server.go:677] "Successfully retrieved node IP(s)" IPs=["192.169.0.41"]
E1010 20:14:35.648714 1 server.go:234] "Kube-proxy configuration may be incomplete or incorrect" err="nodePortAddresses is unset; NodePort connections will be accepted on all local IPs. Consider using `--nodeport-addresses primary`"
I1010 20:14:35.677799 1 server_linux.go:146] "No iptables support for family" ipFamily="IPv6"
I1010 20:14:35.677840 1 server.go:245] "kube-proxy running in single-stack mode" ipFamily="IPv4"
I1010 20:14:35.677857 1 server_linux.go:169] "Using iptables Proxier"
I1010 20:14:35.680025 1 proxier.go:255] "Setting route_localnet=1 to allow node-ports on localhost; to change this either disable iptables.localhostNodePorts (--iptables-localhost-nodeports) or set nodePortAddresses (--nodeport-addresses) to filter loopback addresses" ipFamily="IPv4"
I1010 20:14:35.680601 1 server.go:483] "Version info" version="v1.31.1"
I1010 20:14:35.680629 1 server.go:485] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I1010 20:14:35.682988 1 config.go:199] "Starting service config controller"
I1010 20:14:35.683044 1 shared_informer.go:313] Waiting for caches to sync for service config
I1010 20:14:35.683185 1 config.go:328] "Starting node config controller"
I1010 20:14:35.683264 1 shared_informer.go:313] Waiting for caches to sync for node config
I1010 20:14:35.683694 1 config.go:105] "Starting endpoint slice config controller"
I1010 20:14:35.683734 1 shared_informer.go:313] Waiting for caches to sync for endpoint slice config
I1010 20:14:35.783598 1 shared_informer.go:320] Caches are synced for service config
I1010 20:14:35.783598 1 shared_informer.go:320] Caches are synced for node config
I1010 20:14:35.783782 1 shared_informer.go:320] Caches are synced for endpoint slice config
==> kube-scheduler [550da91d9eb9] <==
E1010 20:17:52.310910 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Service: failed to list *v1.Service: Get \"https://192.169.0.41:8443/api/v1/services?limit=500&resourceVersion=0\": net/http: TLS handshake timeout" logger="UnhandledError"
W1010 20:17:53.450322 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.StatefulSet: Get "https://192.169.0.41:8443/apis/apps/v1/statefulsets?limit=500&resourceVersion=0": net/http: TLS handshake timeout
E1010 20:17:53.450576 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.StatefulSet: failed to list *v1.StatefulSet: Get \"https://192.169.0.41:8443/apis/apps/v1/statefulsets?limit=500&resourceVersion=0\": net/http: TLS handshake timeout" logger="UnhandledError"
W1010 20:17:55.213227 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.CSINode: Get "https://192.169.0.41:8443/apis/storage.k8s.io/v1/csinodes?limit=500&resourceVersion=0": net/http: TLS handshake timeout
E1010 20:17:55.213277 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.CSINode: failed to list *v1.CSINode: Get \"https://192.169.0.41:8443/apis/storage.k8s.io/v1/csinodes?limit=500&resourceVersion=0\": net/http: TLS handshake timeout" logger="UnhandledError"
W1010 20:17:55.793343 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.CSIDriver: Get "https://192.169.0.41:8443/apis/storage.k8s.io/v1/csidrivers?limit=500&resourceVersion=0": net/http: TLS handshake timeout
E1010 20:17:55.793703 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.CSIDriver: failed to list *v1.CSIDriver: Get \"https://192.169.0.41:8443/apis/storage.k8s.io/v1/csidrivers?limit=500&resourceVersion=0\": net/http: TLS handshake timeout" logger="UnhandledError"
W1010 20:17:57.029871 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.ReplicationController: Get "https://192.169.0.41:8443/api/v1/replicationcontrollers?limit=500&resourceVersion=0": dial tcp 192.169.0.41:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.41:36932->192.169.0.41:8443: read: connection reset by peer
E1010 20:17:57.030153 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.ReplicationController: failed to list *v1.ReplicationController: Get \"https://192.169.0.41:8443/api/v1/replicationcontrollers?limit=500&resourceVersion=0\": dial tcp 192.169.0.41:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.41:36932->192.169.0.41:8443: read: connection reset by peer" logger="UnhandledError"
W1010 20:17:57.029990 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.PodDisruptionBudget: Get "https://192.169.0.41:8443/apis/policy/v1/poddisruptionbudgets?limit=500&resourceVersion=0": dial tcp 192.169.0.41:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.41:35380->192.169.0.41:8443: read: connection reset by peer
E1010 20:17:57.030304 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.PodDisruptionBudget: failed to list *v1.PodDisruptionBudget: Get \"https://192.169.0.41:8443/apis/policy/v1/poddisruptionbudgets?limit=500&resourceVersion=0\": dial tcp 192.169.0.41:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.41:35380->192.169.0.41:8443: read: connection reset by peer" logger="UnhandledError"
W1010 20:17:57.030507 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.Node: Get "https://192.169.0.41:8443/api/v1/nodes?limit=500&resourceVersion=0": dial tcp 192.169.0.41:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.41:36940->192.169.0.41:8443: read: connection reset by peer
E1010 20:17:57.030625 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Node: failed to list *v1.Node: Get \"https://192.169.0.41:8443/api/v1/nodes?limit=500&resourceVersion=0\": dial tcp 192.169.0.41:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.41:36940->192.169.0.41:8443: read: connection reset by peer" logger="UnhandledError"
W1010 20:17:57.030845 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.Pod: Get "https://192.169.0.41:8443/api/v1/pods?fieldSelector=status.phase%21%3DSucceeded%2Cstatus.phase%21%3DFailed&limit=500&resourceVersion=0": dial tcp 192.169.0.41:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.41:35394->192.169.0.41:8443: read: connection reset by peer
E1010 20:17:57.030970 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Pod: failed to list *v1.Pod: Get \"https://192.169.0.41:8443/api/v1/pods?fieldSelector=status.phase%21%3DSucceeded%2Cstatus.phase%21%3DFailed&limit=500&resourceVersion=0\": dial tcp 192.169.0.41:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.41:35394->192.169.0.41:8443: read: connection reset by peer" logger="UnhandledError"
W1010 20:17:57.031185 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.PersistentVolume: Get "https://192.169.0.41:8443/api/v1/persistentvolumes?limit=500&resourceVersion=0": dial tcp 192.169.0.41:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.41:35416->192.169.0.41:8443: read: connection reset by peer
E1010 20:17:57.031286 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.PersistentVolume: failed to list *v1.PersistentVolume: Get \"https://192.169.0.41:8443/api/v1/persistentvolumes?limit=500&resourceVersion=0\": dial tcp 192.169.0.41:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.41:35416->192.169.0.41:8443: read: connection reset by peer" logger="UnhandledError"
W1010 20:17:57.031513 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.ReplicaSet: Get "https://192.169.0.41:8443/apis/apps/v1/replicasets?limit=500&resourceVersion=0": dial tcp 192.169.0.41:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.41:35374->192.169.0.41:8443: read: connection reset by peer
E1010 20:17:57.031711 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.ReplicaSet: failed to list *v1.ReplicaSet: Get \"https://192.169.0.41:8443/apis/apps/v1/replicasets?limit=500&resourceVersion=0\": dial tcp 192.169.0.41:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.41:35374->192.169.0.41:8443: read: connection reset by peer" logger="UnhandledError"
W1010 20:17:57.031919 1 reflector.go:561] runtime/asm_amd64.s:1695: failed to list *v1.ConfigMap: Get "https://192.169.0.41:8443/api/v1/namespaces/kube-system/configmaps?fieldSelector=metadata.name%3Dextension-apiserver-authentication&limit=500&resourceVersion=0": dial tcp 192.169.0.41:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.41:35402->192.169.0.41:8443: read: connection reset by peer
E1010 20:17:57.032047 1 reflector.go:158] "Unhandled Error" err="runtime/asm_amd64.s:1695: Failed to watch *v1.ConfigMap: failed to list *v1.ConfigMap: Get \"https://192.169.0.41:8443/api/v1/namespaces/kube-system/configmaps?fieldSelector=metadata.name%3Dextension-apiserver-authentication&limit=500&resourceVersion=0\": dial tcp 192.169.0.41:8443: connect: connection refused - error from a previous attempt: read tcp 192.169.0.41:35402->192.169.0.41:8443: read: connection reset by peer" logger="UnhandledError"
W1010 20:17:58.975309 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.Namespace: Get "https://192.169.0.41:8443/api/v1/namespaces?limit=500&resourceVersion=0": dial tcp 192.169.0.41:8443: connect: connection refused
E1010 20:17:58.975691 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Namespace: failed to list *v1.Namespace: Get \"https://192.169.0.41:8443/api/v1/namespaces?limit=500&resourceVersion=0\": dial tcp 192.169.0.41:8443: connect: connection refused" logger="UnhandledError"
W1010 20:18:00.153372 1 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.PersistentVolumeClaim: Get "https://192.169.0.41:8443/api/v1/persistentvolumeclaims?limit=500&resourceVersion=0": dial tcp 192.169.0.41:8443: connect: connection refused
E1010 20:18:00.153447 1 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.PersistentVolumeClaim: failed to list *v1.PersistentVolumeClaim: Get \"https://192.169.0.41:8443/api/v1/persistentvolumeclaims?limit=500&resourceVersion=0\": dial tcp 192.169.0.41:8443: connect: connection refused" logger="UnhandledError"
==> kube-scheduler [78f0f5ac9315] <==
I1010 20:13:49.473470 1 serving.go:386] Generated self-signed cert in-memory
W1010 20:14:00.072183 1 authentication.go:370] Error looking up in-cluster authentication configuration: Get "https://192.169.0.41:8443/api/v1/namespaces/kube-system/configmaps/extension-apiserver-authentication": net/http: TLS handshake timeout
W1010 20:14:00.072249 1 authentication.go:371] Continuing without authentication configuration. This may treat all requests as anonymous.
W1010 20:14:00.072268 1 authentication.go:372] To require authentication configuration lookup to succeed, set --authentication-tolerate-lookup-failure=false
I1010 20:14:09.058391 1 server.go:167] "Starting Kubernetes Scheduler" version="v1.31.1"
I1010 20:14:09.060344 1 server.go:169] "Golang settings" GOGC="" GOMAXPROCS="" GOTRACEBACK=""
I1010 20:14:09.068534 1 secure_serving.go:213] Serving securely on 127.0.0.1:10259
I1010 20:14:09.071145 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file"
I1010 20:14:09.071184 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
I1010 20:14:09.071314 1 tlsconfig.go:243] "Starting DynamicServingCertificateController"
I1010 20:14:09.172068 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
E1010 20:15:52.429820 1 schedule_one.go:953] "Scheduler cache AssumePod failed" err="pod ecf96d83-fe11-42e0-990c-5d13e0f73bf6(default/busybox-7dff88458-6sct5) is in the cache, so can't be assumed" pod="default/busybox-7dff88458-6sct5"
E1010 20:15:52.432749 1 schedule_one.go:1057] "Error scheduling pod; retrying" err="pod ecf96d83-fe11-42e0-990c-5d13e0f73bf6(default/busybox-7dff88458-6sct5) is in the cache, so can't be assumed" pod="default/busybox-7dff88458-6sct5"
I1010 20:15:52.434989 1 schedule_one.go:1070] "Pod has been assigned to node. Abort adding it back to queue." pod="default/busybox-7dff88458-6sct5" node="ha-626000-m04"
I1010 20:16:16.887889 1 secure_serving.go:258] Stopped listening on 127.0.0.1:10259
I1010 20:16:16.888040 1 tlsconfig.go:258] "Shutting down DynamicServingCertificateController"
I1010 20:16:16.888227 1 configmap_cafile_content.go:226] "Shutting down controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file"
E1010 20:16:16.892715 1 run.go:72] "command failed" err="finished without leader elect"
==> kubelet <==
Oct 10 20:17:56 ha-626000 kubelet[1588]: I1010 20:17:56.519546 1588 scope.go:117] "RemoveContainer" containerID="6f1d7d582b23ead17384e918f7a33c671ead28b4478fc65ec18f4c47b34cb812"
Oct 10 20:17:56 ha-626000 kubelet[1588]: I1010 20:17:56.520173 1588 scope.go:117] "RemoveContainer" containerID="8a44e652b1b7d9348d611ffd8b57b790bc0a44e22f3a1fec0ff10f02e62641bd"
Oct 10 20:17:56 ha-626000 kubelet[1588]: E1010 20:17:56.520296 1588 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-apiserver\" with CrashLoopBackOff: \"back-off 20s restarting failed container=kube-apiserver pod=kube-apiserver-ha-626000_kube-system(794db039b1fbf63946bd1acf3df5a5a0)\"" pod="kube-system/kube-apiserver-ha-626000" podUID="794db039b1fbf63946bd1acf3df5a5a0"
Oct 10 20:17:57 ha-626000 kubelet[1588]: E1010 20:17:57.183793 1588 event.go:368] "Unable to write event (may retry after sleeping)" err="Post \"https://control-plane.minikube.internal:8443/api/v1/namespaces/default/events\": dial tcp 192.169.0.254:8443: connect: no route to host" event="&Event{ObjectMeta:{ha-626000.17fd30b9168ba820 default 0 0001-01-01 00:00:00 +0000 UTC <nil> <nil> map[] map[] [] [] []},InvolvedObject:ObjectReference{Kind:Node,Namespace:,Name:ha-626000,UID:ha-626000,APIVersion:,ResourceVersion:,FieldPath:,},Reason:Starting,Message:Starting kubelet.,Source:EventSource{Component:kubelet,Host:ha-626000,},FirstTimestamp:2024-10-10 20:16:43.4854728 +0000 UTC m=+0.233666422,LastTimestamp:2024-10-10 20:16:43.4854728 +0000 UTC m=+0.233666422,Count:1,Type:Normal,EventTime:0001-01-01 00:00:00 +0000 UTC,Series:nil,Action:,Related:nil,ReportingController:kubelet,ReportingInstance:ha-626000,}"
Oct 10 20:17:57 ha-626000 kubelet[1588]: W1010 20:17:57.183873 1588 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.Service: Get "https://control-plane.minikube.internal:8443/api/v1/services?fieldSelector=spec.clusterIP%21%3DNone&limit=500&resourceVersion=0": dial tcp 192.169.0.254:8443: connect: no route to host
Oct 10 20:17:57 ha-626000 kubelet[1588]: E1010 20:17:57.183947 1588 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Service: failed to list *v1.Service: Get \"https://control-plane.minikube.internal:8443/api/v1/services?fieldSelector=spec.clusterIP%21%3DNone&limit=500&resourceVersion=0\": dial tcp 192.169.0.254:8443: connect: no route to host" logger="UnhandledError"
Oct 10 20:17:57 ha-626000 kubelet[1588]: I1010 20:17:57.539786 1588 scope.go:117] "RemoveContainer" containerID="2170d784f89990cead44a18e60b634457082e2522cf71171f420e3836cd39685"
Oct 10 20:17:57 ha-626000 kubelet[1588]: I1010 20:17:57.540325 1588 scope.go:117] "RemoveContainer" containerID="3afa164adce3e656bdd4ed1f7b7b72d03786a891465ca12364466813a3fc9b37"
Oct 10 20:17:57 ha-626000 kubelet[1588]: E1010 20:17:57.540420 1588 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-controller-manager\" with CrashLoopBackOff: \"back-off 20s restarting failed container=kube-controller-manager pod=kube-controller-manager-ha-626000_kube-system(c7d5462199bb979c6f2346ede78855dd)\"" pod="kube-system/kube-controller-manager-ha-626000" podUID="c7d5462199bb979c6f2346ede78855dd"
Oct 10 20:17:58 ha-626000 kubelet[1588]: I1010 20:17:58.043836 1588 kubelet_node_status.go:72] "Attempting to register node" node="ha-626000"
Oct 10 20:17:58 ha-626000 kubelet[1588]: I1010 20:17:58.422847 1588 scope.go:117] "RemoveContainer" containerID="8a44e652b1b7d9348d611ffd8b57b790bc0a44e22f3a1fec0ff10f02e62641bd"
Oct 10 20:17:58 ha-626000 kubelet[1588]: E1010 20:17:58.422989 1588 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-apiserver\" with CrashLoopBackOff: \"back-off 20s restarting failed container=kube-apiserver pod=kube-apiserver-ha-626000_kube-system(794db039b1fbf63946bd1acf3df5a5a0)\"" pod="kube-system/kube-apiserver-ha-626000" podUID="794db039b1fbf63946bd1acf3df5a5a0"
Oct 10 20:18:00 ha-626000 kubelet[1588]: E1010 20:18:00.256206 1588 kubelet_node_status.go:95] "Unable to register node with API server" err="Post \"https://control-plane.minikube.internal:8443/api/v1/nodes\": dial tcp 192.169.0.254:8443: connect: no route to host" node="ha-626000"
Oct 10 20:18:00 ha-626000 kubelet[1588]: W1010 20:18:00.256195 1588 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.Node: Get "https://control-plane.minikube.internal:8443/api/v1/nodes?fieldSelector=metadata.name%3Dha-626000&limit=500&resourceVersion=0": dial tcp 192.169.0.254:8443: connect: no route to host
Oct 10 20:18:00 ha-626000 kubelet[1588]: E1010 20:18:00.256282 1588 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.Node: failed to list *v1.Node: Get \"https://control-plane.minikube.internal:8443/api/v1/nodes?fieldSelector=metadata.name%3Dha-626000&limit=500&resourceVersion=0\": dial tcp 192.169.0.254:8443: connect: no route to host" logger="UnhandledError"
Oct 10 20:18:00 ha-626000 kubelet[1588]: E1010 20:18:00.256327 1588 controller.go:145] "Failed to ensure lease exists, will retry" err="Get \"https://control-plane.minikube.internal:8443/apis/coordination.k8s.io/v1/namespaces/kube-node-lease/leases/ha-626000?timeout=10s\": dial tcp 192.169.0.254:8443: connect: no route to host" interval="7s"
Oct 10 20:18:00 ha-626000 kubelet[1588]: I1010 20:18:00.706694 1588 scope.go:117] "RemoveContainer" containerID="8a44e652b1b7d9348d611ffd8b57b790bc0a44e22f3a1fec0ff10f02e62641bd"
Oct 10 20:18:00 ha-626000 kubelet[1588]: E1010 20:18:00.706799 1588 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-apiserver\" with CrashLoopBackOff: \"back-off 20s restarting failed container=kube-apiserver pod=kube-apiserver-ha-626000_kube-system(794db039b1fbf63946bd1acf3df5a5a0)\"" pod="kube-system/kube-apiserver-ha-626000" podUID="794db039b1fbf63946bd1acf3df5a5a0"
Oct 10 20:18:02 ha-626000 kubelet[1588]: I1010 20:18:02.998603 1588 scope.go:117] "RemoveContainer" containerID="3afa164adce3e656bdd4ed1f7b7b72d03786a891465ca12364466813a3fc9b37"
Oct 10 20:18:02 ha-626000 kubelet[1588]: E1010 20:18:02.998708 1588 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-controller-manager\" with CrashLoopBackOff: \"back-off 20s restarting failed container=kube-controller-manager pod=kube-controller-manager-ha-626000_kube-system(c7d5462199bb979c6f2346ede78855dd)\"" pod="kube-system/kube-controller-manager-ha-626000" podUID="c7d5462199bb979c6f2346ede78855dd"
Oct 10 20:18:03 ha-626000 kubelet[1588]: W1010 20:18:03.328291 1588 reflector.go:561] k8s.io/client-go/informers/factory.go:160: failed to list *v1.RuntimeClass: Get "https://control-plane.minikube.internal:8443/apis/node.k8s.io/v1/runtimeclasses?limit=500&resourceVersion=0": dial tcp 192.169.0.254:8443: connect: no route to host
Oct 10 20:18:03 ha-626000 kubelet[1588]: E1010 20:18:03.328457 1588 reflector.go:158] "Unhandled Error" err="k8s.io/client-go/informers/factory.go:160: Failed to watch *v1.RuntimeClass: failed to list *v1.RuntimeClass: Get \"https://control-plane.minikube.internal:8443/apis/node.k8s.io/v1/runtimeclasses?limit=500&resourceVersion=0\": dial tcp 192.169.0.254:8443: connect: no route to host" logger="UnhandledError"
Oct 10 20:18:03 ha-626000 kubelet[1588]: E1010 20:18:03.594492 1588 eviction_manager.go:285] "Eviction manager: failed to get summary stats" err="failed to get node info: node \"ha-626000\" not found"
Oct 10 20:18:04 ha-626000 kubelet[1588]: I1010 20:18:04.719697 1588 scope.go:117] "RemoveContainer" containerID="3afa164adce3e656bdd4ed1f7b7b72d03786a891465ca12364466813a3fc9b37"
Oct 10 20:18:04 ha-626000 kubelet[1588]: E1010 20:18:04.719825 1588 pod_workers.go:1301] "Error syncing pod, skipping" err="failed to \"StartContainer\" for \"kube-controller-manager\" with CrashLoopBackOff: \"back-off 20s restarting failed container=kube-controller-manager pod=kube-controller-manager-ha-626000_kube-system(c7d5462199bb979c6f2346ede78855dd)\"" pod="kube-system/kube-controller-manager-ha-626000" podUID="c7d5462199bb979c6f2346ede78855dd"
-- /stdout --
helpers_test.go:254: (dbg) Run: out/minikube-darwin-amd64 status --format={{.APIServer}} -p ha-626000 -n ha-626000
helpers_test.go:254: (dbg) Non-zero exit: out/minikube-darwin-amd64 status --format={{.APIServer}} -p ha-626000 -n ha-626000: exit status 2 (174.792909ms)
-- stdout --
Stopped
-- /stdout --
helpers_test.go:254: status error: exit status 2 (may be ok)
helpers_test.go:256: "ha-626000" apiserver is not running, skipping kubectl commands (state="Stopped")
--- FAIL: TestMultiControlPlane/serial/DegradedAfterClusterRestart (2.94s)